fix

* feat: backup/restore

* feat: backup/restore

* fix

CLAUDE.md

@@ -84,29 +84,15 @@ PostgreSQL with pgvector. Schema managed via Alembic migrations in `hindsight-ap
 
 Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
 
-### Database Backups (IMPORTANT)
-**Before any operation that may affect the database, run a backup:**
-```bash
-docker exec hindsight /backups/backup.sh
-```
-
-Operations requiring backup:
-- Running database migrations
-- Modifying Alembic migration files
-- Rebuilding Docker images
-- Resetting or recreating containers
-- Any schema changes
-- Bulk data operations
-
-Backups are stored in `~/hindsight-backups/` on the host.
-
-To restore:
-```bash
-docker exec -it hindsight /backups/restore.sh <backup-file.sql.gz>
-```
-
 ## Key Conventions
 
+### Code Quality
+**Always run the lint script after making Python or TypeScript/Node changes:**
+```bash
+./scripts/hooks/lint.sh
+```
+This runs the same checks as the pre-commit hook (Ruff for Python, ESLint/Prettier for TypeScript).
+
 ### Memory Banks
 - Each bank is isolated (no cross-bank data access)
 - Banks have dispositions (skepticism, literalism, empathy traits 1-5) affecting reflect
@@ -127,6 +113,29 @@ docker exec -it hindsight /backups/restore.sh <backup-file.sql.gz>
 - Next.js App Router for control plane
 - Tailwind CSS with shadcn/ui components
 
+### Adding New API Configuration Flags
+
+When adding a new environment variable configuration:
+
+1. **config.py** (`hindsight-api/hindsight_api/config.py`):
+   - Add `ENV_*` constant for the environment variable name
+   - Add `DEFAULT_*` constant for the default value
+   - Add field to `HindsightConfig` dataclass
+   - Add initialization in `from_env()` method
+
+2. **main.py** (`hindsight-api/hindsight_api/main.py`):
+   - Add field to the manual `HindsightConfig()` constructor call (search for "CLI override")
+
+3. **Use the config** in code:
+   ```python
+   from ...config import get_config
+   config = get_config()
+   value = config.your_new_field
+   ```
+
+4. **Documentation** (`hindsight-docs/docs/developer/configuration.md`):
+   - Add to appropriate section table with Variable, Description, Default
+
 ## Environment Setup
 
 ```bash

hindsight-api/hindsight_api/admin/__init__.py

@@ -0,0 +1 @@
+# Admin CLI for Hindsight

hindsight-api/hindsight_api/admin/cli.py

@@ -0,0 +1,222 @@
+"""
+Hindsight Admin CLI - backup and restore operations.
+"""
+
+import asyncio
+import io
+import json
+import logging
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import asyncpg
+import typer
+
+from ..config import HindsightConfig
+from ..pg0 import parse_pg0_url, resolve_database_url
+
+
+def _fq_table(table: str, schema: str) -> str:
+    """Get fully-qualified table name with schema prefix."""
+    return f"{schema}.{table}"
+
+
+# Setup logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(message)s",
+)
+logger = logging.getLogger(__name__)
+
+app = typer.Typer(name="hindsight-admin", help="Hindsight administrative commands")
+
+# Tables to backup/restore in dependency order
+# Import must happen in this order due to foreign key constraints
+BACKUP_TABLES = [
+    "banks",
+    "documents",
+    "entities",
+    "chunks",
+    "memory_units",
+    "unit_entities",
+    "entity_cooccurrences",
+    "memory_links",
+]
+
+MANIFEST_VERSION = "1"
+
+
+async def _backup(database_url: str, output_path: Path, schema: str = "public") -> dict[str, Any]:
+    """Backup all tables to a zip file using binary COPY protocol."""
+    conn = await asyncpg.connect(database_url)
+    try:
+        tables: dict[str, Any] = {}
+        manifest: dict[str, Any] = {
+            "version": MANIFEST_VERSION,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+            "schema": schema,
+            "tables": tables,
+        }
+
+        # Use a transaction with REPEATABLE READ isolation to get a consistent
+        # snapshot across all tables. This prevents race conditions where
+        # entity_cooccurrences could reference entities created after the
+        # entities table was backed up.
+        async with conn.transaction(isolation="repeatable_read"):
+            with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zf:
+                for i, table in enumerate(BACKUP_TABLES, 1):
+                    typer.echo(f"  [{i}/{len(BACKUP_TABLES)}] Backing up {table}...", nl=False)
+
+                    buffer = io.BytesIO()
+
+                    # Use binary COPY for exact type preservation
+                    # asyncpg requires schema_name as separate parameter
+                    await conn.copy_from_table(table, schema_name=schema, output=buffer, format="binary")
+
+                    data = buffer.getvalue()
+                    zf.writestr(f"{table}.bin", data)
+
+                    # Get row count for manifest
+                    qualified_table = _fq_table(table, schema)
+                    row_count = await conn.fetchval(f"SELECT COUNT(*) FROM {qualified_table}")
+                    tables[table] = {
+                        "rows": row_count,
+                        "size_bytes": len(data),
+                    }
+
+                    typer.echo(f" {row_count} rows")
+
+                zf.writestr("manifest.json", json.dumps(manifest, indent=2))
+
+        return manifest
+    finally:
+        await conn.close()
+
+
+async def _restore(database_url: str, input_path: Path, schema: str = "public") -> dict[str, Any]:
+    """Restore all tables from a zip file using binary COPY protocol."""
+    conn = await asyncpg.connect(database_url)
+    try:
+        with zipfile.ZipFile(input_path, "r") as zf:
+            # Read and validate manifest
+            manifest: dict[str, Any] = json.loads(zf.read("manifest.json"))
+            if manifest.get("version") != MANIFEST_VERSION:
+                raise ValueError(f"Unsupported backup version: {manifest.get('version')}")
+
+            # Use a transaction for atomic restore - either all tables are
+            # restored or none are, preventing partial/inconsistent state.
+            async with conn.transaction():
+                typer.echo("  Clearing existing data...")
+                # Truncate tables in reverse order (respects FK constraints)
+                for table in reversed(BACKUP_TABLES):
+                    qualified_table = _fq_table(table, schema)
+                    await conn.execute(f"TRUNCATE TABLE {qualified_table} CASCADE")
+
+                # Restore tables in forward order
+                for i, table in enumerate(BACKUP_TABLES, 1):
+                    filename = f"{table}.bin"
+                    if filename not in zf.namelist():
+                        typer.echo(f"  [{i}/{len(BACKUP_TABLES)}] {table}: skipped (not in backup)")
+                        continue
+
+                    expected_rows = manifest["tables"].get(table, {}).get("rows", "?")
+                    typer.echo(f"  [{i}/{len(BACKUP_TABLES)}] Restoring {table}... {expected_rows} rows")
+
+                    data = zf.read(filename)
+                    buffer = io.BytesIO(data)
+                    # asyncpg requires schema_name as separate parameter
+                    await conn.copy_to_table(table, schema_name=schema, source=buffer, format="binary")
+
+                # Refresh materialized view
+                typer.echo("  Refreshing materialized views...")
+                await conn.execute(f"REFRESH MATERIALIZED VIEW {_fq_table('memory_units_bm25', schema)}")
+
+        return manifest
+    finally:
+        await conn.close()
+
+
+async def _run_backup(db_url: str, output: Path, schema: str = "public") -> dict[str, Any]:
+    """Resolve database URL and run backup."""
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+    return await _backup(resolved_url, output, schema)
+
+
+async def _run_restore(db_url: str, input_file: Path, schema: str = "public") -> dict[str, Any]:
+    """Resolve database URL and run restore."""
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+    return await _restore(resolved_url, input_file, schema)
+
+
+@app.command()
+def backup(
+    output: Path = typer.Argument(..., help="Output file path (.zip)"),
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema to backup"),
+):
+    """Backup the Hindsight database to a zip file."""
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    if output.suffix != ".zip":
+        output = output.with_suffix(".zip")
+
+    typer.echo(f"Backing up database (schema: {schema}) to {output}...")
+
+    manifest = asyncio.run(_run_backup(config.database_url, output, schema))
+
+    total_rows = sum(t["rows"] for t in manifest["tables"].values())
+    typer.echo(f"Backed up {total_rows} rows across {len(BACKUP_TABLES)} tables")
+    typer.echo(f"Backup saved to {output}")
+
+
+@app.command()
+def restore(
+    input_file: Path = typer.Argument(..., help="Input backup file (.zip)"),
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema to restore to"),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"),
+):
+    """Restore the database from a backup file. WARNING: This deletes all existing data."""
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    if not input_file.exists():
+        typer.echo(f"Error: File not found: {input_file}", err=True)
+        raise typer.Exit(1)
+
+    if not yes:
+        typer.confirm(
+            "This will DELETE all existing data and replace it with the backup. Continue?",
+            abort=True,
+        )
+
+    typer.echo(f"Restoring database (schema: {schema}) from {input_file}...")
+
+    manifest = asyncio.run(_run_restore(config.database_url, input_file, schema))
+
+    total_rows = sum(t["rows"] for t in manifest["tables"].values())
+    typer.echo(f"Restored {total_rows} rows across {len(BACKUP_TABLES)} tables")
+    typer.echo("Restore complete")
+
+
+def main():
+    app()
+
+
+if __name__ == "__main__":
+    main()

hindsight-api/hindsight_api/config.py

@@ -8,6 +8,11 @@ import logging
 import os
 from dataclasses import dataclass
 
+from dotenv import find_dotenv, load_dotenv
+
+# Load .env file, searching current and parent directories (overrides existing env vars)
+load_dotenv(find_dotenv(usecwd=True), override=True)
+
 logger = logging.getLogger(__name__)
 
 # Environment variable names
@@ -42,6 +47,9 @@ ENV_MCP_INSTRUCTIONS = "HINDSIGHT_API_MCP_INSTRUCTIONS"
 ENV_OBSERVATION_MIN_FACTS = "HINDSIGHT_API_OBSERVATION_MIN_FACTS"
 ENV_OBSERVATION_TOP_ENTITIES = "HINDSIGHT_API_OBSERVATION_TOP_ENTITIES"
 
+# Retain settings
+ENV_RETAIN_MAX_COMPLETION_TOKENS = "HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS"
+
 # Optimization flags
 ENV_SKIP_LLM_VERIFICATION = "HINDSIGHT_API_SKIP_LLM_VERIFICATION"
 ENV_LAZY_RERANKER = "HINDSIGHT_API_LAZY_RERANKER"
@@ -72,6 +80,9 @@ DEFAULT_MCP_LOCAL_BANK_ID = "mcp"
 DEFAULT_OBSERVATION_MIN_FACTS = 5  # Min facts required to generate entity observations
 DEFAULT_OBSERVATION_TOP_ENTITIES = 5  # Max entities to process per retain batch
 
+# Retain settings
+DEFAULT_RETAIN_MAX_COMPLETION_TOKENS = 64000  # Max tokens for fact extraction LLM call
+
 # Default MCP tool descriptions (can be customized via env vars)
 DEFAULT_MCP_RETAIN_DESCRIPTION = """Store important information to long-term memory.
 
@@ -134,6 +145,9 @@ class HindsightConfig:
     observation_min_facts: int
     observation_top_entities: int
 
+    # Retain settings
+    retain_max_completion_tokens: int
+
     # Optimization flags
     skip_llm_verification: bool
     lazy_reranker: bool
@@ -174,6 +188,10 @@ class HindsightConfig:
             observation_top_entities=int(
                 os.getenv(ENV_OBSERVATION_TOP_ENTITIES, str(DEFAULT_OBSERVATION_TOP_ENTITIES))
             ),
+            # Retain settings
+            retain_max_completion_tokens=int(
+                os.getenv(ENV_RETAIN_MAX_COMPLETION_TOKENS, str(DEFAULT_RETAIN_MAX_COMPLETION_TOKENS))
+            ),
         )
 
     def get_llm_base_url(self) -> str:
@@ -220,6 +238,19 @@ class HindsightConfig:
         logger.info(f"Graph retriever: {self.graph_retriever}")
 
 
+# Cached config instance
+_config_cache: HindsightConfig | None = None
+
+
 def get_config() -> HindsightConfig:
-    """Get the current configuration from environment variables."""
-    return HindsightConfig.from_env()
+    """Get the cached configuration, loading from environment on first call."""
+    global _config_cache
+    if _config_cache is None:
+        _config_cache = HindsightConfig.from_env()
+    return _config_cache
+
+
+def clear_config_cache() -> None:
+    """Clear the config cache. Useful for testing or reloading config."""
+    global _config_cache
+    _config_cache = None

hindsight-api/hindsight_api/engine/memory_engine.py

@@ -132,7 +132,7 @@ if TYPE_CHECKING:
 
 from enum import Enum
 
-from ..pg0 import EmbeddedPostgres
+from ..pg0 import EmbeddedPostgres, parse_pg0_url
 from .entity_resolver import EntityResolver
 from .llm_wrapper import LLMConfig
 from .query_analyzer import QueryAnalyzer
@@ -259,31 +259,14 @@ class MemoryEngine(MemoryEngineInterface):
         memory_llm_base_url = memory_llm_base_url or config.get_llm_base_url() or None
         # Track pg0 instance (if used)
         self._pg0: EmbeddedPostgres | None = None
-        self._pg0_instance_name: str | None = None
 
         # Initialize PostgreSQL connection URL
         # The actual URL will be set during initialize() after starting the server
         # Supports: "pg0" (default instance), "pg0://instance-name" (named instance), or regular postgresql:// URL
-        if db_url == "pg0":
-            self._use_pg0 = True
-            self._pg0_instance_name = "hindsight"
-            self._pg0_port = None  # Use default port
-            self.db_url = None
-        elif db_url.startswith("pg0://"):
-            self._use_pg0 = True
-            # Parse instance name and optional port: pg0://instance-name or pg0://instance-name:port
-            url_part = db_url[6:]  # Remove "pg0://"
-            if ":" in url_part:
-                self._pg0_instance_name, port_str = url_part.rsplit(":", 1)
-                self._pg0_port = int(port_str)
-            else:
-                self._pg0_instance_name = url_part or "hindsight"
-                self._pg0_port = None  # Use default port
+        self._use_pg0, self._pg0_instance_name, self._pg0_port = parse_pg0_url(db_url)
+        if self._use_pg0:
             self.db_url = None
         else:
-            self._use_pg0 = False
-            self._pg0_instance_name = None
-            self._pg0_port = None
             self.db_url = db_url
 
         # Set default base URL if not provided

hindsight-api/hindsight_api/engine/retain/fact_extraction.py

@@ -14,6 +14,7 @@ from typing import Literal
 
 from pydantic import BaseModel, ConfigDict, Field, field_validator
 
+from ...config import get_config
 from ..llm_wrapper import LLMConfig, OutputTooLongError
 
 
@@ -109,7 +110,7 @@ class Fact(BaseModel):
 
 
 class CausalRelation(BaseModel):
-    """Causal relationship between facts."""
+    """Causal relationship between facts (legacy - embedded in each fact)."""
 
     target_fact_index: int = Field(
         description="Index of the related fact in the facts array (0-based). "
@@ -131,6 +132,36 @@ class CausalRelation(BaseModel):
     )
 
 
+class TopLevelCausalRelation(BaseModel):
+    """
+    Causal relationship between two facts (top-level schema).
+
+    This is the preferred format - defined AFTER all facts are extracted,
+    allowing the LLM to see the full list of facts before specifying relationships.
+    """
+
+    from_fact_index: int = Field(
+        description="Index of the source fact (0-based). The fact that causes/enables/prevents."
+    )
+    to_fact_index: int = Field(
+        description="Index of the target fact (0-based). The fact that is caused/enabled/prevented."
+    )
+    relation_type: Literal["causes", "caused_by", "enables", "prevents"] = Field(
+        description="Type of causal relationship: "
+        "'causes' = source fact directly causes the target fact, "
+        "'caused_by' = source fact was caused by the target fact, "
+        "'enables' = source fact enables/allows the target fact, "
+        "'prevents' = source fact prevents/blocks the target fact"
+    )
+    strength: float = Field(
+        description="Strength of causal relationship (0.0 to 1.0). "
+        "1.0 = direct/strong causation, 0.5 = moderate, 0.3 = weak/indirect",
+        ge=0.0,
+        le=1.0,
+        default=1.0,
+    )
+
+
 class ExtractedFact(BaseModel):
     """A single extracted fact with 5 required dimensions for comprehensive capture."""
 
@@ -253,9 +284,15 @@ class ExtractedFact(BaseModel):
 
 
 class FactExtractionResponse(BaseModel):
-    """Response containing all extracted facts."""
+    """Response containing all extracted facts and their causal relationships."""
 
     facts: list[ExtractedFact] = Field(description="List of extracted factual statements")
+    causal_relationships: list[TopLevelCausalRelation] | None = Field(
+        default=None,
+        description="Causal relationships between facts. Define these AFTER listing all facts. "
+        "Each relationship specifies from_fact_index -> to_fact_index with a relation type. "
+        "Indices must be valid (0 to N-1 where N is the number of facts).",
+    )
 
 
 def chunk_text(text: str, max_chars: int) -> list[str]:
@@ -572,7 +609,53 @@ WHAT TO EXTRACT vs SKIP
 ══════════════════════════════════════════════════════════════════════════
 
 ✅ EXTRACT: User preferences (ALWAYS as separate facts!), feelings, plans, events, relationships, achievements
-❌ SKIP: Greetings, filler ("thanks", "cool"), purely structural statements"""
+❌ SKIP: Greetings, filler ("thanks", "cool"), purely structural statements
+
+══════════════════════════════════════════════════════════════════════════
+CAUSAL RELATIONSHIPS (CRITICAL - DEFINE AFTER ALL FACTS)
+══════════════════════════════════════════════════════════════════════════
+
+⚠️ IMPORTANT: Causal relationships are defined at the TOP LEVEL, AFTER listing all facts!
+
+The `causal_relationships` array goes at the root of your response (NOT inside each fact).
+This allows you to see all facts first before defining how they relate.
+
+Format:
+```json
+{{
+  "facts": [...all your extracted facts...],
+  "causal_relationships": [
+    {{"from_fact_index": 0, "to_fact_index": 1, "relation_type": "causes", "strength": 0.9}},
+    {{"from_fact_index": 1, "to_fact_index": 2, "relation_type": "enables", "strength": 0.7}}
+  ]
+}}
+```
+
+Relationship types:
+- "causes": Fact A directly causes Fact B (A → B)
+- "caused_by": Fact A was caused by Fact B (A ← B)
+- "enables": Fact A enables/allows Fact B to happen
+- "prevents": Fact A prevents/blocks Fact B from happening
+
+⚠️ INDEX VALIDATION: If you extract N facts (indices 0 to N-1), both from_fact_index and to_fact_index MUST be in range [0, N-1].
+
+Example (Event Date: March 15, 2024):
+Input: "I lost my job in January. Because of that, I couldn't pay rent. So I had to move to a cheaper apartment."
+
+Facts extracted:
+- Fact 0: "User lost their job in January due to layoffs"
+- Fact 1: "User couldn't pay rent because of job loss"
+- Fact 2: "User moved to a cheaper apartment"
+
+Causal relationships (at root level):
+```json
+"causal_relationships": [
+  {{"from_fact_index": 0, "to_fact_index": 1, "relation_type": "causes", "strength": 1.0}},
+  {{"from_fact_index": 1, "to_fact_index": 2, "relation_type": "causes", "strength": 0.9}}
+]
+```
+
+This creates a chain: Job loss (0) → Can't pay rent (1) → Moved to cheaper apartment (2)"""
 
     import logging
 
@@ -583,6 +666,7 @@ WHAT TO EXTRACT vs SKIP
     # Retry logic for JSON validation errors
     max_retries = 2
     last_error = None
+    config = get_config()
 
     # Sanitize input text to prevent Unicode encoding errors (e.g., unpaired surrogates)
     sanitized_chunk = _sanitize_text(chunk)
@@ -608,7 +692,7 @@ Text:
                 response_format=FactExtractionResponse,
                 scope="memory_extract_facts",
                 temperature=0.1,
-                max_completion_tokens=65000,
+                max_completion_tokens=config.retain_max_completion_tokens,
                 skip_validation=True,  # Get raw JSON, we'll validate leniently
             )
 
@@ -631,6 +715,9 @@ Text:
                     return []
 
             raw_facts = extraction_response_json.get("facts", [])
+            # Get top-level causal relationships (new schema)
+            top_level_causal_relations = extraction_response_json.get("causal_relationships", [])
+
             if not raw_facts:
                 logger.debug(
                     f"LLM response missing 'facts' field or returned empty list. "
@@ -641,6 +728,47 @@ Text:
                     f"text: {chunk}"
                 )
 
+            # Build a map from fact index to causal relations (from top-level field)
+            # This converts from_fact_index -> [{target_fact_index, relation_type, strength}]
+            causal_relations_by_fact: dict[int, list[dict]] = {}
+            if top_level_causal_relations:
+                num_facts = len(raw_facts)
+                for rel in top_level_causal_relations:
+                    if not isinstance(rel, dict):
+                        continue
+                    from_idx = rel.get("from_fact_index")
+                    to_idx = rel.get("to_fact_index")
+                    relation_type = rel.get("relation_type")
+                    strength = rel.get("strength", 1.0)
+
+                    # Validate indices
+                    if from_idx is None or to_idx is None or relation_type is None:
+                        logger.warning(f"Skipping malformed top-level causal relation: {rel}")
+                        continue
+                    if from_idx < 0 or from_idx >= num_facts:
+                        logger.warning(
+                            f"Invalid from_fact_index {from_idx} in top-level causal relation "
+                            f"(valid range: 0-{num_facts - 1}). Skipping."
+                        )
+                        continue
+                    if to_idx < 0 or to_idx >= num_facts:
+                        logger.warning(
+                            f"Invalid to_fact_index {to_idx} in top-level causal relation "
+                            f"(valid range: 0-{num_facts - 1}). Skipping."
+                        )
+                        continue
+
+                    # Add to the map for the from_fact_index
+                    if from_idx not in causal_relations_by_fact:
+                        causal_relations_by_fact[from_idx] = []
+                    causal_relations_by_fact[from_idx].append(
+                        {
+                            "target_fact_index": to_idx,
+                            "relation_type": relation_type,
+                            "strength": strength,
+                        }
+                    )
+
             for i, llm_fact in enumerate(raw_facts):
                 # Skip non-dict entries but track them for retry
                 if not isinstance(llm_fact, dict):
@@ -745,19 +873,40 @@ Text:
                     if validated_entities:
                         fact_data["entities"] = validated_entities
 
-                # Add causal relations if present (validate as CausalRelation objects)
-                # Filter out invalid relations (missing required fields)
-                causal_relations = get_value("causal_relations")
-                if causal_relations:
-                    validated_relations = []
-                    for rel in causal_relations:
+                # Add causal relations from both sources:
+                # 1. Top-level causal_relationships (preferred, new schema)
+                # 2. Per-fact causal_relations (legacy, for backward compatibility)
+                validated_relations = []
+
+                # First, add relations from top-level (already validated above)
+                if i in causal_relations_by_fact:
+                    for rel in causal_relations_by_fact[i]:
+                        try:
+                            validated_relations.append(CausalRelation.model_validate(rel))
+                        except Exception as e:
+                            logger.warning(f"Invalid top-level causal relation for fact {i}: {rel}: {e}")
+
+                # Then, add any legacy per-fact relations (with index validation)
+                legacy_causal_relations = get_value("causal_relations")
+                if legacy_causal_relations:
+                    num_facts = len(raw_facts)
+                    for rel in legacy_causal_relations:
                         if isinstance(rel, dict) and "target_fact_index" in rel and "relation_type" in rel:
-                            try:
-                                validated_relations.append(CausalRelation.model_validate(rel))
-                            except Exception as e:
-                                logger.warning(f"Invalid causal relation {rel}: {e}")
-                    if validated_relations:
-                        fact_data["causal_relations"] = validated_relations
+                            target_idx = rel.get("target_fact_index")
+                            # Validate target index for legacy format too
+                            if target_idx is not None and 0 <= target_idx < num_facts:
+                                try:
+                                    validated_relations.append(CausalRelation.model_validate(rel))
+                                except Exception as e:
+                                    logger.warning(f"Invalid causal relation {rel}: {e}")
+                            else:
+                                logger.warning(
+                                    f"Invalid target_fact_index {target_idx} in per-fact causal relation "
+                                    f"from fact {i} (valid range: 0-{num_facts - 1}). Skipping."
+                                )
+
+                if validated_relations:
+                    fact_data["causal_relations"] = validated_relations
 
                 # Always set mentioned_at to the event_date (when the conversation/document occurred)
                 fact_data["mentioned_at"] = event_date.isoformat()

hindsight-api/hindsight_api/main.py

@@ -184,6 +184,7 @@ def main():
             graph_retriever=config.graph_retriever,
             observation_min_facts=config.observation_min_facts,
             observation_top_entities=config.observation_top_entities,
+            retain_max_completion_tokens=config.retain_max_completion_tokens,
             skip_llm_verification=config.skip_llm_verification,
             lazy_reranker=config.lazy_reranker,
         )

hindsight-api/hindsight_api/pg0.py

@@ -132,3 +132,56 @@ async def stop_embedded_postgres() -> None:
     global _default_instance
     if _default_instance:
         await _default_instance.stop()
+
+
+def parse_pg0_url(db_url: str) -> tuple[bool, str | None, int | None]:
+    """
+    Parse a database URL and check if it's a pg0:// embedded database URL.
+
+    Supports:
+    - "pg0" -> default instance "hindsight"
+    - "pg0://instance-name" -> named instance
+    - "pg0://instance-name:port" -> named instance with explicit port
+    - Any other URL (e.g., postgresql://) -> not a pg0 URL
+
+    Args:
+        db_url: The database URL to parse
+
+    Returns:
+        Tuple of (is_pg0, instance_name, port)
+        - is_pg0: True if this is a pg0 URL
+        - instance_name: The instance name (or None if not pg0)
+        - port: The explicit port (or None for auto-assign)
+    """
+    if db_url == "pg0":
+        return True, "hindsight", None
+
+    if db_url.startswith("pg0://"):
+        url_part = db_url[6:]  # Remove "pg0://"
+        if ":" in url_part:
+            instance_name, port_str = url_part.rsplit(":", 1)
+            return True, instance_name or "hindsight", int(port_str)
+        else:
+            return True, url_part or "hindsight", None
+
+    return False, None, None
+
+
+async def resolve_database_url(db_url: str) -> str:
+    """
+    Resolve a database URL, handling pg0:// embedded database URLs.
+
+    If the URL is a pg0:// URL, starts the embedded PostgreSQL and returns
+    the actual postgresql:// connection URL. Otherwise, returns the URL unchanged.
+
+    Args:
+        db_url: Database URL (pg0://, pg0, or postgresql://)
+
+    Returns:
+        The resolved postgresql:// connection URL
+    """
+    is_pg0, instance_name, port = parse_pg0_url(db_url)
+    if is_pg0:
+        pg0 = EmbeddedPostgres(name=instance_name, port=port)
+        return await pg0.ensure_running()
+    return db_url

hindsight-api/pyproject.toml

@@ -38,6 +38,7 @@ dependencies = [
     "dateparser>=1.2.2",
     "google-genai>=1.0.0",
     "anthropic>=0.40.0",
+    "typer>=0.9.0",
 ]
 
 [project.optional-dependencies]
@@ -52,6 +53,7 @@ test = [
 [project.scripts]
 hindsight-api = "hindsight_api.main:main"
 hindsight-local-mcp = "hindsight_api.mcp_local:main"
+hindsight-admin = "hindsight_api.admin.cli:main"
 
 [tool.hatch.build.targets.wheel]
 packages = ["hindsight_api"]
@@ -75,7 +77,7 @@ log_cli = true
 log_cli_level = "INFO"
 log_cli_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"
 log_cli_date_format = "%Y-%m-%d %H:%M:%S"
-addopts = "--timeout 120 -n 8 --durations=10 -v"
+addopts = "--timeout 120 -n 8 --dist loadgroup --durations=10 -v"
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"
 log_auto_indent = true

hindsight-api/tests/test_admin_backup_restore.py

@@ -0,0 +1,208 @@
+"""
+Tests for admin backup and restore functionality.
+
+Note: These tests run sequentially (not in parallel) because they all
+manipulate the same database and do full backup/restore operations.
+"""
+
+import tempfile
+import uuid
+import zipfile
+from pathlib import Path
+
+import pytest
+
+from hindsight_api import RequestContext
+from hindsight_api.admin.cli import _backup, _restore, BACKUP_TABLES
+
+
+# Run these tests sequentially since they do full DB backup/restore
+pytestmark = pytest.mark.xdist_group(name="backup_restore")
+
+
+@pytest.mark.asyncio
+async def test_backup_restore_roundtrip(memory, pg0_db_url, request_context):
+    """Test that backup and restore preserves all data correctly."""
+    # Use unique bank ID to avoid conflicts
+    bank_id = f"test-backup-{uuid.uuid4().hex[:8]}"
+
+    # Create some test data
+    await memory.retain_batch_async(
+        bank_id=bank_id,
+        contents=[
+            {"content": "Alice is a software engineer who loves Python."},
+            {"content": "Bob works with Alice on the backend team."},
+            {"content": "The team uses PostgreSQL for their database."},
+        ],
+        request_context=request_context,
+    )
+
+    # Get counts before backup
+    async with memory._pool.acquire() as conn:
+        counts_before = {}
+        for table in BACKUP_TABLES:
+            counts_before[table] = await conn.fetchval(f"SELECT COUNT(*) FROM {table}")
+
+    # Verify we have data
+    assert counts_before["banks"] > 0
+    assert counts_before["memory_units"] > 0
+
+    # Backup to a temp file
+    with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as f:
+        backup_path = Path(f.name)
+
+    try:
+        manifest = await _backup(pg0_db_url, backup_path)
+
+        # Verify backup file exists and is valid
+        assert backup_path.exists()
+        assert backup_path.stat().st_size > 0
+
+        # Verify manifest
+        assert manifest["version"] == "1"
+        assert "created_at" in manifest
+        for table in BACKUP_TABLES:
+            assert table in manifest["tables"]
+            assert manifest["tables"][table]["rows"] == counts_before[table]
+
+        # Verify zip contents
+        with zipfile.ZipFile(backup_path, "r") as zf:
+            assert "manifest.json" in zf.namelist()
+            for table in BACKUP_TABLES:
+                assert f"{table}.bin" in zf.namelist()
+
+        # Clear all data
+        async with memory._pool.acquire() as conn:
+            for table in reversed(BACKUP_TABLES):
+                await conn.execute(f"TRUNCATE TABLE {table} CASCADE")
+
+        # Verify data is gone
+        async with memory._pool.acquire() as conn:
+            for table in BACKUP_TABLES:
+                count = await conn.fetchval(f"SELECT COUNT(*) FROM {table}")
+                assert count == 0, f"Table {table} should be empty after truncate"
+
+        # Restore from backup
+        await _restore(pg0_db_url, backup_path)
+
+        # Verify counts match original
+        async with memory._pool.acquire() as conn:
+            for table in BACKUP_TABLES:
+                count = await conn.fetchval(f"SELECT COUNT(*) FROM {table}")
+                assert count == counts_before[table], f"Table {table} count mismatch after restore"
+
+        # Verify data content is preserved
+        async with memory._pool.acquire() as conn:
+            texts = await conn.fetch(
+                "SELECT text FROM memory_units WHERE bank_id = $1",
+                bank_id,
+            )
+            text_content = " ".join(r["text"] for r in texts)
+            assert "Alice" in text_content or "software" in text_content
+
+    finally:
+        # Cleanup
+        if backup_path.exists():
+            backup_path.unlink()
+
+
+@pytest.mark.asyncio
+async def test_backup_restore_preserves_all_column_types(memory, pg0_db_url, request_context):
+    """Test that all column types are preserved: vectors, UUIDs, timestamps, JSONB."""
+    # Use unique bank ID
+    bank_id = f"test-types-{uuid.uuid4().hex[:8]}"
+
+    # Create data with meaningful content that will produce facts
+    await memory.retain_batch_async(
+        bank_id=bank_id,
+        contents=[
+            {"content": "John Smith is a senior engineer at Acme Corp since 2020."},
+            {"content": "The project deadline is December 15th 2024."},
+        ],
+        request_context=request_context,
+    )
+
+    # Get original data with all important column types
+    async with memory._pool.acquire() as conn:
+        # memory_units: UUID (id), Vector (embedding), Timestamp (event_date, created_at), JSONB (metadata)
+        original_unit = await conn.fetchrow(
+            """SELECT id, embedding, event_date, created_at, metadata, text
+               FROM memory_units WHERE bank_id = $1 LIMIT 1""",
+            bank_id,
+        )
+
+        # entities: UUID (id), Timestamp (first_seen, last_seen), JSONB (metadata)
+        original_entity = await conn.fetchrow(
+            """SELECT id, first_seen, last_seen, metadata, canonical_name
+               FROM entities WHERE bank_id = $1 LIMIT 1""",
+            bank_id,
+        )
+
+        # banks: JSONB (personality/disposition)
+        original_bank = await conn.fetchrow(
+            "SELECT bank_id, created_at, updated_at FROM banks WHERE bank_id = $1",
+            bank_id,
+        )
+
+    assert original_unit is not None, "Should have created memory units"
+    assert original_unit["embedding"] is not None, "Should have embedding"
+    assert original_unit["id"] is not None, "Should have UUID"
+    assert original_entity is not None, "Should have created entities"
+
+    with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as f:
+        backup_path = Path(f.name)
+
+    try:
+        await _backup(pg0_db_url, backup_path)
+
+        # Clear all data
+        async with memory._pool.acquire() as conn:
+            for table in reversed(BACKUP_TABLES):
+                await conn.execute(f"TRUNCATE TABLE {table} CASCADE")
+
+        await _restore(pg0_db_url, backup_path)
+
+        # Verify all column types are preserved exactly
+        async with memory._pool.acquire() as conn:
+            restored_unit = await conn.fetchrow(
+                """SELECT id, embedding, event_date, created_at, metadata, text
+                   FROM memory_units WHERE bank_id = $1 LIMIT 1""",
+                bank_id,
+            )
+
+            restored_entity = await conn.fetchrow(
+                """SELECT id, first_seen, last_seen, metadata, canonical_name
+                   FROM entities WHERE bank_id = $1 LIMIT 1""",
+                bank_id,
+            )
+
+            restored_bank = await conn.fetchrow(
+                "SELECT bank_id, created_at, updated_at FROM banks WHERE bank_id = $1",
+                bank_id,
+            )
+
+        # Verify memory_units
+        assert restored_unit is not None, "Should have restored memory unit"
+        assert restored_unit["id"] == original_unit["id"], "UUID should match exactly"
+        assert restored_unit["text"] == original_unit["text"], "Text should match"
+        assert list(restored_unit["embedding"]) == list(original_unit["embedding"]), "Vector embedding should match exactly"
+        assert restored_unit["event_date"] == original_unit["event_date"], "Timestamp should match exactly"
+        assert restored_unit["created_at"] == original_unit["created_at"], "Created timestamp should match"
+        assert restored_unit["metadata"] == original_unit["metadata"], "JSONB metadata should match"
+
+        # Verify entities
+        assert restored_entity is not None, "Should have restored entity"
+        assert restored_entity["id"] == original_entity["id"], "Entity UUID should match"
+        assert restored_entity["canonical_name"] == original_entity["canonical_name"], "Entity name should match"
+        assert restored_entity["first_seen"] == original_entity["first_seen"], "Entity first_seen should match"
+        assert restored_entity["last_seen"] == original_entity["last_seen"], "Entity last_seen should match"
+        assert restored_entity["metadata"] == original_entity["metadata"], "Entity metadata should match"
+
+        # Verify banks
+        assert restored_bank is not None, "Should have restored bank"
+        assert restored_bank["bank_id"] == original_bank["bank_id"], "Bank ID should match"
+        assert restored_bank["created_at"] == original_bank["created_at"], "Bank created_at should match"
+
+    finally:
+        if backup_path.exists():
+            backup_path.unlink()

hindsight-api/tests/test_causal_relationships.py

@@ -0,0 +1,222 @@
+"""
+Test suite for causal relationship extraction.
+
+Tests that the fact extraction system correctly identifies and validates
+causal relationships between facts, with valid indices.
+"""
+
+from datetime import datetime
+
+import pytest
+
+from hindsight_api import LLMConfig
+from hindsight_api.engine.retain.fact_extraction import extract_facts_from_text
+
+
+class TestCausalRelationships:
+    """Tests for causal relationship extraction and validation."""
+
+    @pytest.mark.asyncio
+    async def test_causal_chain_extraction(self):
+        """
+        Test that a clear causal chain is extracted with valid relationships.
+
+        Story: Lost job -> couldn't pay rent -> had to move -> found new apartment
+
+        This is a 4-fact causal chain where each fact causes the next.
+        The extracted causal relations should have valid indices (0-3).
+        """
+        text = """
+I lost my job at the tech company in January because of layoffs.
+Because I lost my job, I couldn't pay my rent anymore.
+Since I couldn't afford rent, I had to move out of my apartment.
+After searching for weeks, I finally found a cheaper apartment in Brooklyn.
+"""
+
+        context = "Personal story about housing change"
+        llm_config = LLMConfig.for_memory()
+
+        facts, _ = await extract_facts_from_text(
+            text=text,
+            event_date=datetime(2024, 3, 15),
+            context=context,
+            llm_config=llm_config,
+            agent_name="TestUser"
+        )
+
+        assert len(facts) >= 3, f"Should extract at least 3 facts from the causal chain. Got {len(facts)}"
+
+        # Collect all causal relations from all facts
+        all_causal_relations = []
+        for i, fact in enumerate(facts):
+            if fact.causal_relations:
+                for rel in fact.causal_relations:
+                    all_causal_relations.append({
+                        "from_fact_index": i,
+                        "to_fact_index": rel.target_fact_index,
+                        "relation_type": rel.relation_type,
+                        "strength": rel.strength,
+                        "from_fact_text": fact.fact[:50],
+                    })
+
+        # Verify that ALL causal relation indices are valid
+        num_facts = len(facts)
+        invalid_relations = []
+        for rel in all_causal_relations:
+            if rel["to_fact_index"] < 0 or rel["to_fact_index"] >= num_facts:
+                invalid_relations.append(rel)
+
+        assert len(invalid_relations) == 0, (
+            f"Found {len(invalid_relations)} causal relations with invalid indices! "
+            f"Valid range is 0-{num_facts - 1}. "
+            f"Invalid relations: {invalid_relations}"
+        )
+
+        # Should have at least some causal relations extracted
+        assert len(all_causal_relations) >= 2, (
+            f"Should extract at least 2 causal relationships from this clear chain. "
+            f"Got {len(all_causal_relations)}: {all_causal_relations}"
+        )
+
+        # Verify relation types are valid
+        valid_types = {"causes", "caused_by", "enables", "prevents"}
+        for rel in all_causal_relations:
+            assert rel["relation_type"] in valid_types, (
+                f"Invalid relation_type '{rel['relation_type']}'. Must be one of {valid_types}"
+            )
+
+    @pytest.mark.asyncio
+    async def test_complex_causal_web(self):
+        """
+        Test a more complex scenario with multiple interconnected causes.
+
+        This tests the LLM's ability to identify multiple causal links and
+        ensure all referenced indices exist.
+        """
+        text = """
+The heavy rain caused flooding in the basement.
+The flooding damaged the electrical system.
+Because of the electrical damage, we had to call an electrician.
+The electrician found that the wiring was old and needed replacement.
+We decided to renovate the entire basement while fixing the wiring.
+The renovation took three months and cost $15,000.
+"""
+
+        context = "Home repair story"
+        llm_config = LLMConfig.for_memory()
+
+        facts, _ = await extract_facts_from_text(
+            text=text,
+            event_date=datetime(2024, 6, 1),
+            context=context,
+            llm_config=llm_config,
+            agent_name="TestUser"
+        )
+
+        assert len(facts) >= 4, f"Should extract at least 4 facts. Got {len(facts)}"
+
+        # Validate all causal relation indices
+        num_facts = len(facts)
+        for i, fact in enumerate(facts):
+            if fact.causal_relations:
+                for rel in fact.causal_relations:
+                    assert 0 <= rel.target_fact_index < num_facts, (
+                        f"Fact {i} has causal relation to invalid index {rel.target_fact_index}. "
+                        f"Valid range is 0-{num_facts - 1}. "
+                        f"Fact text: {fact.fact[:80]}..."
+                    )
+
+    @pytest.mark.asyncio
+    async def test_no_self_referencing_causal_relations(self):
+        """
+        Test that facts don't have causal relations pointing to themselves.
+        """
+        text = """
+I started learning Python because I wanted to automate my work tasks.
+Learning Python led me to discover machine learning.
+Machine learning fascinated me so much that I changed my career to data science.
+"""
+
+        context = "Career change story"
+        llm_config = LLMConfig.for_memory()
+
+        facts, _ = await extract_facts_from_text(
+            text=text,
+            event_date=datetime(2024, 1, 1),
+            context=context,
+            llm_config=llm_config,
+            agent_name="TestUser"
+        )
+
+        # Check no fact references itself
+        for i, fact in enumerate(facts):
+            if fact.causal_relations:
+                for rel in fact.causal_relations:
+                    assert rel.target_fact_index != i, (
+                        f"Fact {i} has a self-referencing causal relation! "
+                        f"Fact text: {fact.fact}"
+                    )
+
+    @pytest.mark.asyncio
+    async def test_bidirectional_causal_relationships(self):
+        """
+        Test that bidirectional causal relationships (causes and caused_by)
+        are handled correctly.
+        """
+        text = """
+My promotion at work caused me to move to New York.
+Moving to New York was caused by my promotion at work.
+The new role enabled me to lead a team of engineers.
+"""
+
+        context = "Work promotion story"
+        llm_config = LLMConfig.for_memory()
+
+        facts, _ = await extract_facts_from_text(
+            text=text,
+            event_date=datetime(2024, 2, 15),
+            context=context,
+            llm_config=llm_config,
+            agent_name="TestUser"
+        )
+
+        num_facts = len(facts)
+
+        # Validate all indices
+        for i, fact in enumerate(facts):
+            if fact.causal_relations:
+                for rel in fact.causal_relations:
+                    assert 0 <= rel.target_fact_index < num_facts, (
+                        f"Invalid target_fact_index {rel.target_fact_index} in fact {i}. "
+                        f"Valid range: 0-{num_facts - 1}"
+                    )
+
+    @pytest.mark.asyncio
+    async def test_causal_relation_strength_values(self):
+        """
+        Test that causal relation strength values are within valid range [0.0, 1.0].
+        """
+        text = """
+The stock market crash directly caused the company to lay off employees.
+The layoffs indirectly led to reduced consumer spending in the area.
+Reduced spending somewhat affected local businesses.
+"""
+
+        context = "Economic impact story"
+        llm_config = LLMConfig.for_memory()
+
+        facts, _ = await extract_facts_from_text(
+            text=text,
+            event_date=datetime(2024, 4, 1),
+            context=context,
+            llm_config=llm_config,
+            agent_name="TestUser"
+        )
+
+        for i, fact in enumerate(facts):
+            if fact.causal_relations:
+                for rel in fact.causal_relations:
+                    assert 0.0 <= rel.strength <= 1.0, (
+                        f"Causal relation strength {rel.strength} is outside valid range [0.0, 1.0]. "
+                        f"Fact {i}: {fact.fact[:50]}..."
+                    )

hindsight-clients/python/hindsight_client/hindsight_client.py

@@ -74,6 +74,8 @@ class Hindsight:
         """
         config = hindsight_client_api.Configuration(host=base_url, access_token=api_key)
         self._api_client = hindsight_client_api.ApiClient(config)
+        if api_key:
+            self._api_client.set_default_header("Authorization", f"Bearer {api_key}")
         self._memory_api = memory_api.MemoryApi(self._api_client)
         self._banks_api = banks_api.BanksApi(self._api_client)
 

hindsight-clients/rust/Cargo.lock

@@ -284,9 +284,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.4.12"
+version = "0.4.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386"
+checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -840,9 +840,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.104"
+version = "1.0.105"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0"
+checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7"
 dependencies = [
  "unicode-ident",
 ]
@@ -915,9 +915,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.42"
+version = "1.0.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f"
+checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a"
 dependencies = [
  "proc-macro2",
 ]
@@ -1048,9 +1048,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.35"
+version = "0.23.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f"
+checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b"
 dependencies = [
  "once_cell",
  "rustls-pki-types",
@@ -1208,9 +1208,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.148"
+version = "1.0.149"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3084b546a1dd6289475996f182a22aba973866ea8e8b02c51d9f46b1336a22da"
+checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
 dependencies = [
  "itoa",
  "memchr",
@@ -1641,9 +1641,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
 [[package]]
 name = "url"
-version = "2.5.7"
+version = "2.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b"
+checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"
 dependencies = [
  "form_urlencoded",
  "idna",
@@ -2102,6 +2102,6 @@ dependencies = [
 
 [[package]]
 name = "zmij"
-version = "1.0.10"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30e0d8dffbae3d840f64bda38e28391faef673a7b5a6017840f2a106c8145868"
+checksum = "2fc5a66a20078bf1251bde995aa2fdcc4b800c70b5d92dd2c62abc5c60f679f8"

hindsight-clients/typescript/package.json

@@ -26,7 +26,7 @@
     "directory": "hindsight-clients/typescript"
   },
   "devDependencies": {
-    "@hey-api/openapi-ts": "^0.88.0",
+    "@hey-api/openapi-ts": "0.88.0",
     "@types/jest": "^29.0.0",
     "@types/node": "^20.0.0",
     "jest": "^29.0.0",

hindsight-docs/docs/developer/configuration.md

@@ -183,6 +183,14 @@ Controls when the system generates entity observations (summaries about entities
 | `HINDSIGHT_API_OBSERVATION_MIN_FACTS` | Minimum facts about an entity before generating observations | `5` |
 | `HINDSIGHT_API_OBSERVATION_TOP_ENTITIES` | Max entities to process per retain batch | `5` |
 
+### Retain
+
+Controls the retain (memory ingestion) pipeline.
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS` | Max completion tokens for fact extraction LLM calls | `64000` |
+
 ### Local MCP Server
 
 Configuration for the local MCP server (`hindsight-local-mcp` command).

package.json

@@ -8,5 +8,8 @@
   ],
   "scripts": {
     "prepare": "./scripts/setup-hooks.sh"
+  },
+  "overrides": {
+    "qs": "^6.14.1"
   }
 }

scripts/generate-clients.sh

@@ -51,16 +51,16 @@ echo "=================================================="
 
 RUST_CLIENT_DIR="$CLIENTS_DIR/rust"
 
-# Clean old generated files
+# Clean old generated files (keep Cargo.lock for reproducible builds)
 echo "Cleaning old Rust generated code..."
 rm -rf "$RUST_CLIENT_DIR/target"
-rm -f "$RUST_CLIENT_DIR/Cargo.lock"
 
 # Trigger regeneration by building
+# Use --locked to ensure reproducible builds from committed Cargo.lock
 echo "Regenerating Rust client (via build.rs)..."
 cd "$RUST_CLIENT_DIR"
 cargo clean
-cargo build --release
+cargo build --release --locked
 
 echo "✓ Rust client generated at $RUST_CLIENT_DIR"
 echo ""
@@ -324,9 +324,11 @@ rm -rf "$TYPESCRIPT_CLIENT_DIR/services"
 rm -f "$TYPESCRIPT_CLIENT_DIR/index.ts"
 
 # Generate new client using @hey-api/openapi-ts
+# Use npm run generate to use the locally installed version (pinned in package.json)
+# instead of npx --yes which would fetch the latest version
 echo "Generating from $OPENAPI_SPEC..."
 cd "$TYPESCRIPT_CLIENT_DIR"
-npx --yes @hey-api/openapi-ts
+npm run generate
 
 echo "✓ TypeScript client generated at $TYPESCRIPT_CLIENT_DIR"
 echo ""

uv.lock

@@ -1215,6 +1215,7 @@ dependencies = [
     { name = "tiktoken" },
     { name = "torch" },
     { name = "transformers" },
+    { name = "typer" },
     { name = "uvicorn" },
     { name = "wsproto" },
 ]
@@ -1274,6 +1275,7 @@ requires-dist = [
     { name = "tiktoken", specifier = ">=0.12.0" },
     { name = "torch", specifier = ">=2.0.0" },
     { name = "transformers", specifier = ">=4.30.0,<4.46.0" },
+    { name = "typer", specifier = ">=0.9.0" },
     { name = "uvicorn", specifier = ">=0.38.0" },
     { name = "wsproto", specifier = ">=1.0.0" },
 ]