test(db): pin current production URL shapes as regression guard

Fixes #1216. External PostgreSQL deployments (Cloud SQL, RDS, etc.)
configured with a SQLAlchemy-style URL like
`postgresql+asyncpg://user:pass@host/db?ssl=require` failed in two
places:

1. Five sync `create_engine(database_url)` call sites in migrations.py
   — psycopg2 doesn't understand the asyncpg dialect, and it expects
   `sslmode=require` rather than `ssl=require`.
2. `asyncpg.create_pool(self.db_url)` in memory_engine.py — asyncpg
   doesn't parse the `postgresql+asyncpg://` scheme directly.

Adds a single `to_libpq_url()` helper (urllib.parse-based, idempotent,
safe on passwords containing `+`) and applies it at:

- All five `create_engine()` sites in migrations.py (including the
  run_migrations advisory-lock connection)
- `asyncpg.create_pool()` in memory_engine.py
- The ad-hoc scheme rewrite in alembic/env.py (replaced by the helper)

Existing configs (`pg0`, plain `postgresql://`, `sslmode=require`,
`postgresql+psycopg2://`) are returned byte-identical — no behaviour
change for current users.

.github/workflows/deploy-docs.yml

@@ -34,7 +34,7 @@ jobs:
         env:
           UMAMI_URL: https://analytics.hindsight.vectorize.io
           UMAMI_WEBSITE_ID: ${{ secrets.UMAMI_WEBSITE_ID }}
-      - uses: actions/upload-pages-artifact@v4
+      - uses: actions/upload-pages-artifact@v5
         with:
           path: hindsight-docs/build
   deploy:

.github/workflows/perf-test.yml

@@ -0,0 +1,174 @@
+name: Performance Tests
+
+on:
+  schedule:
+    # Run daily at 06:00 UTC
+    - cron: "0 6 * * *"
+  workflow_dispatch:
+    inputs:
+      scale:
+        description: "Test scale (perf-test)"
+        type: choice
+        options:
+          - tiny
+          - small
+          - medium
+          - large
+        default: large
+      suite:
+        description: "Perf-test suite to run (blank = all)"
+        type: choice
+        options:
+          - ""
+          - retain
+          - recall
+        default: ""
+      locomo_max_conversations:
+        description: "LoComo max conversations (0 = skip, blank = all)"
+        type: number
+        default: 0
+      locomo_skip:
+        description: "Skip LoComo job"
+        type: boolean
+        default: false
+      ref:
+        description: "Git ref to test (branch, tag, or SHA). Defaults to main."
+        type: string
+        default: ""
+
+concurrency:
+  group: perf-test
+  cancel-in-progress: true
+
+jobs:
+  perf-test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ inputs.ref || github.ref }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version-file: ".python-version"
+
+    - name: Cache HuggingFace models
+      uses: actions/cache@v5
+      with:
+        path: ~/.cache/huggingface
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-huggingface-
+
+    - name: Pre-download models
+      working-directory: ./hindsight-api-slim
+      run: |
+        uv run --frozen --all-extras --index-strategy unsafe-best-match python -c "
+        from sentence_transformers import SentenceTransformer
+        print('Downloading embedding model...')
+        SentenceTransformer('BAAI/bge-small-en-v1.5')
+        print('Model downloaded successfully')
+        "
+
+    - name: Install hindsight-dev dependencies
+      run: |
+        cd hindsight-dev && uv sync --frozen --all-extras --index-strategy unsafe-best-match
+
+    - name: Run perf tests
+      run: |
+        SUITE_ARG=""
+        if [ -n "${{ inputs.suite }}" ]; then
+          SUITE_ARG="--suite ${{ inputs.suite }}"
+        fi
+        ./scripts/benchmarks/run-perf-test.sh \
+          --scale ${{ inputs.scale || 'large' }} \
+          $SUITE_ARG \
+          --output perf-results.json
+
+    - name: Upload perf results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: perf-results-${{ github.sha }}
+        path: hindsight-dev/perf-results.json
+        retention-days: 90
+
+  locomo:
+    if: inputs.locomo_skip != true
+    runs-on: ubuntu-latest
+    env:
+      HINDSIGHT_API_LLM_PROVIDER: vertexai
+      HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY: /tmp/gcp-credentials.json
+      HINDSIGHT_API_LLM_MODEL: google/gemini-2.5-flash-lite
+      HINDSIGHT_API_JUDGE_LLM_PROVIDER: vertexai
+      HINDSIGHT_API_JUDGE_LLM_MODEL: google/gemini-2.5-flash-lite
+      HINDSIGHT_API_ANSWER_LLM_PROVIDER: vertexai
+      HINDSIGHT_API_ANSWER_LLM_MODEL: google/gemini-3.1-pro-preview
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ inputs.ref || github.ref }}
+
+    - name: Setup GCP credentials
+      run: |
+        printf '%s' '${{ secrets.GCP_VERTEXAI_CREDENTIALS }}' > /tmp/gcp-credentials.json
+        PROJECT_ID=$(jq -r '.project_id' /tmp/gcp-credentials.json)
+        echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version-file: ".python-version"
+
+    - name: Cache HuggingFace models
+      uses: actions/cache@v5
+      with:
+        path: ~/.cache/huggingface
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-huggingface-
+
+    - name: Pre-download models
+      working-directory: ./hindsight-api-slim
+      run: |
+        uv run --frozen --all-extras --index-strategy unsafe-best-match python -c "
+        from sentence_transformers import SentenceTransformer
+        print('Downloading embedding model...')
+        SentenceTransformer('BAAI/bge-small-en-v1.5')
+        print('Model downloaded successfully')
+        "
+
+    - name: Install hindsight-dev dependencies
+      run: |
+        cd hindsight-dev && uv sync --frozen --all-extras --index-strategy unsafe-best-match
+
+    - name: Run LoComo benchmark
+      run: |
+        MAX_CONV_ARG=""
+        if [ "${{ inputs.locomo_max_conversations }}" != "0" ] && [ -n "${{ inputs.locomo_max_conversations }}" ]; then
+          MAX_CONV_ARG="--max-conversations ${{ inputs.locomo_max_conversations }}"
+        fi
+        uv run python hindsight-dev/benchmarks/locomo/locomo_benchmark.py \
+          --wait-consolidation \
+          $MAX_CONV_ARG
+
+    - name: Upload LoComo results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: locomo-results-${{ github.sha }}
+        path: hindsight-dev/benchmarks/locomo/results/
+        retention-days: 90

.github/workflows/test.yml

@@ -49,6 +49,7 @@ jobs:
       integrations-opencode: ${{ steps.filter.outputs.integrations-opencode }}
       integrations-cloudflare-oauth-proxy: ${{ steps.filter.outputs.integrations-cloudflare-oauth-proxy }}
       integrations-lockfiles: ${{ steps.filter.outputs.integrations-lockfiles }}
+      integrations-openai-agents: ${{ steps.filter.outputs.integrations-openai-agents }}
       dev: ${{ steps.filter.outputs.dev }}
       ci: ${{ steps.filter.outputs.ci }}
       # Secrets are available for internal PRs, pull_request_review, and workflow_dispatch.
@@ -133,6 +134,8 @@ jobs:
             - 'hindsight-integrations/*/package-lock.json'
             - 'hindsight-integrations/*/package.json'
             - 'scripts/check-integration-lockfiles.sh'
+          integrations-openai-agents:
+            - 'hindsight-integrations/openai-agents/**'
           dev:
             - 'hindsight-dev/**'
           ci:
@@ -2043,6 +2046,43 @@ jobs:
       working-directory: ./hindsight-integrations/llamaindex
       run: uv run pytest tests -v
 
+  test-openai-agents-integration:
+    needs: [detect-changes]
+    if: >-
+      github.event_name != 'pull_request_review' &&
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-openai-agents == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version-file: ".python-version"
+
+    - name: Build openai-agents integration
+      working-directory: ./hindsight-integrations/openai-agents
+      run: uv build
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/openai-agents
+      run: uv sync --frozen
+
+    - name: Run tests
+      working-directory: ./hindsight-integrations/openai-agents
+      run: uv run pytest tests -v
+
   test-pip-slim:
     needs: [detect-changes]
     if: >-
@@ -2547,6 +2587,9 @@ jobs:
     - name: Run generate-openapi
       run: ./scripts/generate-openapi.sh
 
+    - name: Run generate-bank-template-schema
+      run: ./scripts/generate-bank-template-schema.sh
+
     - name: Run generate-clients
       run: ./scripts/generate-clients.sh
 
@@ -2566,6 +2609,7 @@ jobs:
           echo ""
           echo "Please run the following commands locally and commit the changes:"
           echo "  ./scripts/generate-openapi.sh"
+          echo "  ./scripts/generate-bank-template-schema.sh"
           echo "  ./scripts/generate-clients.sh"
           echo "  ./scripts/generate-docs-skill.sh"
           echo "  ./scripts/hooks/lint.sh"

.prettierrc.json

@@ -0,0 +1,7 @@
+{
+  "semi": true,
+  "singleQuote": false,
+  "tabWidth": 2,
+  "trailingComma": "es5",
+  "printWidth": 100
+}

CLAUDE.md

@@ -68,8 +68,9 @@ cd hindsight-control-plane && npm run dev
 ./scripts/benchmarks/run-locomo.sh
 
 # Performance benchmarks
+./scripts/benchmarks/run-perf-test.sh                      # System perf (mock LLM + pg0)
+./scripts/benchmarks/run-perf-test.sh --scale tiny          # Quick smoke test
 ./scripts/benchmarks/run-consolidation.sh
-./scripts/benchmarks/run-retain-perf.sh --document <path>  # Requires API server running
 
 # Results viewer
 ./scripts/benchmarks/start-visualizer.sh  # View results at localhost:8001

helm/hindsight/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: hindsight
 description: Hindsight helm chart
 type: application
-version: 0.5.1
-appVersion: "0.5.1"
+version: 0.5.4
+appVersion: "0.5.4"
 keywords:
   - ai
   - memory

hindsight-all-npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vectorize-io/hindsight-all",
-  "version": "0.5.1",
+  "version": "0.5.4",
   "description": "Node.js programmatic lifecycle manager for Hindsight — embeds a local hindsight daemon in a Node application. Pair with @vectorize-io/hindsight-client for memory operations.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

hindsight-all-slim/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hindsight-all-slim"
-version = "0.5.1"
+version = "0.5.4"
 description = "Hindsight: Agent Memory That Works Like Human Memory - Slim All-in-One Bundle"
 readme = "README.md"
 requires-python = ">=3.11"

hindsight-all/hindsight/embedded.py

@@ -71,7 +71,7 @@ class HindsightEmbedded:
         llm_model: Model name to use
         llm_base_url: Optional custom base URL for LLM API
         database_url: Optional database URL override (default: profile-specific pg0)
-        idle_timeout: Seconds before daemon auto-exits when idle (default: 300)
+        idle_timeout: Seconds before daemon auto-exits when idle (default: 0, disabled)
         log_level: Daemon log level (default: "info")
         ui: Whether to start the control plane web UI alongside the daemon (default: False)
         ui_port: Port for the UI. Defaults to daemon_port + 10000.
@@ -86,7 +86,7 @@ class HindsightEmbedded:
         llm_model: str = "openai/gpt-oss-120b",
         llm_base_url: Optional[str] = None,
         database_url: Optional[str] = None,
-        idle_timeout: int = 300,
+        idle_timeout: int = 0,
         log_level: str = "info",
         ui: bool = False,
         ui_port: Optional[int] = None,
@@ -102,7 +102,7 @@ class HindsightEmbedded:
             llm_model: Model name to use
             llm_base_url: Optional custom base URL for LLM API
             database_url: Optional database URL override
-            idle_timeout: Seconds before daemon auto-exits when idle
+            idle_timeout: Seconds before daemon auto-exits when idle (0 = disabled)
             log_level: Daemon log level
             ui: Whether to start the control plane web UI alongside the daemon
             ui_port: Port for the UI (defaults to daemon_port + 10000)
@@ -142,14 +142,37 @@ class HindsightEmbedded:
         self._memories_api: Optional[MemoriesAPI] = None
 
     def _ensure_started(self):
-        """Ensure daemon is running (thread-safe)."""
+        """Ensure daemon is running (thread-safe), restarting if crashed."""
         if self._started and self._client is not None:
-            return
+            if self._manager.is_running(self.profile):
+                return
+            # Daemon crashed — reset state and fall through to restart
+            logger.warning(
+                "Daemon for profile '%s' is no longer responsive, restarting...",
+                self.profile,
+            )
+            try:
+                self._client.close()
+            except Exception:
+                logger.debug("Error closing stale client", exc_info=True)
+            self._client = None
+            self._started = False
 
         with self._lock:
             # Double-check after acquiring lock
             if self._started and self._client is not None:
-                return
+                if self._manager.is_running(self.profile):
+                    return
+                logger.warning(
+                    "Daemon for profile '%s' is no longer responsive (lock path), restarting...",
+                    self.profile,
+                )
+                try:
+                    self._client.close()
+                except Exception:
+                    logger.debug("Error closing stale client", exc_info=True)
+                self._client = None
+                self._started = False
 
             if self._closed:
                 raise RuntimeError(
@@ -253,23 +276,10 @@ class HindsightEmbedded:
         This allows HindsightEmbedded to expose all HindsightClient methods
         without manually wrapping each one.
         """
-        # Ensure server is started before proxying
+        # Ensure server is started (and restart if crashed) before proxying
         self._ensure_started()
 
-        # Get the attribute from the underlying client
-        attr = getattr(self._client, name)
-
-        # If it's a callable, wrap it to ensure server is started
-        # (shouldn't be needed since _ensure_started already called, but defensive)
-        if callable(attr):
-
-            def wrapper(*args, **kwargs):
-                self._ensure_started()
-                return attr(*args, **kwargs)
-
-            return wrapper
-
-        return attr
+        return getattr(self._client, name)
 
     def __enter__(self):
         """Context manager entry - ensures server is started."""
@@ -394,11 +404,8 @@ class HindsightEmbedded:
         """
         Get the underlying Hindsight client for direct access.
 
-        WARNING: Using this property directly means daemon restarts won't be
-        handled automatically. Prefer using the API namespaces (banks, mental_models,
-        directives, memories) or direct method calls on HindsightEmbedded instead.
-
-        Ensures daemon is started before returning the client.
+        Ensures daemon is started (and restarts it if it has crashed) before
+        returning the client.
 
         Returns:
             Hindsight: The underlying client instance
@@ -409,9 +416,8 @@ class HindsightEmbedded:
 
             embedded = HindsightEmbedded(profile="myapp", ...)
 
-            # Direct access (not recommended - daemon crashes won't be handled)
             client = embedded.client
-            banks = client.list_banks()  # If daemon crashes, this will fail
+            banks = client.list_banks()
             ```
         """
         self._ensure_started()
@@ -425,8 +431,13 @@ class HindsightEmbedded:
 
     @property
     def is_running(self) -> bool:
-        """Check if the client is initialized."""
-        return self._started and not self._closed and self._client is not None
+        """Check if the client is initialized and the daemon is responsive."""
+        return (
+            self._started
+            and not self._closed
+            and self._client is not None
+            and self._manager.is_running(self.profile)
+        )
 
     @property
     def ui_url(self) -> str:

hindsight-all/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-all"
-version = "0.5.1"
+version = "0.5.4"
 description = "Hindsight: Agent Memory That Works Like Human Memory - All-in-One Bundle"
 readme = "README.md"
 requires-python = ">=3.11"

hindsight-all/tests/test_embedded.py

@@ -401,3 +401,42 @@ def test_embedded_ui_flag(llm_config):
 
     finally:
         client.close()
+
+
+def test_embedded_daemon_crash_recovery(llm_config):
+    """
+    Test that HindsightEmbedded recovers when the daemon crashes.
+
+    Simulates a crash by stopping the daemon, then verifies
+    that the next operation transparently restarts it.
+    """
+    profile = f"test_crash_{uuid.uuid4().hex[:8]}"
+    bank_id = f"bank_{uuid.uuid4().hex[:8]}"
+
+    client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
+
+    try:
+        # Start daemon and store a memory
+        result = client.retain(bank_id=bank_id, content="Before crash")
+        assert result.success, "Initial retain should succeed"
+        assert client.is_running, "Daemon should be running"
+
+        original_url = client.url
+
+        # Simulate daemon crash by stopping it
+        client._manager.stop(client.profile)
+        assert not client._manager.is_running(client.profile), (
+            "Daemon should be stopped after simulated crash"
+        )
+
+        # Next operation should transparently restart the daemon
+        result2 = client.retain(bank_id=bank_id, content="After crash recovery")
+        assert result2.success, "Retain after crash recovery should succeed"
+        assert client.is_running, "Daemon should be running again after recovery"
+
+        # Verify recall still works
+        recall_result = client.recall(bank_id=bank_id, query="crash")
+        assert isinstance(recall_result.results, list), "Recall should return results"
+
+    finally:
+        client.close()

hindsight-api-slim/hindsight_api/__init__.py

@@ -46,4 +46,4 @@ __all__ = [
     "RemoteTEICrossEncoder",
     "LLMConfig",
 ]
-__version__ = "0.5.1"
+__version__ = "0.5.4"

hindsight-api-slim/hindsight_api/admin/cli.py

@@ -375,6 +375,140 @@ def decommission_worker(
         typer.echo(f"No tasks found for worker '{worker_id}'")
 
 
+async def _decommission_all_workers(db_url: str, schema: str = "public") -> list[dict[str, Any]]:
+    """Release all processing tasks from all workers, setting them back to pending status."""
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+
+    conn = await asyncpg.connect(resolved_url)
+    try:
+        table = _fq_table("async_operations", schema)
+        rows = await conn.fetch(
+            f"""
+            UPDATE {table}
+            SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
+            WHERE status = 'processing'
+            RETURNING operation_id, worker_id, operation_type
+            """,
+        )
+        return [dict(r) for r in rows]
+    finally:
+        await conn.close()
+
+
+@app.command(name="decommission-workers")
+def decommission_workers(
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema"),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"),
+):
+    """Release all processing tasks from all workers (sets status back to pending).
+
+    Use this command to recover from situations where one or more workers have crashed
+    or been removed without graceful shutdown. All tasks currently in 'processing' status
+    will be released back to the queue regardless of which worker owns them.
+    """
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    if not yes:
+        typer.confirm(
+            "This will release ALL processing tasks from ALL workers back to pending. Continue?",
+            abort=True,
+        )
+
+    typer.echo(f"Decommissioning all workers (schema: {schema})...")
+
+    released = asyncio.run(_decommission_all_workers(config.database_url, schema))
+
+    if released:
+        # Group by worker_id for summary
+        by_worker: dict[str, int] = {}
+        for row in released:
+            wid = row["worker_id"] or "unknown"
+            by_worker[wid] = by_worker.get(wid, 0) + 1
+
+        typer.echo(f"Released {len(released)} task(s):")
+        for wid, count in by_worker.items():
+            typer.echo(f"  {wid}: {count} task(s)")
+    else:
+        typer.echo("No processing tasks found")
+
+
+async def _worker_status(db_url: str, schema: str = "public") -> list[dict[str, Any]]:
+    """Get all processing tasks grouped by worker with their last update time."""
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+
+    conn = await asyncpg.connect(resolved_url)
+    try:
+        table = _fq_table("async_operations", schema)
+        rows = await conn.fetch(
+            f"""
+            SELECT worker_id, operation_id, operation_type, bank_id,
+                   claimed_at, updated_at,
+                   now() - claimed_at AS running_for,
+                   now() - updated_at AS last_update_ago
+            FROM {table}
+            WHERE status = 'processing'
+            ORDER BY worker_id, claimed_at
+            """,
+        )
+        return [dict(r) for r in rows]
+    finally:
+        await conn.close()
+
+
+@app.command(name="worker-status")
+def worker_status(
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema"),
+):
+    """Show all currently processing tasks grouped by worker.
+
+    Displays each worker's active tasks with operation type, bank, how long
+    the task has been running, and when it was last updated. Useful for
+    identifying dead workers with orphaned tasks.
+    """
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    rows = asyncio.run(_worker_status(config.database_url, schema))
+
+    if not rows:
+        typer.echo("No processing tasks found")
+        return
+
+    # Group by worker_id
+    by_worker: dict[str, list[dict[str, Any]]] = {}
+    for row in rows:
+        wid = row["worker_id"] or "unknown"
+        by_worker.setdefault(wid, []).append(row)
+
+    typer.echo(f"Processing tasks across {len(by_worker)} worker(s):\n")
+    for wid, tasks in by_worker.items():
+        typer.echo(f"Worker: {wid} ({len(tasks)} task(s))")
+        for task in tasks:
+            op_id = str(task["operation_id"])[:8]
+            running_for = task["running_for"]
+            last_update = task["last_update_ago"]
+            typer.echo(
+                f"  {op_id}  {task['operation_type']:<20s} bank={task['bank_id']}"
+                f"  running={running_for}  last_update={last_update} ago"
+            )
+        typer.echo("")
+
+
 def main():
     app()
 

hindsight-api-slim/hindsight_api/alembic/env.py

@@ -12,6 +12,7 @@ from dotenv import load_dotenv
 from sqlalchemy import engine_from_config, pool
 
 # Import your models here
+from hindsight_api.db_url import to_libpq_url
 from hindsight_api.models import Base
 
 
@@ -65,11 +66,11 @@ def get_database_url() -> str:
                 "Set HINDSIGHT_API_DATABASE_URL environment variable or pass database_url to run_migrations()."
             )
 
-    # For migrations, use psycopg2 (sync driver) to avoid pgbouncer prepared statement issues
-    if database_url.startswith("postgresql+asyncpg://"):
-        database_url = database_url.replace("postgresql+asyncpg://", "postgresql://", 1)
-    elif database_url.startswith("postgres+asyncpg://"):
-        database_url = database_url.replace("postgres+asyncpg://", "postgresql://", 1)
+    # For migrations, use the sync psycopg2 driver (avoids pgbouncer prepared
+    # statement issues and is required since create_engine is the sync API).
+    # Also translates ?ssl=require (SQLAlchemy asyncpg style) to ?sslmode=require
+    # (libpq style) for external-PostgreSQL deployments.
+    database_url = to_libpq_url(database_url)
 
     # Update config with processed URL for engine_from_config to use
     config.set_main_option("sqlalchemy.url", database_url)

hindsight-api-slim/hindsight_api/alembic/versions/2eee35aa3cfc_case_insensitive_entities_trgm_index.py

@@ -4,8 +4,8 @@ The previous GIN trigram index on canonical_name was case-sensitive, causing
 "Alice" and "alice" to have different trigram sets. This recreates it on
 LOWER(canonical_name) so the % operator matches case-insensitively.
 
-Revision ID: d6e7f8a9b0c1
-Revises: c5d6e7f8a9b0
+Revision ID: 2eee35aa3cfc
+Revises: d6e7f8a9b0c1
 Create Date: 2026-03-31
 """
 
@@ -13,8 +13,8 @@ from collections.abc import Sequence
 
 from alembic import context, op
 
-revision: str = "d6e7f8a9b0c1"
-down_revision: str | Sequence[str] | None = "c5d6e7f8a9b0"
+revision: str = "2eee35aa3cfc"
+down_revision: str | Sequence[str] | None = "d6e7f8a9b0c1"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 

hindsight-api-slim/hindsight_api/alembic/versions/8c6fa6f7230b_merge_v053_divergent_heads.py

@@ -0,0 +1,40 @@
+"""Merge divergent migration heads for v0.5.3
+
+v0.5.3 shipped with two migration heads that were never unified:
+
+  * ``c4x5y6z7a8b9`` — delta-refresh chain
+    (``add_last_refreshed_source_query`` ->
+     ``add_structured_content_to_mental_models`` ->
+     ``backsweep_orphan_observations_v2``)
+
+  * ``h3i4j5k6l7m8`` — per-bank vector indexes / audit log chain
+    (the ``merge_heads_and_add_unit_entities_index`` subtree)
+
+Both fork from ``z1u2v3w4x5y6``. Upgrades from v0.5.2 still succeed — the
+walker applies the three c4x5 revisions and leaves the database stamped at
+both heads — but the result is a split DAG: ``alembic upgrade head``
+(singular) is ambiguous, and any future migration has to pick one head as
+its parent, orphaning the other.
+
+This revision linearises the DAG into a single head. It has no schema
+effect.
+
+Revision ID: 8c6fa6f7230b
+Revises: c4x5y6z7a8b9, h3i4j5k6l7m8
+Create Date: 2026-04-18
+"""
+
+from collections.abc import Sequence
+
+revision: str = "8c6fa6f7230b"
+down_revision: str | Sequence[str] | None = ("c4x5y6z7a8b9", "h3i4j5k6l7m8")
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    pass
+
+
+def downgrade() -> None:
+    pass

hindsight-api-slim/hindsight_api/alembic/versions/a2v3w4x5y6z7_add_last_refreshed_source_query.py

@@ -0,0 +1,38 @@
+"""Add last_refreshed_source_query column to mental_models
+
+Revision ID: a2v3w4x5y6z7
+Revises: z1u2v3w4x5y6
+Create Date: 2026-04-15
+
+Tracks the source_query that was used during the most recent refresh.
+Used by delta-mode refresh to detect when the query has changed: if it has,
+delta mode falls back to a full regeneration because the surgical-edit
+assumption (same topic, new facts) no longer holds.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "a2v3w4x5y6z7"
+down_revision: str | Sequence[str] | None = "z1u2v3w4x5y6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD COLUMN IF NOT EXISTS last_refreshed_source_query TEXT
+    """)
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS last_refreshed_source_query")

hindsight-api-slim/hindsight_api/alembic/versions/a4b5c6d7e8f9_fix_per_bank_vector_index_type.py

@@ -1,7 +1,7 @@
 """Fix per-bank vector indexes to match configured extension
 
 Revision ID: a4b5c6d7e8f9
-Revises: d6e7f8a9b0c1
+Revises: 2eee35aa3cfc
 Create Date: 2026-04-01
 
 Migration d5e6f7a8b9c0 hardcoded HNSW when creating per-bank partial vector
@@ -21,7 +21,7 @@ from alembic import context, op
 from sqlalchemy import text
 
 revision: str = "a4b5c6d7e8f9"
-down_revision: str | Sequence[str] | None = "d6e7f8a9b0c1"
+down_revision: str | Sequence[str] | None = "2eee35aa3cfc"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 

hindsight-api-slim/hindsight_api/alembic/versions/b3w4x5y6z7a8_add_structured_content_to_mental_models.py

@@ -0,0 +1,44 @@
+"""Add structured_content JSONB column to mental_models
+
+Revision ID: b3w4x5y6z7a8
+Revises: a2v3w4x5y6z7
+Create Date: 2026-04-16
+
+Stores the structured representation of a mental model document (sections,
+blocks). The plain ``content`` column remains the rendered markdown shown to
+users. ``structured_content`` is the source of truth for delta-mode refreshes:
+each refresh applies a list of typed operations to the structured doc, then
+re-renders to markdown — so unchanged sections come through byte-identical
+without an LLM round-trip.
+
+Nullable: existing markdown-only mental models continue to work in full mode;
+the column is populated lazily the first time a model is refreshed in delta
+mode.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "b3w4x5y6z7a8"
+down_revision: str | Sequence[str] | None = "a2v3w4x5y6z7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD COLUMN IF NOT EXISTS structured_content JSONB
+    """)
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS structured_content")

hindsight-api-slim/hindsight_api/alembic/versions/c4x5y6z7a8b9_backsweep_orphan_observations_v2.py

@@ -0,0 +1,66 @@
+"""backsweep_orphan_observations_v2
+
+Re-run of Pass 2 from migration ``g7h8i9j0k1l2_backsweep_orphan_observations``
+to sweep observations that became orphaned between then and now.
+
+Why we need it again:
+  ``fact_storage.handle_document_tracking`` (the retain/upsert path) deleted
+  the existing document via the FK cascade — which removes the source
+  ``memory_units`` — but never invalidated the observations derived from
+  them.  Only the explicit ``MemoryEngine.delete_document`` API called
+  ``_delete_stale_observations_for_memories``.  Every document re-ingest
+  therefore left orphan observations whose ``source_memory_ids`` arrays
+  pointed at IDs that no longer existed in ``memory_units``.
+
+  ``handle_document_tracking`` now calls the same cleanup helper before the
+  cascade, so no new orphans will accumulate going forward.  This migration
+  cleans up the historical residue.
+
+Identical to Pass 2 of g7h8i9j0k1l2.  Pass 1 (memory_units whose bank is
+gone) is intentionally not re-run; that scenario has no fresh source.
+
+Revision ID: c4x5y6z7a8b9
+Revises: b3w4x5y6z7a8
+Create Date: 2026-04-16
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "c4x5y6z7a8b9"
+down_revision: str | Sequence[str] | None = "b3w4x5y6z7a8"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    mu = f"{schema}memory_units"
+
+    # Delete observations whose every source_memory_id refers to a now-deleted
+    # memory_unit (or the array is empty).  Observations with at least one
+    # surviving source are left alone — the consolidation engine will refresh
+    # their text on the next pass.
+    op.execute(
+        f"""
+        DELETE FROM {mu} orphan
+        WHERE orphan.fact_type = 'observation'
+          AND NOT EXISTS (
+              SELECT 1
+              FROM {mu} src
+              WHERE src.id = ANY(orphan.source_memory_ids)
+                AND src.bank_id = orphan.bank_id
+          )
+        """
+    )
+
+
+def downgrade() -> None:
+    # Deleted rows cannot be restored.
+    pass

hindsight-api-slim/hindsight_api/alembic/versions/d6e7f8a9b0c1_drop_documents_metadata_column.py

@@ -0,0 +1,39 @@
+"""Drop unused metadata column from documents table
+
+Revision ID: d6e7f8a9b0c1
+Revises: c2d3e4f5g6h7, c5d6e7f8a9b0
+Create Date: 2026-03-30
+
+The metadata column on documents was always stored as an empty dict {}.
+Actual document metadata is stored inside retain_params.metadata.
+
+This migration was originally shipped in v0.4.22, then its file was deleted
+in v0.5.0 (and its revision ID accidentally reused by 2eee35aa3cfc).
+Restoring the file so that databases stamped at this revision can upgrade
+cleanly to v0.5.x+.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "d6e7f8a9b0c1"
+down_revision: str | Sequence[str] | None = ("c2d3e4f5g6h7", "c5d6e7f8a9b0")
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}documents DROP COLUMN IF EXISTS metadata")
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}documents ADD COLUMN IF NOT EXISTS metadata jsonb DEFAULT '{{}}'")

hindsight-api-slim/hindsight_api/alembic/versions/h3i4j5k6l7m8_merge_heads_and_add_unit_entities_index.py

@@ -1,7 +1,7 @@
 """Merge 3 migration heads and add unit_entities composite index
 
 Revision ID: h3i4j5k6l7m8
-Revises: a4b5c6d7e8f9, c2d3e4f5g6h7, g2h3i4j5k6l7
+Revises: a4b5c6d7e8f9, g2h3i4j5k6l7
 Create Date: 2026-04-07
 
 Merges three unmerged migration heads into one, and adds a composite index
@@ -14,7 +14,7 @@ from collections.abc import Sequence
 from alembic import context, op
 
 revision: str = "h3i4j5k6l7m8"
-down_revision: str | Sequence[str] | None = ("a4b5c6d7e8f9", "c2d3e4f5g6h7", "g2h3i4j5k6l7")
+down_revision: str | Sequence[str] | None = ("a4b5c6d7e8f9", "g2h3i4j5k6l7")
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 

hindsight-api-slim/hindsight_api/api/http.py

@@ -294,6 +294,44 @@ class EntityListResponse(BaseModel):
     offset: int
 
 
+class EntityGraphResponse(BaseModel):
+    """Response model for entity co-occurrence graph endpoint."""
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "nodes": [
+                    {"data": {"id": "uuid-1", "label": "Alice", "mentionCount": 12, "color": "#42a5f5"}},
+                    {"data": {"id": "uuid-2", "label": "Google", "mentionCount": 8, "color": "#42a5f5"}},
+                ],
+                "edges": [
+                    {
+                        "data": {
+                            "id": "uuid-1-uuid-2",
+                            "source": "uuid-1",
+                            "target": "uuid-2",
+                            "linkType": "cooccurrence",
+                            "weight": 5,
+                            "color": "#ffd700",
+                            "lineStyle": "solid",
+                            "lastCooccurred": "2024-02-01T14:00:00Z",
+                        }
+                    }
+                ],
+                "total_entities": 2,
+                "total_edges": 1,
+                "limit": 1000,
+            }
+        }
+    )
+
+    nodes: list[dict[str, Any]]
+    edges: list[dict[str, Any]]
+    total_entities: int
+    total_edges: int
+    limit: int
+
+
 class EntityDetailResponse(BaseModel):
     """Response model for entity detail endpoint."""
 
@@ -1416,6 +1454,7 @@ class BankStatsResponse(BaseModel):
                 "failed_operations": 0,
                 "last_consolidated_at": "2024-01-15T10:30:00Z",
                 "pending_consolidation": 0,
+                "failed_consolidation": 0,
                 "total_observations": 45,
             }
         }
@@ -1431,12 +1470,41 @@ class BankStatsResponse(BaseModel):
     links_breakdown: dict[str, dict[str, int]]
     pending_operations: int
     failed_operations: int
+    operations_by_status: dict[str, int] = Field(
+        default_factory=dict,
+        description="Async operations grouped by status (pending, in_progress, completed, failed, cancelled).",
+    )
     # Consolidation stats
     last_consolidated_at: str | None = Field(default=None, description="When consolidation last ran (ISO format)")
     pending_consolidation: int = Field(default=0, description="Number of memories not yet processed into observations")
+    failed_consolidation: int = Field(
+        default=0,
+        description="Number of source memories (world/experience) whose consolidation permanently failed and can be retried via the consolidation recovery endpoint.",
+    )
     total_observations: int = Field(default=0, description="Total number of observations")
 
 
+class MemoryTimeseriesBucket(BaseModel):
+    """One bucket in the memory ingestion time-series."""
+
+    time: str = Field(description="Bucket start timestamp in ISO-8601 (UTC).")
+    world: int = Field(default=0, description="World-fact memories ingested in this bucket.")
+    experience: int = Field(default=0, description="Experience memories ingested in this bucket.")
+    observation: int = Field(default=0, description="Observations recorded in this bucket.")
+
+
+class MemoriesTimeseriesResponse(BaseModel):
+    """Time-series of memory ingestion bucketed by time and fact type."""
+
+    bank_id: str
+    period: str = Field(description="One of: 1h, 12h, 1d, 7d, 30d, 90d.")
+    trunc: str = Field(description="Bucket granularity: minute, hour, day.")
+    buckets: list[MemoryTimeseriesBucket] = Field(
+        default_factory=list,
+        description="Per-bucket counts, always returned fully padded for the requested period.",
+    )
+
+
 # Mental Model models
 
 
@@ -1493,6 +1561,16 @@ class UpdateDirectiveRequest(BaseModel):
 class MentalModelTrigger(BaseModel):
     """Trigger settings for a mental model."""
 
+    mode: Literal["full", "delta"] = Field(
+        default="full",
+        description=(
+            "Refresh mode. 'full' (default) regenerates the mental model content from scratch on each refresh. "
+            "'delta' performs surgical edits against the existing content: unchanged sections are preserved "
+            "byte-for-byte, stale content is removed, new content is added. If the mental model has no existing "
+            "content, or if the source_query has changed since the last refresh, delta mode falls back to a full "
+            "regeneration automatically."
+        ),
+    )
     refresh_after_consolidation: bool = Field(
         default=False,
         description="If true, refresh this mental model after observations consolidation (real-time mode)",
@@ -1526,6 +1604,27 @@ class MentalModelTrigger(BaseModel):
             "Supports nested and/or/not expressions for complex tag-based scoping."
         ),
     )
+    include_chunks: bool | None = Field(
+        default=None,
+        description=(
+            "Override whether the internal recall used during refresh returns raw chunk text. "
+            "None means use the bank/global config default (recall_include_chunks)."
+        ),
+    )
+    recall_max_tokens: int | None = Field(
+        default=None,
+        description=(
+            "Override the token budget for facts returned by the internal recall during refresh. "
+            "None means use the bank/global config default (recall_max_tokens)."
+        ),
+    )
+    recall_chunks_max_tokens: int | None = Field(
+        default=None,
+        description=(
+            "Override the token budget for raw chunks returned by the internal recall during refresh. "
+            "None means use the bank/global config default (recall_chunks_max_tokens)."
+        ),
+    )
 
     @field_validator("fact_types")
     @classmethod
@@ -1555,6 +1654,14 @@ class MentalModelResponse(BaseModel):
         default=None,
         description="Full reflect API response payload including based_on facts and observations",
     )
+    is_stale: bool | None = Field(
+        default=None,
+        description=(
+            "True when new memories matching this mental model's tag/fact_type scope have been "
+            "ingested since last_refreshed_at, or consolidation has pending items. Only populated "
+            "when detail=full."
+        ),
+    )
 
 
 class MentalModelListResponse(BaseModel):
@@ -1673,6 +1780,61 @@ class BankTemplateConfig(BaseModel):
     entities_allow_free_form: bool | None = Field(
         default=None, description="Allow entities outside the label vocabulary"
     )
+    retain_default_strategy: str | None = Field(
+        default=None, description="Name of the default retain strategy (key into retain_strategies map)"
+    )
+    retain_strategies: dict | None = Field(
+        default=None, description="Map of retain strategy name to per-strategy config dict"
+    )
+    retain_chunk_batch_size: int | None = Field(
+        default=None, description="Max chunks per streaming batch (0 disables batching)"
+    )
+    mcp_enabled_tools: list[str] | None = Field(
+        default=None, description="MCP tool allowlist for this bank (None = all tools)"
+    )
+    consolidation_llm_batch_size: int | None = Field(
+        default=None, description="LLM batch size for observation consolidation"
+    )
+    consolidation_source_facts_max_tokens: int | None = Field(
+        default=None, description="Max tokens of source facts per consolidation batch"
+    )
+    consolidation_source_facts_max_tokens_per_observation: int | None = Field(
+        default=None, description="Max tokens of source facts per observation"
+    )
+    max_observations_per_scope: int | None = Field(
+        default=None, description="Max observations to retain per consolidation scope"
+    )
+    reflect_source_facts_max_tokens: int | None = Field(
+        default=None, description="Max tokens of source facts per reflect call"
+    )
+    llm_gemini_safety_settings: list | None = Field(
+        default=None, description="Per-bank Gemini/VertexAI safety filter settings"
+    )
+    recall_budget_function: str | None = Field(
+        default=None, description="Recall budget mapping function: 'fixed' or 'adaptive'"
+    )
+    recall_budget_fixed_low: int | None = Field(
+        default=None, description="Fixed thinking_budget for budget=low (function='fixed')"
+    )
+    recall_budget_fixed_mid: int | None = Field(
+        default=None, description="Fixed thinking_budget for budget=mid (function='fixed')"
+    )
+    recall_budget_fixed_high: int | None = Field(
+        default=None, description="Fixed thinking_budget for budget=high (function='fixed')"
+    )
+    recall_budget_adaptive_low: float | None = Field(
+        default=None, description="Ratio of max_tokens for budget=low (function='adaptive')"
+    )
+    recall_budget_adaptive_mid: float | None = Field(
+        default=None, description="Ratio of max_tokens for budget=mid (function='adaptive')"
+    )
+    recall_budget_adaptive_high: float | None = Field(
+        default=None, description="Ratio of max_tokens for budget=high (function='adaptive')"
+    )
+    recall_budget_min: int | None = Field(default=None, description="Floor for the adaptive function (after clamping)")
+    recall_budget_max: int | None = Field(
+        default=None, description="Ceiling for the adaptive function (after clamping)"
+    )
 
     def get_config_updates(self) -> dict[str, Any]:
         """Return only the fields that were explicitly set (non-None)."""
@@ -1957,6 +2119,8 @@ class OperationResponse(BaseModel):
                 "created_at": "2024-01-15T10:30:00Z",
                 "status": "pending",
                 "error_message": None,
+                "retry_count": 0,
+                "next_retry_at": None,
             }
         }
     )
@@ -1968,6 +2132,20 @@ class OperationResponse(BaseModel):
     created_at: str
     status: str
     error_message: str | None
+    retry_count: int | None = Field(
+        default=None,
+        description="Number of times this operation has been retried after failure.",
+    )
+    next_retry_at: str | None = Field(
+        default=None,
+        description=(
+            "When the worker will next attempt this operation. For a pending "
+            "operation, a value in the future indicates the task is waiting "
+            "rather than available for immediate pickup — for example, an "
+            "extension may have raised DeferOperation to park the task until "
+            "some backpressure window opens. Always null for completed tasks."
+        ),
+    )
 
 
 class ConsolidationResponse(BaseModel):
@@ -2077,6 +2255,19 @@ class OperationStatusResponse(BaseModel):
     updated_at: str | None = None
     completed_at: str | None = None
     error_message: str | None = None
+    retry_count: int | None = Field(
+        default=None,
+        description="Number of times this operation has been retried after failure.",
+    )
+    next_retry_at: str | None = Field(
+        default=None,
+        description=(
+            "When the worker will next attempt this operation. For a pending "
+            "operation, a value in the future indicates the task is parked "
+            "(e.g. by an extension raising DeferOperation) rather than awaiting "
+            "immediate pickup."
+        ),
+    )
     result_metadata: dict[str, Any] | None = Field(
         default=None,
         description="Internal metadata for debugging. Structure may change without notice. Not for production use.",
@@ -2084,6 +2275,10 @@ class OperationStatusResponse(BaseModel):
     child_operations: list[ChildOperationStatus] | None = Field(
         default=None, description="Child operations for batch operations (if applicable)"
     )
+    task_payload: dict[str, Any] | None = Field(
+        default=None,
+        description="Raw task payload (params the operation was submitted with). Only populated when include_payload=true.",
+    )
 
 
 class AsyncOperationSubmitResponse(BaseModel):
@@ -2420,7 +2615,7 @@ def create_app(
                 schema=schema,
                 tenant_extension=memory._tenant_extension,
                 max_slots=config.worker_max_slots,
-                consolidation_max_slots=config.worker_consolidation_max_slots,
+                slot_reservations=config.worker_slot_reservations,
             )
             poller_task = asyncio.create_task(poller.run())
             logging.info(f"Worker poller started (worker_id={worker_id})")
@@ -2767,6 +2962,7 @@ def _register_routes(app: FastAPI):
         bank_id: str,
         type: str | None = None,
         q: str | None = None,
+        consolidation_state: str | None = None,
         limit: int = 100,
         offset: int = 0,
         request_context: RequestContext = Depends(get_request_context),
@@ -2781,6 +2977,8 @@ def _register_routes(app: FastAPI):
             bank_id: Memory Bank ID (from path)
             type: Filter by fact type (world, experience, opinion)
             q: Search query for full-text search (searches text and context)
+            consolidation_state: Filter by consolidation state for source memories
+                (world/experience). One of 'failed', 'pending', or 'done'.
             limit: Maximum number of results (default: 100)
             offset: Offset for pagination (default: 0)
         """
@@ -2789,11 +2987,14 @@ def _register_routes(app: FastAPI):
                 bank_id=bank_id,
                 fact_type=type,
                 search_query=q,
+                consolidation_state=consolidation_state,
                 limit=limit,
                 offset=offset,
                 request_context=request_context,
             )
             return data
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
         except OperationValidationError as e:
             raise HTTPException(status_code=e.status_code, detail=e.reason)
         except (AuthenticationError, HTTPException):
@@ -3248,8 +3449,10 @@ def _register_routes(app: FastAPI):
                 links_breakdown=links_breakdown,
                 pending_operations=ops.get("pending", 0),
                 failed_operations=ops.get("failed", 0),
+                operations_by_status=ops,
                 last_consolidated_at=stats["last_consolidated_at"],
                 pending_consolidation=stats["pending_consolidation"],
+                failed_consolidation=stats.get("failed_consolidation", 0),
                 total_observations=stats["total_observations"],
             )
         except OperationValidationError as e:
@@ -3263,6 +3466,35 @@ def _register_routes(app: FastAPI):
             logger.error(f"Error in /v1/default/banks/{bank_id}/stats: {error_detail}")
             raise HTTPException(status_code=500, detail=str(e))
 
+    @app.get(
+        "/v1/default/banks/{bank_id}/stats/memories-timeseries",
+        response_model=MemoriesTimeseriesResponse,
+        summary="Memory ingestion time-series",
+        description="Memories ingested over a period, bucketed by time and broken down by fact type.",
+        operation_id="get_memories_timeseries",
+        tags=["Banks"],
+    )
+    async def api_memories_timeseries(
+        bank_id: str,
+        period: str = "7d",
+        request_context: RequestContext = Depends(get_request_context),
+    ):
+        try:
+            data = await app.state.memory.get_memories_timeseries(
+                bank_id, period=period, request_context=request_context
+            )
+            return MemoriesTimeseriesResponse(**data)
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in /v1/default/banks/{bank_id}/stats/memories-timeseries: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
     @app.get(
         "/v1/default/banks/{bank_id}/entities",
         response_model=EntityListResponse,
@@ -3299,6 +3531,36 @@ def _register_routes(app: FastAPI):
             logger.error(f"Error in /v1/default/banks/{bank_id}/entities: {error_detail}")
             raise HTTPException(status_code=500, detail=str(e))
 
+    @app.get(
+        "/v1/default/banks/{bank_id}/entities/graph",
+        response_model=EntityGraphResponse,
+        summary="Get entity co-occurrence graph",
+        description="Return a graph of entities (nodes) and their co-occurrences (edges) for visualization.",
+        operation_id="get_entity_graph",
+        tags=["Entities"],
+    )
+    async def api_entity_graph(
+        bank_id: str,
+        limit: int = Query(default=1000, description="Maximum number of co-occurrence edges to return"),
+        min_count: int = Query(default=1, description="Minimum cooccurrence_count to include an edge"),
+        request_context: RequestContext = Depends(get_request_context),
+    ):
+        """Return entity co-occurrence graph for a bank."""
+        try:
+            return await app.state.memory.get_entity_graph(
+                bank_id, limit=limit, min_count=min_count, request_context=request_context
+            )
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in /v1/default/banks/{bank_id}/entities/graph: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
     @app.get(
         "/v1/default/banks/{bank_id}/entities/{entity_id}",
         response_model=EntityDetailResponse,
@@ -4168,7 +4430,13 @@ def _register_routes(app: FastAPI):
         tags=["Operations"],
     )
     async def api_get_operation_status(
-        bank_id: str, operation_id: str, request_context: RequestContext = Depends(get_request_context)
+        bank_id: str,
+        operation_id: str,
+        include_payload: bool = Query(
+            default=False,
+            description="Include the raw task payload (submission params) in the response. May be large.",
+        ),
+        request_context: RequestContext = Depends(get_request_context),
     ):
         """Get the status of an async operation."""
         try:
@@ -4178,7 +4446,9 @@ def _register_routes(app: FastAPI):
             except ValueError:
                 raise HTTPException(status_code=400, detail=f"Invalid operation_id format: {operation_id}")
 
-            result = await app.state.memory.get_operation_status(bank_id, operation_id, request_context=request_context)
+            result = await app.state.memory.get_operation_status(
+                bank_id, operation_id, request_context=request_context, include_payload=include_payload
+            )
             return OperationStatusResponse(**result)
         except OperationValidationError as e:
             raise HTTPException(status_code=e.status_code, detail=e.reason)

hindsight-api-slim/hindsight_api/config.py

@@ -177,6 +177,7 @@ ENV_EMBEDDINGS_TEI_URL = "HINDSIGHT_API_EMBEDDINGS_TEI_URL"
 ENV_EMBEDDINGS_OPENAI_API_KEY = "HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY"
 ENV_EMBEDDINGS_OPENAI_MODEL = "HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL"
 ENV_EMBEDDINGS_OPENAI_BASE_URL = "HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL"
+ENV_EMBEDDINGS_OPENAI_BATCH_SIZE = "HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"
 
 # Gemini/Vertex AI embeddings configuration
 ENV_EMBEDDINGS_GEMINI_API_KEY = "HINDSIGHT_API_EMBEDDINGS_GEMINI_API_KEY"
@@ -238,6 +239,7 @@ ENV_RERANKER_LOCAL_BATCH_SIZE = "HINDSIGHT_API_RERANKER_LOCAL_BATCH_SIZE"
 ENV_RERANKER_TEI_URL = "HINDSIGHT_API_RERANKER_TEI_URL"
 ENV_RERANKER_TEI_BATCH_SIZE = "HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE"
 ENV_RERANKER_TEI_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT"
+ENV_RERANKER_TEI_HTTP_TIMEOUT = "HINDSIGHT_API_RERANKER_TEI_HTTP_TIMEOUT"
 ENV_RERANKER_MAX_CANDIDATES = "HINDSIGHT_API_RERANKER_MAX_CANDIDATES"
 ENV_RERANKER_FLASHRANK_MODEL = "HINDSIGHT_API_RERANKER_FLASHRANK_MODEL"
 ENV_RERANKER_FLASHRANK_CACHE_DIR = "HINDSIGHT_API_RERANKER_FLASHRANK_CACHE_DIR"
@@ -265,6 +267,7 @@ ENV_PORT = "HINDSIGHT_API_PORT"
 ENV_BASE_PATH = "HINDSIGHT_API_BASE_PATH"
 ENV_LOG_LEVEL = "HINDSIGHT_API_LOG_LEVEL"
 ENV_LOG_FORMAT = "HINDSIGHT_API_LOG_FORMAT"
+ENV_LOG_JSON_FIELDS = "HINDSIGHT_API_LOG_JSON_FIELDS"
 ENV_WORKERS = "HINDSIGHT_API_WORKERS"
 ENV_MCP_ENABLED = "HINDSIGHT_API_MCP_ENABLED"
 ENV_MCP_ENABLED_TOOLS = "HINDSIGHT_API_MCP_ENABLED_TOOLS"
@@ -333,12 +336,14 @@ ENV_FILE_DELETE_AFTER_RETAIN = "HINDSIGHT_API_FILE_DELETE_AFTER_RETAIN"
 # Observations settings (consolidated knowledge from facts)
 ENV_ENABLE_OBSERVATIONS = "HINDSIGHT_API_ENABLE_OBSERVATIONS"
 ENV_CONSOLIDATION_BATCH_SIZE = "HINDSIGHT_API_CONSOLIDATION_BATCH_SIZE"
+ENV_CONSOLIDATION_MAX_MEMORIES_PER_ROUND = "HINDSIGHT_API_CONSOLIDATION_MAX_MEMORIES_PER_ROUND"
 ENV_CONSOLIDATION_LLM_BATCH_SIZE = "HINDSIGHT_API_CONSOLIDATION_LLM_BATCH_SIZE"
 ENV_CONSOLIDATION_MAX_TOKENS = "HINDSIGHT_API_CONSOLIDATION_MAX_TOKENS"
 ENV_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS = "HINDSIGHT_API_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS"
 ENV_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION = (
     "HINDSIGHT_API_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION"
 )
+ENV_CONSOLIDATION_MAX_ATTEMPTS = "HINDSIGHT_API_CONSOLIDATION_MAX_ATTEMPTS"
 ENV_OBSERVATIONS_MISSION = "HINDSIGHT_API_OBSERVATIONS_MISSION"
 ENV_MAX_OBSERVATIONS_PER_SCOPE = "HINDSIGHT_API_MAX_OBSERVATIONS_PER_SCOPE"
 ENV_ENABLE_OBSERVATION_HISTORY = "HINDSIGHT_API_ENABLE_OBSERVATION_HISTORY"
@@ -370,6 +375,7 @@ ENV_DB_POOL_MIN_SIZE = "HINDSIGHT_API_DB_POOL_MIN_SIZE"
 ENV_DB_POOL_MAX_SIZE = "HINDSIGHT_API_DB_POOL_MAX_SIZE"
 ENV_DB_COMMAND_TIMEOUT = "HINDSIGHT_API_DB_COMMAND_TIMEOUT"
 ENV_DB_ACQUIRE_TIMEOUT = "HINDSIGHT_API_DB_ACQUIRE_TIMEOUT"
+ENV_DB_STATEMENT_TIMEOUT = "HINDSIGHT_API_DB_STATEMENT_TIMEOUT"
 
 # Worker configuration (distributed task processing)
 ENV_WORKER_ENABLED = "HINDSIGHT_API_WORKER_ENABLED"
@@ -378,7 +384,18 @@ ENV_WORKER_POLL_INTERVAL_MS = "HINDSIGHT_API_WORKER_POLL_INTERVAL_MS"
 ENV_WORKER_MAX_RETRIES = "HINDSIGHT_API_WORKER_MAX_RETRIES"
 ENV_WORKER_HTTP_PORT = "HINDSIGHT_API_WORKER_HTTP_PORT"
 ENV_WORKER_MAX_SLOTS = "HINDSIGHT_API_WORKER_MAX_SLOTS"
-ENV_WORKER_CONSOLIDATION_MAX_SLOTS = "HINDSIGHT_API_WORKER_CONSOLIDATION_MAX_SLOTS"
+
+# Per-operation-type slot reservations. Each entry maps an operation_type
+# (as stored in async_operations.operation_type) to its env var and default.
+# Adding a new operation type here is the ONLY change needed to make it
+# reservable via env var — config fields, from_env(), and the
+# worker_slot_reservations property all derive from this dict.
+WORKER_SLOT_RESERVATION_TYPES: dict[str, tuple[str, int]] = {
+    "consolidation": ("HINDSIGHT_API_WORKER_CONSOLIDATION_MAX_SLOTS", 2),
+    "retain": ("HINDSIGHT_API_WORKER_RETAIN_MAX_SLOTS", 0),
+    "file_convert_retain": ("HINDSIGHT_API_WORKER_FILE_CONVERT_RETAIN_MAX_SLOTS", 0),
+    "refresh_mental_model": ("HINDSIGHT_API_WORKER_REFRESH_MENTAL_MODEL_MAX_SLOTS", 0),
+}
 ENV_RETAIN_MAX_CONCURRENT = "HINDSIGHT_API_RETAIN_MAX_CONCURRENT"
 
 # Reflect agent settings
@@ -387,6 +404,20 @@ ENV_REFLECT_MAX_CONTEXT_TOKENS = "HINDSIGHT_API_REFLECT_MAX_CONTEXT_TOKENS"
 ENV_REFLECT_WALL_TIMEOUT = "HINDSIGHT_API_REFLECT_WALL_TIMEOUT"
 ENV_REFLECT_MISSION = "HINDSIGHT_API_REFLECT_MISSION"
 ENV_REFLECT_SOURCE_FACTS_MAX_TOKENS = "HINDSIGHT_API_REFLECT_SOURCE_FACTS_MAX_TOKENS"
+ENV_RECALL_INCLUDE_CHUNKS = "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS"
+ENV_RECALL_MAX_TOKENS = "HINDSIGHT_API_RECALL_MAX_TOKENS"
+ENV_RECALL_CHUNKS_MAX_TOKENS = "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS"
+
+# Recall budget mapping (budget enum -> thinking_budget integer)
+ENV_RECALL_BUDGET_FUNCTION = "HINDSIGHT_API_RECALL_BUDGET_FUNCTION"
+ENV_RECALL_BUDGET_FIXED_LOW = "HINDSIGHT_API_RECALL_BUDGET_FIXED_LOW"
+ENV_RECALL_BUDGET_FIXED_MID = "HINDSIGHT_API_RECALL_BUDGET_FIXED_MID"
+ENV_RECALL_BUDGET_FIXED_HIGH = "HINDSIGHT_API_RECALL_BUDGET_FIXED_HIGH"
+ENV_RECALL_BUDGET_ADAPTIVE_LOW = "HINDSIGHT_API_RECALL_BUDGET_ADAPTIVE_LOW"
+ENV_RECALL_BUDGET_ADAPTIVE_MID = "HINDSIGHT_API_RECALL_BUDGET_ADAPTIVE_MID"
+ENV_RECALL_BUDGET_ADAPTIVE_HIGH = "HINDSIGHT_API_RECALL_BUDGET_ADAPTIVE_HIGH"
+ENV_RECALL_BUDGET_MIN = "HINDSIGHT_API_RECALL_BUDGET_MIN"
+ENV_RECALL_BUDGET_MAX = "HINDSIGHT_API_RECALL_BUDGET_MAX"
 
 # Audit log settings
 ENV_AUDIT_LOG_ENABLED = "HINDSIGHT_API_AUDIT_LOG_ENABLED"
@@ -432,7 +463,7 @@ DEFAULT_LLAMACPP_NO_GRAMMAR = False  # True = disable JSON grammar enforcement (
 DEFAULT_LLAMACPP_EXTRA_ARGS = None  # Space-separated extra CLI args for llama.cpp server
 
 DEFAULT_LLM_MAX_CONCURRENT = 32
-DEFAULT_LLM_MAX_RETRIES = 10  # Max retry attempts for LLM API calls
+DEFAULT_LLM_MAX_RETRIES = 3  # Max retry attempts for LLM API calls
 DEFAULT_LLM_INITIAL_BACKOFF = 1.0  # Initial backoff in seconds for retry exponential backoff
 DEFAULT_LLM_MAX_BACKOFF = 60.0  # Max backoff cap in seconds for retry exponential backoff
 DEFAULT_LLM_TIMEOUT = 120.0  # seconds
@@ -450,6 +481,7 @@ DEFAULT_EMBEDDINGS_LOCAL_MODEL = "BAAI/bge-small-en-v1.5"
 DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU = False  # Force CPU mode for local embeddings (avoids MPS/XPC issues on macOS)
 DEFAULT_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE = False  # Security: disabled by default, required for some models
 DEFAULT_EMBEDDINGS_OPENAI_MODEL = "text-embedding-3-small"
+DEFAULT_EMBEDDINGS_OPENAI_BATCH_SIZE = 100
 DEFAULT_EMBEDDINGS_GEMINI_MODEL = "gemini-embedding-001"
 DEFAULT_EMBEDDINGS_GEMINI_OUTPUT_DIMENSIONALITY = 768
 DEFAULT_EMBEDDING_DIMENSION = 384
@@ -466,6 +498,7 @@ DEFAULT_RERANKER_LOCAL_BUCKET_BATCHING = False  # Length-sorted bucket batching:
 DEFAULT_RERANKER_LOCAL_BATCH_SIZE = 32  # Batch size for local reranker predict() calls
 DEFAULT_RERANKER_TEI_BATCH_SIZE = 128
 DEFAULT_RERANKER_TEI_MAX_CONCURRENT = 8
+DEFAULT_RERANKER_TEI_HTTP_TIMEOUT = 30.0  # HTTP timeout for TEI reranker requests (seconds)
 DEFAULT_RERANKER_MAX_CANDIDATES = 300
 DEFAULT_RERANKER_FLASHRANK_MODEL = "ms-marco-MiniLM-L-12-v2"  # Best balance of speed and quality
 DEFAULT_RERANKER_FLASHRANK_CACHE_DIR = None  # Use default cache directory
@@ -551,7 +584,11 @@ DEFAULT_FILE_DELETE_AFTER_RETAIN = True  # Delete file bytes after retain (saves
 DEFAULT_ENABLE_OBSERVATIONS = True  # Observations enabled by default
 DEFAULT_ENABLE_OBSERVATION_HISTORY = True  # Observation history tracking enabled by default
 DEFAULT_ENABLE_MENTAL_MODEL_HISTORY = True  # Mental model history tracking enabled by default
+DEFAULT_CONSOLIDATION_MAX_ATTEMPTS = 3  # Outer retry attempts for consolidation LLM batch calls
 DEFAULT_CONSOLIDATION_BATCH_SIZE = 50  # Memories to load per batch (internal memory optimization)
+DEFAULT_CONSOLIDATION_MAX_MEMORIES_PER_ROUND = (
+    100  # Max memories per consolidation round (0 = unlimited). Limits how long one bank holds a worker slot.
+)
 DEFAULT_CONSOLIDATION_LLM_BATCH_SIZE = 8  # Facts per LLM call (1 = no batching; >1 = batch mode)
 DEFAULT_CONSOLIDATION_MAX_TOKENS = 512  # Max tokens for recall when finding related observations
 DEFAULT_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS = (
@@ -571,6 +608,7 @@ DEFAULT_DB_POOL_MIN_SIZE = 5
 DEFAULT_DB_POOL_MAX_SIZE = 100
 DEFAULT_DB_COMMAND_TIMEOUT = 60  # seconds
 DEFAULT_DB_ACQUIRE_TIMEOUT = 30  # seconds
+DEFAULT_DB_STATEMENT_TIMEOUT = 600  # seconds (Postgres statement_timeout applied on every pool connection; 0 disables)
 
 # Worker configuration (distributed task processing)
 DEFAULT_WORKER_ENABLED = True  # API runs worker by default (standalone mode)
@@ -579,7 +617,6 @@ DEFAULT_WORKER_POLL_INTERVAL_MS = 500  # Poll database every 500ms
 DEFAULT_WORKER_MAX_RETRIES = 3  # Max retries before marking task failed
 DEFAULT_WORKER_HTTP_PORT = 8889  # HTTP port for worker metrics/health
 DEFAULT_WORKER_MAX_SLOTS = 10  # Total concurrent tasks per worker
-DEFAULT_WORKER_CONSOLIDATION_MAX_SLOTS = 2  # Max concurrent consolidation tasks per worker
 DEFAULT_RETAIN_MAX_CONCURRENT = 4  # Max concurrent retain DB phases (HNSW reads + writes). Limits I/O contention.
 
 # Reflect agent settings
@@ -587,6 +624,25 @@ DEFAULT_REFLECT_MAX_ITERATIONS = 10  # Max tool call iterations before forcing r
 DEFAULT_REFLECT_MAX_CONTEXT_TOKENS = 100_000  # Max accumulated context tokens before forcing final prompt
 DEFAULT_REFLECT_WALL_TIMEOUT = 300  # Wall-clock timeout in seconds for the entire reflect operation (5 minutes)
 DEFAULT_REFLECT_SOURCE_FACTS_MAX_TOKENS = -1  # Token budget for source facts in search_observations (-1 = disabled)
+DEFAULT_RECALL_INCLUDE_CHUNKS = True  # Whether internal recall (e.g. mental model refresh) returns raw chunks
+DEFAULT_RECALL_MAX_TOKENS = 2048  # Token budget for facts returned by internal recall
+DEFAULT_RECALL_CHUNKS_MAX_TOKENS = 1000  # Token budget for raw chunks returned by internal recall
+
+# Recall budget mapping
+# "fixed": thinking_budget = recall_budget_fixed_<level> (preserves legacy behavior)
+# "adaptive": thinking_budget = round(max_tokens * recall_budget_adaptive_<level>),
+#             clamped to [recall_budget_min, recall_budget_max]
+RECALL_BUDGET_FUNCTIONS = ("fixed", "adaptive")
+DEFAULT_RECALL_BUDGET_FUNCTION = "fixed"
+DEFAULT_RECALL_BUDGET_FIXED_LOW = 100
+DEFAULT_RECALL_BUDGET_FIXED_MID = 300
+DEFAULT_RECALL_BUDGET_FIXED_HIGH = 1000
+# Adaptive defaults chosen to roughly match fixed defaults at max_tokens=4096
+DEFAULT_RECALL_BUDGET_ADAPTIVE_LOW = 0.025
+DEFAULT_RECALL_BUDGET_ADAPTIVE_MID = 0.075
+DEFAULT_RECALL_BUDGET_ADAPTIVE_HIGH = 0.25
+DEFAULT_RECALL_BUDGET_MIN = 20  # Floor for the adaptive function
+DEFAULT_RECALL_BUDGET_MAX = 2000  # Ceiling for the adaptive function
 
 # Disposition defaults (None = not set, fall back to bank DB value or 3)
 DEFAULT_DISPOSITION_SKEPTICISM = None
@@ -649,6 +705,10 @@ class JsonFormatter(logging.Formatter):
         logging.CRITICAL: "CRITICAL",
     }
 
+    def __init__(self, allowed_fields: frozenset[str] | None = None):
+        super().__init__()
+        self._allowed_fields = allowed_fields
+
     def format(self, record: logging.LogRecord) -> str:
         log_entry = {
             "severity": self.SEVERITY_MAP.get(record.levelno, "DEFAULT"),
@@ -657,10 +717,20 @@ class JsonFormatter(logging.Formatter):
             "logger": record.name,
         }
 
+        # Lazy import to avoid circular dependency (engine imports from config).
+        from hindsight_api.engine.memory_engine import _current_schema
+
+        tenant = _current_schema.get()
+        if tenant:
+            log_entry["tenant"] = tenant
+
         # Add exception info if present
         if record.exc_info:
             log_entry["exception"] = self.formatException(record.exc_info)
 
+        if self._allowed_fields is not None:
+            log_entry = {k: v for k, v in log_entry.items() if k in self._allowed_fields}
+
         return json.dumps(log_entry)
 
 
@@ -669,6 +739,25 @@ def _parse_str_list(value: str) -> list[str]:
     return [v.strip() for v in value.split(",") if v.strip()]
 
 
+def _parse_positive_int(name: str, raw: str | None, default: int) -> int:
+    """
+    Parse an env var that must be a positive integer (>= 1).
+
+    Falls back to ``default`` when unset/empty. Raises ValueError on non-integer
+    or non-positive values so misconfiguration fails fast instead of triggering
+    infinite loops or zero-step range() calls downstream.
+    """
+    if raw is None or raw == "":
+        return default
+    try:
+        parsed = int(raw)
+    except ValueError as e:
+        raise ValueError(f"{name} must be an integer, got {raw!r}") from e
+    if parsed < 1:
+        raise ValueError(f"{name} must be >= 1, got {parsed}")
+    return parsed
+
+
 def _validate_extraction_mode(mode: str) -> str:
     """Validate and normalize extraction mode."""
     mode_lower = mode.lower()
@@ -681,6 +770,18 @@ def _validate_extraction_mode(mode: str) -> str:
     return mode_lower
 
 
+def _validate_recall_budget_function(function: str) -> str:
+    """Validate and normalize recall budget function."""
+    function_lower = function.lower()
+    if function_lower not in RECALL_BUDGET_FUNCTIONS:
+        logger.warning(
+            f"Invalid recall budget function '{function}', must be one of {RECALL_BUDGET_FUNCTIONS}. "
+            f"Defaulting to '{DEFAULT_RECALL_BUDGET_FUNCTION}'."
+        )
+        return DEFAULT_RECALL_BUDGET_FUNCTION
+    return function_lower
+
+
 def _get_default_model_for_provider(provider: str) -> str:
     """Get the default model for a given provider."""
     return PROVIDER_DEFAULT_MODELS.get(provider.lower(), DEFAULT_LLM_MODEL)
@@ -820,6 +921,7 @@ class HindsightConfig:
     reranker_tei_url: str | None
     reranker_tei_batch_size: int
     reranker_tei_max_concurrent: int
+    reranker_tei_http_timeout: float
     reranker_max_candidates: int
     reranker_cohere_api_key: str | None
     reranker_cohere_model: str
@@ -849,6 +951,7 @@ class HindsightConfig:
     base_path: str
     log_level: str
     log_format: str
+    log_json_fields: list[str] | None  # None = all fields; explicit list = allowlist
     mcp_enabled: bool
     mcp_enabled_tools: list[str] | None  # None = all tools; explicit list = allowlist
     mcp_stateless: bool  # True = stateless HTTP (POST-only); False = stateful (supports GET/SSE)
@@ -907,10 +1010,12 @@ class HindsightConfig:
     enable_observation_history: bool
     enable_mental_model_history: bool
     consolidation_batch_size: int
+    consolidation_max_memories_per_round: int
     consolidation_llm_batch_size: int
     consolidation_max_tokens: int
     consolidation_source_facts_max_tokens: int
     consolidation_source_facts_max_tokens_per_observation: int
+    consolidation_max_attempts: int
     observations_mission: str | None
     max_observations_per_scope: int
 
@@ -925,6 +1030,25 @@ class HindsightConfig:
     reflect_mission: str | None
     reflect_source_facts_max_tokens: int
 
+    # Recall settings (used by internal recall, e.g. during mental model refresh)
+    recall_include_chunks: bool
+    recall_max_tokens: int
+    recall_chunks_max_tokens: int
+
+    # Recall budget mapping: how the Budget enum (LOW/MID/HIGH) maps to thinking_budget integer.
+    # function="fixed": use the recall_budget_fixed_* values directly (legacy behavior).
+    # function="adaptive": compute round(max_tokens * recall_budget_adaptive_*),
+    #                      clamped to [recall_budget_min, recall_budget_max].
+    recall_budget_function: str
+    recall_budget_fixed_low: int
+    recall_budget_fixed_mid: int
+    recall_budget_fixed_high: int
+    recall_budget_adaptive_low: float
+    recall_budget_adaptive_mid: float
+    recall_budget_adaptive_high: float
+    recall_budget_min: int
+    recall_budget_max: int
+
     # Disposition settings (hierarchical - can be overridden per bank; None = fall back to DB)
     disposition_skepticism: int | None
     disposition_literalism: int | None
@@ -942,6 +1066,7 @@ class HindsightConfig:
     db_pool_max_size: int
     db_command_timeout: int
     db_acquire_timeout: int
+    db_statement_timeout: int
 
     # Worker configuration (distributed task processing)
     worker_enabled: bool
@@ -950,7 +1075,7 @@ class HindsightConfig:
     worker_max_retries: int
     worker_http_port: int
     worker_max_slots: int
-    worker_consolidation_max_slots: int
+    worker_slot_reservations: dict[str, int]
     retain_max_concurrent: int
 
     # Reflect agent settings
@@ -977,6 +1102,10 @@ class HindsightConfig:
     webhook_event_types: list[str]  # Event types to deliver globally
     webhook_delivery_poll_interval_seconds: int  # How often the delivery worker polls
 
+    # Defaulted fields (source-compatible additions — existing direct constructor callers keep working).
+    # Keep at the end of the dataclass; Python forbids non-default fields after default fields.
+    embeddings_openai_batch_size: int = DEFAULT_EMBEDDINGS_OPENAI_BATCH_SIZE
+
     # Class-level sets for configuration categorization
 
     # CREDENTIAL_FIELDS: Never exposed via API, never configurable per-tenant/bank
@@ -1031,6 +1160,7 @@ class HindsightConfig:
         # Consolidation settings
         "enable_observations",
         "consolidation_llm_batch_size",
+        "consolidation_max_memories_per_round",
         "consolidation_source_facts_max_tokens",
         "consolidation_source_facts_max_tokens_per_observation",
         "observations_mission",
@@ -1038,6 +1168,20 @@ class HindsightConfig:
         # Reflect settings
         "reflect_mission",
         "reflect_source_facts_max_tokens",
+        # Recall settings (used by internal recall, e.g. mental model refresh)
+        "recall_include_chunks",
+        "recall_max_tokens",
+        "recall_chunks_max_tokens",
+        # Recall budget mapping (Budget enum -> thinking_budget integer)
+        "recall_budget_function",
+        "recall_budget_fixed_low",
+        "recall_budget_fixed_mid",
+        "recall_budget_fixed_high",
+        "recall_budget_adaptive_low",
+        "recall_budget_adaptive_mid",
+        "recall_budget_adaptive_high",
+        "recall_budget_min",
+        "recall_budget_max",
         # Disposition settings
         "disposition_skepticism",
         "disposition_literalism",
@@ -1144,6 +1288,16 @@ class HindsightConfig:
                 f"provider: {self.retain_llm_provider or self.llm_provider})"
             )
 
+        # Validate that sum of per-operation slot reservations does not exceed max_slots
+        total_reserved = sum(self.worker_slot_reservations.values())
+        if total_reserved > self.worker_max_slots:
+            reservation_details = ", ".join(f"{k}={v}" for k, v in self.worker_slot_reservations.items() if v > 0)
+            raise ValueError(
+                f"Sum of per-operation slot reservations ({total_reserved}: {reservation_details}) "
+                f"exceeds worker_max_slots ({self.worker_max_slots}). "
+                f"Reduce reservations or increase HINDSIGHT_API_WORKER_MAX_SLOTS."
+            )
+
     @classmethod
     def from_env(cls) -> "HindsightConfig":
         """Create configuration from environment variables."""
@@ -1270,6 +1424,11 @@ class HindsightConfig:
             in ("true", "1"),
             embeddings_tei_url=os.getenv(ENV_EMBEDDINGS_TEI_URL),
             embeddings_openai_base_url=os.getenv(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None,
+            embeddings_openai_batch_size=_parse_positive_int(
+                ENV_EMBEDDINGS_OPENAI_BATCH_SIZE,
+                os.getenv(ENV_EMBEDDINGS_OPENAI_BATCH_SIZE),
+                DEFAULT_EMBEDDINGS_OPENAI_BATCH_SIZE,
+            ),
             # Cohere embeddings (with backward-compatible fallback to shared API key)
             embeddings_cohere_api_key=os.getenv(ENV_EMBEDDINGS_COHERE_API_KEY) or os.getenv(ENV_COHERE_API_KEY),
             embeddings_cohere_model=os.getenv(ENV_EMBEDDINGS_COHERE_MODEL, DEFAULT_EMBEDDINGS_COHERE_MODEL),
@@ -1338,6 +1497,9 @@ class HindsightConfig:
             reranker_tei_max_concurrent=int(
                 os.getenv(ENV_RERANKER_TEI_MAX_CONCURRENT, str(DEFAULT_RERANKER_TEI_MAX_CONCURRENT))
             ),
+            reranker_tei_http_timeout=float(
+                os.getenv(ENV_RERANKER_TEI_HTTP_TIMEOUT, str(DEFAULT_RERANKER_TEI_HTTP_TIMEOUT))
+            ),
             reranker_max_candidates=int(os.getenv(ENV_RERANKER_MAX_CANDIDATES, str(DEFAULT_RERANKER_MAX_CANDIDATES))),
             # Cohere reranker (with backward-compatible fallback to shared API key)
             reranker_cohere_api_key=os.getenv(ENV_RERANKER_COHERE_API_KEY) or os.getenv(ENV_COHERE_API_KEY),
@@ -1382,6 +1544,7 @@ class HindsightConfig:
             base_path=os.getenv(ENV_BASE_PATH, DEFAULT_BASE_PATH),
             log_level=os.getenv(ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL),
             log_format=os.getenv(ENV_LOG_FORMAT, DEFAULT_LOG_FORMAT).lower(),
+            log_json_fields=_parse_str_list(os.getenv(ENV_LOG_JSON_FIELDS, "")) or None,
             mcp_enabled=os.getenv(ENV_MCP_ENABLED, str(DEFAULT_MCP_ENABLED)).lower() == "true",
             mcp_enabled_tools=[t.strip() for t in os.getenv(ENV_MCP_ENABLED_TOOLS).split(",") if t.strip()]
             if os.getenv(ENV_MCP_ENABLED_TOOLS)
@@ -1474,6 +1637,12 @@ class HindsightConfig:
             consolidation_batch_size=int(
                 os.getenv(ENV_CONSOLIDATION_BATCH_SIZE, str(DEFAULT_CONSOLIDATION_BATCH_SIZE))
             ),
+            consolidation_max_memories_per_round=int(
+                os.getenv(
+                    ENV_CONSOLIDATION_MAX_MEMORIES_PER_ROUND,
+                    str(DEFAULT_CONSOLIDATION_MAX_MEMORIES_PER_ROUND),
+                )
+            ),
             consolidation_llm_batch_size=int(
                 os.getenv(ENV_CONSOLIDATION_LLM_BATCH_SIZE, str(DEFAULT_CONSOLIDATION_LLM_BATCH_SIZE))
             ),
@@ -1489,6 +1658,9 @@ class HindsightConfig:
                     str(DEFAULT_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION),
                 )
             ),
+            consolidation_max_attempts=int(
+                os.getenv(ENV_CONSOLIDATION_MAX_ATTEMPTS, str(DEFAULT_CONSOLIDATION_MAX_ATTEMPTS))
+            ),
             observations_mission=os.getenv(ENV_OBSERVATIONS_MISSION) or DEFAULT_OBSERVATIONS_MISSION,
             max_observations_per_scope=int(
                 os.getenv(ENV_MAX_OBSERVATIONS_PER_SCOPE, str(DEFAULT_MAX_OBSERVATIONS_PER_SCOPE))
@@ -1502,6 +1674,7 @@ class HindsightConfig:
             db_pool_max_size=int(os.getenv(ENV_DB_POOL_MAX_SIZE, str(DEFAULT_DB_POOL_MAX_SIZE))),
             db_command_timeout=int(os.getenv(ENV_DB_COMMAND_TIMEOUT, str(DEFAULT_DB_COMMAND_TIMEOUT))),
             db_acquire_timeout=int(os.getenv(ENV_DB_ACQUIRE_TIMEOUT, str(DEFAULT_DB_ACQUIRE_TIMEOUT))),
+            db_statement_timeout=int(os.getenv(ENV_DB_STATEMENT_TIMEOUT, str(DEFAULT_DB_STATEMENT_TIMEOUT))),
             # Worker configuration
             worker_enabled=os.getenv(ENV_WORKER_ENABLED, str(DEFAULT_WORKER_ENABLED)).lower() == "true",
             worker_id=os.getenv(ENV_WORKER_ID) or DEFAULT_WORKER_ID,
@@ -1509,9 +1682,11 @@ class HindsightConfig:
             worker_max_retries=int(os.getenv(ENV_WORKER_MAX_RETRIES, str(DEFAULT_WORKER_MAX_RETRIES))),
             worker_http_port=int(os.getenv(ENV_WORKER_HTTP_PORT, str(DEFAULT_WORKER_HTTP_PORT))),
             worker_max_slots=int(os.getenv(ENV_WORKER_MAX_SLOTS, str(DEFAULT_WORKER_MAX_SLOTS))),
-            worker_consolidation_max_slots=int(
-                os.getenv(ENV_WORKER_CONSOLIDATION_MAX_SLOTS, str(DEFAULT_WORKER_CONSOLIDATION_MAX_SLOTS))
-            ),
+            worker_slot_reservations={
+                op_type: int(os.getenv(env_var, str(default)))
+                for op_type, (env_var, default) in WORKER_SLOT_RESERVATION_TYPES.items()
+                if int(os.getenv(env_var, str(default))) > 0
+            },
             retain_max_concurrent=int(os.getenv(ENV_RETAIN_MAX_CONCURRENT, str(DEFAULT_RETAIN_MAX_CONCURRENT))),
             # Reflect agent settings
             reflect_max_iterations=int(os.getenv(ENV_REFLECT_MAX_ITERATIONS, str(DEFAULT_REFLECT_MAX_ITERATIONS))),
@@ -1523,6 +1698,31 @@ class HindsightConfig:
             reflect_source_facts_max_tokens=int(
                 os.getenv(ENV_REFLECT_SOURCE_FACTS_MAX_TOKENS, str(DEFAULT_REFLECT_SOURCE_FACTS_MAX_TOKENS))
             ),
+            recall_include_chunks=os.getenv(ENV_RECALL_INCLUDE_CHUNKS, str(DEFAULT_RECALL_INCLUDE_CHUNKS)).lower()
+            in ("true", "1", "yes"),
+            recall_max_tokens=int(os.getenv(ENV_RECALL_MAX_TOKENS, str(DEFAULT_RECALL_MAX_TOKENS))),
+            recall_chunks_max_tokens=int(
+                os.getenv(ENV_RECALL_CHUNKS_MAX_TOKENS, str(DEFAULT_RECALL_CHUNKS_MAX_TOKENS))
+            ),
+            recall_budget_function=_validate_recall_budget_function(
+                os.getenv(ENV_RECALL_BUDGET_FUNCTION, DEFAULT_RECALL_BUDGET_FUNCTION)
+            ),
+            recall_budget_fixed_low=int(os.getenv(ENV_RECALL_BUDGET_FIXED_LOW, str(DEFAULT_RECALL_BUDGET_FIXED_LOW))),
+            recall_budget_fixed_mid=int(os.getenv(ENV_RECALL_BUDGET_FIXED_MID, str(DEFAULT_RECALL_BUDGET_FIXED_MID))),
+            recall_budget_fixed_high=int(
+                os.getenv(ENV_RECALL_BUDGET_FIXED_HIGH, str(DEFAULT_RECALL_BUDGET_FIXED_HIGH))
+            ),
+            recall_budget_adaptive_low=float(
+                os.getenv(ENV_RECALL_BUDGET_ADAPTIVE_LOW, str(DEFAULT_RECALL_BUDGET_ADAPTIVE_LOW))
+            ),
+            recall_budget_adaptive_mid=float(
+                os.getenv(ENV_RECALL_BUDGET_ADAPTIVE_MID, str(DEFAULT_RECALL_BUDGET_ADAPTIVE_MID))
+            ),
+            recall_budget_adaptive_high=float(
+                os.getenv(ENV_RECALL_BUDGET_ADAPTIVE_HIGH, str(DEFAULT_RECALL_BUDGET_ADAPTIVE_HIGH))
+            ),
+            recall_budget_min=int(os.getenv(ENV_RECALL_BUDGET_MIN, str(DEFAULT_RECALL_BUDGET_MIN))),
+            recall_budget_max=int(os.getenv(ENV_RECALL_BUDGET_MAX, str(DEFAULT_RECALL_BUDGET_MAX))),
             # Disposition settings (None = fall back to DB value)
             disposition_skepticism=int(os.getenv(ENV_DISPOSITION_SKEPTICISM))
             if os.getenv(ENV_DISPOSITION_SKEPTICISM)
@@ -1613,7 +1813,8 @@ class HindsightConfig:
         handler.setLevel(self.get_python_log_level())
 
         if self.log_format == "json":
-            handler.setFormatter(JsonFormatter())
+            allowed = frozenset(self.log_json_fields) if self.log_json_fields else None
+            handler.setFormatter(JsonFormatter(allowed_fields=allowed))
         else:
             handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s"))
 

hindsight-api-slim/hindsight_api/config_resolver.py

@@ -15,7 +15,12 @@ from typing import Any
 
 import asyncpg
 
-from hindsight_api.config import HindsightConfig, _get_raw_config, normalize_config_dict
+from hindsight_api.config import (
+    RECALL_BUDGET_FUNCTIONS,
+    HindsightConfig,
+    _get_raw_config,
+    normalize_config_dict,
+)
 from hindsight_api.engine.memory_engine import fq_table
 from hindsight_api.extensions.tenant import TenantExtension
 from hindsight_api.models import RequestContext
@@ -256,6 +261,9 @@ class ConfigResolver:
                     "Strategy names must not be empty strings. Remove entries with empty names before saving."
                 )
 
+        # Validate recall budget fields
+        _validate_recall_budget_updates(normalized_updates)
+
         # Merge with existing config (JSONB || operator)
         async with self.pool.acquire() as conn:
             await conn.execute(
@@ -292,6 +300,53 @@ class ConfigResolver:
         logger.info(f"Reset bank config for {bank_id} to defaults")
 
 
+_RECALL_BUDGET_FIXED_KEYS = (
+    "recall_budget_fixed_low",
+    "recall_budget_fixed_mid",
+    "recall_budget_fixed_high",
+)
+_RECALL_BUDGET_ADAPTIVE_KEYS = (
+    "recall_budget_adaptive_low",
+    "recall_budget_adaptive_mid",
+    "recall_budget_adaptive_high",
+)
+
+
+def _validate_recall_budget_updates(updates: dict[str, Any]) -> None:
+    """Validate recall budget config updates. Raises ValueError on invalid input."""
+    if "recall_budget_function" in updates:
+        function = updates["recall_budget_function"]
+        if not isinstance(function, str) or function.lower() not in RECALL_BUDGET_FUNCTIONS:
+            raise ValueError(
+                f"recall_budget_function must be one of {sorted(RECALL_BUDGET_FUNCTIONS)}, got {function!r}"
+            )
+
+    for key in _RECALL_BUDGET_FIXED_KEYS:
+        if key in updates:
+            value = updates[key]
+            if not isinstance(value, int) or isinstance(value, bool) or value < 1:
+                raise ValueError(f"{key} must be a positive integer, got {value!r}")
+
+    for key in _RECALL_BUDGET_ADAPTIVE_KEYS:
+        if key in updates:
+            value = updates[key]
+            if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
+                raise ValueError(f"{key} must be a positive number, got {value!r}")
+
+    for key in ("recall_budget_min", "recall_budget_max"):
+        if key in updates:
+            value = updates[key]
+            if not isinstance(value, int) or isinstance(value, bool) or value < 1:
+                raise ValueError(f"{key} must be a positive integer, got {value!r}")
+
+    if "recall_budget_min" in updates and "recall_budget_max" in updates:
+        if updates["recall_budget_min"] > updates["recall_budget_max"]:
+            raise ValueError(
+                f"recall_budget_min ({updates['recall_budget_min']}) must be <= "
+                f"recall_budget_max ({updates['recall_budget_max']})"
+            )
+
+
 def apply_strategy(config: HindsightConfig, strategy_name: str) -> HindsightConfig:
     """
     Apply a named retain strategy's overrides on top of a resolved config.

hindsight-api-slim/hindsight_api/db_url.py

@@ -0,0 +1,58 @@
+"""Database URL normalization.
+
+Hindsight accepts SQLAlchemy-style URLs like ``postgresql+asyncpg://...?ssl=require``
+for its async engine, but the same string cannot be handed directly to synchronous
+SQLAlchemy (psycopg2) or to :func:`asyncpg.create_pool`, which both expect a
+libpq-compatible URL (``postgresql://...?sslmode=require``).
+
+:func:`to_libpq_url` performs that translation. It is idempotent and safe to
+apply to URLs that are already libpq-compatible, to the ``pg0`` embedded-PG
+marker, or to any non-PostgreSQL string (returned unchanged).
+"""
+
+from __future__ import annotations
+
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+_ASYNCPG_SCHEMES = ("postgresql+asyncpg", "postgres+asyncpg")
+_POSTGRES_SCHEMES = ("postgresql", "postgres") + _ASYNCPG_SCHEMES
+
+
+def to_libpq_url(url: str) -> str:
+    """Normalize a PostgreSQL URL for libpq-style consumers.
+
+    Accepts a SQLAlchemy URL (``postgresql+asyncpg://...``) or a plain libpq
+    URL and returns a form suitable for:
+
+    - :func:`sqlalchemy.create_engine` (sync / psycopg2)
+    - :func:`asyncpg.create_pool`
+
+    Transformations:
+
+    - ``postgresql+asyncpg`` / ``postgres+asyncpg`` / ``postgres`` → ``postgresql``
+    - Query param ``ssl=<mode>`` → ``sslmode=<mode>`` (SQLAlchemy's asyncpg
+      dialect uses ``ssl=``; libpq uses ``sslmode=``)
+
+    Any non-PostgreSQL input (e.g. the ``pg0`` embedded-PG marker, a sqlite
+    URL, an empty string) is returned unchanged. Already-normalized URLs are
+    returned unchanged.
+    """
+    if not url or "://" not in url:
+        return url
+
+    parts = urlsplit(url)
+    if parts.scheme not in _POSTGRES_SCHEMES:
+        return url
+
+    new_scheme = "postgresql"
+
+    new_query_pairs = [
+        ("sslmode", v) if k == "ssl" else (k, v)
+        for k, v in parse_qsl(parts.query, keep_blank_values=True)
+    ]
+    new_query = urlencode(new_query_pairs)
+
+    if new_scheme == parts.scheme and new_query == parts.query:
+        return url
+
+    return urlunsplit((new_scheme, parts.netloc, parts.path, new_query, parts.fragment))

hindsight-api-slim/hindsight_api/engine/consolidation/consolidator.py

@@ -42,6 +42,34 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+async def _filter_live_source_memories(
+    conn: "Connection",
+    bank_id: str,
+    source_memory_ids: list[uuid.UUID],
+) -> list[uuid.UUID]:
+    """Return only the source memory ids that still exist in the bank.
+
+    Uses FOR SHARE to block concurrent deletes from removing a row between the
+    check and the subsequent insert/update. Combined with the delete path running
+    its stale-observation sweep *after* deleting the source row, this closes the
+    race window where consolidation would otherwise produce an orphan observation.
+    """
+    if not source_memory_ids:
+        return []
+    rows = await conn.fetch(
+        f"""
+        SELECT id
+        FROM {fq_table("memory_units")}
+        WHERE id = ANY($1::uuid[]) AND bank_id = $2
+        FOR SHARE
+        """,
+        source_memory_ids,
+        bank_id,
+    )
+    live = {row["id"] for row in rows}
+    return [mid for mid in source_memory_ids if mid in live]
+
+
 class _CreateAction(BaseModel):
     text: str
     source_fact_ids: list[str]  # memory UUIDs from the NEW FACTS list
@@ -219,6 +247,7 @@ async def run_consolidation_job(
 
     perf = ConsolidationPerfLog(bank_id)
     max_memories_per_batch = config.consolidation_batch_size
+    max_memories_per_round = config.consolidation_max_memories_per_round
     llm_batch_size = max(1, config.consolidation_llm_batch_size)
 
     # Check if consolidation is enabled
@@ -281,8 +310,17 @@ async def run_consolidation_job(
     # Track all unique tags from consolidated memories for mental model refresh filtering
     consolidated_tags: set[str] = set()
 
+    round_limit_enabled = max_memories_per_round > 0
+    round_remaining = max_memories_per_round if round_limit_enabled else float("inf")
+    hit_round_limit = False
+
     llm_batch_num = 0
     while True:
+        # Cap fetch size by remaining round budget
+        fetch_limit = (
+            min(max_memories_per_batch, int(round_remaining)) if round_limit_enabled else max_memories_per_batch
+        )
+
         # Fetch next batch of unconsolidated memories
         async with pool.acquire() as conn:
             t0 = time.time()
@@ -299,7 +337,7 @@ async def run_consolidation_job(
                 LIMIT $2
                 """,
                 bank_id,
-                max_memories_per_batch,
+                fetch_limit,
             )
             perf.record_timing("fetch_memories", time.time() - t0)
 
@@ -524,6 +562,25 @@ async def run_consolidation_job(
                 f" | avg={llm_batch_time / len(llm_batch):.3f}s/memory"
             )
 
+        # Update round budget after processing this DB fetch batch
+        if round_limit_enabled:
+            round_remaining -= len(memories)
+            if round_remaining <= 0:
+                hit_round_limit = True
+                break
+
+    # Re-submit consolidation if we hit the round limit and there's likely more work
+    if hit_round_limit:
+        remaining = total_count - stats["memories_processed"]
+        logger.info(
+            f"[CONSOLIDATION] bank={bank_id} hit round limit of {max_memories_per_round} memories,"
+            f" ~{remaining} remaining. Re-queuing consolidation."
+        )
+        try:
+            await memory_engine.submit_async_consolidation(bank_id=bank_id, request_context=request_context)
+        except Exception as e:
+            logger.warning(f"[CONSOLIDATION] bank={bank_id} failed to re-queue consolidation: {e}")
+
     # Build summary
     perf.log(
         f"[3] Results: {stats['memories_processed']} memories -> "
@@ -552,16 +609,21 @@ async def run_consolidation_job(
     if timing_parts:
         perf.log(f"[4] Timing breakdown: {', '.join(timing_parts)}")
 
-    # Trigger mental model refreshes for models with refresh_after_consolidation=true
-    # SECURITY: Only refresh mental models with matching tags (or all if no tags were consolidated)
-    mental_models_refreshed = await _trigger_mental_model_refreshes(
-        memory_engine=memory_engine,
-        bank_id=bank_id,
-        request_context=request_context,
-        consolidated_tags=list(consolidated_tags) if consolidated_tags else None,
-        perf=perf,
-    )
-    stats["mental_models_refreshed"] = mental_models_refreshed
+    # Trigger mental model refreshes only on the final round (when all memories are processed).
+    # If we hit the round limit and re-queued, skip MM refresh — the next round will handle it.
+    if hit_round_limit:
+        stats["mental_models_refreshed"] = 0
+        logger.info(f"[CONSOLIDATION] bank={bank_id} skipping mental model refresh (round limit hit, re-queued)")
+    else:
+        # SECURITY: Only refresh mental models with matching tags (or all if no tags were consolidated)
+        mental_models_refreshed = await _trigger_mental_model_refreshes(
+            memory_engine=memory_engine,
+            bank_id=bank_id,
+            request_context=request_context,
+            consolidated_tags=list(consolidated_tags) if consolidated_tags else None,
+            perf=perf,
+        )
+        stats["mental_models_refreshed"] = mental_models_refreshed
 
     perf.flush()
 
@@ -593,17 +655,15 @@ async def _trigger_mental_model_refreshes(
     """
     pool = memory_engine._pool
 
-    # Find mental models with refresh_after_consolidation=true
-    # SECURITY: Control which mental models get refreshed based on tags
+    # Find mental models with refresh_after_consolidation=true that are actually stale.
+    # The tag filter on the SELECT enforces the security boundary (never look outside the
+    # relevant tag scope); compute_mental_model_is_stale then verifies that new memories
+    # in the MM's scope really were ingested since its last refresh.
     async with pool.acquire() as conn:
         if consolidated_tags:
-            # Tagged memories were consolidated - refresh:
-            # 1. Mental models with overlapping tags (security boundary)
-            # 2. Untagged mental models (they're "global" and available to all contexts)
-            # DO NOT refresh mental models with different tags
-            rows = await conn.fetch(
+            candidates = await conn.fetch(
                 f"""
-                SELECT id, name, tags
+                SELECT id, name, tags, last_refreshed_at, trigger
                 FROM {fq_table("mental_models")}
                 WHERE bank_id = $1
                   AND (trigger->>'refresh_after_consolidation')::boolean = true
@@ -616,11 +676,9 @@ async def _trigger_mental_model_refreshes(
                 consolidated_tags,
             )
         else:
-            # Untagged memories were consolidated - only refresh untagged mental models
-            # SECURITY: Tagged mental models are NOT refreshed when untagged memories are consolidated
-            rows = await conn.fetch(
+            candidates = await conn.fetch(
                 f"""
-                SELECT id, name, tags
+                SELECT id, name, tags, last_refreshed_at, trigger
                 FROM {fq_table("mental_models")}
                 WHERE bank_id = $1
                   AND (trigger->>'refresh_after_consolidation')::boolean = true
@@ -629,6 +687,11 @@ async def _trigger_mental_model_refreshes(
                 bank_id,
             )
 
+        rows = []
+        for candidate in candidates:
+            if await memory_engine.compute_mental_model_is_stale(conn, bank_id, candidate):
+                rows.append(candidate)
+
     if not rows:
         return 0
 
@@ -889,6 +952,15 @@ async def _execute_update_action(
         logger.debug(f"Update skipped: observation {observation_id} not found in recall results")
         return
 
+    live_source_memory_ids = await _filter_live_source_memories(conn, bank_id, source_memory_ids)
+    if not live_source_memory_ids:
+        logger.debug(
+            f"Update skipped: all {len(source_memory_ids)} source memories for observation "
+            f"{observation_id} were deleted concurrently"
+        )
+        return
+    source_memory_ids = live_source_memory_ids
+
     from ...config import get_config
 
     history_entry = {
@@ -1131,14 +1203,16 @@ async def _consolidate_batch_with_llm(
     memories: list[dict[str, Any]],
     union_observations: "list[MemoryFact]",
     union_source_facts: "dict[str, MemoryFact]",
-    config: Any = None,
+    config: Any,
     remaining_observation_slots: int | None = None,
     max_observations_per_scope: int = -1,
 ) -> _BatchLLMResult:
     """Single LLM call for a batch of facts against a pooled set of observations."""
+    if config is None:
+        raise ValueError("config is required for _consolidate_batch_with_llm")
     if union_observations:
         obs_list = _build_observations_for_llm(union_observations, union_source_facts)
-        observations_text = json.dumps(obs_list, indent=2)
+        observations_text = json.dumps(obs_list, indent=2, ensure_ascii=False)
     else:
         observations_text = "[]"
 
@@ -1172,8 +1246,7 @@ async def _consolidate_batch_with_llm(
                 f"(out of {max_observations_per_scope}). Prefer UPDATE over CREATE when possible."
             )
 
-    observations_mission = config.observations_mission if config is not None else None
-    prompt_template = build_batch_consolidation_prompt(observations_mission, observation_capacity_note)
+    prompt_template = build_batch_consolidation_prompt(config.observations_mission, observation_capacity_note)
     prompt = prompt_template.format(
         facts_text=facts_lines,
         observations_text=observations_text,
@@ -1182,15 +1255,29 @@ async def _consolidate_batch_with_llm(
     # Use a constrained response model when observation limit is active
     response_model = _build_response_model(max_creates=remaining_observation_slots)
 
-    max_attempts = 3
+    max_attempts = config.consolidation_max_attempts
+    inner_max_retries = config.consolidation_llm_max_retries
     last_exc: Exception | None = None
+    # Pre-compute a stable identifier set for the batch so failure logs name the
+    # exact memories whose consolidation is failing — without this, an opaque
+    # "LLM batch call failed" line gives operators no way to find the offending
+    # input until adaptive bisection narrows the batch down to a single memory.
+    memory_ids = [str(m.get("id")) for m in memories]
+    if len(memory_ids) <= 5:
+        ids_label = ", ".join(memory_ids)
+    else:
+        ids_label = f"{', '.join(memory_ids[:3])}, ... +{len(memory_ids) - 3} more"
+    batch_label = f"{len(memory_ids)} memories [{ids_label}]"
     for attempt in range(1, max_attempts + 1):
         try:
-            response: _ConsolidationBatchResponse = await llm_config.call(
-                messages=[{"role": "user", "content": prompt}],
-                response_format=response_model,
-                scope="consolidation",
-            )
+            call_kwargs: dict[str, Any] = {
+                "messages": [{"role": "user", "content": prompt}],
+                "response_format": response_model,
+                "scope": "consolidation",
+            }
+            if inner_max_retries is not None:
+                call_kwargs["max_retries"] = inner_max_retries
+            response: _ConsolidationBatchResponse = await llm_config.call(**call_kwargs)
             # Defensive truncation: some LLM providers may not enforce JSON schema max_length
             creates = response.creates
             if remaining_observation_slots is not None and remaining_observation_slots >= 0:
@@ -1209,10 +1296,13 @@ async def _consolidate_batch_with_llm(
             )
         except Exception as exc:
             last_exc = exc
-            logger.warning(f"[CONSOLIDATION] LLM batch call failed (attempt {attempt}/{max_attempts}): {exc}")
+            logger.warning(
+                f"[CONSOLIDATION] LLM batch call failed (attempt {attempt}/{max_attempts}) for {batch_label}: {exc}"
+            )
 
     logger.error(
-        f"[CONSOLIDATION] LLM batch call failed after {max_attempts} attempts, skipping batch. Last error: {last_exc}"
+        f"[CONSOLIDATION] LLM batch call failed after {max_attempts} attempts for {batch_label}, "
+        f"skipping batch. Last error: {last_exc}"
     )
     return _BatchLLMResult(obs_count=len(union_observations), prompt_chars=len(prompt), failed=True)
 
@@ -1231,6 +1321,12 @@ async def _create_observation_directly(
     perf: ConsolidationPerfLog | None = None,
 ) -> dict[str, Any]:
     """Create an observation from one or more source memories with pre-processed text."""
+    live_source_memory_ids = await _filter_live_source_memories(conn, bank_id, source_memory_ids)
+    if not live_source_memory_ids:
+        logger.debug(f"Create skipped: all {len(source_memory_ids)} source memories were deleted concurrently")
+        return {"action": "skipped", "reason": "sources_deleted"}
+    source_memory_ids = live_source_memory_ids
+
     # Generate embedding for the observation (convert to string for pgvector)
     t0 = time.time()
     embeddings = await embedding_utils.generate_embeddings_batch(memory_engine.embeddings, [observation_text])

hindsight-api-slim/hindsight_api/engine/cross_encoder.py

@@ -33,6 +33,7 @@ from ..config import (
     DEFAULT_RERANKER_SILICONFLOW_BASE_URL,
     DEFAULT_RERANKER_SILICONFLOW_MODEL,
     DEFAULT_RERANKER_TEI_BATCH_SIZE,
+    DEFAULT_RERANKER_TEI_HTTP_TIMEOUT,
     DEFAULT_RERANKER_TEI_MAX_CONCURRENT,
     DEFAULT_RERANKER_ZEROENTROPY_MODEL,
     ENV_RERANKER_COHERE_API_KEY,
@@ -48,6 +49,7 @@ from ..config import (
     ENV_RERANKER_PROVIDER,
     ENV_RERANKER_SILICONFLOW_API_KEY,
     ENV_RERANKER_TEI_BATCH_SIZE,
+    ENV_RERANKER_TEI_HTTP_TIMEOUT,
     ENV_RERANKER_TEI_MAX_CONCURRENT,
     ENV_RERANKER_TEI_URL,
     ENV_RERANKER_ZEROENTROPY_API_KEY,
@@ -1282,6 +1284,7 @@ class JinaMLXCrossEncoder(CrossEncoderModel):
     def _load_model(self) -> None:
         """Download (if needed) and load the MLX reranker. Runs in a thread."""
         import os
+        import threading
 
         from huggingface_hub import snapshot_download
 
@@ -1297,6 +1300,10 @@ class JinaMLXCrossEncoder(CrossEncoderModel):
             model_path=model_path,
             projector_path=os.path.join(model_path, "projector.safetensors"),
         )
+        # MLX Metal GPU ops are not thread-safe — concurrent calls to
+        # Device::end_encoding() crash with SIGSEGV (NULL deref).
+        # Serialize all reranker inference through this lock.
+        self._mlx_lock = threading.Lock()
         logger.info("Reranker: jina-mlx provider initialized")
 
     def _predict_sync(self, pairs: list[tuple[str, str]]) -> list[float]:
@@ -1310,13 +1317,14 @@ class JinaMLXCrossEncoder(CrossEncoderModel):
 
         all_scores = [0.0] * len(pairs)
 
-        for query, indexed_docs in query_groups.items():
-            docs = [doc for _, doc in indexed_docs]
-            indices = [idx for idx, _ in indexed_docs]
-            results = self._reranker.rerank(query, docs)
-            for result in results:
-                original_idx = result["index"]
-                all_scores[indices[original_idx]] = result["relevance_score"]
+        with self._mlx_lock:
+            for query, indexed_docs in query_groups.items():
+                docs = [doc for _, doc in indexed_docs]
+                indices = [idx for idx, _ in indexed_docs]
+                results = self._reranker.rerank(query, docs)
+                for result in results:
+                    original_idx = result["index"]
+                    all_scores[indices[original_idx]] = result["relevance_score"]
 
         return all_scores
 
@@ -1506,6 +1514,7 @@ def create_cross_encoder_from_env() -> CrossEncoderModel:
             raise ValueError(f"{ENV_RERANKER_TEI_URL} is required when {ENV_RERANKER_PROVIDER} is 'tei'")
         return RemoteTEICrossEncoder(
             base_url=url,
+            timeout=config.reranker_tei_http_timeout,
             batch_size=config.reranker_tei_batch_size,
             max_concurrent=config.reranker_tei_max_concurrent,
         )

hindsight-api-slim/hindsight_api/engine/embeddings.py

@@ -1100,7 +1100,12 @@ def create_embeddings_from_env() -> Embeddings:
             )
         model = os.environ.get(ENV_EMBEDDINGS_OPENAI_MODEL, DEFAULT_EMBEDDINGS_OPENAI_MODEL)
         base_url = os.environ.get(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None
-        return OpenAIEmbeddings(api_key=api_key, model=model, base_url=base_url)
+        return OpenAIEmbeddings(
+            api_key=api_key,
+            model=model,
+            base_url=base_url,
+            batch_size=config.embeddings_openai_batch_size,
+        )
     elif provider == "openrouter":
         api_key = config.embeddings_openrouter_api_key
         if not api_key:
@@ -1112,6 +1117,7 @@ def create_embeddings_from_env() -> Embeddings:
             api_key=api_key,
             model=config.embeddings_openrouter_model,
             base_url="https://openrouter.ai/api/v1",
+            batch_size=config.embeddings_openai_batch_size,
         )
     elif provider == "cohere":
         api_key = config.embeddings_cohere_api_key

hindsight-api-slim/hindsight_api/engine/providers/anthropic_llm.py

@@ -153,7 +153,7 @@ class AnthropicLLM(LLMInterface):
         # Add JSON schema instruction if response_format is provided
         if response_format is not None and hasattr(response_format, "model_json_schema"):
             schema = response_format.model_json_schema()
-            schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}"
+            schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2, ensure_ascii=False)}"
             if system_prompt:
                 system_prompt += schema_msg
             else:

hindsight-api-slim/hindsight_api/engine/providers/claude_code_llm.py

@@ -171,7 +171,7 @@ class ClaudeCodeLLM(LLMInterface):
         if response_format is not None and hasattr(response_format, "model_json_schema"):
             schema = response_format.model_json_schema()
             schema_instruction = (
-                f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}\n\n"
+                f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2, ensure_ascii=False)}\n\n"
                 "Respond with ONLY the JSON, no markdown formatting."
             )
             user_content += schema_instruction

hindsight-api-slim/hindsight_api/engine/providers/codex_llm.py

@@ -205,7 +205,7 @@ class CodexLLM(LLMInterface):
         # Add JSON schema instruction if response_format is provided
         if response_format is not None and hasattr(response_format, "model_json_schema"):
             schema = response_format.model_json_schema()
-            schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}"
+            schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2, ensure_ascii=False)}"
             system_instruction += schema_msg
 
         # gpt-5.2-codex only supports "detailed" reasoning summary

hindsight-api-slim/hindsight_api/engine/providers/gemini_llm.py

@@ -175,7 +175,7 @@ class GeminiLLM(LLMInterface):
         Args:
             messages: List of message dicts with 'role' and 'content'.
             response_format: Optional Pydantic model for structured output.
-            max_completion_tokens: Maximum tokens in response (not supported by Gemini).
+            max_completion_tokens: Maximum tokens in response (mapped to Gemini's max_output_tokens).
             temperature: Sampling temperature (0.0-2.0).
             scope: Scope identifier for tracking.
             max_retries: Maximum retry attempts.
@@ -212,7 +212,7 @@ class GeminiLLM(LLMInterface):
         # Add JSON schema instruction if response_format is provided
         if response_format is not None and hasattr(response_format, "model_json_schema"):
             schema = response_format.model_json_schema()
-            schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}"
+            schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2, ensure_ascii=False)}"
             if system_instruction:
                 system_instruction += schema_msg
             else:
@@ -227,6 +227,11 @@ class GeminiLLM(LLMInterface):
             config_kwargs["response_schema"] = response_format
         if temperature is not None:
             config_kwargs["temperature"] = temperature
+        # Gemini's equivalent of OpenAI-style max_completion_tokens is max_output_tokens.
+        # Without it the model can produce arbitrarily long responses, ignoring the
+        # caller's intended cap (e.g. mental_models max_tokens during refresh).
+        if max_completion_tokens is not None:
+            config_kwargs["max_output_tokens"] = max_completion_tokens
 
         # Apply safety settings: context var (per-request bank override) takes precedence over instance default
         effective_safety_settings = _safety_settings_ctx.get()
@@ -401,7 +406,7 @@ class GeminiLLM(LLMInterface):
         Args:
             messages: List of message dicts. Can include tool results with role='tool'.
             tools: List of tool definitions in OpenAI format.
-            max_completion_tokens: Maximum tokens (not supported by Gemini).
+            max_completion_tokens: Maximum tokens (mapped to Gemini's max_output_tokens).
             temperature: Sampling temperature.
             scope: Scope identifier for tracking.
             max_retries: Maximum retry attempts.
@@ -493,6 +498,10 @@ class GeminiLLM(LLMInterface):
             config_kwargs["system_instruction"] = system_instruction
         if temperature is not None:
             config_kwargs["temperature"] = temperature
+        # See note in `call`: Gemini's max_output_tokens is the equivalent of
+        # OpenAI-style max_completion_tokens.
+        if max_completion_tokens is not None:
+            config_kwargs["max_output_tokens"] = max_completion_tokens
 
         # Map OpenAI-style tool_choice to Gemini FunctionCallingConfig
         if tool_choice == "required":

hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py

@@ -60,6 +60,32 @@ def _strip_code_fences(content: str) -> str:
         return content
 
 
+def _summarize_status_error(e: APIStatusError, body_max: int = 400) -> str:
+    """Render an APIStatusError with status code + truncated response body.
+
+    Without this, retry loops only log "API error after N attempts" with the
+    bare exception message — losing the provider's actual error payload, which
+    is the only thing that explains *why* a request failed (rate limit reason,
+    invalid tool schema, model overloaded, etc.).
+    """
+    body: Any = getattr(e, "body", None)
+    if body is None:
+        try:
+            body = e.response.text
+        except Exception:
+            body = None
+    if isinstance(body, (dict, list)):
+        try:
+            body_str = json.dumps(body, default=str, ensure_ascii=False)
+        except Exception:
+            body_str = str(body)
+    else:
+        body_str = str(body or "").strip()
+    if len(body_str) > body_max:
+        body_str = body_str[:body_max] + "...TRUNCATED"
+    return f"HTTP {e.status_code}: {body_str or '<no body>'}"
+
+
 class OpenAICompatibleLLM(LLMInterface):
     """
     LLM provider for OpenAI-compatible APIs.
@@ -339,9 +365,7 @@ class OpenAICompatibleLLM(LLMInterface):
             else:
                 # Soft enforcement: add schema to prompt and use json_object mode
                 if schema is not None:
-                    schema_msg = (
-                        f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}"
-                    )
+                    schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2, ensure_ascii=False)}"
 
                     if call_params["messages"] and call_params["messages"][0].get("role") == "system":
                         first_msg = call_params["messages"][0]
@@ -550,12 +574,19 @@ class OpenAICompatibleLLM(LLMInterface):
 
                 last_exception = e
                 if attempt < max_retries:
+                    logger.warning(
+                        f"APIStatusError ({self.provider}/{self.model}, scope={scope}, "
+                        f"attempt {attempt + 1}/{max_retries + 1}): {_summarize_status_error(e)}"
+                    )
                     backoff = min(initial_backoff * (2**attempt), max_backoff)
                     jitter = backoff * 0.2 * (2 * (time.time() % 1) - 1)
                     sleep_time = backoff + jitter
                     await asyncio.sleep(sleep_time)
                 else:
-                    logger.error(f"API error after {max_retries + 1} attempts: {str(e)}")
+                    logger.error(
+                        f"API error after {max_retries + 1} attempts ({self.provider}/{self.model}, "
+                        f"scope={scope}): {_summarize_status_error(e)}"
+                    )
                     raise
 
             except Exception:
@@ -706,18 +737,41 @@ class OpenAICompatibleLLM(LLMInterface):
 
             except APIConnectionError as e:
                 last_exception = e
+                status_code = getattr(e, "status_code", None) or getattr(
+                    getattr(e, "response", None), "status_code", None
+                )
                 if attempt < max_retries:
+                    logger.warning(
+                        f"APIConnectionError in tool call ({self.provider}/{self.model}, scope={scope}, "
+                        f"attempt {attempt + 1}/{max_retries + 1}, HTTP {status_code}): {str(e)[:200]}"
+                    )
                     await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff))
                     continue
+                logger.error(
+                    f"Connection error in tool call after {max_retries + 1} attempts "
+                    f"({self.provider}/{self.model}, scope={scope}): {str(e)}"
+                )
                 raise
 
             except APIStatusError as e:
                 if e.status_code in (401, 403):
+                    logger.error(
+                        f"Auth error in tool call (HTTP {e.status_code}, {self.provider}/{self.model}), "
+                        f"not retrying: {_summarize_status_error(e)}"
+                    )
                     raise
                 last_exception = e
                 if attempt < max_retries:
+                    logger.warning(
+                        f"APIStatusError in tool call ({self.provider}/{self.model}, scope={scope}, "
+                        f"attempt {attempt + 1}/{max_retries + 1}): {_summarize_status_error(e)}"
+                    )
                     await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff))
                     continue
+                logger.error(
+                    f"API error in tool call after {max_retries + 1} attempts "
+                    f"({self.provider}/{self.model}, scope={scope}): {_summarize_status_error(e)}"
+                )
                 raise
 
             except Exception:
@@ -765,6 +819,7 @@ class OpenAICompatibleLLM(LLMInterface):
             "model": self.model,
             "messages": messages,
             "stream": False,
+            "think": False,  # Disable thinking for reasoning models (qwen3.5, etc.)
         }
 
         # Add schema as format parameter for structured output
@@ -919,7 +974,7 @@ class OpenAICompatibleLLM(LLMInterface):
         logger.info(f"Submitting batch with {len(requests)} requests to {self.provider}")
 
         # Format requests as JSONL
-        jsonl_content = "\n".join(json.dumps(req) for req in requests)
+        jsonl_content = "\n".join(json.dumps(req, ensure_ascii=False) for req in requests)
 
         # Upload file to provider (wrap in BytesIO with filename)
         file_bytes = io.BytesIO(jsonl_content.encode("utf-8"))

hindsight-api-slim/hindsight_api/engine/reflect/agent.py

@@ -17,7 +17,12 @@ from typing import TYPE_CHECKING, Any, Awaitable, Callable
 import tiktoken
 
 from .models import DirectiveInfo, LLMCall, ReflectAgentResult, TokenUsageSummary, ToolCall
-from .prompts import FINAL_SYSTEM_PROMPT, _extract_directive_rules, build_final_prompt, build_system_prompt_for_tools
+from .prompts import (
+    _extract_directive_rules,
+    build_final_prompt,
+    build_final_system_prompt,
+    build_system_prompt_for_tools,
+)
 from .tools_schema import get_reflect_tools
 
 
@@ -186,7 +191,7 @@ async def _generate_structured_output(
         DynamicModel = create_model("StructuredResponse", **fields)
 
         # Include the full schema in the prompt for better LLM guidance
-        schema_str = json.dumps(response_schema, indent=2)
+        schema_str = json.dumps(response_schema, indent=2, ensure_ascii=False)
 
         # Build field descriptions for the prompt
         field_descriptions = []
@@ -446,7 +451,7 @@ async def run_reflect_agent(
             llm_start = time.time()
             response, usage = await llm_config.call(
                 messages=[
-                    {"role": "system", "content": FINAL_SYSTEM_PROMPT},
+                    {"role": "system", "content": build_final_system_prompt(bank_profile.get("mission"))},
                     {"role": "user", "content": prompt},
                 ],
                 scope="reflect",
@@ -503,7 +508,7 @@ async def run_reflect_agent(
             llm_start = time.time()
             response, usage = await llm_config.call(
                 messages=[
-                    {"role": "system", "content": FINAL_SYSTEM_PROMPT},
+                    {"role": "system", "content": build_final_system_prompt(bank_profile.get("mission"))},
                     {"role": "user", "content": prompt},
                 ],
                 scope="reflect",
@@ -606,7 +611,7 @@ async def run_reflect_agent(
             llm_start = time.time()
             response, usage = await llm_config.call(
                 messages=[
-                    {"role": "system", "content": FINAL_SYSTEM_PROMPT},
+                    {"role": "system", "content": build_final_system_prompt(bank_profile.get("mission"))},
                     {"role": "user", "content": prompt},
                 ],
                 scope="reflect",
@@ -649,9 +654,57 @@ async def run_reflect_agent(
 
         # No tool calls - LLM wants to respond with text
         if not result.tool_calls:
-            if result.content:
+            # When directives are present but no evidence has been gathered,
+            # the LLM tends to echo directive content verbatim as its answer.
+            # Fall through to the final-prompt path which doesn't include
+            # directives and handles "no data" gracefully.
+            has_gathered_evidence = (
+                bool(available_memory_ids) or bool(available_mental_model_ids) or bool(available_observation_ids)
+            )
+            directive_leak_risk = directives and not has_gathered_evidence
+            if result.content and not directive_leak_risk:
                 answer = _clean_answer_text(result.content.strip())
 
+                # The call_with_tools call above is intentionally uncapped so the
+                # LLM has headroom to emit tool-call JSON plus any intermediate
+                # reasoning. But when the LLM short-circuits and returns text
+                # directly, that text becomes the user-visible final answer and
+                # must respect max_tokens like the forced-final paths do. If it
+                # overshoots, run one extra capped call to rewrite it within
+                # the cap.
+                if max_tokens is not None and len(_TIKTOKEN_ENCODING.encode(answer)) > max_tokens:
+                    rewrite_start = time.time()
+                    rewritten, rewrite_usage = await llm_config.call(
+                        messages=[
+                            {
+                                "role": "system",
+                                "content": (
+                                    "Rewrite the user's text so it fits within the requested token "
+                                    "budget. Preserve the key facts and structure; drop lower-priority "
+                                    "detail. Respond with the rewritten text only, no preamble."
+                                ),
+                            },
+                            {
+                                "role": "user",
+                                "content": f"Target budget: {max_tokens} tokens.\n\nText to rewrite:\n{answer}",
+                            },
+                        ],
+                        scope="reflect",
+                        max_completion_tokens=max_tokens,
+                        return_usage=True,
+                    )
+                    total_input_tokens += rewrite_usage.input_tokens
+                    total_output_tokens += rewrite_usage.output_tokens
+                    llm_trace.append(
+                        {
+                            "scope": "final_rewrite",
+                            "duration_ms": int((time.time() - rewrite_start) * 1000),
+                            "input_tokens": rewrite_usage.input_tokens,
+                            "output_tokens": rewrite_usage.output_tokens,
+                        }
+                    )
+                    answer = _clean_answer_text(rewritten.strip())
+
                 # Generate structured output if schema provided
                 structured_output = None
                 if response_schema and answer:
@@ -679,7 +732,7 @@ async def run_reflect_agent(
             llm_start = time.time()
             response, usage = await llm_config.call(
                 messages=[
-                    {"role": "system", "content": FINAL_SYSTEM_PROMPT},
+                    {"role": "system", "content": build_final_system_prompt(bank_profile.get("mission"))},
                     {"role": "user", "content": prompt},
                 ],
                 scope="reflect",
@@ -743,7 +796,8 @@ async def run_reflect_agent(
                         "content": json.dumps(
                             {
                                 "error": "You must search for information first. Use search_mental_models(), search_observations(), or recall() before providing your final answer."
-                            }
+                            },
+                            ensure_ascii=False,
                         ),
                     }
                 )
@@ -805,7 +859,8 @@ async def run_reflect_agent(
                         "content": json.dumps(
                             {
                                 "error": f"Tool '{_normalize_tool_name(tc.name)}' is not available. Use only the tools provided to you."
-                            }
+                            },
+                            ensure_ascii=False,
                         ),
                     }
                 )
@@ -876,7 +931,7 @@ async def run_reflect_agent(
                         "role": "tool",
                         "tool_call_id": tc.id,
                         "name": tc.name,  # Required by Gemini
-                        "content": json.dumps(output, default=str),
+                        "content": json.dumps(output, default=str, ensure_ascii=False),
                     }
                 )
 
@@ -899,7 +954,7 @@ async def run_reflect_agent(
                 )
 
                 try:
-                    output_chars = len(json.dumps(output))
+                    output_chars = len(json.dumps(output, ensure_ascii=False))
                 except (TypeError, ValueError):
                     output_chars = len(str(output))
 
@@ -936,7 +991,7 @@ def _tool_call_to_dict(tc: "LLMToolCall") -> dict[str, Any]:
         "type": "function",
         "function": {
             "name": tc.name,
-            "arguments": json.dumps(tc.arguments),
+            "arguments": json.dumps(tc.arguments, ensure_ascii=False),
         },
     }
     if tc.thought_signature is not None:
@@ -1034,7 +1089,7 @@ async def _execute_tool_with_timing(
         # Set attributes
         span.set_attribute("hindsight.tool.name", normalized_name)
         span.set_attribute("hindsight.tool.id", tc.id)
-        span.set_attribute("hindsight.tool.arguments", json.dumps(tc.arguments))
+        span.set_attribute("hindsight.tool.arguments", json.dumps(tc.arguments, ensure_ascii=False))
 
         try:
             result = await _execute_tool(

hindsight-api-slim/hindsight_api/engine/reflect/delta_ops.py

@@ -0,0 +1,307 @@
+"""Delta operations for structured mental models.
+
+The LLM's job during a delta refresh is to emit a list of these operations,
+each targeting an existing section (by id) or referencing a position relative
+to one.  ``apply_operations`` validates and applies each op in turn against a
+copy of the document; invalid ops (unknown ``section_id``, out-of-range
+``block_index``, malformed payloads) are dropped with a debug-friendly reason.
+
+Sections and blocks not mentioned by any op are physically copied through
+unchanged — there is no LLM-mediated re-emission of unchanged text, so prose
+drift is structurally impossible.
+
+Why operations and not "output the new structured doc":
+- "Output the new doc" still asks the LLM to *generate* every section's
+  blocks, including ones it didn't intend to modify, which gives it the same
+  opportunity to drift.
+- Operations make the no-change case mechanical: zero ops → identical doc.
+- Operations are auditable: each refresh produces a log of exactly what
+  changed, useful for debugging the LLM's behaviour and explaining diffs.
+
+Failure modes are by design conservative: an operation list that fails to
+parse against the Pydantic schema, or an LLM that returns invalid ops, results
+in zero changes — the document stays as-is. The structure can only get better
+or stay the same per refresh, never get worse.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Annotated, Any, Literal, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .structured_doc import (
+    Block,
+    Section,
+    StructuredDocument,
+    make_unique_id,
+    slugify_heading,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# Op payloads ---------------------------------------------------------------
+
+
+class _OpBase(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class AppendBlockOp(_OpBase):
+    """Add a new block at the end of an existing section."""
+
+    op: Literal["append_block"] = "append_block"
+    section_id: str
+    block: Block
+
+
+class InsertBlockOp(_OpBase):
+    """Insert a new block at ``index`` in an existing section.
+
+    ``index`` may equal ``len(section.blocks)`` (append) but not be greater.
+    """
+
+    op: Literal["insert_block"] = "insert_block"
+    section_id: str
+    index: int = Field(ge=0)
+    block: Block
+
+
+class ReplaceBlockOp(_OpBase):
+    """Replace the block at ``index`` of an existing section."""
+
+    op: Literal["replace_block"] = "replace_block"
+    section_id: str
+    index: int = Field(ge=0)
+    block: Block
+
+
+class RemoveBlockOp(_OpBase):
+    """Remove the block at ``index`` of an existing section."""
+
+    op: Literal["remove_block"] = "remove_block"
+    section_id: str
+    index: int = Field(ge=0)
+
+
+class AddSectionOp(_OpBase):
+    """Add a brand-new section.
+
+    ``after_section_id`` is optional; when omitted the new section is appended
+    at the end. ``new_id`` is optional; when omitted we slugify the heading
+    and disambiguate against existing IDs.
+    """
+
+    op: Literal["add_section"] = "add_section"
+    heading: str
+    level: int = Field(default=2, ge=1, le=6)
+    blocks: list[Block] = Field(default_factory=list)
+    after_section_id: str | None = None
+    new_id: str | None = None
+
+
+class RemoveSectionOp(_OpBase):
+    """Remove an entire section by id."""
+
+    op: Literal["remove_section"] = "remove_section"
+    section_id: str
+
+
+class ReplaceSectionBlocksOp(_OpBase):
+    """Replace all blocks of a section in one go.
+
+    Used when most of a section's contents are stale and rebuilding it as a
+    unit is clearer than emitting many block-level ops. The section's heading
+    and id are preserved.
+    """
+
+    op: Literal["replace_section_blocks"] = "replace_section_blocks"
+    section_id: str
+    blocks: list[Block] = Field(default_factory=list)
+
+
+class RenameSectionOp(_OpBase):
+    """Rename a section's heading. The id is unchanged so future ops still resolve."""
+
+    op: Literal["rename_section"] = "rename_section"
+    section_id: str
+    new_heading: str
+
+
+Operation = Annotated[
+    Union[
+        AppendBlockOp,
+        InsertBlockOp,
+        ReplaceBlockOp,
+        RemoveBlockOp,
+        AddSectionOp,
+        RemoveSectionOp,
+        ReplaceSectionBlocksOp,
+        RenameSectionOp,
+    ],
+    Field(discriminator="op"),
+]
+
+
+class DeltaOperationList(BaseModel):
+    """Container for the operations produced by an LLM delta call."""
+
+    model_config = ConfigDict(extra="forbid")
+    operations: list[Operation] = Field(default_factory=list)
+
+
+# Application ---------------------------------------------------------------
+
+
+class AppliedDelta(BaseModel):
+    """Outcome of applying a list of operations to a document."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    document: StructuredDocument
+    applied: list[dict[str, Any]] = Field(default_factory=list)
+    skipped: list[dict[str, Any]] = Field(default_factory=list)
+
+    @property
+    def changed(self) -> bool:
+        return len(self.applied) > 0
+
+
+def _op_summary(op: Operation) -> dict[str, Any]:
+    """Compact dict suitable for the audit trail."""
+    data = op.model_dump()
+    return {k: v for k, v in data.items() if k != "block" and k != "blocks"} | {
+        "op": data["op"],
+    }
+
+
+def apply_operations(
+    doc: StructuredDocument,
+    operations: list[Operation],
+) -> AppliedDelta:
+    """Apply a list of operations to a document, returning a new document.
+
+    The original document is never mutated. Invalid operations (unknown
+    section, out-of-range index, name collision when adding a section) are
+    skipped and recorded in ``skipped`` with a ``reason`` string.
+    """
+    new_doc = doc.model_copy(deep=True)
+    applied: list[dict[str, Any]] = []
+    skipped: list[dict[str, Any]] = []
+
+    def skip(op: Operation, reason: str) -> None:
+        entry = _op_summary(op)
+        entry["reason"] = reason
+        skipped.append(entry)
+        logger.debug(f"[STRUCTURED_DELTA] skipping op {entry}")
+
+    for op in operations:
+        if isinstance(op, AppendBlockOp):
+            section = new_doc.section_by_id(op.section_id)
+            if section is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            section.blocks.append(op.block)
+            applied.append(_op_summary(op))
+            continue
+
+        if isinstance(op, InsertBlockOp):
+            section = new_doc.section_by_id(op.section_id)
+            if section is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            if op.index > len(section.blocks):
+                skip(
+                    op,
+                    f"index out of range: {op.index} > {len(section.blocks)}",
+                )
+                continue
+            section.blocks.insert(op.index, op.block)
+            applied.append(_op_summary(op))
+            continue
+
+        if isinstance(op, ReplaceBlockOp):
+            section = new_doc.section_by_id(op.section_id)
+            if section is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            if op.index >= len(section.blocks):
+                skip(
+                    op,
+                    f"index out of range: {op.index} >= {len(section.blocks)}",
+                )
+                continue
+            section.blocks[op.index] = op.block
+            applied.append(_op_summary(op))
+            continue
+
+        if isinstance(op, RemoveBlockOp):
+            section = new_doc.section_by_id(op.section_id)
+            if section is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            if op.index >= len(section.blocks):
+                skip(
+                    op,
+                    f"index out of range: {op.index} >= {len(section.blocks)}",
+                )
+                continue
+            section.blocks.pop(op.index)
+            applied.append(_op_summary(op))
+            continue
+
+        if isinstance(op, AddSectionOp):
+            existing_ids = {s.id for s in new_doc.sections}
+            base_id = op.new_id or slugify_heading(op.heading)
+            section_id = make_unique_id(base_id, existing_ids)
+            new_section = Section(
+                id=section_id,
+                heading=op.heading,
+                level=op.level,
+                blocks=list(op.blocks),
+            )
+            if op.after_section_id is None:
+                new_doc.sections.append(new_section)
+            else:
+                idx = new_doc.section_index(op.after_section_id)
+                if idx is None:
+                    skip(op, f"unknown after_section_id: {op.after_section_id}")
+                    continue
+                new_doc.sections.insert(idx + 1, new_section)
+            entry = _op_summary(op)
+            entry["assigned_id"] = section_id
+            applied.append(entry)
+            continue
+
+        if isinstance(op, RemoveSectionOp):
+            idx = new_doc.section_index(op.section_id)
+            if idx is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            new_doc.sections.pop(idx)
+            applied.append(_op_summary(op))
+            continue
+
+        if isinstance(op, ReplaceSectionBlocksOp):
+            section = new_doc.section_by_id(op.section_id)
+            if section is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            section.blocks = list(op.blocks)
+            applied.append(_op_summary(op))
+            continue
+
+        if isinstance(op, RenameSectionOp):
+            section = new_doc.section_by_id(op.section_id)
+            if section is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            section.heading = op.new_heading
+            applied.append(_op_summary(op))
+            continue
+
+        skip(op, f"unhandled op type: {type(op).__name__}")  # pragma: no cover
+
+    return AppliedDelta(document=new_doc, applied=applied, skipped=skipped)

hindsight-api-slim/hindsight_api/engine/reflect/prompts.py

@@ -18,6 +18,9 @@ _TIKTOKEN_ENCODING = tiktoken.get_encoding("cl100k_base")
 # The remainder covers the system prompt, question, bank context, and output tokens.
 _FINAL_PROMPT_CONTEXT_FRACTION = 0.8
 
+_DEFAULT_ROLE = "You are a reflection agent that answers questions by reasoning over retrieved memories."
+_DEFAULT_FINAL_ROLE = "You are a thoughtful assistant that synthesizes answers from retrieved memories."
+
 
 def _extract_directive_rules(directives: list[dict[str, Any]]) -> list[str]:
     """Extract directive rules as a list of strings."""
@@ -133,7 +136,9 @@ def build_system_prompt_for_tools(
 
     parts.extend(
         [
-            "You are a reflection agent that answers questions by reasoning over retrieved memories.",
+            mission.strip() if mission else _DEFAULT_ROLE,
+            "",
+            "Answer the user's question by reasoning over retrieved memories.",
             "",
         ]
     )
@@ -369,7 +374,7 @@ def build_agent_prompt(
             output = entry["output"]
             # Format as proper JSON for LLM readability
             try:
-                output_str = json.dumps(output, indent=2, default=str)
+                output_str = json.dumps(output, indent=2, default=str, ensure_ascii=False)
             except (TypeError, ValueError):
                 output_str = str(output)
             parts.append(f"\n### Call {i}: {tool}\n```json\n{output_str}\n```")
@@ -444,7 +449,7 @@ def build_final_prompt(
             tool = entry["tool"]
             output = entry["output"]
             try:
-                output_str = json.dumps(output, indent=2, default=str)
+                output_str = json.dumps(output, indent=2, default=str, ensure_ascii=False)
             except (TypeError, ValueError):
                 output_str = str(output)
             block = f"\n### From {tool}:\n```json\n{output_str}\n```"
@@ -479,9 +484,9 @@ def build_final_prompt(
     return "\n".join(parts)
 
 
-FINAL_SYSTEM_PROMPT = """CRITICAL: You MUST ONLY use information from retrieved tool results. NEVER make up names, people, events, or entities.
+_FINAL_SYSTEM_PROMPT_BASE = """CRITICAL: You MUST ONLY use information from retrieved tool results. NEVER make up names, people, events, or entities.
 
-You are a thoughtful assistant that synthesizes answers from retrieved memories.
+{role_section}
 
 Your approach:
 - Reason over the retrieved memories to answer the question
@@ -508,3 +513,213 @@ CRITICAL: Output ONLY the final synthesized answer. Do NOT include:
 Just provide the direct answer with proper markdown formatting.
 
 CRITICAL: This is a NON-CONVERSATIONAL system. NEVER ask follow-up questions, offer to search again, suggest alternatives, or end with anything like "Would you like me to..." or "Let me know if...". The user cannot reply. Your answer must be complete and self-contained."""
+
+
+def build_final_system_prompt(mission: str | None = None) -> str:
+    """Build the final synthesis system prompt, using mission as role when set."""
+    role_section = mission.strip() if mission else _DEFAULT_FINAL_ROLE
+    return _FINAL_SYSTEM_PROMPT_BASE.format(role_section=role_section)
+
+
+# Backward-compatible constant for non-identity missions
+FINAL_SYSTEM_PROMPT = build_final_system_prompt()
+
+
+STRUCTURED_DELTA_SYSTEM_PROMPT = """You are integrating *new information* into an existing structured document.
+
+You will be given:
+1. TOPIC — the question this document answers. Content that does not help
+   answer this question is OFF-TOPIC and should be removed.
+2. CURRENT DOCUMENT (JSON) — the existing structured mental model. Each section
+   has a stable ``id``, a ``heading``, a ``level`` (1..6), and an ordered list
+   of ``blocks``. Blocks are typed: ``paragraph``, ``bullet_list``,
+   ``ordered_list``, or ``code``.
+3. NEW INFORMATION SYNTHESIS (markdown) — a synthesis showing how the new facts
+   relate to the document's topic. Use it to understand context and relevance,
+   but do NOT copy its formatting or wording wholesale.
+4. SUPPORTING FACTS — observations and facts created since the last refresh.
+   These are genuinely new — they were NOT available when the current document
+   was written.
+
+Your task: output a JSON object ``{"operations": [...]}``. Applied to CURRENT
+DOCUMENT, the operations must produce a document that best answers the TOPIC
+by integrating the new facts.
+
+RULES
+- These facts are NEW since the last refresh. The existing document already
+  captures all prior information from earlier refreshes. Your job is to
+  integrate the new facts into the existing document.
+- **Preserve existing content**: The current document was built from prior facts
+  that you cannot see. Do NOT remove or replace existing sections just because
+  the new facts do not reference them. Only remove content when the new facts
+  explicitly contradict or supersede it.
+- **Merge overlapping topics**: When new facts cover topics that overlap with
+  existing sections, merge the new information INTO the existing section
+  rather than creating duplicates. When new facts provide more specific or
+  authoritative guidance on a topic already covered generically, update the
+  existing content to reflect the more specific guidance.
+- **Preserve examples**: Concrete examples, before/after pairs, sample sentences,
+  and illustrative ✅/❌ comparisons are MORE valuable than abstract rules.
+  When facts contain examples, include them. Never drop an example to make
+  room for an abstract restatement of the same point.
+- Operations target sections by ``section_id`` (use the ``id`` field of the
+  section in CURRENT DOCUMENT, NOT the heading). Block operations target
+  blocks by ``index`` (0-based, against the section's current block list).
+- **Add** new content with ``append_block``, ``insert_block``, or ``add_section``
+  when facts introduce information not yet covered. Prefer extending an
+  existing section over creating a new one.
+- **Update** existing content with ``replace_block`` or ``replace_section_blocks``
+  when new facts provide corrections, updates, or more specific information
+  about topics already in the document.
+- **Remove** content with ``remove_block`` or ``remove_section`` ONLY when
+  the new facts explicitly contradict or supersede it.
+- NEVER emit operations whose only effect is to reword unchanged content.
+- NEVER emit operations to "normalize" formatting (numbered → bulleted, casing
+  changes, paragraph → list, etc).
+- Every operation MUST be justifiable by a specific fact in SUPPORTING FACTS.
+- Output ``{"operations": []}`` only if the new facts are already reflected
+  in the document (e.g., from a concurrent update).
+
+ALLOWED OPERATIONS (each line shows the JSON shape)
+- ``{"op": "append_block", "section_id": "...", "block": {...}}``
+- ``{"op": "insert_block", "section_id": "...", "index": N, "block": {...}}``
+- ``{"op": "replace_block", "section_id": "...", "index": N, "block": {...}}``
+- ``{"op": "remove_block", "section_id": "...", "index": N}``
+- ``{"op": "add_section", "heading": "...", "level": 2, "blocks": [...], "after_section_id": "..."}``
+- ``{"op": "remove_section", "section_id": "..."}``
+- ``{"op": "replace_section_blocks", "section_id": "...", "blocks": [...]}``
+- ``{"op": "rename_section", "section_id": "...", "new_heading": "..."}``
+
+Block shapes
+- ``{"type": "paragraph", "text": "..."}``
+- ``{"type": "bullet_list", "items": ["...", "..."]}``
+- ``{"type": "ordered_list", "items": ["...", "..."]}``
+- ``{"type": "code", "language": "json", "text": "..."}``
+
+OUTPUT FORMAT
+Return ONLY a single JSON object on its own, with no prose before or after,
+no markdown code fences, no commentary. The object must have exactly one
+top-level key, ``operations``, whose value is an array of operation objects
+(empty array when nothing changes).
+
+Examples
+- No changes needed → ``{"operations": []}``
+- Add one bullet to an existing "Members" section →
+  ``{"operations": [{"op": "append_block", "section_id": "members",
+  "block": {"type": "bullet_list", "items": ["Carol — junior engineer"]}}]}``
+- Replace a paragraph that has been corrected by new facts →
+  ``{"operations": [{"op": "replace_block", "section_id": "overview",
+  "index": 0, "block": {"type": "paragraph", "text": "Updated summary."}}]}``
+- Remove an obsolete block →
+  ``{"operations": [{"op": "remove_block", "section_id": "status", "index": 2}]}``"""
+
+
+def build_structured_delta_prompt(
+    *,
+    current_document_json: str,
+    candidate_markdown: str,
+    supporting_facts: list[dict[str, Any]],
+    source_query: str,
+    max_output_tokens: int | None = None,
+) -> str:
+    """Build the user prompt for a structured-delta mental model refresh.
+
+    The LLM's job is to emit operations against ``current_document_json``;
+    the surrounding ``candidate_markdown`` and ``supporting_facts`` are
+    references for *what new information exists*, not templates to mimic.
+
+    ``max_output_tokens`` is surfaced in the prompt so the model can keep its
+    op list within the provider's response cap. The actual cap is enforced by
+    the caller; this is just an advisory anchor — without it the model often
+    returns op lists whose JSON gets truncated mid-string.
+    """
+    fact_lines: list[str] = []
+    for f in supporting_facts:
+        fid = f.get("id", "")
+        text = (f.get("text") or "").strip().replace("\n", " ")
+        ftype = f.get("type", "")
+        fact_lines.append(f"- [{ftype}:{fid}] {text}")
+    facts_block = "\n".join(fact_lines) if fact_lines else "(no supporting facts retrieved)"
+
+    budget_hint = ""
+    if max_output_tokens is not None:
+        budget_hint = (
+            f"\n\n## Output budget\n"
+            f"Your JSON response must fit within ~{max_output_tokens} tokens. If you "
+            "would need more than this to express every change, prefer the highest-"
+            "leverage edits first (a few ``replace_section_blocks`` ops over many "
+            "block-level ops) so the response always parses as valid JSON."
+        )
+
+    return (
+        f"## Topic\n{source_query}\n\n"
+        f"## CURRENT DOCUMENT (apply ops to this; reference section ids as listed)\n"
+        f"```json\n{current_document_json}\n```\n\n"
+        f"## NEW INFORMATION SYNTHESIS (context for how new facts relate to the topic)\n"
+        f"```markdown\n{candidate_markdown}\n```\n\n"
+        f"## SUPPORTING FACTS (new since last refresh — integrate these)\n{facts_block}"
+        f"{budget_hint}\n\n"
+        "## Task\n"
+        "Output a JSON object matching the operations schema. Integrate the new "
+        "supporting facts into CURRENT DOCUMENT. Add, update, or remove content "
+        "as needed. Preserve unchanged sections and blocks by not mentioning them."
+    )
+
+
+DELTA_SYSTEM_PROMPT = """You are performing a surgical delta update to an existing mental model document.
+
+You will be given:
+1. CURRENT DOCUMENT: the existing mental model content (markdown).
+2. CANDIDATE UPDATE: a freshly generated synthesis based on the latest retrieved memories.
+3. SUPPORTING FACTS: the observations and facts that support the CANDIDATE UPDATE.
+
+Your task: produce an updated version of the CURRENT DOCUMENT that reflects the new reality, with the MINIMUM possible changes.
+
+ABSOLUTE RULES:
+- Preserve unchanged content BYTE-FOR-BYTE. If a sentence, heading, bullet, code block, or section is still accurate according to the CANDIDATE UPDATE and SUPPORTING FACTS, copy it verbatim — same wording, same punctuation, same whitespace, same markdown structure.
+- Do NOT reformat, rephrase, or re-style content that is still accurate. No "light edits for clarity", no reordering for flow, no synonym swaps.
+- Remove content that is contradicted by the CANDIDATE UPDATE or SUPPORTING FACTS (stale content).
+- Add new content ONLY when the SUPPORTING FACTS contain information not already in the CURRENT DOCUMENT.
+- When adding new content, prefer appending to an existing relevant section. Creating a new section is acceptable when the new information does not fit any existing section.
+- When creating a new section, match the heading style, tone, and formatting conventions used in the CURRENT DOCUMENT.
+- Every assertion in your output MUST be grounded in either (a) the CURRENT DOCUMENT (preserved) or (b) the SUPPORTING FACTS. Never introduce outside knowledge.
+- If nothing in the SUPPORTING FACTS contradicts or extends the CURRENT DOCUMENT, return the CURRENT DOCUMENT UNCHANGED, character for character.
+
+OUTPUT FORMAT:
+- Output ONLY the updated markdown document. No preamble, no explanation, no diff markers, no commentary.
+- Do not wrap the output in code fences unless the CURRENT DOCUMENT itself was entirely a code fence."""
+
+
+def build_delta_prompt(
+    *,
+    current_content: str,
+    candidate_content: str,
+    supporting_facts: list[dict[str, Any]],
+    source_query: str,
+) -> str:
+    """Build the user prompt for a delta-mode mental model refresh.
+
+    Args:
+        current_content: The existing mental model content (to preserve as much as possible).
+        candidate_content: Fresh synthesis from the reflect agent reflecting new reality.
+        supporting_facts: Flat list of fact dicts (id, text, type) supporting the candidate.
+        source_query: The mental model's source query, for topical framing.
+    """
+    fact_lines: list[str] = []
+    for f in supporting_facts:
+        fid = f.get("id", "")
+        text = (f.get("text") or "").strip().replace("\n", " ")
+        ftype = f.get("type", "")
+        fact_lines.append(f"- [{ftype}:{fid}] {text}")
+    facts_block = "\n".join(fact_lines) if fact_lines else "(no supporting facts retrieved)"
+
+    return (
+        f"## Topic\n{source_query}\n\n"
+        f"## CURRENT DOCUMENT\n```markdown\n{current_content}\n```\n\n"
+        f"## CANDIDATE UPDATE\n```markdown\n{candidate_content}\n```\n\n"
+        f"## SUPPORTING FACTS\n{facts_block}\n\n"
+        "## Task\n"
+        "Produce the updated mental model document by applying the minimum necessary changes "
+        "to CURRENT DOCUMENT so that it reflects CANDIDATE UPDATE and SUPPORTING FACTS. "
+        "Preserve unchanged content byte-for-byte. Output only the final markdown."
+    )

hindsight-api-slim/hindsight_api/engine/reflect/structured_doc.py

@@ -0,0 +1,301 @@
+"""Structured representation of a mental model document.
+
+Why this exists
+---------------
+Storing mental models as raw markdown forces every refresh to round-trip prose
+through an LLM, which then drifts on stylistic details (numbered vs bulleted
+lists, casing, separator lines, paraphrasing) even when instructed to preserve
+content byte-for-byte. The intrinsic mechanism of an LLM is to *generate* the
+next token from a gestalt of the input — not to copy tokens verbatim — so any
+"preserve unchanged content" instruction is fundamentally a soft constraint.
+
+The fix is to give the LLM no opportunity to drift on unchanged content. We
+keep an authoritative structured representation of the document; the markdown
+shown to users is a deterministic render of that structure. Delta refreshes
+emit *operations* against the structure (see ``delta_ops.py``); sections and
+blocks not mentioned by any operation are physically untouched.
+
+Schema (v1)
+-----------
+A document is an ordered list of ``Section``s.  Each section has:
+- ``id``    : stable slug derived from ``heading`` (used as the operation
+              target across refreshes; surviving renames is a separate
+              concern handled by an explicit ``rename`` op).
+- ``heading``: the markdown heading text (without the ``#`` prefix).
+- ``level`` : 1 (``#``) … 6 (``######``).  Default 2.
+- ``blocks``: ordered list of typed blocks — paragraph, bullet_list,
+              ordered_list, code.
+
+The schema is intentionally narrow: it covers what real mental-model documents
+actually contain (the kind a coding agent writes for itself or a user writes as
+a "skill" doc).  Tables, images, and raw HTML are out of scope until needed.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Annotated, Literal, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+# Blocks ---------------------------------------------------------------------
+
+
+class ParagraphBlock(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    type: Literal["paragraph"] = "paragraph"
+    text: str
+
+
+class BulletListBlock(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    type: Literal["bullet_list"] = "bullet_list"
+    items: list[str] = Field(default_factory=list)
+
+
+class OrderedListBlock(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    type: Literal["ordered_list"] = "ordered_list"
+    items: list[str] = Field(default_factory=list)
+
+
+class CodeBlock(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    type: Literal["code"] = "code"
+    language: str = ""
+    text: str
+
+
+Block = Annotated[
+    Union[ParagraphBlock, BulletListBlock, OrderedListBlock, CodeBlock],
+    Field(discriminator="type"),
+]
+
+
+# Section / Document ---------------------------------------------------------
+
+
+class Section(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    id: str
+    heading: str
+    level: int = Field(default=2, ge=1, le=6)
+    blocks: list[Block] = Field(default_factory=list)
+
+
+class StructuredDocument(BaseModel):
+    """Top-level structured representation of a mental model."""
+
+    model_config = ConfigDict(extra="forbid")
+    version: Literal[1] = 1
+    sections: list[Section] = Field(default_factory=list)
+
+    def section_by_id(self, section_id: str) -> Section | None:
+        for s in self.sections:
+            if s.id == section_id:
+                return s
+        return None
+
+    def section_index(self, section_id: str) -> int | None:
+        for i, s in enumerate(self.sections):
+            if s.id == section_id:
+                return i
+        return None
+
+
+# Slug helpers ---------------------------------------------------------------
+
+_SLUG_RX = re.compile(r"[^a-z0-9]+")
+
+
+def slugify_heading(heading: str) -> str:
+    """Stable, deterministic slug from a heading.
+
+    "Stop Conditions" -> "stop-conditions"
+    "Inputs and Context" -> "inputs-and-context"
+    """
+    slug = _SLUG_RX.sub("-", heading.strip().lower()).strip("-")
+    return slug or "section"
+
+
+def make_unique_id(base: str, existing: set[str]) -> str:
+    """Disambiguate by appending -2, -3, … if the slug is already in use."""
+    if base not in existing:
+        return base
+    i = 2
+    while f"{base}-{i}" in existing:
+        i += 1
+    return f"{base}-{i}"
+
+
+# Renderer -------------------------------------------------------------------
+
+
+def render_block(block: Block) -> str:
+    """Render a single block to markdown. No trailing newline."""
+    if isinstance(block, ParagraphBlock):
+        return block.text.rstrip()
+    if isinstance(block, BulletListBlock):
+        return "\n".join(f"- {item.rstrip()}" for item in block.items)
+    if isinstance(block, OrderedListBlock):
+        return "\n".join(f"{i + 1}. {item.rstrip()}" for i, item in enumerate(block.items))
+    if isinstance(block, CodeBlock):
+        fence_lang = block.language or ""
+        return f"```{fence_lang}\n{block.text}\n```"
+    raise TypeError(f"Unknown block type: {type(block)!r}")
+
+
+def render_section(section: Section) -> str:
+    """Render a section: heading + blank line + blocks separated by blank lines."""
+    parts = ["#" * section.level + " " + section.heading.strip()]
+    for block in section.blocks:
+        parts.append("")  # blank line before each block
+        parts.append(render_block(block))
+    return "\n".join(parts)
+
+
+def render_document(doc: StructuredDocument) -> str:
+    """Render the whole document. Sections separated by a single blank line.
+
+    The output is byte-stable: same structured input always produces the same
+    markdown, modulo the inherent ordering of sections/blocks/items.
+    """
+    if not doc.sections:
+        return ""
+    return "\n\n".join(render_section(s) for s in doc.sections) + "\n"
+
+
+# Parser ---------------------------------------------------------------------
+#
+# The parser is intentionally lenient: it accepts the markdown produced by
+# our own renderer (round-trip-safe) and the markdown an LLM tends to produce
+# for mental-model documents.  It is *not* a general CommonMark parser — it
+# does not need to be.  When it cannot classify a block it falls back to a
+# paragraph so that no content is silently dropped.
+
+_HEADING_RX = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
+_BULLET_RX = re.compile(r"^\s*[-*+]\s+(.*)$")
+_ORDERED_RX = re.compile(r"^\s*\d+[.)]\s+(.*)$")
+_FENCE_RX = re.compile(r"^```([A-Za-z0-9_+-]*)\s*$")
+
+
+def _strip_separators(lines: list[str]) -> list[str]:
+    """Drop horizontal-rule lines (`---`, `***`) used as section separators.
+
+    Our renderer never emits these, but LLM output frequently includes them
+    between sections; treating them as blank lines avoids parsing them as
+    paragraphs.
+    """
+    return ["" if re.fullmatch(r"\s*([-*_])\1{2,}\s*", line) else line for line in lines]
+
+
+def _split_blocks(lines: list[str]) -> list[list[str]]:
+    """Group consecutive non-blank lines into block chunks."""
+    chunks: list[list[str]] = []
+    current: list[str] = []
+    in_fence = False
+    for line in lines:
+        if _FENCE_RX.match(line):
+            current.append(line)
+            in_fence = not in_fence
+            continue
+        if in_fence:
+            current.append(line)
+            continue
+        if line.strip() == "":
+            if current:
+                chunks.append(current)
+                current = []
+        else:
+            current.append(line)
+    if current:
+        chunks.append(current)
+    return chunks
+
+
+def _parse_block(chunk: list[str]) -> Block:
+    """Parse a single non-empty chunk into a block."""
+    if chunk and _FENCE_RX.match(chunk[0]):
+        m = _FENCE_RX.match(chunk[0])
+        lang = m.group(1) if m else ""
+        body_lines = chunk[1:]
+        if body_lines and _FENCE_RX.match(body_lines[-1]):
+            body_lines = body_lines[:-1]
+        return CodeBlock(language=lang, text="\n".join(body_lines))
+
+    if all(_BULLET_RX.match(line) for line in chunk):
+        items = []
+        for line in chunk:
+            m = _BULLET_RX.match(line)
+            assert m is not None
+            items.append(m.group(1).strip())
+        return BulletListBlock(items=items)
+
+    if all(_ORDERED_RX.match(line) for line in chunk):
+        items = []
+        for line in chunk:
+            m = _ORDERED_RX.match(line)
+            assert m is not None
+            items.append(m.group(1).strip())
+        return OrderedListBlock(items=items)
+
+    return ParagraphBlock(text=" ".join(line.strip() for line in chunk).strip())
+
+
+def parse_markdown(markdown: str) -> StructuredDocument:
+    """Best-effort parse of a markdown document into the structured schema.
+
+    Sections are introduced by ATX headings (``#``..``######``).  Anything
+    before the first heading is wrapped into an implicit "Overview" section
+    so we never silently drop user content.  Section IDs are unique slugs of
+    their headings.
+    """
+    raw_lines = (markdown or "").splitlines()
+    lines = _strip_separators(raw_lines)
+
+    sections: list[Section] = []
+    used_ids: set[str] = set()
+    pending: list[str] = []
+    current: Section | None = None
+
+    def flush_pending_into(section: Section) -> None:
+        if not pending:
+            return
+        for chunk in _split_blocks(pending):
+            section.blocks.append(_parse_block(chunk))
+        pending.clear()
+
+    for line in lines:
+        m = _HEADING_RX.match(line)
+        if m:
+            if current is not None:
+                flush_pending_into(current)
+                sections.append(current)
+            elif pending:
+                # Content before the first heading: wrap in implicit section.
+                base = "overview"
+                section_id = make_unique_id(base, used_ids)
+                used_ids.add(section_id)
+                implicit = Section(id=section_id, heading="Overview", level=2)
+                flush_pending_into(implicit)
+                sections.append(implicit)
+            level = len(m.group(1))
+            heading = m.group(2).strip()
+            section_id = make_unique_id(slugify_heading(heading), used_ids)
+            used_ids.add(section_id)
+            current = Section(id=section_id, heading=heading, level=level)
+        else:
+            pending.append(line)
+
+    if current is not None:
+        flush_pending_into(current)
+        sections.append(current)
+    elif pending:
+        base = "overview"
+        section_id = make_unique_id(base, used_ids)
+        used_ids.add(section_id)
+        implicit = Section(id=section_id, heading="Overview", level=2)
+        flush_pending_into(implicit)
+        sections.append(implicit)
+
+    return StructuredDocument(sections=sections)

hindsight-api-slim/hindsight_api/engine/reflect/tools.py

@@ -23,6 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 async def tool_search_mental_models(
+    memory_engine: "MemoryEngine",
     conn: "Connection",
     bank_id: str,
     query: str,
@@ -32,7 +33,6 @@ async def tool_search_mental_models(
     tags_match: str = "any",
     tag_groups: "list | None" = None,
     exclude_ids: list[str] | None = None,
-    pending_consolidation: int = 0,
 ) -> dict[str, Any]:
     """
     Search user-curated mental models by semantic similarity.
@@ -82,7 +82,7 @@ async def tool_search_mental_models(
         f"""
         SELECT
             id, name, content,
-            tags, created_at, last_refreshed_at,
+            tags, created_at, last_refreshed_at, trigger,
             1 - (embedding <=> $2::vector) as relevance
         FROM {fq_table("mental_models")}
         WHERE bank_id = $1 AND embedding IS NOT NULL {filters}
@@ -99,10 +99,9 @@ async def tool_search_mental_models(
         if last_refreshed_at and last_refreshed_at.tzinfo is None:
             last_refreshed_at = last_refreshed_at.replace(tzinfo=timezone.utc)
 
-        # A mental model is stale when there are memories that haven't been consolidated yet —
-        # the same signal used for observations staleness.
-        is_stale = pending_consolidation > 0
-        staleness_reason = f"{pending_consolidation} memories pending consolidation" if is_stale else None
+        # Per-MM staleness: new in-scope memories since last refresh (includes pending).
+        is_stale = await memory_engine.compute_mental_model_is_stale(conn, bank_id, row)
+        staleness_reason = "new in-scope memories ingested since last refresh" if is_stale else None
 
         mental_models.append(
             {
@@ -136,6 +135,8 @@ async def tool_search_observations(
     last_consolidated_at: datetime | None = None,
     pending_consolidation: int = 0,
     source_facts_max_tokens: int = -1,
+    created_after: datetime | None = None,
+    created_before: datetime | None = None,
 ) -> dict[str, Any]:
     """
     Search consolidated observations using recall.
@@ -179,6 +180,8 @@ async def tool_search_observations(
         tags_match=tags_match,
         tag_groups=tag_groups,
         include_source_facts=include_source_facts,
+        created_after=created_after,
+        created_before=created_before,
         _connection_budget=1,
         _quiet=True,
         **recall_kwargs,
@@ -214,6 +217,9 @@ async def tool_recall(
     connection_budget: int = 1,
     max_chunk_tokens: int = 1000,
     fact_types: list[str] | None = None,
+    include_chunks: bool = True,
+    created_after: datetime | None = None,
+    created_before: datetime | None = None,
 ) -> dict[str, Any]:
     """
     Search memories using TEMPR retrieval.
@@ -230,15 +236,15 @@ async def tool_recall(
         tags: Filter by tags (includes untagged memories)
         tags_match: How to match tags - "any" (OR), "all" (AND), or "exact"
         connection_budget: Max DB connections for this recall (default 1 for internal ops)
-        max_chunk_tokens: Maximum tokens for raw source chunk text (default 1000, always included)
+        max_chunk_tokens: Maximum tokens for raw source chunk text (default 1000)
         fact_types: Optional filter for fact types to retrieve. Defaults to ["experience", "world"].
+        include_chunks: Whether to fetch raw chunk text alongside facts (default True).
 
     Returns:
-        Dict with list of matching memories including raw chunk text
+        Dict with list of matching memories including raw chunk text (when include_chunks)
     """
     # Only world/experience are valid for raw recall (observation is handled by search_observations)
     recall_fact_type = [ft for ft in (fact_types or ["experience", "world"]) if ft in ("world", "experience")]
-    include_chunks = True
     internal_ctx = replace(request_context, internal=True)
     result = await memory_engine.recall_async(
         bank_id=bank_id,
@@ -250,6 +256,8 @@ async def tool_recall(
         tags=tags,
         tags_match=tags_match,
         tag_groups=tag_groups,
+        created_after=created_after,
+        created_before=created_before,
         _connection_budget=connection_budget,
         _quiet=True,  # Suppress logging for internal operations
         include_chunks=include_chunks,

hindsight-api-slim/hindsight_api/engine/retain/embedding_utils.py

@@ -47,6 +47,16 @@ async def generate_embeddings_batch(embeddings_backend, texts: list[str]) -> lis
             embeddings_backend.encode,
             texts,
         )
-        return embeddings
     except Exception as e:
         raise Exception(f"Failed to generate batch embeddings: {str(e)}")
+
+    # Guarantee 1:1 alignment with input texts. A silent length mismatch here
+    # propagates downstream as zip() drops items, eventually surfacing as an
+    # IndexError in retain mapping (see issue #1037).
+    if len(embeddings) != len(texts):
+        raise RuntimeError(
+            f"Embeddings backend returned {len(embeddings)} vectors for {len(texts)} input texts; "
+            "expected exact 1:1 alignment"
+        )
+
+    return embeddings

hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py

@@ -7,6 +7,7 @@ Handles insertion of facts into the database.
 import json
 import logging
 import uuid
+from datetime import datetime
 
 from ...config import get_config
 from ..memory_engine import fq_table
@@ -224,6 +225,85 @@ async def ensure_bank_exists(conn, bank_id: str) -> None:
         await create_bank_vector_indexes(conn, bank_id, str(internal_id))
 
 
+async def delete_stale_observations_for_memories(
+    conn,
+    bank_id: str,
+    fact_ids: "list[str | uuid.UUID]",
+) -> int:
+    """Delete observations whose source memories are about to be removed.
+
+    Mirrors the cleanup performed by ``MemoryEngine.delete_document`` so that
+    every code path that removes ``memory_units`` also removes the
+    observations derived from them. Without this, ingesting a fresh version
+    of a document via the retain pipeline (which does a full-replace
+    ``DELETE FROM documents`` cascade) used to leave orphan observations
+    pointing at memory IDs that no longer existed.
+
+    For each observation referencing any of ``fact_ids``:
+    1. Delete the observation row (its text is stale once even one source
+       memory disappears).
+    2. Reset ``consolidated_at = NULL`` on the surviving source memories so
+       they get re-consolidated under fresh observations on the next run.
+
+    Must be called within an active transaction, before the source memories
+    are deleted.
+
+    Returns the number of observations deleted.
+    """
+    if not fact_ids:
+        return 0
+
+    fact_uuids = [uuid.UUID(str(fid)) if not isinstance(fid, uuid.UUID) else fid for fid in fact_ids]
+
+    affected_obs = await conn.fetch(
+        f"""
+        SELECT id, source_memory_ids
+        FROM {fq_table("memory_units")}
+        WHERE bank_id = $1
+          AND fact_type = 'observation'
+          AND source_memory_ids && $2::uuid[]
+        """,
+        bank_id,
+        fact_uuids,
+    )
+
+    if not affected_obs:
+        return 0
+
+    deleted_set = {str(uid) for uid in fact_uuids}
+    obs_ids = [obs["id"] for obs in affected_obs]
+    seen_remaining: set[str] = set()
+    remaining_source_ids: list[uuid.UUID] = []
+    for obs in affected_obs:
+        for src_id in obs["source_memory_ids"] or []:
+            src_str = str(src_id)
+            if src_str not in deleted_set and src_str not in seen_remaining:
+                remaining_source_ids.append(src_id)
+                seen_remaining.add(src_str)
+
+    await conn.execute(
+        f"DELETE FROM {fq_table('memory_units')} WHERE id = ANY($1::uuid[])",
+        obs_ids,
+    )
+
+    if remaining_source_ids:
+        await conn.execute(
+            f"""
+            UPDATE {fq_table("memory_units")}
+            SET consolidated_at = NULL
+            WHERE id = ANY($1::uuid[])
+              AND fact_type IN ('experience', 'world')
+            """,
+            remaining_source_ids,
+        )
+
+    logger.info(
+        f"[OBSERVATIONS] Deleted {len(obs_ids)} observations, reset {len(remaining_source_ids)} "
+        f"source memories for re-consolidation in bank {bank_id}"
+    )
+    return len(obs_ids)
+
+
 async def handle_document_tracking(
     conn,
     bank_id: str,
@@ -254,17 +334,58 @@ async def handle_document_tracking(
     combined_content = _sanitize_text(combined_content) or ""
     content_hash = hashlib.sha256(combined_content.encode()).hexdigest()
 
-    # Delete old document first (cascades to units and links)
-    # Only delete on the first batch to avoid deleting data we just inserted
+    # Delete old document first (cascades to units and links).
+    # Only delete on the first batch to avoid deleting data we just inserted.
+    # Before the cascade, fan out to delete observations derived from the
+    # outgoing memory_units — otherwise the FK ON DELETE CASCADE removes the
+    # source memory_units but leaves observation rows pointing at IDs that
+    # no longer exist (consolidated_at on co-source memories also stays
+    # frozen). Same cleanup the explicit ``delete_document`` API performs.
+    preserved_created_at = None
     if is_first_batch:
-        await conn.fetchval(
-            f"DELETE FROM {fq_table('documents')} WHERE id = $1 AND bank_id = $2 RETURNING id",
+        existing_unit_rows = await conn.fetch(
+            f"""
+            SELECT id FROM {fq_table("memory_units")}
+            WHERE document_id = $1 AND fact_type IN ('experience', 'world')
+            """,
+            document_id,
+        )
+        existing_unit_ids = [row["id"] for row in existing_unit_rows]
+        if existing_unit_ids:
+            invalidated = await delete_stale_observations_for_memories(conn, bank_id, existing_unit_ids)
+            if invalidated:
+                logger.info(
+                    f"[RETAIN] Document {document_id} re-ingested: invalidated "
+                    f"{invalidated} observation(s) derived from {len(existing_unit_ids)} outgoing memory_units"
+                )
+        # Explicitly delete memory_units by document_id BEFORE deleting the
+        # document row. The CASCADE from documents→chunks→memory_units only
+        # catches units that have a non-NULL chunk_id FK. Units with chunk_id=NULL
+        # (e.g. from partial writes or edge cases) would survive the cascade.
+        # This explicit delete ensures complete cleanup.
+        await conn.execute(
+            f"DELETE FROM {fq_table('memory_units')} WHERE document_id = $1 AND bank_id = $2",
+            document_id,
+            bank_id,
+        )
+        # Capture created_at before deletion so re-ingestion preserves it.
+        preserved_created_at = await conn.fetchval(
+            f"DELETE FROM {fq_table('documents')} WHERE id = $1 AND bank_id = $2 RETURNING created_at",
             document_id,
             bank_id,
         )
 
     # Insert document (or update if exists from concurrent operations)
-    await _upsert_document_row(conn, bank_id, document_id, combined_content, content_hash, retain_params, document_tags)
+    await _upsert_document_row(
+        conn,
+        bank_id,
+        document_id,
+        combined_content,
+        content_hash,
+        retain_params,
+        document_tags,
+        preserved_created_at=preserved_created_at,
+    )
 
 
 async def upsert_document_metadata(
@@ -297,12 +418,19 @@ async def _upsert_document_row(
     content_hash: str,
     retain_params: dict | None = None,
     document_tags: list[str] | None = None,
+    preserved_created_at: datetime | None = None,
 ) -> None:
-    """Insert or update a document row."""
+    """Insert or update a document row.
+
+    When ``preserved_created_at`` is provided, it is used for ``created_at`` on
+    INSERT so that re-ingesting a document (which deletes + inserts the row)
+    keeps the original creation timestamp. ``updated_at`` is always set to
+    ``NOW()`` on both INSERT and the ON CONFLICT UPDATE branch.
+    """
     await conn.execute(
         f"""
-        INSERT INTO {fq_table("documents")} (id, bank_id, original_text, content_hash, retain_params, tags)
-        VALUES ($1, $2, $3, $4, $5, $6)
+        INSERT INTO {fq_table("documents")} (id, bank_id, original_text, content_hash, retain_params, tags, created_at, updated_at)
+        VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, NOW()), NOW())
         ON CONFLICT (id, bank_id) DO UPDATE
         SET original_text = EXCLUDED.original_text,
             content_hash = EXCLUDED.content_hash,
@@ -316,6 +444,7 @@ async def _upsert_document_row(
         content_hash,
         json.dumps(retain_params) if retain_params else None,
         document_tags or [],
+        preserved_created_at,
     )
 
 

hindsight-api-slim/hindsight_api/engine/retain/link_utils.py

@@ -812,7 +812,6 @@ async def compute_semantic_links_ann(
                 bank_id,
                 fact_type,
                 top_k,
-                timeout=300,  # ANN on large banks can take minutes
             )
             logger.debug(f"[ANN] fact_type={fact_type}: {len(ft_rows)} rows in {time_mod.time() - t_query:.3f}s")
             rows.extend(ft_rows)

hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py

@@ -16,7 +16,7 @@ from typing import Any
 
 from ...worker.stage import set_stage
 from ..db_utils import acquire_with_retry
-from ..memory_engine import fq_table
+from ..memory_engine import count_tokens, fq_table
 from . import bank_utils
 
 
@@ -25,6 +25,32 @@ def utcnow():
     return datetime.now(UTC)
 
 
+def _merge_processed_content_tokens(a: int | None, b: int | None) -> int | None:
+    """Combine the processed-content-tokens signal across sub-results.
+
+    Semantics (see RetainResult.processed_content_tokens):
+      * None means "this part of the retain did not go through chunk-level
+        dedup" — i.e. the entire submitted payload was processed. If any
+        sub-result is None, the aggregate is None so callers conservatively
+        bill the full content.
+      * Otherwise, accumulate the int values.
+    """
+    if a is None or b is None:
+        return None
+    return a + b
+
+
+def _count_delta_content_tokens(delta_contents: list["RetainContent"]) -> int:
+    """Sum content + context tokens across the chunk items that were
+    actually fed into the extraction pipeline on a partial-delta retain.
+    """
+    total = 0
+    for c in delta_contents:
+        total += count_tokens(c.content or "")
+        total += count_tokens(c.context or "")
+    return total
+
+
 def parse_datetime_flexible(value: Any) -> datetime:
     """
     Parse a datetime value that could be either a datetime object or an ISO string.
@@ -72,7 +98,6 @@ from . import (
 from .types import (
     ChunkMetadata,
     EntityResolutionResult,
-    ExtractedFact,
     Phase1Result,
     Phase3Context,
     ProcessedFact,
@@ -302,8 +327,11 @@ async def _insert_facts_and_links(
         causal_link_count = await link_creation.create_causal_links_batch(conn, bank_id, unit_ids, processed_facts)
         log_buffer.append(f"  Causal links: {causal_link_count} links in {time.time() - step_start:.3f}s")
 
-    # Map results back to original content items
-    result_unit_ids = _map_results_to_contents(contents, extracted_facts, unit_ids if unit_ids else [])
+    # Map results back to original content items. Use processed_facts (not
+    # extracted_facts) because unit_ids has 1:1 alignment with processed_facts —
+    # any upstream drop between extraction and processing would otherwise cause
+    # an IndexError (see issue #1037).
+    result_unit_ids = _map_results_to_contents(contents, processed_facts, unit_ids if unit_ids else [])
 
     if outbox_callback:
         await outbox_callback(conn)
@@ -415,13 +443,21 @@ async def retain_batch(
     schema: str | None = None,
     outbox_callback: Callable[["asyncpg.Connection"], Awaitable[None]] | None = None,
     db_semaphore: "asyncio.Semaphore | None" = None,
-) -> tuple[list[list[str]], TokenUsage]:
+) -> tuple[list[list[str]], TokenUsage, int | None]:
     """
     Process a batch of content through the retain pipeline.
 
     Supports delta retain: when upserting a document that already has chunks,
     only re-processes chunks whose content has changed. Unchanged chunks keep
     their existing facts, entities, and links.
+
+    Returns a three-tuple of:
+      * per-content-item unit ID lists
+      * aggregate LLM token usage
+      * processed_content_tokens — content+context tokens that actually went
+        through extraction after chunk-level dedup, or ``None`` if this path
+        didn't dedup (caller should treat as "bill full submitted content").
+        See ``RetainResult.processed_content_tokens`` for details.
     """
     start_time = time.time()
     total_chars = sum(len(item.get("content", "")) for item in contents_dicts)
@@ -461,8 +497,9 @@ async def retain_batch(
             # Process each group and merge results back in original order
             result_unit_ids: list[list[str]] = [[] for _ in contents_dicts]
             total_usage = TokenUsage()
+            total_processed_tokens: int | None = 0
             for doc_key, (group_dicts, group_contents) in groups.items():
-                group_ids, group_usage = await retain_batch(
+                group_ids, group_usage, group_processed = await retain_batch(
                     pool=pool,
                     embeddings_model=embeddings_model,
                     llm_config=llm_config,
@@ -484,11 +521,12 @@ async def retain_batch(
                     if group_idx < len(group_ids):
                         result_unit_ids[orig_idx] = group_ids[group_idx]
                 total_usage = total_usage + group_usage
-            return result_unit_ids, total_usage
+                total_processed_tokens = _merge_processed_content_tokens(total_processed_tokens, group_processed)
+            return result_unit_ids, total_usage, total_processed_tokens
 
     # Resolve effective document_id early so both delta and streaming paths
-    # can find existing chunks from a prior attempt. On retry, the generated
-    # document_id is recovered from operation result_metadata.
+    # can find existing chunks from a prior attempt. On retry, a generated
+    # document_id is recovered from operation result_metadata.document_ids[0].
     effective_doc_id = document_id
     if not effective_doc_id:
         doc_ids = {item.get("document_id") for item in contents_dicts if item.get("document_id")}
@@ -507,26 +545,41 @@ async def retain_batch(
                         if isinstance(row["result_metadata"], dict)
                         else json.loads(row["result_metadata"])
                     )
-                    effective_doc_id = meta.get("generated_document_id")
+                    recovered = meta.get("document_ids") or []
+                    if recovered:
+                        effective_doc_id = recovered[0]
         except Exception:
             pass
     if not effective_doc_id:
         effective_doc_id = str(uuid.uuid4())
-        # Persist so retries reuse the same document_id
-        if operation_id:
-            try:
-                async with acquire_with_retry(pool) as conn:
-                    await conn.execute(
-                        f"""
-                        UPDATE {fq_table("async_operations")}
-                        SET result_metadata = result_metadata || $1::jsonb, updated_at = now()
-                        WHERE operation_id = $2
-                        """,
-                        json.dumps({"generated_document_id": effective_doc_id}),
-                        uuid.UUID(operation_id),
-                    )
-            except Exception:
-                logger.warning("Failed to persist generated document_id", exc_info=True)
+
+    # Record effective_doc_id on the operation (idempotent set-append). Captures
+    # both user-provided and generated ids so the operation shows every document
+    # it touched, and lets retries reuse the same generated id.
+    if operation_id:
+        try:
+            async with acquire_with_retry(pool) as conn:
+                await conn.execute(
+                    f"""
+                    UPDATE {fq_table("async_operations")}
+                    SET result_metadata = jsonb_set(
+                        COALESCE(result_metadata, '{{}}'::jsonb),
+                        '{{document_ids}}',
+                        CASE
+                            WHEN COALESCE(result_metadata->'document_ids', '[]'::jsonb) @> $1::jsonb
+                                THEN result_metadata->'document_ids'
+                            ELSE COALESCE(result_metadata->'document_ids', '[]'::jsonb) || $1::jsonb
+                        END,
+                        true
+                    ),
+                    updated_at = now()
+                    WHERE operation_id = $2
+                    """,
+                    json.dumps([effective_doc_id]),
+                    uuid.UUID(operation_id),
+                )
+        except Exception:
+            logger.warning("Failed to persist document_id", exc_info=True)
 
     # --- Append mode: prepend existing document content to new content ---
     # When update_mode="append", fetch the existing document text and prepend it
@@ -557,6 +610,31 @@ async def retain_batch(
                 f"[append] Prepended {len(existing_text):,} chars from existing document {effective_doc_id}"
             )
 
+    # --- Stale-request check (best-effort, before LLM extraction) ---
+    # If the document was already updated by a more recent retain (updated_at > our
+    # start_time), skip this request entirely to avoid overwriting newer content
+    # (e.g. a longer conversation) with older data. This is an optimization — the
+    # real correctness guarantee comes from the FOR UPDATE + content_hash check
+    # inside each batch TXN (see _run_mini_batch_db_work).
+    async with acquire_with_retry(pool) as conn:
+        doc_row = await conn.fetchrow(
+            f"SELECT updated_at FROM {fq_table('documents')} WHERE id = $1 AND bank_id = $2",
+            effective_doc_id,
+            bank_id,
+        )
+    if doc_row and doc_row["updated_at"]:
+        doc_updated = doc_row["updated_at"].timestamp()
+        if doc_updated > start_time:
+            log_buffer.append(
+                f"[stale] Skipping retain: document {effective_doc_id} was updated at "
+                f"{doc_row['updated_at'].isoformat()} (after this request started at "
+                f"{datetime.fromtimestamp(start_time, tz=UTC).isoformat()})"
+            )
+            logger.info("\n" + "\n".join(log_buffer) + "\n")
+            # No new content was processed — report 0 so callers can skip
+            # billing cleanly instead of falling back to full-content billing.
+            return [[] for _ in contents], TokenUsage(), 0
+
     # --- Delta retain: check if we can skip unchanged chunks ---
     if is_first_batch:
         delta_result = await _try_delta_retain(
@@ -713,7 +791,6 @@ async def _run_final_semantic_ann(
         async with ann_semaphore:
             t0 = time.time()
             async with acquire_with_retry(pool) as conn:
-                await conn.execute("SET statement_timeout = '300s'")
                 ann_links = await compute_semantic_links_ann(
                     conn,
                     bank_id,
@@ -726,7 +803,6 @@ async def _run_final_semantic_ann(
                 if ann_links:
                     await _bulk_insert_links(conn, ann_links, bank_id=bank_id)
                 chunk_link_counts[chunk_idx] = len(ann_links)
-                await conn.execute("RESET statement_timeout")
             logger.info(
                 f"[streaming] Final ANN chunk {chunk_idx + 1}/{num_chunks}: "
                 f"{len(ann_links)} links in {time.time() - t0:.3f}s"
@@ -792,25 +868,27 @@ async def _streaming_retain_batch(
     # Default template for metadata (context, event_date, etc.) when content list is empty.
     _default_content = RetainContent(content="")
 
-    # Load existing chunk hashes BEFORE document tracking to detect recovery.
-    # If chunks exist AND the document content hash matches, this is a retry of
-    # the same content — preserve existing data. If content differs, this is an
-    # update — cascade-delete old data and start fresh.
+    # ---------------------------------------------------------------------------
+    # Recovery detection (read-only, before LLM extraction)
+    # ---------------------------------------------------------------------------
+    # Check if this is a retry of the same content (crash recovery). If the
+    # document exists with a matching content_hash and has committed chunks,
+    # the producer can skip already-extracted chunks to avoid duplicate work.
     existing_chunk_hashes: set[str] = set()
     combined_content = "\n".join([c.get("content", "") for c in contents_dicts])
-    new_content_hash = hashlib.sha256(combined_content.encode()).hexdigest()
+    # Sanitize before hashing to match what handle_document_tracking stores
+    sanitized_content = fact_extraction._sanitize_text(combined_content) or ""
+    new_content_hash = hashlib.sha256(sanitized_content.encode()).hexdigest()
     is_recovery = False
 
     try:
         async with acquire_with_retry(pool) as conn:
-            # Check if document exists with matching content hash
             doc_row = await conn.fetchrow(
                 f"SELECT content_hash FROM {fq_table('documents')} WHERE id = $1 AND bank_id = $2",
                 effective_doc_id,
                 bank_id,
             )
             if doc_row and doc_row["content_hash"] == new_content_hash:
-                # Same content — load chunk hashes for recovery skip
                 existing_rows = await chunk_storage.load_existing_chunks(conn, bank_id, effective_doc_id)
                 existing_chunk_hashes = {c.content_hash for c in existing_rows if c.content_hash}
                 if existing_chunk_hashes:
@@ -822,24 +900,22 @@ async def _streaming_retain_batch(
     except Exception:
         pass  # If we can't load, just process all chunks
 
-    # Create/update the document row.
+    # ---------------------------------------------------------------------------
+    # Document tracking is DEFERRED to the first consumer batch TXN.
+    # ---------------------------------------------------------------------------
+    # Previously, document tracking (cascade-delete old data + insert doc row)
+    # ran in a separate transaction BEFORE LLM extraction. This left a gap
+    # between the cascade-delete and the first chunk write, allowing concurrent
+    # requests to interleave and produce duplicates.
+    #
+    # Now, document tracking runs atomically inside the first batch's write TXN,
+    # using SELECT ... FOR UPDATE on the document row for serialization across
+    # workers. Each batch TXN also verifies document ownership via content_hash
+    # to detect when a concurrent request has taken over the document.
+    # See _run_mini_batch_db_work() for the implementation.
     retain_params, merged_tags = _build_retain_params(contents_dicts, document_tags)
-    async with acquire_with_retry(pool) as conn:
-        async with conn.transaction():
-            if is_recovery:
-                # Recovery: same content, partially committed — preserve existing data
-                await fact_storage.upsert_document_metadata(
-                    conn, bank_id, effective_doc_id, combined_content, retain_params, merged_tags
-                )
-                log_buffer.append(
-                    f"[streaming] Document {effective_doc_id} updated (recovery, preserving existing chunks)"
-                )
-            else:
-                # Fresh or update: cascade-delete old data if document exists
-                await fact_storage.handle_document_tracking(
-                    conn, bank_id, effective_doc_id, combined_content, is_first_batch, retain_params, merged_tags
-                )
-                log_buffer.append(f"[streaming] Document {effective_doc_id} tracked (full content)")
+    # Track whether document tracking has been done (by the first batch)
+    doc_tracking_done = [False]
 
     # ---------------------------------------------------------------------------
     # Producer-consumer pipeline: LLM extraction runs concurrently with DB writes
@@ -852,6 +928,10 @@ async def _streaming_retain_batch(
 
     # Shared mutable state for the producer to report skipped chunks and usage
     producer_error: list[BaseException] = []
+    # Set to True by _run_mini_batch_db_work when a concurrent request takes
+    # over the document (content_hash mismatch). The consumer checks this and
+    # stops processing further batches.
+    pipeline_aborted: list[bool] = [False]
 
     # ---- LLM Producer ----
     # Fires all chunk extractions as concurrent tasks (bounded by the LLM
@@ -910,17 +990,15 @@ async def _streaming_retain_batch(
     # Phase 1 (entity resolution) -> Phase 2 (write txn) -> Phase 3 (ANN fire-and-forget).
     async def _db_consumer() -> None:
         batch: list[tuple] = []
-        global_chunk_offset = 0
         consumer_batch_idx = 0
 
         while True:
             item = await chunk_queue.get()
             if item is None:
                 # Process any remaining items
-                if batch:
+                if batch and not pipeline_aborted[0]:
                     await _process_db_batch(
                         batch,
-                        global_chunk_offset,
                         consumer_batch_idx,
                         is_last=True,
                     )
@@ -929,19 +1007,24 @@ async def _streaming_retain_batch(
             batch.append(item)
 
             if len(batch) >= chunk_batch_size:
+                if pipeline_aborted[0]:
+                    # Another request took over the document — discard this batch
+                    log_buffer.append(
+                        f"[streaming] Consumer: discarding batch of {len(batch)} chunks "
+                        f"(pipeline aborted due to concurrent takeover)"
+                    )
+                    batch = []
+                    continue
                 await _process_db_batch(
                     batch,
-                    global_chunk_offset,
                     consumer_batch_idx,
                     is_last=False,
                 )
-                global_chunk_offset += len(batch)
                 consumer_batch_idx += 1
                 batch = []
 
     async def _process_db_batch(
         batch: list[tuple],
-        global_chunk_offset: int,
         consumer_batch_idx: int,
         is_last: bool,
     ) -> None:
@@ -955,15 +1038,17 @@ async def _streaming_retain_batch(
 
         for global_idx, content, extracted, processed, chunk_meta, usage in batch:
             content_idx_in_batch = len(batch_contents)
-            # Adjust chunk indices to global offsets and remap content_index
+            # Adjust chunk indices to use the original global position (global_idx)
+            # so that chunk_id = {bank}_{doc}_{chunk_index} is deterministic regardless
+            # of task completion order. content_index is batch-relative for result grouping.
             for fact in extracted:
                 fact.content_index = content_idx_in_batch
                 if fact.chunk_index is not None:
-                    fact.chunk_index = global_chunk_offset + content_idx_in_batch
+                    fact.chunk_index = global_idx
             for pf in processed:
                 pf.content_index = content_idx_in_batch
             for cm in chunk_meta:
-                cm.chunk_index = global_chunk_offset + content_idx_in_batch
+                cm.chunk_index = global_idx
 
             batch_contents.append(content)
             batch_extracted.extend(extracted)
@@ -975,6 +1060,46 @@ async def _streaming_retain_batch(
         total_usage = total_usage + batch_usage
 
         if not batch_extracted:
+            # Even with 0 facts, the first batch must still run document tracking
+            # (cascade-delete + insert doc row) to establish ownership and prevent
+            # concurrent requests from interleaving. Later batches can safely skip.
+            if not doc_tracking_done[0]:
+                async with acquire_with_retry(pool) as conn:
+                    async with conn.transaction():
+                        await conn.execute(
+                            f"INSERT INTO {fq_table('documents')} (id, bank_id, original_text, content_hash) "
+                            f"VALUES ($1, $2, '', '__pending__') "
+                            f"ON CONFLICT (id, bank_id) DO NOTHING",
+                            effective_doc_id,
+                            bank_id,
+                        )
+                        await conn.fetchval(
+                            f"SELECT content_hash FROM {fq_table('documents')} "
+                            f"WHERE id = $1 AND bank_id = $2 FOR UPDATE",
+                            effective_doc_id,
+                            bank_id,
+                        )
+                        if is_recovery:
+                            await fact_storage.upsert_document_metadata(
+                                conn,
+                                bank_id,
+                                effective_doc_id,
+                                combined_content,
+                                retain_params,
+                                merged_tags,
+                            )
+                        else:
+                            await fact_storage.handle_document_tracking(
+                                conn,
+                                bank_id,
+                                effective_doc_id,
+                                combined_content,
+                                is_first_batch,
+                                retain_params,
+                                merged_tags,
+                            )
+                        doc_tracking_done[0] = True
+                        log_buffer.append(f"[streaming] Document {effective_doc_id} tracked (0 facts in first batch)")
             log_buffer.append(
                 f"[streaming] Consumer batch {consumer_batch_idx + 1}: "
                 f"0 facts extracted from {len(batch)} chunks, skipping"
@@ -1005,10 +1130,92 @@ async def _streaming_retain_batch(
 
             logger.info(f"[streaming] Phase 1 (entity resolution): {time.time() - p1_start:.3f}s")
 
-            # Phase 2 — Write transaction (within-batch semantic links only)
+            # Phase 2 — Write transaction
+            # -----------------------------------------------------------------
+            # Concurrent-safety via row-level locking:
+            #
+            # The streaming pipeline splits work across multiple batch TXNs.
+            # Without protection, two concurrent retains for the same document
+            # can interleave: Request A writes batch1, Request B cascade-deletes
+            # A's doc and writes its own batch1, then A's batch2 adds stale data
+            # on top of B's → duplicates.
+            #
+            # To prevent this, every batch TXN:
+            #   1. SELECT ... FOR UPDATE on the document row — serializes all
+            #      writers for this document at the DB level (works across workers).
+            #   2. Check content_hash — if it doesn't match ours, another request
+            #      took over the document → abort remaining batches.
+            #   3. First batch only: run handle_document_tracking (cascade-delete
+            #      old data + insert doc row) atomically with the first chunk write.
+            #      This eliminates the gap between "delete old" and "insert new"
+            #      that previously allowed interleaving.
+            # -----------------------------------------------------------------
+
             p2_start = time.time()
+            batch_result_ids = None
+            phase3_ctx = None
             async with acquire_with_retry(pool) as conn:
                 async with conn.transaction():
+                    # --- Document ownership gate ---
+                    # Lock the document row to serialize all concurrent writers.
+                    # SELECT ... FOR UPDATE doesn't lock non-existent rows, so we
+                    # first ensure the row exists with a lightweight upsert, THEN lock it.
+                    # The content_hash='__pending__' placeholder is immediately overwritten
+                    # by handle_document_tracking or upsert_document_metadata below.
+                    await conn.execute(
+                        f"INSERT INTO {fq_table('documents')} (id, bank_id, original_text, content_hash) "
+                        f"VALUES ($1, $2, '', '__pending__') "
+                        f"ON CONFLICT (id, bank_id) DO NOTHING",
+                        effective_doc_id,
+                        bank_id,
+                    )
+                    existing_hash = await conn.fetchval(
+                        f"SELECT content_hash FROM {fq_table('documents')} WHERE id = $1 AND bank_id = $2 FOR UPDATE",
+                        effective_doc_id,
+                        bank_id,
+                    )
+
+                    if not doc_tracking_done[0]:
+                        # --- First batch: document tracking (atomic with chunk write) ---
+                        if is_recovery:
+                            await fact_storage.upsert_document_metadata(
+                                conn,
+                                bank_id,
+                                effective_doc_id,
+                                combined_content,
+                                retain_params,
+                                merged_tags,
+                            )
+                            log_buffer.append(
+                                f"[streaming] Document {effective_doc_id} updated "
+                                f"(recovery, preserving existing chunks)"
+                            )
+                        else:
+                            await fact_storage.handle_document_tracking(
+                                conn,
+                                bank_id,
+                                effective_doc_id,
+                                combined_content,
+                                is_first_batch,
+                                retain_params,
+                                merged_tags,
+                            )
+                            log_buffer.append(f"[streaming] Document {effective_doc_id} tracked (full content)")
+                        doc_tracking_done[0] = True
+                    else:
+                        # --- Later batches: verify we still own the document ---
+                        # If another request took over (cascade-deleted our doc and
+                        # inserted its own), the content_hash won't match ours.
+                        if existing_hash is not None and existing_hash != new_content_hash:
+                            log_buffer.append(
+                                f"[streaming] Document {effective_doc_id} taken over by "
+                                f"concurrent request (hash mismatch) — aborting remaining batches"
+                            )
+                            logger.info("\n" + "\n".join(log_buffer) + "\n")
+                            # Signal the consumer to stop processing further batches
+                            pipeline_aborted[0] = True
+                            return
+
                     # Store chunks with correct global indices
                     step_start = time.time()
                     chunk_id_map = {}
@@ -1050,11 +1257,14 @@ async def _streaming_retain_batch(
                 logger.info(f"[streaming] Phase 2 (write txn): {time.time() - p2_start:.3f}s")
 
                 # Best-effort: entity viz + stats (fast, not semantic ANN)
-                try:
-                    await entity_resolver.flush_pending_stats()
-                    await _build_and_insert_entity_links_phase3(pool, entity_resolver, bank_id, phase3_ctx, log_buffer)
-                except Exception:
-                    logger.warning(f"Phase 3 stats (consumer batch {consumer_batch_idx + 1}) failed", exc_info=True)
+                if phase3_ctx is not None:
+                    try:
+                        await entity_resolver.flush_pending_stats()
+                        await _build_and_insert_entity_links_phase3(
+                            pool, entity_resolver, bank_id, phase3_ctx, log_buffer
+                        )
+                    except Exception:
+                        logger.warning(f"Phase 3 stats (consumer batch {consumer_batch_idx + 1}) failed", exc_info=True)
 
             logger.info(
                 f"[streaming] Consumer batch {consumer_batch_idx + 1} total "
@@ -1062,8 +1272,9 @@ async def _streaming_retain_batch(
             )
 
             # Collect unit_ids from this batch
-            for content_ids in batch_result_ids:
-                all_unit_ids.extend(content_ids)
+            if batch_result_ids:
+                for content_ids in batch_result_ids:
+                    all_unit_ids.extend(content_ids)
 
         if db_semaphore is not None:
             async with db_semaphore:
@@ -1106,6 +1317,47 @@ async def _streaming_retain_batch(
         if producer_error:
             raise producer_error[0]
 
+        # If no batch was processed (e.g. zero facts extracted from gibberish
+        # content, or all chunks skipped in recovery), the document row was
+        # never created by the first batch TXN. Create it now so the document
+        # is tracked regardless of extraction results.
+        if not doc_tracking_done[0] and not pipeline_aborted[0]:
+            async with acquire_with_retry(pool) as conn:
+                async with conn.transaction():
+                    await conn.execute(
+                        f"INSERT INTO {fq_table('documents')} (id, bank_id, original_text, content_hash) "
+                        f"VALUES ($1, $2, '', '__pending__') "
+                        f"ON CONFLICT (id, bank_id) DO NOTHING",
+                        effective_doc_id,
+                        bank_id,
+                    )
+                    await conn.fetchval(
+                        f"SELECT content_hash FROM {fq_table('documents')} WHERE id = $1 AND bank_id = $2 FOR UPDATE",
+                        effective_doc_id,
+                        bank_id,
+                    )
+                    if is_recovery:
+                        await fact_storage.upsert_document_metadata(
+                            conn,
+                            bank_id,
+                            effective_doc_id,
+                            combined_content,
+                            retain_params,
+                            merged_tags,
+                        )
+                    else:
+                        await fact_storage.handle_document_tracking(
+                            conn,
+                            bank_id,
+                            effective_doc_id,
+                            combined_content,
+                            is_first_batch,
+                            retain_params,
+                            merged_tags,
+                        )
+                    doc_tracking_done[0] = True
+                    log_buffer.append(f"[streaming] Document {effective_doc_id} tracked (no facts extracted)")
+
         # Mark facts as committed in operation metadata (crash recovery checkpoint)
         if operation_id and all_unit_ids:
             try:
@@ -1142,16 +1394,31 @@ async def _streaming_retain_batch(
     # This replaces per-batch within-batch + fire-and-forget ANN with a single
     # efficient pass after all facts are in the database.
     # ---------------------------------------------------------------------------
-    if all_unit_ids:
+    if all_unit_ids and not pipeline_aborted[0]:
         ann_start = time.time()
-        await _run_final_semantic_ann(pool, bank_id, all_unit_ids, log_buffer)
+        try:
+            await _run_final_semantic_ann(pool, bank_id, all_unit_ids, log_buffer)
+        except Exception:
+            # ANN pass is best-effort. FK violations can occur if a concurrent
+            # retain cascade-deleted our units between the batch commit and here.
+            logger.warning(
+                f"[streaming] Final ANN pass failed for document {effective_doc_id} "
+                f"(units may have been superseded by concurrent retain)",
+                exc_info=True,
+            )
         log_buffer.append(f"[streaming] Final ANN pass: {time.time() - ann_start:.3f}s for {len(all_unit_ids)} units")
 
     total_time = time.time() - start_time
     log_buffer.append(f"{'=' * 60}")
-    log_buffer.append(
-        f"STREAMING RETAIN COMPLETE: {len(all_unit_ids)} units across {num_batches} batches in {total_time:.3f}s"
-    )
+    if pipeline_aborted[0]:
+        log_buffer.append(
+            f"STREAMING RETAIN ABORTED: document {effective_doc_id} was taken over by "
+            f"a concurrent request after {total_time:.3f}s — data from this request was discarded"
+        )
+    else:
+        log_buffer.append(
+            f"STREAMING RETAIN COMPLETE: {len(all_unit_ids)} units across {num_batches} batches in {total_time:.3f}s"
+        )
     log_buffer.append(f"Document: {effective_doc_id}")
     log_buffer.append(f"{'=' * 60}")
     logger.info("\n" + "\n".join(log_buffer) + "\n")
@@ -1159,7 +1426,10 @@ async def _streaming_retain_batch(
     # Map all unit_ids back to the original content items.
     # For streaming mode with a single document, all units belong to content 0.
     result_unit_ids = [all_unit_ids] + [[] for _ in contents[1:]]
-    return result_unit_ids, total_usage
+    # The streaming path doesn't compute per-chunk content-hash dedup in
+    # a way that lets us report a partial-processed tokens count — signal
+    # ``None`` so callers bill against the full submitted payload.
+    return result_unit_ids, total_usage, None
 
 
 # ---------------------------------------------------------------------------
@@ -1187,10 +1457,15 @@ async def _try_delta_retain(
     schema,
     outbox_callback,
     db_semaphore: "asyncio.Semaphore | None" = None,
-):
+) -> tuple[list[list[str]], TokenUsage, int | None] | None:
     """
     Attempt delta retain for a document upsert. Returns result tuple if delta
     was performed, or None to fall back to full retain.
+
+    When a result tuple is returned, the third element is the content+context
+    token count for the chunks that actually went through extraction
+    (``0`` if the submission matched prior content exactly and nothing was
+    re-extracted).
     """
     # Need a single document_id
     effective_doc_id = document_id
@@ -1200,9 +1475,17 @@ async def _try_delta_retain(
             return None
         effective_doc_id = doc_ids.pop()
 
-    # Load existing chunks
+    # Load existing chunks and snapshot the document's content_hash. This is
+    # outside the write TXN, so a concurrent retain could modify the document
+    # between this read and the write. The write TXN verifies the hash hasn't
+    # changed; if it has, we fall back to streaming (which has full protection).
     async with acquire_with_retry(pool) as conn:
         existing_chunks = await chunk_storage.load_existing_chunks(conn, bank_id, effective_doc_id)
+        doc_hash_at_load = await conn.fetchval(
+            f"SELECT content_hash FROM {fq_table('documents')} WHERE id = $1 AND bank_id = $2",
+            effective_doc_id,
+            bank_id,
+        )
 
     if not existing_chunks:
         return None
@@ -1310,8 +1593,28 @@ async def _try_delta_retain(
         )
 
         # PHASE 2 — Core Write Transaction (atomic)
+        # Lock the document row and verify ownership. Delta loaded existing
+        # chunks OUTSIDE this TXN, so a concurrent retain may have cascade-deleted
+        # and replaced the document since then. If the content_hash changed,
+        # the chunk state we based our delta diff on is stale — abort.
         async with acquire_with_retry(pool) as conn:
             async with conn.transaction():
+                current_hash = await conn.fetchval(
+                    f"SELECT content_hash FROM {fq_table('documents')} WHERE id = $1 AND bank_id = $2 FOR UPDATE",
+                    effective_doc_id,
+                    bank_id,
+                )
+                # Verify the document hasn't been replaced since we loaded chunks.
+                # Compare the current hash against what we snapshotted at load time.
+                if current_hash is not None and doc_hash_at_load is not None and current_hash != doc_hash_at_load:
+                    log_buffer.append(
+                        f"[delta] Document {effective_doc_id} was modified by concurrent request "
+                        f"since chunks were loaded — aborting delta, falling back to full retain"
+                    )
+                    logger.info("\n" + "\n".join(log_buffer) + "\n")
+                    # Return None to fall back to streaming (which has full FOR UPDATE protection)
+                    return None
+
                 # Update document metadata (no delete)
                 step_start = time.time()
                 combined_content = "\n".join([c.get("content", "") for c in contents_dicts])
@@ -1421,7 +1724,12 @@ async def _try_delta_retain(
             await _run_delta_db_work()
     else:
         await _run_delta_db_work()
-    return result_unit_ids, usage
+    # Count content + context tokens that actually went through extraction.
+    # ``delta_contents`` holds the per-chunk RetainContent items for the
+    # changed/new chunks (see ``_build_delta_contents``) — i.e. exactly what
+    # the LLM pipeline saw this call. Unchanged chunks contribute zero.
+    processed_tokens = _count_delta_content_tokens(delta_contents)
+    return result_unit_ids, usage, processed_tokens
 
 
 async def _delta_metadata_only(
@@ -1438,6 +1746,12 @@ async def _delta_metadata_only(
     """Handle the case where no chunks changed — just update document metadata and tags."""
     async with acquire_with_retry(pool) as conn:
         async with conn.transaction():
+            # Lock the document row to serialize with concurrent retains
+            await conn.fetchval(
+                f"SELECT content_hash FROM {fq_table('documents')} WHERE id = $1 AND bank_id = $2 FOR UPDATE",
+                document_id,
+                bank_id,
+            )
             combined_content = "\n".join([c.get("content", "") for c in contents_dicts])
             retain_params, merged_tags = _build_retain_params(contents_dicts, document_tags)
             await fact_storage.upsert_document_metadata(
@@ -1455,7 +1769,11 @@ async def _delta_metadata_only(
     total_time = time.time() - start_time
     log_buffer.append(f"DELTA RETAIN (no changes): metadata updated in {total_time:.3f}s")
     logger.info("\n" + "\n".join(log_buffer) + "\n")
-    return [[] for _ in contents], TokenUsage()
+    # Nothing went through the extraction pipeline — report 0 processed
+    # content tokens so callers can bill accordingly (a caller that's been
+    # told ``0`` knows the retain was a pure metadata update and should
+    # charge nothing for content).
+    return [[] for _ in contents], TokenUsage(), 0
 
 
 # ---------------------------------------------------------------------------
@@ -1550,12 +1868,19 @@ def _build_delta_contents(
 
 def _map_results_to_contents(
     contents: list[RetainContent],
-    extracted_facts: list[ExtractedFact],
+    processed_facts: list[ProcessedFact],
     unit_ids: list[str],
 ) -> list[list[str]]:
-    """Map created unit IDs back to original content items."""
+    """Map created unit IDs back to original content items.
+
+    `processed_facts` and `unit_ids` must have the same length: each unit_id
+    corresponds to the processed_fact at the same index.
+    """
+    if len(processed_facts) != len(unit_ids):
+        raise ValueError(f"processed_facts ({len(processed_facts)}) and unit_ids ({len(unit_ids)}) length mismatch")
+
     facts_by_content: dict[int, list[int]] = {i: [] for i in range(len(contents))}
-    for i, fact in enumerate(extracted_facts):
+    for i, fact in enumerate(processed_facts):
         # Normalize content_index: some LLM providers return 1-indexed values.
         # Clamp to valid range to prevent KeyError.
         idx = fact.content_index
@@ -1564,12 +1889,8 @@ def _map_results_to_contents(
         facts_by_content[idx].append(i)
 
     result_unit_ids = []
-    unit_idx = 0
     for content_index in range(len(contents)):
-        content_unit_ids = []
-        for _ in facts_by_content[content_index]:
-            content_unit_ids.append(unit_ids[unit_idx])
-            unit_idx += 1
+        content_unit_ids = [unit_ids[i] for i in facts_by_content[content_index]]
         result_unit_ids.append(content_unit_ids)
 
     return result_unit_ids

hindsight-api-slim/hindsight_api/engine/search/graph_retrieval.py

@@ -8,6 +8,7 @@ of the recall pipeline.
 
 import logging
 from abc import ABC, abstractmethod
+from datetime import datetime
 
 from .tags import TagGroup, TagsMatch
 from .types import GraphRetrievalTimings, RetrievalResult
@@ -45,6 +46,8 @@ class GraphRetriever(ABC):
         tags: list[str] | None = None,  # Visibility scope tags for filtering
         tags_match: TagsMatch = "any",  # How to match tags: 'any' (OR) or 'all' (AND)
         tag_groups: list[TagGroup] | None = None,  # Compound boolean tag filter groups
+        created_after: datetime | None = None,  # Only include memory_units created after this time
+        created_before: datetime | None = None,  # Only include memory_units created before this time
     ) -> tuple[list[RetrievalResult], GraphRetrievalTimings | None]:
         """
         Retrieve relevant facts via graph traversal.

hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py

@@ -28,6 +28,8 @@ import asyncio
 import logging
 import math
 import time
+from datetime import datetime
+from typing import Any
 
 from ...config import get_config
 from ..db_utils import acquire_with_retry
@@ -49,6 +51,8 @@ async def _find_semantic_seeds(
     tags: list[str] | None = None,
     tags_match: TagsMatch = "any",
     tag_groups: list[TagGroup] | None = None,
+    created_after: datetime | None = None,
+    created_before: datetime | None = None,
 ) -> list[RetrievalResult]:
     """Find semantic seeds via embedding search."""
     from .tags import build_tag_groups_where_clause, build_tags_where_clause_simple
@@ -56,10 +60,24 @@ async def _find_semantic_seeds(
     tags_clause = build_tags_where_clause_simple(tags, 6, match=tags_match)
     tag_groups_param_start = 6 + (1 if tags else 0)
     groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start)
+
+    _next_idx = tag_groups_param_start + len(groups_params)
+    created_range_clause = ""
+    created_range_params: list[Any] = []
+    if created_after is not None:
+        created_range_params.append(created_after)
+        created_range_clause += f" AND updated_at > ${_next_idx}"
+        _next_idx += 1
+    if created_before is not None:
+        created_range_params.append(created_before)
+        created_range_clause += f" AND updated_at < ${_next_idx}"
+        _next_idx += 1
+
     params = [query_embedding_str, bank_id, fact_type, threshold, limit]
     if tags:
         params.append(tags)
     params.extend(groups_params)
+    params.extend(created_range_params)
 
     rows = await conn.fetch(
         f"""
@@ -73,6 +91,7 @@ async def _find_semantic_seeds(
           AND (1 - (embedding <=> $1::vector)) >= $4
           {tags_clause}
           {groups_clause}
+          {created_range_clause}
         ORDER BY embedding <=> $1::vector
         LIMIT $5
         """,
@@ -121,6 +140,8 @@ class LinkExpansionRetriever(GraphRetriever):
         tags: list[str] | None = None,
         tags_match: TagsMatch = "any",
         tag_groups: list[TagGroup] | None = None,
+        created_after: "datetime | None" = None,
+        created_before: "datetime | None" = None,
     ) -> tuple[list[RetrievalResult], GraphRetrievalTimings | None]:
         """
         Retrieve facts by expanding links from seeds.
@@ -159,6 +180,8 @@ class LinkExpansionRetriever(GraphRetriever):
                     tags=tags,
                     tags_match=tags_match,
                     tag_groups=tag_groups,
+                    created_after=created_after,
+                    created_before=created_before,
                 )
                 timings.seeds_time = time.time() - seeds_start
                 logger.debug(

hindsight-api-slim/hindsight_api/engine/search/retrieval.py

@@ -13,7 +13,7 @@ import logging
 import re
 from dataclasses import dataclass, field
 from datetime import UTC, datetime
-from typing import Optional
+from typing import Any, Optional
 
 from ...config import get_config
 from ..db_utils import acquire_with_retry
@@ -98,6 +98,8 @@ async def retrieve_semantic_bm25_combined(
     tags: list[str] | None = None,
     tags_match: TagsMatch = "any",
     tag_groups: list[TagGroup] | None = None,
+    created_after: datetime | None = None,
+    created_before: datetime | None = None,
 ) -> dict[str, tuple[list[RetrievalResult], list[RetrievalResult]]]:
     """
     Combined semantic + BM25 retrieval for multiple fact types in a single query.
@@ -163,6 +165,21 @@ async def retrieve_semantic_bm25_combined(
     tag_groups_param_start = tags_param_idx + (1 if tags else 0)
     groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start)
 
+    # --- created_at time range filter (appended after tags/groups) ---
+    # Param indices are computed relative to the final params list built below,
+    # so we pre-compute the next available index after all preceding params.
+    _next_idx = tag_groups_param_start + len(groups_params)
+    created_range_clause = ""
+    created_range_params: list[Any] = []
+    if created_after is not None:
+        created_range_params.append(created_after)
+        created_range_clause += f" AND updated_at > ${_next_idx}"
+        _next_idx += 1
+    if created_before is not None:
+        created_range_params.append(created_before)
+        created_range_clause += f" AND updated_at < ${_next_idx}"
+        _next_idx += 1
+
     # --- Semantic UNION ALL arms (one per fact_type) ---
     # Each arm has its own ORDER BY embedding <=> $1 LIMIT {hnsw_fetch}, which
     # lets the planner use the partial HNSW index for that fact_type.
@@ -180,6 +197,7 @@ async def retrieve_semantic_bm25_combined(
             f"   AND (1 - (embedding <=> $1::vector)) >= 0.3"
             f"   {tags_clause}"
             f"   {groups_clause}"
+            f"   {created_range_clause}"
             f" ORDER BY embedding <=> $1::vector"
             f" LIMIT {hnsw_fetch})"
         )
@@ -220,6 +238,7 @@ async def retrieve_semantic_bm25_combined(
                 f"   {bm25_where_filter}"
                 f"   {tags_clause}"
                 f"   {groups_clause}"
+                f"   {created_range_clause}"
                 f" ORDER BY {bm25_order_by}"
                 f" LIMIT $3)"
             )
@@ -233,6 +252,7 @@ async def retrieve_semantic_bm25_combined(
     if tags:
         params.append(tags)
     params.extend(groups_params)
+    params.extend(created_range_params)
 
     rows = await conn.fetch(query, *params)
 
@@ -266,6 +286,8 @@ async def retrieve_temporal_combined(
     tags: list[str] | None = None,
     tags_match: TagsMatch = "any",
     tag_groups: list[TagGroup] | None = None,
+    created_after: datetime | None = None,
+    created_before: datetime | None = None,
 ) -> dict[str, list[RetrievalResult]]:
     """
     Temporal retrieval for multiple fact types in a single query.
@@ -299,10 +321,25 @@ async def retrieve_temporal_combined(
     tags_clause = build_tags_where_clause_simple(tags, 7, match=tags_match)
     tag_groups_param_start = 7 + (1 if tags else 0)
     groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start)
+
+    # created_at time range filter (after tags/groups)
+    _next_idx = tag_groups_param_start + len(groups_params)
+    created_range_clause = ""
+    created_range_params: list[Any] = []
+    if created_after is not None:
+        created_range_params.append(created_after)
+        created_range_clause += f" AND updated_at > ${_next_idx}"
+        _next_idx += 1
+    if created_before is not None:
+        created_range_params.append(created_before)
+        created_range_clause += f" AND updated_at < ${_next_idx}"
+        _next_idx += 1
+
     params: list = [query_emb_str, bank_id, fact_types, start_date, end_date, semantic_threshold]
     if tags:
         params.append(tags)
     params.extend(groups_params)
+    params.extend(created_range_params)
 
     # Two-phase entry point query:
     # Phase 1 (date_ranked): rank by date only — no embedding computation — for all units in
@@ -334,6 +371,7 @@ async def retrieve_temporal_combined(
               )
               {tags_clause}
               {groups_clause}
+              {created_range_clause}
         ),
         sim_ranked AS (
             SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.proof_count, mu.document_id, mu.chunk_id, mu.tags, mu.metadata,
@@ -536,6 +574,8 @@ async def retrieve_all_fact_types_parallel(
     tags: list[str] | None = None,
     tags_match: TagsMatch = "any",
     tag_groups: list[TagGroup] | None = None,
+    created_after: datetime | None = None,
+    created_before: datetime | None = None,
 ) -> MultiFactTypeRetrievalResult:
     """
     Optimized retrieval for multiple fact types using batched queries.
@@ -594,6 +634,8 @@ async def retrieve_all_fact_types_parallel(
             tags=tags,
             tags_match=tags_match,
             tag_groups=tag_groups,
+            created_after=created_after,
+            created_before=created_before,
         )
         semantic_bm25_time = time.time() - semantic_bm25_start
 
@@ -613,6 +655,8 @@ async def retrieve_all_fact_types_parallel(
                 tags=tags,
                 tags_match=tags_match,
                 tag_groups=tag_groups,
+                created_after=created_after,
+                created_before=created_before,
             )
             temporal_time = time.time() - temporal_start
 
@@ -636,6 +680,8 @@ async def retrieve_all_fact_types_parallel(
             tags=tags,
             tags_match=tags_match,
             tag_groups=tag_groups,
+            created_after=created_after,
+            created_before=created_before,
         )
         return ft, results, time.time() - graph_start, graph_timing
 

hindsight-api-slim/hindsight_api/engine/search/think_utils.py

@@ -73,7 +73,7 @@ def format_facts_for_prompt(facts: list[MemoryFact]) -> str:
 
         formatted.append(fact_obj)
 
-    return json.dumps(formatted, indent=2)
+    return json.dumps(formatted, indent=2, ensure_ascii=False)
 
 
 def format_entity_summaries_for_prompt(entities: dict) -> str:

hindsight-api-slim/hindsight_api/engine/task_backend.py

@@ -2,7 +2,8 @@
 Task backend for distributed task processing.
 
 This provides an abstraction for task storage and execution:
-- BrokerTaskBackend: Uses PostgreSQL as broker (production)
+- BrokerTaskBackend: Uses PostgreSQL as broker (production API servers)
+- WorkerTaskBackend: No-op submit_task (production workers — child tasks are polled)
 - SyncTaskBackend: Executes tasks immediately (testing/embedded)
 """
 
@@ -125,6 +126,33 @@ class SyncTaskBackend(TaskBackend):
         logger.debug("SyncTaskBackend shutdown")
 
 
+class WorkerTaskBackend(TaskBackend):
+    """
+    Task backend for worker processes.
+
+    Workers execute tasks directly via the poller (claim → execute), so they
+    don't need submit_task to run anything.  When engine code running *inside*
+    a worker-executed task calls submit_task (e.g. retain triggers consolidation),
+    the row has already been INSERTed into async_operations with task_payload by
+    _submit_async_operation — so submit_task is a no-op.  The new task will be
+    picked up by a worker on the next poll cycle instead of being executed inline,
+    which avoids blocking the parent task.
+    """
+
+    async def initialize(self):
+        self._initialized = True
+        logger.debug("WorkerTaskBackend initialized")
+
+    async def submit_task(self, task_dict: dict[str, Any]):
+        """No-op: the row already exists in async_operations; a worker will claim it."""
+        task_type = task_dict.get("type", "unknown")
+        logger.debug(f"WorkerTaskBackend: submit_task no-op for {task_type} (will be picked up by poller)")
+
+    async def shutdown(self):
+        self._initialized = False
+        logger.debug("WorkerTaskBackend shutdown")
+
+
 class BrokerTaskBackend(TaskBackend):
     """
     Task backend using PostgreSQL as broker.
@@ -193,17 +221,21 @@ class BrokerTaskBackend(TaskBackend):
         table = fq_table("async_operations", schema)
 
         if operation_id:
-            # Update existing operation with task payload
+            # Callers now include task_payload in the same INSERT that creates the
+            # async_operations row (see MemoryEngine._submit_async_operation). The
+            # WHERE clause guards against overwriting that payload — the UPDATE is a
+            # no-op when the row is already claimable, and only fills in a NULL payload
+            # for any legacy caller that still creates the row first.
             await pool.execute(
                 f"""
                 UPDATE {table}
                 SET task_payload = $1::jsonb, updated_at = now()
-                WHERE operation_id = $2
+                WHERE operation_id = $2 AND task_payload IS NULL
                 """,
                 payload_json,
                 operation_id,
             )
-            logger.debug(f"Updated task payload for operation {operation_id}")
+            logger.debug(f"submit_task UPDATE for operation {operation_id} (no-op if payload already set)")
         else:
             # Insert new operation (for tasks without pre-created records)
             # e.g., access_count_update tasks

hindsight-api-slim/hindsight_api/extensions/__init__.py

@@ -55,6 +55,7 @@ from hindsight_api.extensions.tenant import (
     TenantExtension,
 )
 from hindsight_api.models import RequestContext
+from hindsight_api.worker.exceptions import DeferOperation
 
 __all__ = [
     # Base
@@ -68,6 +69,7 @@ __all__ = [
     # MCP Extension
     "MCPExtension",
     # Operation Validator - Core
+    "DeferOperation",
     "OperationValidationError",
     "OperationValidatorExtension",
     "RecallContext",

hindsight-api-slim/hindsight_api/extensions/operation_validator.py

@@ -176,6 +176,22 @@ class RetainResult:
     llm_input_tokens: int | None = None
     llm_output_tokens: int | None = None
     llm_total_tokens: int | None = None
+    # Content tokens the retain pipeline actually processed, after
+    # chunk-level content-hash deduplication. Semantics:
+    #   None — no dedup signal available (e.g. a first-time retain or a
+    #          path that doesn't compute it). Callers that care about
+    #          "what was actually new on this retain" should treat None
+    #          as "the full submitted content was processed."
+    #   0    — the entire submission was a duplicate of prior content
+    #          (all chunks matched by content_hash); nothing went
+    #          through LLM extraction.
+    #   N>0  — only N tokens of content + context went through the
+    #          extraction pipeline. The remainder was dedup'd against
+    #          existing chunks.
+    # This is the basis most billing/metering extensions want to use
+    # when the customer's client resubmits growing payloads to the same
+    # document_id (e.g. a session transcript appended to on each turn).
+    processed_content_tokens: int | None = None
 
 
 @dataclass
@@ -376,6 +392,16 @@ class OperationValidatorExtension(Extension, ABC):
         2. [operation executes]
         3. on_*_complete (post-operation)
 
+    Outcomes for `validate_*` hooks:
+        - accept: return `ValidationResult.accept()` (or `accept_with(...)`)
+        - reject: return `ValidationResult.reject(reason, status_code)`
+          (raises `OperationValidationError` upstream)
+        - defer: raise `DeferOperation(exec_date, reason)` from
+          `hindsight_api.worker.exceptions` to requeue the task for a
+          future time without bumping `retry_count`. Worker-only — do
+          not raise from `validate_recall` / `validate_reflect` in
+          synchronous HTTP request paths, where it surfaces as a 500.
+
     Supported operations:
         - retain, recall, reflect (core memory operations)
         - consolidate (mental models consolidation)

hindsight-api-slim/hindsight_api/mcp_tools.py

@@ -2854,6 +2854,44 @@ def _register_get_bank_stats(mcp: FastMCP, memory: MemoryEngine, config: MCPTool
             return f'{{"error": "{e}"}}'
 
 
+async def _do_update_bank(
+    memory: MemoryEngine,
+    target_bank: str,
+    request_context: RequestContext,
+    *,
+    name: str | None = None,
+    mission: str | None = None,
+    config_updates: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Shared implementation for update_bank MCP tool variants.
+
+    Args:
+        name: Display name (stored in banks table).
+        mission: Deprecated alias for reflect_mission — mapped into config_updates.
+        config_updates: Arbitrary config overrides passed to config_resolver.update_bank_config().
+            Supports all configurable fields (retain_mission, disposition_*, etc.).
+            The config resolver validates keys and rejects non-configurable/credential fields.
+    """
+    # Update display name via engine (stored in DB banks table)
+    if name is not None:
+        await memory.update_bank(
+            target_bank,
+            name=name,
+            request_context=request_context,
+        )
+
+    # Merge deprecated mission alias into config_updates as reflect_mission
+    effective_config: dict[str, Any] = dict(config_updates) if config_updates else {}
+    if mission is not None and "reflect_mission" not in effective_config:
+        effective_config["reflect_mission"] = mission
+
+    if effective_config:
+        await memory._config_resolver.update_bank_config(target_bank, effective_config, request_context)
+
+    # Return updated profile
+    return await memory.get_bank_profile(target_bank, request_context=request_context)
+
+
 def _register_update_bank(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig) -> None:
     """Register the update_bank tool."""
 
@@ -2863,16 +2901,37 @@ def _register_update_bank(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsCo
         async def update_bank(
             name: str | None = None,
             mission: str | None = None,
+            config_updates: dict[str, Any] | None = None,
             bank_id: str | None = None,
         ) -> str:
             """
-            Update a memory bank's metadata.
+            Update a memory bank's configuration.
 
-            Changes the name or mission of an existing bank.
+            Updates the bank's name and/or any bank-level configuration fields.
+            Only provided fields will be updated; omitted fields remain unchanged.
 
             Args:
-                name: New human-friendly name for the bank
-                mission: New mission describing who the agent is and what they're trying to accomplish
+                name: Human-friendly display name for the bank.
+                mission: Deprecated alias for config_updates.reflect_mission.
+                config_updates: Dictionary of configuration fields to update. Supports all
+                    bank-configurable fields including:
+                    - reflect_mission: Mission/context for Reflect operations.
+                    - retain_mission: Steers what gets extracted during retain().
+                    - retain_extraction_mode: 'concise' (default), 'verbose', or 'custom'.
+                    - retain_custom_instructions: Custom extraction prompt (active when mode is 'custom').
+                    - retain_chunk_size: Maximum token size for each content chunk.
+                    - retain_chunk_batch_size: Number of chunks to process in parallel.
+                    - enable_observations: Toggle observation consolidation after retain().
+                    - observations_mission: Controls observation synthesis rules.
+                    - disposition_skepticism: Critical evaluation level (1-5).
+                    - disposition_literalism: Literal vs. abstract interpretation (1-5).
+                    - disposition_empathy: Emotional context consideration (1-5).
+                    - entity_labels: Controlled vocabulary for entity classification.
+                    - entities_allow_free_form: Allow labels outside entity_labels.
+                    - recall_include_chunks: Include raw chunks in recall results.
+                    - recall_max_tokens: Max tokens for recall results.
+                    - mcp_enabled_tools: Tool allowlist for this bank.
+                    Any configurable field name is accepted (use Python field names).
                 bank_id: Optional bank (defaults to session bank). Use for cross-bank operations.
             """
             try:
@@ -2880,14 +2939,16 @@ def _register_update_bank(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsCo
                 if target_bank is None:
                     return '{"error": "No bank_id configured"}'
 
-                result = await memory.update_bank(
+                result = await _do_update_bank(
+                    memory,
                     target_bank,
+                    _get_request_context(config),
                     name=name,
                     mission=mission,
-                    request_context=_get_request_context(config),
+                    config_updates=config_updates,
                 )
                 return json.dumps(result, indent=2, default=str)
-            except OperationValidationError as e:
+            except (OperationValidationError, ValueError) as e:
                 logger.warning(f"Operation rejected: {e}")
                 return json.dumps({"error": str(e)})
             except Exception as e:
@@ -2900,29 +2961,52 @@ def _register_update_bank(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsCo
         async def update_bank(
             name: str | None = None,
             mission: str | None = None,
+            config_updates: dict[str, Any] | None = None,
         ) -> dict:
             """
-            Update this memory bank's metadata.
+            Update this memory bank's configuration.
 
-            Changes the name or mission of the bank.
+            Updates the bank's name and/or any bank-level configuration fields.
+            Only provided fields will be updated; omitted fields remain unchanged.
 
             Args:
-                name: New human-friendly name for the bank
-                mission: New mission describing who the agent is and what they're trying to accomplish
+                name: Human-friendly display name for the bank.
+                mission: Deprecated alias for config_updates.reflect_mission.
+                config_updates: Dictionary of configuration fields to update. Supports all
+                    bank-configurable fields including:
+                    - reflect_mission: Mission/context for Reflect operations.
+                    - retain_mission: Steers what gets extracted during retain().
+                    - retain_extraction_mode: 'concise' (default), 'verbose', or 'custom'.
+                    - retain_custom_instructions: Custom extraction prompt (active when mode is 'custom').
+                    - retain_chunk_size: Maximum token size for each content chunk.
+                    - retain_chunk_batch_size: Number of chunks to process in parallel.
+                    - enable_observations: Toggle observation consolidation after retain().
+                    - observations_mission: Controls observation synthesis rules.
+                    - disposition_skepticism: Critical evaluation level (1-5).
+                    - disposition_literalism: Literal vs. abstract interpretation (1-5).
+                    - disposition_empathy: Emotional context consideration (1-5).
+                    - entity_labels: Controlled vocabulary for entity classification.
+                    - entities_allow_free_form: Allow labels outside entity_labels.
+                    - recall_include_chunks: Include raw chunks in recall results.
+                    - recall_max_tokens: Max tokens for recall results.
+                    - mcp_enabled_tools: Tool allowlist for this bank.
+                    Any configurable field name is accepted (use Python field names).
             """
             try:
                 target_bank = config.bank_id_resolver()
                 if target_bank is None:
                     return {"error": "No bank_id configured"}
 
-                result = await memory.update_bank(
+                result = await _do_update_bank(
+                    memory,
                     target_bank,
+                    _get_request_context(config),
                     name=name,
                     mission=mission,
-                    request_context=_get_request_context(config),
+                    config_updates=config_updates,
                 )
                 return result
-            except OperationValidationError as e:
+            except (OperationValidationError, ValueError) as e:
                 logger.warning(f"Operation rejected: {e}")
                 return {"error": str(e)}
             except Exception as e:

hindsight-api-slim/hindsight_api/migrations.py

@@ -27,6 +27,7 @@ from alembic.config import Config
 from alembic.script.revision import ResolutionError
 from sqlalchemy import Connection, create_engine, text
 
+from .db_url import to_libpq_url
 from .utils import mask_network_location
 
 logger = logging.getLogger(__name__)
@@ -220,7 +221,7 @@ def run_migrations(
     # ineffective when the app URL goes through a pooler.  Configure
     # HINDSIGHT_API_MIGRATION_DATABASE_URL to the direct PostgreSQL endpoint
     # (e.g. hindsight-pg-rw) to restore correct locking behaviour.
-    migration_url = migration_database_url or database_url
+    migration_url = to_libpq_url(migration_database_url or database_url)
 
     try:
         # Determine script location
@@ -450,7 +451,7 @@ def check_migration_status(
             return None, None
 
         # Get current revision from database
-        engine = create_engine(database_url)
+        engine = create_engine(to_libpq_url(database_url))
         with engine.connect() as connection:
             context = MigrationContext.configure(connection)
             current_rev = context.get_current_revision()
@@ -624,7 +625,7 @@ def ensure_embedding_dimension(
     """
     schema_name = schema or "public"
 
-    engine = create_engine(database_url)
+    engine = create_engine(to_libpq_url(database_url))
     with engine.connect() as conn:
         # Check if memory_units table exists (proxy for schema being initialized)
         table_exists = conn.execute(
@@ -673,7 +674,7 @@ def ensure_vector_extension(
     """
     schema_name = schema or "public"
 
-    engine = create_engine(database_url)
+    engine = create_engine(to_libpq_url(database_url))
     with engine.connect() as conn:
         # Detect which vector extension should be used
         target_ext = _detect_vector_extension(conn, vector_extension)
@@ -894,7 +895,7 @@ def ensure_text_search_extension(
     """
     schema_name = schema or "public"
 
-    engine = create_engine(database_url)
+    engine = create_engine(to_libpq_url(database_url))
     with engine.connect() as conn:
         # Tables with search_vector columns to check
         tables_to_check = [

hindsight-api-slim/hindsight_api/models.py

@@ -24,6 +24,12 @@ class RequestContext:
     mcp_authenticated: bool = False  # True when MCP transport auth already validated (skips tenant re-auth)
     user_initiated: bool = False  # True for async operations that originated from a user request
     allowed_bank_ids: list[str] | None = None  # None = unrestricted (all banks)
+    # Number of times this task has been retried. Populated by the worker
+    # from async_operations.retry_count before dispatching to a task handler;
+    # 0 for sync/HTTP requests and for the first worker attempt. Useful for
+    # validators that want exponential backoff on repeated failures (e.g.
+    # "defer for 2^retry_count minutes") without querying the DB themselves.
+    retry_count: int = 0
 
 
 from pgvector.sqlalchemy import Vector
@@ -62,7 +68,6 @@ class Document(Base):
     bank_id: Mapped[str] = mapped_column(Text, primary_key=True)
     original_text: Mapped[str | None] = mapped_column(Text)
     content_hash: Mapped[str | None] = mapped_column(Text)
-    doc_metadata: Mapped[dict] = mapped_column("metadata", JSONB, server_default=sql_text("'{}'::jsonb"))
     created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now())
     updated_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now())
 

hindsight-api-slim/hindsight_api/worker/exceptions.py

@@ -7,3 +7,24 @@ class RetryTaskAt(Exception):
     def __init__(self, retry_at: datetime, message: str = ""):
         self.retry_at = retry_at
         super().__init__(message)
+
+
+class DeferOperation(Exception):
+    """Raise from an extension hook (or task handler) to requeue the
+    operation for execution at a later time, without counting as a retry.
+
+    Unlike `RetryTaskAt`, this is not a failure: `retry_count` is not
+    incremented and `error_message` is not written. Use this for
+    backpressure / "not yet, try later" decisions made before or during
+    task execution (e.g. quota windows, warming dependencies, upstream
+    rate limits).
+
+    Worker-only: raising this from a hook called in HTTP request context
+    (e.g. `validate_recall` for a synchronous recall) will surface as an
+    unhandled 500 — there is no queue to defer to.
+    """
+
+    def __init__(self, exec_date: datetime, reason: str = ""):
+        self.exec_date = exec_date
+        self.reason = reason
+        super().__init__(reason)

hindsight-api-slim/hindsight_api/worker/main.py

@@ -18,7 +18,7 @@ import sys
 import warnings
 
 from ..config import get_config
-from ..engine.task_backend import SyncTaskBackend
+from ..engine.task_backend import WorkerTaskBackend
 from .poller import WorkerPoller
 
 # Filter deprecation warnings from third-party libraries
@@ -164,7 +164,11 @@ def main():
     print(f"  Poll interval: {args.poll_interval}ms")
     print(f"  Max retries: {args.max_retries}")
     print(f"  Max slots: {config.worker_max_slots}")
-    print(f"  Consolidation max slots: {config.worker_consolidation_max_slots}")
+    reservations = config.worker_slot_reservations
+    reservations_str = ", ".join(f"{k}={v}" for k, v in reservations.items()) if reservations else "none"
+    shared_pool = max(0, config.worker_max_slots - sum(reservations.values()))
+    print(f"  Slot reservations: {reservations_str}")
+    print(f"  Shared pool: {shared_pool}")
     print(f"  HTTP server: {args.http_host}:{args.http_port}")
     print()
 
@@ -191,11 +195,13 @@ def main():
             logger.info(f"Loaded operation validator: {operation_validator.__class__.__name__}")
 
         # Initialize MemoryEngine
-        # Workers use SyncTaskBackend because they execute tasks directly,
-        # they don't need to store tasks (they poll from DB)
+        # Workers use WorkerTaskBackend: submit_task is a no-op because the
+        # row already exists in async_operations.  Child tasks (e.g. consolidation
+        # triggered by retain) will be picked up by the poller on the next cycle
+        # instead of being executed inline, which avoids blocking the parent task.
         memory = MemoryEngine(
             run_migrations=False,  # Workers don't run migrations
-            task_backend=SyncTaskBackend(),
+            task_backend=WorkerTaskBackend(),
             tenant_extension=tenant_extension,
             operation_validator=operation_validator,
         )
@@ -222,7 +228,7 @@ def main():
             schema=schema,
             tenant_extension=tenant_extension,
             max_slots=config.worker_max_slots,
-            consolidation_max_slots=config.worker_consolidation_max_slots,
+            slot_reservations=config.worker_slot_reservations,
         )
 
         # Create the HTTP app for metrics/health

hindsight-api-slim/hindsight_api/worker/poller.py

@@ -15,7 +15,7 @@ from collections.abc import Awaitable, Callable
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any
 
-from .exceptions import RetryTaskAt
+from .exceptions import DeferOperation, RetryTaskAt
 from .stage import StageHolder, bind_holder
 
 if TYPE_CHECKING:
@@ -70,6 +70,21 @@ class ClaimedTask:
     schema: str | None
 
 
+@dataclass
+class SlotAvailability:
+    """Available slot capacity across reserved and shared pools.
+
+    Each operation type with a reservation has its own reserved pool.
+    The shared pool (max_slots - sum of reservations) is usable by any type.
+    """
+
+    reserved: dict[str, int]
+    """Per-operation-type remaining reserved capacity."""
+
+    shared: int
+    """Remaining shared pool capacity (usable by any operation type)."""
+
+
 class WorkerPoller:
     """
     Polls PostgreSQL for pending tasks and executes them.
@@ -89,7 +104,7 @@ class WorkerPoller:
         schema: str | None = None,
         tenant_extension: "TenantExtension | None" = None,
         max_slots: int = 10,
-        consolidation_max_slots: int = 2,
+        slot_reservations: dict[str, int] | None = None,
     ):
         """
         Initialize the worker poller.
@@ -103,7 +118,10 @@ class WorkerPoller:
             tenant_extension: Extension for dynamic multi-tenant discovery. If None, creates a
                             DefaultTenantExtension with the configured schema.
             max_slots: Maximum concurrent tasks per worker
-            consolidation_max_slots: Maximum concurrent consolidation tasks per worker
+            slot_reservations: Per-operation-type reserved slot counts (e.g. {"consolidation": 2,
+                "retain": 3}). Reserved slots guarantee capacity for that operation type.
+                Remaining slots (max_slots - sum of reservations) form a shared pool usable
+                by any operation type. Defaults to {"consolidation": 2} if None.
         """
         self._pool = pool
         self._worker_id = worker_id
@@ -119,7 +137,9 @@ class WorkerPoller:
             tenant_extension = DefaultTenantExtension(config=config)
         self._tenant_extension = tenant_extension
         self._max_slots = max_slots
-        self._consolidation_max_slots = consolidation_max_slots
+        self._slot_reservations: dict[str, int] = (
+            slot_reservations if slot_reservations is not None else {"consolidation": 2}
+        )
         self._shutdown = asyncio.Event()
         self._current_tasks: set[asyncio.Task] = set()
         self._in_flight_count = 0
@@ -130,6 +150,9 @@ class WorkerPoller:
         self._active_tasks: dict[str, ActiveTaskInfo] = {}
         # Track in-flight tasks by operation type
         self._in_flight_by_type: dict[str, int] = {}
+        # Rotation offset for per-tenant fair claiming. Advances past the last
+        # schema we serviced so a busy tenant can't monopolize the poll order.
+        self._next_schema_idx: int = 0
 
     async def _get_schemas(self) -> list[str | None]:
         """Get list of schemas to poll. Returns [None] for default schema (no prefix)."""
@@ -139,29 +162,91 @@ class WorkerPoller:
         # Convert default schema to None for SQL compatibility (no prefix), keep others as-is
         return [t.schema if t.schema != DEFAULT_DATABASE_SCHEMA else None for t in tenants]
 
-    async def _get_available_slots(self) -> tuple[int, int]:
+    async def _scan_active_schemas(self, schemas: list[str | None]) -> set[str | None]:
+        """Find which schemas have pending work.
+
+        Tries a server-side PL/pgSQL function first (single DB round-trip,
+        ~200ms for 1400+ schemas). Falls back to per-schema Python EXISTS
+        queries if the function is not installed (~4ms each).
+
+        The server-side function should be installed in the ``public``
+        schema as::
+
+            CREATE OR REPLACE FUNCTION public.schemas_with_pending_work()
+            RETURNS SETOF text AS $$
+            DECLARE
+              r RECORD; has_work BOOLEAN;
+            BEGIN
+              FOR r IN SELECT nspname FROM pg_namespace
+                       WHERE nspname LIKE 'tenant_%' LOOP
+                BEGIN
+                  EXECUTE format(
+                    'SELECT EXISTS(SELECT 1 FROM %I.async_operations '
+                    'WHERE status = ''pending'' '
+                    'AND task_payload IS NOT NULL LIMIT 1)',
+                    r.nspname) INTO has_work;
+                  IF has_work THEN RETURN NEXT r.nspname; END IF;
+                EXCEPTION WHEN OTHERS THEN NULL;
+                END;
+              END LOOP;
+            END $$ LANGUAGE plpgsql STABLE;
+
+        In hindsight-cloud deployments this is installed by a Helm hook
+        job alongside ``total_pending_tasks()``.
+        """
+        async with self._pool.acquire() as conn:
+            try:
+                rows = await conn.fetch("SELECT * FROM schemas_with_pending_work()")
+                return {r[0] for r in rows}
+            except Exception:
+                pass
+
+            # Fallback: per-schema EXISTS checks from Python
+            active: set[str | None] = set()
+            for schema in schemas:
+                table = fq_table("async_operations", schema)
+                try:
+                    has_work = await conn.fetchval(
+                        f"SELECT EXISTS(SELECT 1 FROM {table} "
+                        f"WHERE status = 'pending' AND task_payload IS NOT NULL LIMIT 1)"
+                    )
+                    if has_work:
+                        active.add(schema)
+                except Exception:
+                    pass
+            return active
+
+    async def _get_available_slots(self) -> SlotAvailability:
         """
         Calculate available slots for claiming tasks.
 
-        Consolidation has a reserved pool of ``consolidation_max_slots`` within
-        ``max_slots``. Non-consolidation tasks may use at most
-        ``max_slots - consolidation_max_slots`` slots, leaving the remainder
-        always available for consolidation. This prevents consolidation from
-        being starved when retain throughput continuously saturates the queue.
+        Each operation type can have reserved slots (via ``slot_reservations``).
+        Reserved slots guarantee capacity for that type — they cannot be used by
+        other types. The remaining slots (``max_slots - sum(reservations)``) form
+        a shared pool usable by any operation type on a first-come basis.
 
-        Returns:
-            (non_consolidation_available, consolidation_available) tuple
+        When an operation type's in-flight count exceeds its reservation, the
+        excess tasks are considered to be using shared pool slots.
         """
         async with self._in_flight_lock:
             total_in_flight = self._in_flight_count
-            consolidation_in_flight = self._in_flight_by_type.get("consolidation", 0)
+            in_flight_snapshot = dict(self._in_flight_by_type)
 
-        non_consolidation_in_flight = max(0, total_in_flight - consolidation_in_flight)
-        non_consolidation_max = max(0, self._max_slots - self._consolidation_max_slots)
-        non_consolidation_available = max(0, non_consolidation_max - non_consolidation_in_flight)
-        consolidation_available = max(0, self._consolidation_max_slots - consolidation_in_flight)
+        # Per-type reserved availability
+        reserved_available: dict[str, int] = {}
+        tasks_in_reserved = 0
+        for op_type, reserved in self._slot_reservations.items():
+            in_flight = in_flight_snapshot.get(op_type, 0)
+            reserved_available[op_type] = max(0, reserved - in_flight)
+            tasks_in_reserved += min(reserved, in_flight)
 
-        return non_consolidation_available, consolidation_available
+        # Shared pool: total slots minus reservations minus tasks using shared slots
+        sum_reservations = sum(self._slot_reservations.values())
+        shared_pool_size = max(0, self._max_slots - sum_reservations)
+        tasks_in_shared = max(0, total_in_flight - tasks_in_reserved)
+        shared_available = max(0, shared_pool_size - tasks_in_shared)
+
+        return SlotAvailability(reserved=reserved_available, shared=shared_available)
 
     async def wait_for_active_tasks(self, timeout: float = 10.0) -> bool:
         """
@@ -192,47 +277,119 @@ class WorkerPoller:
     async def claim_batch(self) -> list[ClaimedTask]:
         """
         Claim pending tasks atomically across all tenant schemas,
-        respecting slot limits (total and consolidation).
+        respecting per-operation-type slot reservations and shared pool limits.
 
         Uses FOR UPDATE SKIP LOCKED to ensure no conflicts with other workers.
 
+        Schema iteration is round-robin to prevent one busy tenant from
+        starving others. Each poll starts at ``self._next_schema_idx`` and
+        wraps around the full list. First pass caps at 1 claim per pool per
+        schema so every tenant with pending work gets a fair chance; a second
+        pass backfills remaining slots from any schema when there's spare
+        capacity. After the call, the offset advances past the last
+        schema we serviced (or by 1 if nothing was claimed) so the next
+        poll starts at a different position.
+
         Returns:
             List of ClaimedTask objects containing operation_id, task_dict, and schema
         """
-        # Calculate available slots (independent pools after reservation)
-        non_consolidation_available, consolidation_available = await self._get_available_slots()
+        # Calculate available slots (per-type reserved + shared pool)
+        availability = await self._get_available_slots()
 
-        if non_consolidation_available <= 0 and consolidation_available <= 0:
+        if all(v <= 0 for v in availability.reserved.values()) and availability.shared <= 0:
             return []
 
         schemas = await self._get_schemas()
+        if not schemas:
+            return []
+
+        # Scan: find which schemas have pending work using a lightweight
+        # EXISTS check (no locks). Then only claim from those schemas
+        # using the expensive FOR UPDATE SKIP LOCKED query.
+        active_schemas = await self._scan_active_schemas(schemas)
+
+        if not active_schemas:
+            self._next_schema_idx = (self._next_schema_idx + 1) % len(schemas)
+            return []
+
+        # Build rotation list from active schemas only, preserving their
+        # original positions for correct offset advancement.
+        all_indexed = list(enumerate(schemas))
+        active_indexed = [(i, s) for i, s in all_indexed if s in active_schemas]
+
+        # Rotate so no tenant is always first.
+        start = self._next_schema_idx % len(schemas)
+        rotated = [x for x in active_indexed if x[0] >= start] + [x for x in active_indexed if x[0] < start]
+
         all_tasks: list[ClaimedTask] = []
-        remaining_non_consolidation = non_consolidation_available
-        remaining_consolidation = consolidation_available
+        remaining_reserved = dict(availability.reserved)
+        remaining_shared = availability.shared
+        last_serviced_idx: int | None = None
+        schemas_with_work: list[tuple[int, str | None]] = []
 
-        for schema in schemas:
-            if remaining_non_consolidation <= 0 and remaining_consolidation <= 0:
-                break
-
-            tasks = await self._claim_batch_for_schema(schema, remaining_non_consolidation, remaining_consolidation)
+        def _has_capacity() -> bool:
+            return any(v > 0 for v in remaining_reserved.values()) or remaining_shared > 0
 
+        def _account_tasks(tasks: list[ClaimedTask]) -> None:
+            nonlocal remaining_shared
             for task in tasks:
                 op_type = task.task_dict.get("operation_type", "unknown")
-                if op_type == "consolidation":
-                    remaining_consolidation -= 1
+                if op_type in remaining_reserved and remaining_reserved[op_type] > 0:
+                    remaining_reserved[op_type] -= 1
                 else:
-                    remaining_non_consolidation -= 1
+                    remaining_shared -= 1
+
+        # Pass 1: fairness pass — iterate only active schemas, cap at
+        # 1 claim per pool per schema.
+        for orig_idx, schema in rotated:
+            if not _has_capacity():
+                break
+
+            fair_reserved = {t: min(1, v) for t, v in remaining_reserved.items() if v > 0}
+            fair_shared = min(1, remaining_shared) if remaining_shared > 0 else 0
+            tasks = await self._claim_batch_for_schema(schema, fair_reserved, fair_shared)
+
+            _account_tasks(tasks)
+
+            if tasks:
+                last_serviced_idx = orig_idx
+                schemas_with_work.append((orig_idx, schema))
 
             all_tasks.extend(tasks)
 
+        # Pass 2: capacity pass — fill remaining slots from schemas
+        # that had work in pass 1 only.
+        if _has_capacity() and schemas_with_work:
+            for orig_idx, schema in schemas_with_work:
+                if not _has_capacity():
+                    break
+
+                tasks = await self._claim_batch_for_schema(
+                    schema, {t: v for t, v in remaining_reserved.items() if v > 0}, remaining_shared
+                )
+
+                _account_tasks(tasks)
+
+                if tasks:
+                    last_serviced_idx = orig_idx
+
+                all_tasks.extend(tasks)
+
+        # Advance offset past the last schema we serviced, or by 1 if
+        # nothing was claimed (so we don't keep re-hitting an empty head).
+        if last_serviced_idx is not None:
+            self._next_schema_idx = (last_serviced_idx + 1) % len(schemas)
+        else:
+            self._next_schema_idx = (start + 1) % len(schemas)
+
         return all_tasks
 
     async def _claim_batch_for_schema(
-        self, schema: str | None, non_consolidation_limit: int, consolidation_limit: int
+        self, schema: str | None, reserved_limits: dict[str, int], shared_limit: int
     ) -> list[ClaimedTask]:
-        """Claim tasks from a specific schema respecting slot limits."""
+        """Claim tasks from a specific schema respecting per-type and shared slot limits."""
         try:
-            return await self._claim_batch_for_schema_inner(schema, non_consolidation_limit, consolidation_limit)
+            return await self._claim_batch_for_schema_inner(schema, reserved_limits, shared_limit)
         except Exception as e:
             # Format schema for logging: custom schemas in quotes, None as-is
             schema_display = f'"{schema}"' if schema else str(schema)
@@ -240,67 +397,173 @@ class WorkerPoller:
             return []
 
     async def _claim_batch_for_schema_inner(
-        self, schema: str | None, non_consolidation_limit: int, consolidation_limit: int
+        self, schema: str | None, reserved_limits: dict[str, int], shared_limit: int
     ) -> list[ClaimedTask]:
-        """Inner implementation for claiming tasks from a specific schema with slot limits.
+        """Inner implementation for claiming tasks from a specific schema.
 
-        Non-consolidation and consolidation pools are independent: each is bounded by
-        its own limit and they do not borrow from each other.
+        Claims happen in two phases:
+        1. Reserved pools: one query per operation type that has reserved slots.
+           Consolidation queries always include bank-serialization (no two consolidation
+           tasks for the same bank simultaneously).
+        2. Shared pool: remaining capacity is filled by any operation type. Two queries
+           are used (non-consolidation + consolidation with bank serialization) to
+           preserve consolidation's bank-serialization constraint.
+
+        Within the same transaction, rows locked by earlier queries are excluded from
+        later queries via ``operation_id != ALL($excluded)`` since ``FOR UPDATE SKIP
+        LOCKED`` only skips rows locked by *other* transactions.
         """
         table = fq_table("async_operations", schema)
 
         async with self._pool.acquire() as conn:
             async with conn.transaction():
-                # 1. Claim non-consolidation tasks
-                non_consolidation_rows = []
-                if non_consolidation_limit > 0:
-                    non_consolidation_rows = await conn.fetch(
-                        f"""
-                        SELECT operation_id, task_payload, retry_count
-                        FROM {table}
-                        WHERE status = 'pending'
-                          AND task_payload IS NOT NULL
-                          AND operation_type != 'consolidation'
-                          AND (next_retry_at IS NULL OR next_retry_at <= NOW())
-                        ORDER BY created_at
-                        LIMIT $1
-                        FOR UPDATE SKIP LOCKED
-                        """,
-                        non_consolidation_limit,
-                    )
+                all_rows: list[Any] = []
+                claimed_ids: list[Any] = []
 
-                # 2. Claim consolidation tasks from their reserved pool
-                consolidation_rows = []
-                if consolidation_limit > 0:
-                    consolidation_rows = await conn.fetch(
-                        f"""
-                        SELECT operation_id, task_payload, retry_count
-                        FROM {table} AS pending
-                        WHERE status = 'pending'
-                          AND task_payload IS NOT NULL
-                          AND operation_type = 'consolidation'
-                          AND (next_retry_at IS NULL OR next_retry_at <= NOW())
-                          AND NOT EXISTS (
-                              SELECT 1 FROM {table} AS processing
-                              WHERE processing.bank_id = pending.bank_id
-                                AND processing.operation_type = 'consolidation'
-                                AND processing.status = 'processing'
-                          )
-                        ORDER BY created_at
-                        LIMIT $1
-                        FOR UPDATE SKIP LOCKED
-                        """,
-                        consolidation_limit,
-                    )
+                # --- Phase 1: claim from reserved pools ---
+                for op_type, limit in reserved_limits.items():
+                    if limit <= 0:
+                        continue
 
-                tagged_rows = [(row, False) for row in non_consolidation_rows] + [
-                    (row, True) for row in consolidation_rows
-                ]
+                    if op_type == "consolidation":
+                        rows = await conn.fetch(
+                            f"""
+                            SELECT operation_id, operation_type, task_payload, retry_count
+                            FROM {table} AS pending
+                            WHERE status = 'pending'
+                              AND task_payload IS NOT NULL
+                              AND operation_type = 'consolidation'
+                              AND (next_retry_at IS NULL OR next_retry_at <= NOW())
+                              AND NOT EXISTS (
+                                  SELECT 1 FROM {table} AS processing
+                                  WHERE processing.bank_id = pending.bank_id
+                                    AND processing.operation_type = 'consolidation'
+                                    AND processing.status = 'processing'
+                              )
+                            ORDER BY created_at
+                            LIMIT $1
+                            FOR UPDATE SKIP LOCKED
+                            """,
+                            limit,
+                        )
+                    else:
+                        rows = await conn.fetch(
+                            f"""
+                            SELECT operation_id, operation_type, task_payload, retry_count
+                            FROM {table}
+                            WHERE status = 'pending'
+                              AND task_payload IS NOT NULL
+                              AND operation_type = $1
+                              AND (next_retry_at IS NULL OR next_retry_at <= NOW())
+                            ORDER BY created_at
+                            LIMIT $2
+                            FOR UPDATE SKIP LOCKED
+                            """,
+                            op_type,
+                            limit,
+                        )
 
-                if not tagged_rows:
+                    for row in rows:
+                        claimed_ids.append(row["operation_id"])
+                        all_rows.append(row)
+
+                # --- Phase 2: claim from shared pool ---
+                remaining_shared = shared_limit
+                if remaining_shared > 0:
+                    # 2a. Non-consolidation tasks (any type except consolidation)
+                    if claimed_ids:
+                        rows = await conn.fetch(
+                            f"""
+                            SELECT operation_id, operation_type, task_payload, retry_count
+                            FROM {table}
+                            WHERE status = 'pending'
+                              AND task_payload IS NOT NULL
+                              AND operation_type != 'consolidation'
+                              AND (next_retry_at IS NULL OR next_retry_at <= NOW())
+                              AND operation_id != ALL($1::uuid[])
+                            ORDER BY created_at
+                            LIMIT $2
+                            FOR UPDATE SKIP LOCKED
+                            """,
+                            claimed_ids,
+                            remaining_shared,
+                        )
+                    else:
+                        rows = await conn.fetch(
+                            f"""
+                            SELECT operation_id, operation_type, task_payload, retry_count
+                            FROM {table}
+                            WHERE status = 'pending'
+                              AND task_payload IS NOT NULL
+                              AND operation_type != 'consolidation'
+                              AND (next_retry_at IS NULL OR next_retry_at <= NOW())
+                            ORDER BY created_at
+                            LIMIT $1
+                            FOR UPDATE SKIP LOCKED
+                            """,
+                            remaining_shared,
+                        )
+
+                    for row in rows:
+                        claimed_ids.append(row["operation_id"])
+                        all_rows.append(row)
+                    remaining_shared -= len(rows)
+
+                    # 2b. Consolidation tasks (with bank-serialization constraint)
+                    if remaining_shared > 0:
+                        if claimed_ids:
+                            rows = await conn.fetch(
+                                f"""
+                                SELECT operation_id, operation_type, task_payload, retry_count
+                                FROM {table} AS pending
+                                WHERE status = 'pending'
+                                  AND task_payload IS NOT NULL
+                                  AND operation_type = 'consolidation'
+                                  AND (next_retry_at IS NULL OR next_retry_at <= NOW())
+                                  AND operation_id != ALL($1::uuid[])
+                                  AND NOT EXISTS (
+                                      SELECT 1 FROM {table} AS processing
+                                      WHERE processing.bank_id = pending.bank_id
+                                        AND processing.operation_type = 'consolidation'
+                                        AND processing.status = 'processing'
+                                  )
+                                ORDER BY created_at
+                                LIMIT $2
+                                FOR UPDATE SKIP LOCKED
+                                """,
+                                claimed_ids,
+                                remaining_shared,
+                            )
+                        else:
+                            rows = await conn.fetch(
+                                f"""
+                                SELECT operation_id, operation_type, task_payload, retry_count
+                                FROM {table} AS pending
+                                WHERE status = 'pending'
+                                  AND task_payload IS NOT NULL
+                                  AND operation_type = 'consolidation'
+                                  AND (next_retry_at IS NULL OR next_retry_at <= NOW())
+                                  AND NOT EXISTS (
+                                      SELECT 1 FROM {table} AS processing
+                                      WHERE processing.bank_id = pending.bank_id
+                                        AND processing.operation_type = 'consolidation'
+                                        AND processing.status = 'processing'
+                                  )
+                                ORDER BY created_at
+                                LIMIT $1
+                                FOR UPDATE SKIP LOCKED
+                                """,
+                                remaining_shared,
+                            )
+
+                        for row in rows:
+                            claimed_ids.append(row["operation_id"])
+                            all_rows.append(row)
+
+                if not all_rows:
                     return []
 
-                operation_ids = [row["operation_id"] for row, _ in tagged_rows]
+                operation_ids = [row["operation_id"] for row in all_rows]
                 await conn.execute(
                     f"""
                     UPDATE {table}
@@ -312,15 +575,15 @@ class WorkerPoller:
                 )
 
                 result = []
-                for row, is_consolidation in tagged_rows:
+                for row in all_rows:
                     task_dict = json.loads(row["task_payload"])
                     task_dict["_retry_count"] = row["retry_count"]
                     task_dict["_operation_id"] = str(row["operation_id"])
-                    # The DB row knows the operation_type, but the JSON payload may not
-                    # carry it. Inject it so in-flight tracking and slot accounting
-                    # (which key off task_dict["operation_type"]) work correctly.
-                    if is_consolidation:
-                        task_dict["operation_type"] = "consolidation"
+                    # The DB column is authoritative for operation_type — inject it
+                    # into task_dict so in-flight tracking and slot accounting work.
+                    db_op_type = row["operation_type"]
+                    if db_op_type:
+                        task_dict["operation_type"] = db_op_type
                     result.append(
                         ClaimedTask(
                             operation_id=str(row["operation_id"]),
@@ -461,6 +724,25 @@ class WorkerPoller:
         )
         logger.warning(f"Task {operation_id} scheduled for retry at {retry_at}: {error_message}")
 
+    async def _defer_operation(self, operation_id: str, exec_date: "Any", reason: str, schema: str | None):
+        """Reset task to pending for re-pickup at exec_date without counting as a retry.
+
+        Unlike `_schedule_retry`, this does not bump `retry_count` and does not
+        populate `error_message` — defer is intentional backpressure, not a failure.
+        """
+        table = fq_table("async_operations", schema)
+        await self._pool.execute(
+            f"""
+            UPDATE {table}
+            SET status = 'pending', next_retry_at = $2, worker_id = NULL, claimed_at = NULL,
+                updated_at = now()
+            WHERE operation_id = $1
+            """,
+            operation_id,
+            exec_date,
+        )
+        logger.info(f"Task {operation_id} deferred until {exec_date}: {reason}")
+
     async def execute_task(self, task: ClaimedTask):
         """Execute a single task as a background job (fire-and-forget)."""
         task_type = task.task_dict.get("type", "unknown")
@@ -532,6 +814,8 @@ class WorkerPoller:
                 task.task_dict["_schema"] = task.schema
             await self._executor(task.task_dict)
             logger.debug(f"Task {task.operation_id} execution finished")
+        except DeferOperation as e:
+            await self._defer_operation(task.operation_id, e.exec_date, e.reason, task.schema)
         except RetryTaskAt as e:
             await self._schedule_retry(task.operation_id, e.retry_at, str(e), task.schema)
         except Exception as e:
@@ -670,9 +954,13 @@ class WorkerPoller:
         """
         await self.recover_own_tasks()
 
+        reservations_str = (
+            ", ".join(f"{k}={v}" for k, v in self._slot_reservations.items()) if self._slot_reservations else "none"
+        )
+        shared_pool = max(0, self._max_slots - sum(self._slot_reservations.values()))
         logger.info(
             f"Worker {self._worker_id} starting polling loop "
-            f"(max_slots={self._max_slots}, consolidation_max_slots={self._consolidation_max_slots})"
+            f"(max_slots={self._max_slots}, reservations=[{reservations_str}], shared_pool={shared_pool})"
         )
 
         while not self._shutdown.is_set():
@@ -791,11 +1079,19 @@ class WorkerPoller:
                 in_flight_by_type = dict(self._in_flight_by_type)
                 active_tasks = dict(self._active_tasks)
 
-            consolidation_count = in_flight_by_type.get("consolidation", 0)
-            non_consolidation_in_flight = max(0, in_flight - consolidation_count)
-            non_consolidation_max = max(0, self._max_slots - self._consolidation_max_slots)
-            available_slots = max(0, non_consolidation_max - non_consolidation_in_flight)
-            available_consolidation_slots = max(0, self._consolidation_max_slots - consolidation_count)
+            # Compute per-type reserved availability and shared pool
+            tasks_in_reserved = 0
+            reserved_parts = []
+            for op_type, reserved in self._slot_reservations.items():
+                type_in_flight = in_flight_by_type.get(op_type, 0)
+                type_available = max(0, reserved - type_in_flight)
+                tasks_in_reserved += min(reserved, type_in_flight)
+                reserved_parts.append(f"{op_type}={type_in_flight}/{reserved}(avail={type_available})")
+            sum_reservations = sum(self._slot_reservations.values())
+            shared_pool_size = max(0, self._max_slots - sum_reservations)
+            tasks_in_shared = max(0, in_flight - tasks_in_reserved)
+            shared_available = max(0, shared_pool_size - tasks_in_shared)
+            reserved_str = ", ".join(reserved_parts) if reserved_parts else "none"
 
             # Build local processing breakdown (aggregate counts)
             task_groups: dict[tuple[str, str], int] = {}
@@ -812,13 +1108,42 @@ class WorkerPoller:
             schemas = await self._get_schemas()
             global_pending = 0
             all_worker_counts: dict[str, int] = {}
+            # operation_type -> aggregated bucket counts across schemas
+            pending_breakdown: dict[str, dict[str, int]] = {}
 
             async with self._pool.acquire() as conn:
                 for schema in schemas:
                     table = fq_table("async_operations", schema)
 
-                    row = await conn.fetchrow(f"SELECT COUNT(*) as count FROM {table} WHERE status = 'pending'")
-                    global_pending += row["count"] if row else 0
+                    # Bucket pending rows by the same predicates the claim query
+                    # filters on, so an operator can see why pending > 0 but
+                    # nothing is being claimed (orphaned batch_retain parents,
+                    # retry backoff, etc.).
+                    breakdown_rows = await conn.fetch(
+                        f"""
+                        SELECT
+                            operation_type,
+                            COUNT(*) AS total,
+                            COUNT(*) FILTER (WHERE task_payload IS NULL) AS payload_null,
+                            COUNT(*) FILTER (
+                                WHERE next_retry_at IS NOT NULL AND next_retry_at > now()
+                            ) AS retry_blocked,
+                            COUNT(*) FILTER (WHERE worker_id IS NOT NULL) AS assigned
+                        FROM {table}
+                        WHERE status = 'pending'
+                        GROUP BY operation_type
+                        """
+                    )
+                    for br in breakdown_rows:
+                        op_type = br["operation_type"] or "unknown"
+                        bucket = pending_breakdown.setdefault(
+                            op_type, {"total": 0, "payload_null": 0, "retry_blocked": 0, "assigned": 0}
+                        )
+                        bucket["total"] += br["total"]
+                        bucket["payload_null"] += br["payload_null"]
+                        bucket["retry_blocked"] += br["retry_blocked"]
+                        bucket["assigned"] += br["assigned"]
+                        global_pending += br["total"]
 
                     worker_rows = await conn.fetch(
                         f"""
@@ -847,8 +1172,9 @@ class WorkerPoller:
             schemas_str = ", ".join(s if s else "default" for s in schemas)
             logger.info(
                 f"[WORKER_STATS] worker={self._worker_id} "
-                f"slots={in_flight}/{self._max_slots} (consolidation={consolidation_count}/{self._consolidation_max_slots}) | "
-                f"available={available_slots} (consolidation={available_consolidation_slots}) | "
+                f"slots={in_flight}/{self._max_slots} | "
+                f"reserved: [{reserved_str}] | "
+                f"shared={tasks_in_shared}/{shared_pool_size}(avail={shared_available}) | "
                 f"global: pending={global_pending} (schemas: {schemas_str}) | "
                 f"others: {others_str} | "
                 f"pool: {pool_str} | "
@@ -856,6 +1182,13 @@ class WorkerPoller:
                 f"my_active: {processing_str}"
             )
 
+            # Pending breakdown - explains why pending rows aren't being claimed
+            # (orphaned batch_retain parents have payload_null > 0, retry storms
+            # show up as retry_blocked, etc.). Skip when nothing is pending so
+            # the line doesn't add noise on idle deployments.
+            if global_pending > 0:
+                self._log_pending_breakdown(pending_breakdown)
+
             # Per-task lines, sorted oldest-first so stuck tasks bubble to the top.
             self._log_per_task_lines(active_tasks, now=time.monotonic())
 
@@ -895,7 +1228,14 @@ class WorkerPoller:
             min_size = pool.get_min_size() if hasattr(pool, "get_min_size") else None
             max_size = pool.get_max_size() if hasattr(pool, "get_max_size") else None
             queue = getattr(pool, "_queue", None)
-            waiters = queue.qsize() if queue is not None and hasattr(queue, "qsize") else None
+            # asyncpg's _queue is a LifoQueue pre-filled to max_size with
+            # PoolConnectionHolder objects. qsize() therefore counts *available
+            # holders*, not callers waiting on the pool — the previous "waiters"
+            # label here was the opposite of what it suggested. The actual count
+            # of awaiters is len(_queue._getters), nonzero only when qsize()==0.
+            free_holders = queue.qsize() if queue is not None and hasattr(queue, "qsize") else None
+            getters = getattr(queue, "_getters", None) if queue is not None else None
+            pending_acquires = len(getters) if getters is not None else None
 
             parts = [f"size={size}"]
             if min_size is not None and max_size is not None:
@@ -903,13 +1243,44 @@ class WorkerPoller:
             if free is not None:
                 parts.append(f"idle={free}")
                 parts.append(f"in_use={size - free}")
-            if waiters is not None:
-                parts.append(f"waiters={waiters}")
+            if free_holders is not None:
+                parts.append(f"free_holders={free_holders}")
+            if pending_acquires is not None:
+                parts.append(f"pending_acquires={pending_acquires}")
             return " ".join(parts)
         except Exception as e:
             logger.debug(f"Pool stats unavailable: {e}")
             return "unavailable"
 
+    def _log_pending_breakdown(self, breakdown: dict[str, dict[str, int]]) -> None:
+        """Emit one [PENDING_BREAKDOWN] line bucketing pending rows by claimability.
+
+        Each bucket mirrors a predicate in the claim query:
+          * payload_null   - row has no task_payload (e.g. batch_retain parent
+                             whose reconciliation never fired); claim query
+                             skips it forever
+          * retry_blocked  - next_retry_at is still in the future
+          * assigned       - worker_id already set; another worker owns it
+
+        ``claimable`` is the residual that *should* be picked up on the next
+        poll. If ``claimable > 0`` while workers report free slots, the bug is
+        somewhere else (lock contention, tenant discovery, etc.) - this line
+        narrows the search.
+        """
+        if not breakdown:
+            return
+
+        parts = []
+        for op_type in sorted(breakdown):
+            b = breakdown[op_type]
+            claimable = b["total"] - b["payload_null"] - b["retry_blocked"] - b["assigned"]
+            parts.append(
+                f"{op_type}: total={b['total']} claimable={claimable} "
+                f"payload_null={b['payload_null']} retry_blocked={b['retry_blocked']} "
+                f"assigned={b['assigned']}"
+            )
+        logger.info(f"[PENDING_BREAKDOWN] {' | '.join(parts)}")
+
     def _log_per_task_lines(self, active_tasks: dict[str, ActiveTaskInfo], now: float) -> None:
         """Emit one [WORKER_TASK] line per in-flight task and dump stuck stacks.
 

hindsight-api-slim/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-api-slim"
-version = "0.5.1"
+version = "0.5.4"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"

hindsight-api-slim/tests/test_alembic_dag.py

@@ -0,0 +1,47 @@
+"""Graph-level sanity checks for the Alembic migration DAG.
+
+These tests do not touch a database; they only parse the revision files on
+disk, so they are cheap to run in CI and catch DAG accidents (divergent
+heads, unreachable revisions) at merge time instead of at deploy time.
+"""
+
+from pathlib import Path
+
+from alembic.config import Config
+from alembic.script import ScriptDirectory
+
+
+def _script_directory() -> ScriptDirectory:
+    cfg = Config()
+    script_location = Path(__file__).parent.parent / "hindsight_api" / "alembic"
+    cfg.set_main_option("script_location", str(script_location))
+    return ScriptDirectory.from_config(cfg)
+
+
+def test_single_head() -> None:
+    """The DAG must have exactly one head.
+
+    A second head means a branch was added without a merge revision, which
+    makes ``alembic upgrade head`` (singular) ambiguous and forces the next
+    migration author to orphan whichever head they don't pick as parent.
+    v0.5.3 shipped in exactly that state; this test would have caught it.
+
+    Fix for a new head: ``alembic merge heads -m "<reason>"``.
+    """
+    script = _script_directory()
+    heads = script.get_heads()
+    assert len(heads) == 1, (
+        f"Alembic has {len(heads)} heads ({heads}); expected exactly 1. "
+        "Unify them with ``alembic merge heads -m '<reason>'``."
+    )
+
+
+def test_single_base() -> None:
+    """The DAG must have exactly one base (the initial schema).
+
+    Multiple bases mean disconnected migration trees, which can only happen
+    through manual file edits.
+    """
+    script = _script_directory()
+    bases = script.get_bases()
+    assert len(bases) == 1, f"Alembic has {len(bases)} bases ({bases}); expected exactly 1."

hindsight-api-slim/tests/test_async_batch_retain.py

@@ -433,3 +433,309 @@ async def test_config_retain_batch_tokens_respected(memory, request_context):
     # Even small batches use parent-child pattern now (simpler code path)
     assert "child_operations" in status
     assert status["result_metadata"]["num_sub_batches"] == 1
+
+
+async def _child_metadata(memory, bank_id: str, parent_operation_id: str, request_context):
+    """Fetch the first child operation's result_metadata for a parent batch_retain."""
+    parent = await memory.get_operation_status(
+        bank_id=bank_id,
+        operation_id=parent_operation_id,
+        request_context=request_context,
+    )
+    assert parent["status"] == "completed", parent
+    assert parent["child_operations"], "expected at least one child operation"
+    child_id = parent["child_operations"][0]["operation_id"]
+    child = await memory.get_operation_status(
+        bank_id=bank_id,
+        operation_id=child_id,
+        request_context=request_context,
+    )
+    return child["result_metadata"]
+
+
+@pytest.mark.asyncio
+async def test_retain_records_user_provided_document_ids(memory, request_context):
+    """User-supplied document_ids land in child op result_metadata.document_ids."""
+    bank_id = "test_doc_ids_user_supplied"
+    d1 = str(uuid.uuid4())
+    d2 = str(uuid.uuid4())
+    contents = [
+        {"content": "User-supplied doc one content.", "document_id": d1},
+        {"content": "User-supplied doc two content.", "document_id": d2},
+    ]
+
+    result = await memory.submit_async_retain(
+        bank_id=bank_id,
+        contents=contents,
+        request_context=request_context,
+    )
+    await asyncio.sleep(0.2)
+
+    meta = await _child_metadata(memory, bank_id, result["operation_id"], request_context)
+    assert "document_ids" in meta, meta
+    assert set(meta["document_ids"]) == {d1, d2}
+
+
+@pytest.mark.asyncio
+async def test_retain_records_generated_document_id(memory, request_context):
+    """With no document_ids supplied, retain records the single generated id."""
+    bank_id = "test_doc_ids_generated"
+    contents = [
+        {"content": "Generated doc item one."},
+        {"content": "Generated doc item two."},
+    ]
+
+    result = await memory.submit_async_retain(
+        bank_id=bank_id,
+        contents=contents,
+        request_context=request_context,
+    )
+    await asyncio.sleep(0.2)
+
+    meta = await _child_metadata(memory, bank_id, result["operation_id"], request_context)
+    assert "document_ids" in meta, meta
+    assert isinstance(meta["document_ids"], list)
+    assert len(meta["document_ids"]) == 1
+    # Must be a valid UUID string (generated by the orchestrator)
+    uuid.UUID(meta["document_ids"][0])
+
+
+@pytest.mark.asyncio
+async def test_retain_records_shared_document_id_once(memory, request_context):
+    """Items sharing one document_id record it exactly once (idempotent set-append)."""
+    bank_id = "test_doc_ids_shared"
+    shared = str(uuid.uuid4())
+    # Duplicate per-item doc_ids are rejected up front, so shared-doc mode
+    # is exercised by a single item carrying the id.
+    contents = [{"content": "Shared doc, chunk A.", "document_id": shared}]
+
+    result = await memory.submit_async_retain(
+        bank_id=bank_id,
+        contents=contents,
+        request_context=request_context,
+    )
+    await asyncio.sleep(0.2)
+
+    meta = await _child_metadata(memory, bank_id, result["operation_id"], request_context)
+    assert meta.get("document_ids") == [shared]
+
+
+@pytest.mark.asyncio
+async def test_get_operation_status_include_payload(memory, request_context):
+    """include_payload=True returns the original submission payload; default omits it."""
+    bank_id = "test_include_payload"
+    contents = [{"content": "Payload roundtrip test item."}]
+
+    result = await memory.submit_async_retain(
+        bank_id=bank_id,
+        contents=contents,
+        request_context=request_context,
+    )
+    await asyncio.sleep(0.2)
+
+    parent = await memory.get_operation_status(
+        bank_id=bank_id,
+        operation_id=result["operation_id"],
+        request_context=request_context,
+    )
+    child_id = parent["child_operations"][0]["operation_id"]
+
+    # Default: no payload
+    without = await memory.get_operation_status(
+        bank_id=bank_id,
+        operation_id=child_id,
+        request_context=request_context,
+    )
+    assert without.get("task_payload") is None
+
+    # With flag: payload populated
+    with_payload = await memory.get_operation_status(
+        bank_id=bank_id,
+        operation_id=child_id,
+        request_context=request_context,
+        include_payload=True,
+    )
+    payload = with_payload.get("task_payload")
+    assert payload is not None, with_payload
+    assert payload.get("bank_id") == bank_id
+    assert payload.get("contents")
+    assert payload["contents"][0]["content"] == "Payload roundtrip test item."
+
+
+@pytest.mark.asyncio
+async def test_operation_status_exposes_retry_count_and_next_retry_at(memory, request_context):
+    """get_operation_status and list_operations return retry_count and next_retry_at.
+
+    Consumers need these to distinguish a freshly-queued pending task from
+    one that's parked for a future retry (e.g. because an extension raised
+    DeferOperation). Without them, "pending" is ambiguous and callers can't
+    render a helpful "deferred until X" state.
+    """
+    from datetime import datetime, timedelta, timezone
+
+    bank_id = "test_retry_fields"
+    result = await memory.submit_async_retain(
+        bank_id=bank_id,
+        contents=[{"content": "retry-fields test item"}],
+        request_context=request_context,
+    )
+    await asyncio.sleep(0.1)
+    parent_id = result["operation_id"]
+    child_id = None
+
+    # Get the child op (the batch_retain parent holds a single child in the
+    # sync/simplified path used by SyncTaskBackend tests).
+    parent_status = await memory.get_operation_status(
+        bank_id=bank_id, operation_id=parent_id, request_context=request_context,
+    )
+    assert "retry_count" in parent_status
+    assert "next_retry_at" in parent_status
+    assert parent_status["retry_count"] == 0
+    # Completed tasks should have next_retry_at cleared on the row (or the
+    # status field doesn't include it meaningfully), so we don't assert a
+    # specific value here — only that the key is present.
+    if parent_status.get("child_operations"):
+        child_id = parent_status["child_operations"][0]["operation_id"]
+
+    # list_operations also exposes both fields
+    listed = await memory.list_operations(
+        bank_id=bank_id, request_context=request_context, limit=10, offset=0,
+    )
+    assert listed["operations"], listed
+    for op in listed["operations"]:
+        assert "retry_count" in op
+        assert "next_retry_at" in op
+        assert isinstance(op["retry_count"], int)
+
+    # Simulate a deferred op: set next_retry_at to 15 min in the future for
+    # the child row directly in the DB, then fetch via the API and confirm
+    # the value round-trips as an ISO-8601 string.
+    if child_id:
+        pool = await memory._get_pool()
+        future = datetime.now(timezone.utc) + timedelta(minutes=15)
+        await pool.execute(
+            "UPDATE async_operations SET status = 'pending', next_retry_at = $1, retry_count = 2 WHERE operation_id = $2",
+            future,
+            uuid.UUID(child_id),
+        )
+        fetched = await memory.get_operation_status(
+            bank_id=bank_id, operation_id=child_id, request_context=request_context,
+        )
+        assert fetched["retry_count"] == 2
+        assert fetched["next_retry_at"] is not None
+        # Round-trip tolerance: within 1 second.
+        parsed = datetime.fromisoformat(fetched["next_retry_at"])
+        assert abs((parsed - future).total_seconds()) < 1.0
+
+
+@pytest.mark.asyncio
+async def test_request_context_retry_count_propagated_to_validator(memory_no_llm_verify, request_context):
+    """_handle_batch_retain forwards the task's _retry_count as
+    RequestContext.retry_count, so validator extensions can compute
+    exponential backoff without querying async_operations themselves.
+    """
+    from hindsight_api.extensions import (
+        OperationValidatorExtension,
+        RecallContext,
+        ReflectContext,
+        RetainContext,
+        ValidationResult,
+    )
+
+    captured: dict[str, int] = {"retry_count": -1}
+
+    class CapturingValidator(OperationValidatorExtension):
+        def __init__(self):
+            super().__init__({})
+
+        async def validate_retain(self, ctx: RetainContext) -> ValidationResult:
+            captured["retry_count"] = ctx.request_context.retry_count
+            return ValidationResult.accept()
+
+        async def validate_recall(self, ctx: RecallContext) -> ValidationResult:
+            return ValidationResult.accept()
+
+        async def validate_reflect(self, ctx: ReflectContext) -> ValidationResult:
+            return ValidationResult.accept()
+
+    memory_no_llm_verify._operation_validator = CapturingValidator()
+
+    bank_id = f"test-retry-propagate-{uuid.uuid4().hex[:8]}"
+    pool = await memory_no_llm_verify._get_pool()
+    await _ensure_bank(pool, bank_id)
+
+    task_dict = {
+        "type": "batch_retain",
+        "bank_id": bank_id,
+        "contents": [{"content": "retry-propagate test"}],
+        "_tenant_id": "default",
+        "_retry_count": 3,  # simulate 3rd retry
+    }
+    await memory_no_llm_verify._handle_batch_retain(task_dict)
+
+    assert captured["retry_count"] == 3, (
+        "Validator should see retry_count=3 from task_dict['_retry_count']; "
+        f"got {captured['retry_count']}"
+    )
+
+    # Default (missing _retry_count key) must surface as 0, not raise.
+    captured["retry_count"] = -1
+    task_dict_no_retry = {
+        "type": "batch_retain",
+        "bank_id": bank_id,
+        "contents": [{"content": "retry-propagate default test"}],
+        "_tenant_id": "default",
+    }
+    await memory_no_llm_verify._handle_batch_retain(task_dict_no_retry)
+    assert captured["retry_count"] == 0
+
+
+@pytest.mark.asyncio
+async def test_submit_async_operation_leaves_claimable_row_when_submit_task_fails(memory):
+    """Regression for the crash-window orphan bug fixed in #1091.
+
+    Previously, _submit_async_operation INSERTed the async_operations row without
+    task_payload, then called submit_task as a separate step to fill it in. If
+    submit_task failed (crash, timeout, dropped connection) after the INSERT
+    committed, the row was left with task_payload IS NULL and became permanently
+    stuck because the worker claim query filters on task_payload IS NOT NULL.
+
+    With the atomic INSERT, even if submit_task raises afterwards the row is born
+    claimable. This test simulates the crash by forcing submit_task to raise.
+    """
+    bank_id = f"test_orphan_prevention_{uuid.uuid4().hex[:8]}"
+    pool = await memory._get_pool()
+    await _ensure_bank(pool, bank_id)
+
+    async def failing_submit_task(_task_dict):
+        raise RuntimeError("Simulated crash between INSERT and submit_task")
+
+    memory._task_backend.submit_task = failing_submit_task  # type: ignore[method-assign]
+
+    with pytest.raises(RuntimeError, match="Simulated crash"):
+        await memory._submit_async_operation(
+            bank_id=bank_id,
+            operation_type="retain",
+            task_type="batch_retain",
+            task_payload={"contents": [{"content": "hello", "document_id": "d1"}]},
+        )
+
+    rows = await pool.fetch(
+        """
+        SELECT status, task_payload
+        FROM async_operations
+        WHERE bank_id = $1 AND operation_type = 'retain'
+        """,
+        bank_id,
+    )
+    assert len(rows) == 1, f"Expected exactly one retain row for bank_id={bank_id}, got {len(rows)}"
+    row = rows[0]
+    assert row["status"] == "pending"
+    assert row["task_payload"] is not None, (
+        "task_payload must be set atomically by the INSERT — a NULL here means "
+        "the worker claim query (task_payload IS NOT NULL) will never pick this row up"
+    )
+    payload = json.loads(row["task_payload"])
+    assert payload["type"] == "batch_retain"
+    assert payload["bank_id"] == bank_id
+    assert payload["contents"] == [{"content": "hello", "document_id": "d1"}]

hindsight-api-slim/tests/test_bank_stats.py

@@ -0,0 +1,242 @@
+"""
+Tests for the bank stats endpoint and the memories-timeseries endpoint.
+
+Covers the new fields exposed by GET /v1/default/banks/{bank_id}/stats
+(operations_by_status) and the new endpoint
+GET /v1/default/banks/{bank_id}/stats/memories-timeseries.
+"""
+import uuid
+from datetime import datetime
+
+import httpx
+import pytest
+import pytest_asyncio
+
+from hindsight_api.api import create_app
+
+
+@pytest_asyncio.fixture
+async def api_client(memory):
+    app = create_app(memory, initialize_memory=False)
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
+        yield client
+
+
+@pytest.fixture
+def test_bank_id():
+    return f"stats_test_{datetime.now().timestamp()}"
+
+
+async def _insert_memory(memory, bank_id: str, text: str, *, failed: bool = False) -> str:
+    """Insert a single experience memory, optionally marked as consolidation-failed."""
+    mem_id = uuid.uuid4()
+    async with memory._pool.acquire() as conn:
+        await conn.execute(
+            """
+            INSERT INTO memory_units (id, bank_id, text, fact_type, created_at, consolidation_failed_at)
+            VALUES ($1, $2, $3, 'experience', now(), CASE WHEN $4 THEN now() ELSE NULL END)
+            """,
+            mem_id,
+            bank_id,
+            text,
+            failed,
+        )
+    return str(mem_id)
+
+
+@pytest.mark.asyncio
+async def test_bank_stats_exposes_operations_by_status(api_client, test_bank_id):
+    """/stats should return operations_by_status with all finished operations."""
+    try:
+        # Kick off a retain so at least one completed operation exists.
+        response = await api_client.post(
+            f"/v1/default/banks/{test_bank_id}/memories",
+            json={"items": [{"content": "Alice is a software engineer.", "context": "team"}]},
+        )
+        assert response.status_code == 200
+
+        response = await api_client.get(f"/v1/default/banks/{test_bank_id}/stats")
+        assert response.status_code == 200
+        stats = response.json()
+
+        assert "operations_by_status" in stats
+        assert isinstance(stats["operations_by_status"], dict)
+        # A synchronous retain finishes as "completed".
+        assert stats["operations_by_status"].get("completed", 0) >= 1
+        # pending/failed counters should still be present as scalar mirrors.
+        assert stats["pending_operations"] == stats["operations_by_status"].get("pending", 0)
+        assert stats["failed_operations"] == stats["operations_by_status"].get("failed", 0)
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "period,expected_count,expected_trunc",
+    [
+        ("1h", 60, "minute"),
+        ("12h", 12, "hour"),
+        ("1d", 24, "hour"),
+        ("7d", 7, "day"),
+        ("30d", 30, "day"),
+        ("90d", 90, "day"),
+    ],
+)
+async def test_memories_timeseries_periods(
+    api_client, test_bank_id, period, expected_count, expected_trunc
+):
+    """Every period must return the full expected bucket count and trunc."""
+    try:
+        response = await api_client.post(
+            f"/v1/default/banks/{test_bank_id}/memories",
+            json={"items": [{"content": "Bob works on infrastructure.", "context": "team"}]},
+        )
+        assert response.status_code == 200
+
+        response = await api_client.get(
+            f"/v1/default/banks/{test_bank_id}/stats/memories-timeseries",
+            params={"period": period},
+        )
+        assert response.status_code == 200
+        body = response.json()
+
+        assert body["bank_id"] == test_bank_id
+        assert body["period"] == period
+        assert body["trunc"] == expected_trunc
+        assert len(body["buckets"]) == expected_count
+
+        for bucket in body["buckets"]:
+            assert "time" in bucket
+            assert bucket["world"] >= 0
+            assert bucket["experience"] >= 0
+            assert bucket["observation"] >= 0
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+async def test_memories_timeseries_invalid_period_falls_back(api_client, test_bank_id):
+    """An unknown period must fall back to the 7d default."""
+    try:
+        response = await api_client.get(
+            f"/v1/default/banks/{test_bank_id}/stats/memories-timeseries",
+            params={"period": "nonsense"},
+        )
+        assert response.status_code == 200
+        body = response.json()
+        assert body["period"] == "7d"
+        assert body["trunc"] == "day"
+        assert len(body["buckets"]) == 7
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+async def test_memories_timeseries_empty_bank_returns_zero_filled_buckets(
+    api_client, test_bank_id
+):
+    """A bank with no memories must still return the full zero-filled bucket set."""
+    try:
+        # Ensure the bank exists.
+        response = await api_client.get(f"/v1/default/banks/{test_bank_id}/profile")
+        assert response.status_code == 200
+
+        response = await api_client.get(
+            f"/v1/default/banks/{test_bank_id}/stats/memories-timeseries",
+            params={"period": "7d"},
+        )
+        assert response.status_code == 200
+        body = response.json()
+        assert len(body["buckets"]) == 7
+        for bucket in body["buckets"]:
+            assert bucket["world"] == 0
+            assert bucket["experience"] == 0
+            assert bucket["observation"] == 0
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+async def test_memories_timeseries_reflects_retained_memories(api_client, test_bank_id):
+    """Freshly-retained memories must show up in today's bucket counts."""
+    try:
+        response = await api_client.post(
+            f"/v1/default/banks/{test_bank_id}/memories",
+            json={
+                "items": [
+                    {"content": "Alice is a software engineer.", "context": "team"},
+                    {"content": "Bob works on infrastructure.", "context": "team"},
+                ]
+            },
+        )
+        assert response.status_code == 200
+
+        response = await api_client.get(
+            f"/v1/default/banks/{test_bank_id}/stats/memories-timeseries",
+            params={"period": "7d"},
+        )
+        assert response.status_code == 200
+        body = response.json()
+        totals = sum(b["world"] + b["experience"] + b["observation"] for b in body["buckets"])
+        assert totals >= 2, "expected at least two memories across all buckets"
+
+        # Those memories should land in the most-recent bucket.
+        latest = body["buckets"][-1]
+        assert latest["world"] + latest["experience"] + latest["observation"] >= 2
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+async def test_bank_stats_reports_failed_consolidation(api_client, memory, test_bank_id):
+    """/stats must surface the count of memories with consolidation_failed_at set."""
+    try:
+        await _insert_memory(memory, test_bank_id, "Alice failed 1.", failed=True)
+        await _insert_memory(memory, test_bank_id, "Alice failed 2.", failed=True)
+        await _insert_memory(memory, test_bank_id, "Alice pending.", failed=False)
+
+        response = await api_client.get(f"/v1/default/banks/{test_bank_id}/stats")
+        assert response.status_code == 200
+        stats = response.json()
+
+        assert stats["failed_consolidation"] == 2
+        # The two failed memories also count as "not-yet-consolidated".
+        assert stats["pending_consolidation"] >= 3
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+async def test_list_memories_filter_by_consolidation_state_failed(api_client, memory, test_bank_id):
+    """?consolidation_state=failed returns only memories with consolidation_failed_at set."""
+    try:
+        failed_id = await _insert_memory(memory, test_bank_id, "Broken item.", failed=True)
+        await _insert_memory(memory, test_bank_id, "Healthy item.", failed=False)
+
+        response = await api_client.get(
+            f"/v1/default/banks/{test_bank_id}/memories/list",
+            params={"consolidation_state": "failed"},
+        )
+        assert response.status_code == 200
+        body = response.json()
+
+        ids = [item["id"] for item in body["items"]]
+        assert failed_id in ids
+        assert body["total"] == 1
+        assert body["items"][0]["consolidation_failed_at"] is not None
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+async def test_list_memories_filter_by_consolidation_state_rejects_unknown(api_client, test_bank_id):
+    """An invalid consolidation_state value must return a 400 (not 500)."""
+    try:
+        response = await api_client.get(
+            f"/v1/default/banks/{test_bank_id}/memories/list",
+            params={"consolidation_state": "bogus"},
+        )
+        assert response.status_code == 400
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")

hindsight-api-slim/tests/test_bank_template_configurable_fields.py

@@ -0,0 +1,115 @@
+"""Verify that BankTemplateConfig exposes every hierarchical field that
+_CONFIGURABLE_FIELDS already accepts at the engine layer.
+
+This test guards the fix for the gap described in the upstream PR title
+"fix(bank-template): align BankTemplateConfig with _CONFIGURABLE_FIELDS".
+Each new field is POSTed through /v1/default/banks/{id}/import and then
+read back via the bank-config endpoint; assertion is that the applied
+value round-trips through the engine.
+
+Runs via: uv run pytest tests/test_bank_template_configurable_fields.py -v
+
+The api_client fixture (shared with tests/test_bank_templates.py) wraps
+create_app(memory, initialize_memory=False) in an httpx.ASGITransport
+with base_url http://test — in-process, no network, no tenant extension.
+Copy the fixture inline here so the test file does not depend on a
+conftest we do not ship in the patch.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+import httpx
+import pytest
+import pytest_asyncio
+
+from hindsight_api.api import create_app
+from hindsight_api.api.http import BankTemplateConfig
+
+# Each tuple is (field_name, applied_value). Values chosen to differ
+# visibly from defaults so round-trip bugs surface.
+NEW_FIELDS: list[tuple[str, object]] = [
+    ("retain_default_strategy", "strategy-a"),
+    ("retain_strategies", {"strategy-a": {"mode": "concise", "max_tokens": 512}}),
+    ("retain_chunk_batch_size", 7),
+    ("mcp_enabled_tools", ["list_banks", "get_bank_profile"]),
+    ("consolidation_llm_batch_size", 11),
+    ("consolidation_source_facts_max_tokens", 2048),
+    ("consolidation_source_facts_max_tokens_per_observation", 256),
+    ("max_observations_per_scope", 13),
+    ("reflect_source_facts_max_tokens", 4096),
+    ("llm_gemini_safety_settings", [{"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"}]),
+    ("recall_budget_function", "adaptive"),
+    ("recall_budget_fixed_low", 50),
+    ("recall_budget_fixed_mid", 250),
+    ("recall_budget_fixed_high", 800),
+    ("recall_budget_adaptive_low", 0.05),
+    ("recall_budget_adaptive_mid", 0.1),
+    ("recall_budget_adaptive_high", 0.4),
+    ("recall_budget_min", 30),
+    ("recall_budget_max", 1500),
+]
+
+
+@pytest_asyncio.fixture
+async def api_client(memory):
+    """Matches the fixture in tests/test_bank_templates.py — in-process
+    ASGI test client, no tenant extension, no auth."""
+    app = create_app(memory, initialize_memory=False)
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
+        yield client
+
+
+@pytest.fixture
+def bank_id():
+    return f"tmpl_config_{datetime.now().timestamp()}"
+
+
+def test_bank_template_config_declares_every_configurable_field():
+    """Pydantic-level guard: every field in NEW_FIELDS must be a declared
+    attribute of BankTemplateConfig so get_config_updates() picks it up."""
+    declared = set(BankTemplateConfig.model_fields.keys())
+    missing = [name for name, _ in NEW_FIELDS if name not in declared]
+    assert not missing, f"BankTemplateConfig missing fields: {missing}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("field_name,applied_value", NEW_FIELDS, ids=[n for n, _ in NEW_FIELDS])
+async def test_new_field_round_trips_through_import(
+    api_client: httpx.AsyncClient,
+    bank_id: str,
+    field_name: str,
+    applied_value: object,
+):
+    """POST a minimal manifest with one new field set, then read bank
+    config back and assert the value made it through.
+
+    Bank config response shape per upstream's test_import_applies_config:
+    top-level keys are resolved hierarchical config; per-bank overrides
+    live under config["overrides"][<field>]. Assert on the override slot.
+    """
+    unique_bank_id = f"{bank_id}_{field_name}"
+    manifest = {
+        "version": "1",
+        "bank": {field_name: applied_value},
+    }
+
+    resp = await api_client.post(
+        f"/v1/default/banks/{unique_bank_id}/import",
+        json=manifest,
+    )
+    assert resp.status_code == 200, resp.text
+
+    # Read bank config back — field must reflect the applied value
+    # under the "overrides" slot, matching upstream's own test shape.
+    read = await api_client.get(f"/v1/default/banks/{unique_bank_id}/config")
+    assert read.status_code == 200, read.text
+    config = read.json()
+    overrides = config.get("overrides", {})
+    assert overrides.get(field_name) == applied_value, (
+        f"round-trip mismatch for {field_name}: "
+        f"sent {applied_value!r}, got {overrides.get(field_name)!r} "
+        f"(full overrides: {overrides!r})"
+    )

hindsight-api-slim/tests/test_batch_api.py

@@ -416,7 +416,7 @@ async def test_worker_batch_recovery(memory, request_context):
             schema=schema,
             tenant_extension=tenant_extension,
             max_slots=5,
-            consolidation_max_slots=2,
+            slot_reservations={"consolidation": 2},
         )
 
         # Run recovery

hindsight-api-slim/tests/test_consolidation.py

@@ -1515,6 +1515,7 @@ class TestHierarchicalRetrieval:
         async with memory._pool.acquire() as conn:
             query_embedding = memory.embeddings.encode(["What does John like?"])[0]
             mental_model_result = await tool_search_mental_models(
+                memory_engine=memory,
                 conn=conn,
                 bank_id=bank_id,
                 query="What does John like?",
@@ -1576,6 +1577,7 @@ class TestHierarchicalRetrieval:
         async with memory._pool.acquire() as conn:
             query_embedding = memory.embeddings.encode(["Where does Sarah work?"])[0]
             mental_model_result = await tool_search_mental_models(
+                memory_engine=memory,
                 conn=conn,
                 bank_id=bank_id,
                 query="Where does Sarah work?",

hindsight-api-slim/tests/test_consolidation_retry_budget.py

@@ -0,0 +1,100 @@
+"""Tests for consolidation retry budget configurability (issue #1042)."""
+
+import pytest
+
+from unittest.mock import AsyncMock, MagicMock
+
+from hindsight_api.engine.consolidation.consolidator import _consolidate_batch_with_llm
+
+
+@pytest.fixture
+def mock_llm_config():
+    llm = AsyncMock()
+    response = MagicMock()
+    response.creates = []
+    response.updates = []
+    response.deletes = []
+    llm.call.return_value = response
+    return llm
+
+
+@pytest.fixture
+def mock_config():
+    config = MagicMock()
+    config.observations_mission = None
+    config.consolidation_max_attempts = 3
+    config.consolidation_llm_max_retries = None
+    return config
+
+
+class TestConsolidationRetryBudget:
+    @pytest.mark.asyncio
+    async def test_config_is_required(self, mock_llm_config):
+        """Passing config=None raises — it's a programmer error, not a runtime fallback."""
+        with pytest.raises(ValueError, match="config is required"):
+            await _consolidate_batch_with_llm(
+                llm_config=mock_llm_config,
+                memories=[{"id": "m1", "text": "test"}],
+                union_observations=[],
+                union_source_facts={},
+                config=None,
+            )
+
+    @pytest.mark.asyncio
+    async def test_configurable_max_attempts(self, mock_llm_config, mock_config):
+        """consolidation_max_attempts controls the outer retry loop."""
+        mock_config.consolidation_max_attempts = 5
+        mock_llm_config.call.side_effect = RuntimeError("fail")
+        result = await _consolidate_batch_with_llm(
+            llm_config=mock_llm_config,
+            memories=[{"id": "m1", "text": "test"}],
+            union_observations=[],
+            union_source_facts={},
+            config=mock_config,
+        )
+        assert result.failed
+        assert mock_llm_config.call.call_count == 5
+
+    @pytest.mark.asyncio
+    async def test_max_retries_threaded_to_call(self, mock_llm_config, mock_config):
+        """consolidation_llm_max_retries is passed to llm_config.call()."""
+        mock_config.consolidation_llm_max_retries = 3
+        await _consolidate_batch_with_llm(
+            llm_config=mock_llm_config,
+            memories=[{"id": "m1", "text": "test"}],
+            union_observations=[],
+            union_source_facts={},
+            config=mock_config,
+        )
+        assert mock_llm_config.call.call_args.kwargs.get("max_retries") == 3
+
+    @pytest.mark.asyncio
+    async def test_max_retries_not_passed_when_none(self, mock_llm_config, mock_config):
+        """When consolidation_llm_max_retries is None, max_retries is not passed."""
+        mock_config.consolidation_llm_max_retries = None
+        await _consolidate_batch_with_llm(
+            llm_config=mock_llm_config,
+            memories=[{"id": "m1", "text": "test"}],
+            union_observations=[],
+            union_source_facts={},
+            config=mock_config,
+        )
+        assert "max_retries" not in mock_llm_config.call.call_args.kwargs
+
+    @pytest.mark.asyncio
+    async def test_reduced_budget_limits_total_calls(self, mock_llm_config, mock_config):
+        """Setting both to low values caps total failure attempts."""
+        mock_config.consolidation_max_attempts = 2
+        mock_config.consolidation_llm_max_retries = 2
+        mock_llm_config.call.side_effect = RuntimeError("upstream 503")
+        result = await _consolidate_batch_with_llm(
+            llm_config=mock_llm_config,
+            memories=[{"id": "m1", "text": "test"}],
+            union_observations=[],
+            union_source_facts={},
+            config=mock_config,
+        )
+        assert result.failed
+        assert mock_llm_config.call.call_count == 2
+        for call_args in mock_llm_config.call.call_args_list:
+            assert call_args.kwargs.get("max_retries") == 2

hindsight-api-slim/tests/test_consolidation_round_limit.py

@@ -0,0 +1,146 @@
+"""Integration tests for consolidation_max_memories_per_round config."""
+
+import uuid
+from unittest.mock import patch
+
+import pytest
+
+from hindsight_api.config import _get_raw_config
+from hindsight_api.engine.consolidation.consolidator import run_consolidation_job
+from hindsight_api.engine.memory_engine import MemoryEngine
+
+
+@pytest.fixture(autouse=True)
+def enable_observations():
+    config = _get_raw_config()
+    original = config.enable_observations
+    config.enable_observations = True
+    yield
+    config.enable_observations = original
+
+
+def _make_config(**overrides):
+    raw = _get_raw_config()
+    return type(raw)(
+        **{
+            **{f: getattr(raw, f) for f in raw.__dataclass_fields__},
+            **overrides,
+        }
+    )
+
+
+@pytest.mark.asyncio
+async def test_round_limit_caps_processed_memories(memory: MemoryEngine, request_context):
+    """When max_memories_per_round is set, consolidation processes at most that many memories
+    and re-submits itself for the remaining backlog."""
+    bank_id = f"test-round-limit-{uuid.uuid4().hex[:8]}"
+    await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
+
+    # Disable consolidation during retain so we build up a backlog
+    fake_config_no_obs = _make_config(enable_observations=False)
+
+    with patch.object(memory._config_resolver, "resolve_full_config", return_value=fake_config_no_obs):
+        for i in range(6):
+            await memory.retain_async(
+                bank_id=bank_id,
+                content=f"Fact number {i}: The user enjoys activity {i} on weekends.",
+                request_context=request_context,
+            )
+
+    # Verify we have unconsolidated memories
+    async with memory._pool.acquire() as conn:
+        unconsolidated = await conn.fetchval(
+            """
+            SELECT COUNT(*) FROM memory_units
+            WHERE bank_id = $1 AND consolidated_at IS NULL
+              AND consolidation_failed_at IS NULL AND fact_type IN ('experience', 'world')
+            """,
+            bank_id,
+        )
+    assert unconsolidated >= 6, f"Expected at least 6 unconsolidated memories, got {unconsolidated}"
+
+    # Run consolidation with a round limit of 3
+    round_limit = 3
+    fake_config = _make_config(consolidation_max_memories_per_round=round_limit)
+
+    with (
+        patch.object(memory._config_resolver, "resolve_full_config", return_value=fake_config),
+        patch.object(memory, "submit_async_consolidation") as mock_requeue,
+    ):
+        result = await run_consolidation_job(
+            memory_engine=memory,
+            bank_id=bank_id,
+            request_context=request_context,
+        )
+
+    assert result["status"] == "completed"
+    assert result["memories_processed"] <= round_limit
+
+    # Must have re-queued consolidation for remaining work
+    mock_requeue.assert_called_once_with(bank_id=bank_id, request_context=request_context)
+
+    # Mental model refresh should be skipped on intermediate round
+    assert result.get("mental_models_refreshed", 0) == 0
+
+    # Verify some memories are still unconsolidated
+    async with memory._pool.acquire() as conn:
+        still_unconsolidated = await conn.fetchval(
+            """
+            SELECT COUNT(*) FROM memory_units
+            WHERE bank_id = $1 AND consolidated_at IS NULL
+              AND consolidation_failed_at IS NULL AND fact_type IN ('experience', 'world')
+            """,
+            bank_id,
+        )
+    assert still_unconsolidated > 0, "Some memories should still be unconsolidated after hitting round limit"
+
+    await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_unlimited_round_processes_all(memory: MemoryEngine, request_context):
+    """When max_memories_per_round is 0 (unlimited), all memories are processed without re-queue."""
+    bank_id = f"test-unlimited-{uuid.uuid4().hex[:8]}"
+    await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
+
+    # Disable consolidation during retain
+    fake_config_no_obs = _make_config(enable_observations=False)
+
+    with patch.object(memory._config_resolver, "resolve_full_config", return_value=fake_config_no_obs):
+        for i in range(4):
+            await memory.retain_async(
+                bank_id=bank_id,
+                content=f"Fact {i}: The user visited city {i} last year.",
+                request_context=request_context,
+            )
+
+    # Run consolidation with unlimited round (0)
+    fake_config = _make_config(consolidation_max_memories_per_round=0)
+
+    with (
+        patch.object(memory._config_resolver, "resolve_full_config", return_value=fake_config),
+        patch.object(memory, "submit_async_consolidation") as mock_requeue,
+    ):
+        result = await run_consolidation_job(
+            memory_engine=memory,
+            bank_id=bank_id,
+            request_context=request_context,
+        )
+
+    assert result["status"] == "completed"
+    # Should NOT re-queue
+    mock_requeue.assert_not_called()
+
+    # All memories should be consolidated
+    async with memory._pool.acquire() as conn:
+        still_unconsolidated = await conn.fetchval(
+            """
+            SELECT COUNT(*) FROM memory_units
+            WHERE bank_id = $1 AND consolidated_at IS NULL
+              AND consolidation_failed_at IS NULL AND fact_type IN ('experience', 'world')
+            """,
+            bank_id,
+        )
+    assert still_unconsolidated == 0
+
+    await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_db_url.py

@@ -0,0 +1,141 @@
+"""Tests for ``hindsight_api.db_url.to_libpq_url``.
+
+Covers backward compatibility (existing configs must pass through unchanged)
+and the two transformations needed to support external PostgreSQL deployments
+that use SQLAlchemy-style ``postgresql+asyncpg://...?ssl=require`` URLs:
+
+1. strip the ``+asyncpg`` dialect suffix,
+2. rename the ``ssl=`` query parameter to ``sslmode=``.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from hindsight_api.db_url import to_libpq_url
+
+
+class TestPassthrough:
+    """Inputs that must be returned unchanged — protects existing configs."""
+
+    @pytest.mark.parametrize(
+        "url",
+        [
+            "pg0",
+            "",
+            "postgresql://user:pass@host:5432/db",
+            "postgresql://user:pass@host:5432/db?sslmode=require",
+            "postgresql://user:pass@host/db?sslmode=verify-full&connect_timeout=10",
+            "sqlite:///./test.db",
+            "postgresql+psycopg2://user:pass@host/db",
+        ],
+    )
+    def test_unchanged(self, url: str) -> None:
+        assert to_libpq_url(url) == url
+
+
+class TestSchemeNormalization:
+    def test_asyncpg_scheme_stripped(self) -> None:
+        assert (
+            to_libpq_url("postgresql+asyncpg://user:pass@host:5432/db")
+            == "postgresql://user:pass@host:5432/db"
+        )
+
+    def test_postgres_asyncpg_scheme_normalized(self) -> None:
+        assert (
+            to_libpq_url("postgres+asyncpg://user:pass@host/db")
+            == "postgresql://user:pass@host/db"
+        )
+
+    def test_bare_postgres_scheme_normalized_to_postgresql(self) -> None:
+        assert to_libpq_url("postgres://user:pass@host/db") == "postgresql://user:pass@host/db"
+
+
+class TestSslParamRename:
+    def test_ssl_require_to_sslmode_require(self) -> None:
+        assert (
+            to_libpq_url("postgresql+asyncpg://user:pass@host:5432/db?ssl=require")
+            == "postgresql://user:pass@host:5432/db?sslmode=require"
+        )
+
+    @pytest.mark.parametrize("mode", ["disable", "allow", "prefer", "require", "verify-ca", "verify-full"])
+    def test_all_ssl_modes_translated(self, mode: str) -> None:
+        result = to_libpq_url(f"postgresql+asyncpg://h/d?ssl={mode}")
+        assert result == f"postgresql://h/d?sslmode={mode}"
+
+    def test_ssl_rename_on_libpq_url(self) -> None:
+        """Someone accidentally using SQLAlchemy-style ssl= on a libpq URL is also fixed."""
+        assert to_libpq_url("postgresql://h/d?ssl=require") == "postgresql://h/d?sslmode=require"
+
+    def test_ssl_param_preserved_among_other_params(self) -> None:
+        result = to_libpq_url(
+            "postgresql+asyncpg://h/d?ssl=require&application_name=hindsight&connect_timeout=10"
+        )
+        assert result.startswith("postgresql://h/d?")
+        # Query order should be preserved; ssl renamed, others untouched.
+        assert "sslmode=require" in result
+        assert "application_name=hindsight" in result
+        assert "connect_timeout=10" in result
+        assert "ssl=" not in result.split("?", 1)[1].replace("sslmode=", "")
+
+    def test_sslmode_not_double_renamed(self) -> None:
+        """An already-correct sslmode= param must not be altered."""
+        assert (
+            to_libpq_url("postgresql+asyncpg://h/d?sslmode=require")
+            == "postgresql://h/d?sslmode=require"
+        )
+
+
+class TestProductionConfigs:
+    """Regression guard: current production URL shapes must pass through unchanged.
+
+    These are the exact shapes currently set for HINDSIGHT_API_DATABASE_URL,
+    HINDSIGHT_API_CONTROL_DATABASE_URL and HINDSIGHT_API_MIGRATION_DATABASE_URL
+    in production. The helper must be a pure no-op for them so this change is
+    truly backward-compatible.
+    """
+
+    @pytest.mark.parametrize(
+        "url",
+        [
+            "postgresql://app:[email protected]:5432/appdb?sslmode=disable",
+            "postgresql://app:[email protected]:5432/appdb_control?sslmode=disable",
+            "postgresql://app:[email protected]:5432/appdb?sslmode=disable",
+        ],
+    )
+    def test_prod_urls_object_identical(self, url: str) -> None:
+        # Not just equal — must be the exact same object (early-out path),
+        # guaranteeing no parse/reassembly and no subtle mutation.
+        assert to_libpq_url(url) is url
+
+
+class TestEdgeCases:
+    def test_idempotent(self) -> None:
+        original = "postgresql+asyncpg://user:pass@host:5432/db?ssl=require"
+        once = to_libpq_url(original)
+        twice = to_libpq_url(once)
+        assert once == twice
+
+    def test_password_with_plus_is_preserved(self) -> None:
+        """A naive str.replace('+asyncpg', ...) would corrupt passwords containing '+'.
+
+        urllib.parse operates on the parsed scheme only, so this stays safe.
+        """
+        url = "postgresql+asyncpg://user:pa%2Bsswd@host/db?ssl=require"
+        result = to_libpq_url(url)
+        assert result == "postgresql://user:pa%2Bsswd@host/db?sslmode=require"
+
+    def test_password_literal_asyncpg_in_password(self) -> None:
+        """Even a password that literally contains '+asyncpg' must survive."""
+        url = "postgresql+asyncpg://user:my%2Basyncpgpass@host/db"
+        result = to_libpq_url(url)
+        assert result == "postgresql://user:my%2Basyncpgpass@host/db"
+
+    def test_url_without_query_string(self) -> None:
+        assert (
+            to_libpq_url("postgresql+asyncpg://user:pass@host/db")
+            == "postgresql://user:pass@host/db"
+        )
+
+    def test_url_with_port_and_path_only(self) -> None:
+        assert to_libpq_url("postgresql+asyncpg://host:5432/db") == "postgresql://host:5432/db"

hindsight-api-slim/tests/test_delta_editorial_fusion.py

@@ -0,0 +1,191 @@
+"""Integration test: delta mental model fuses generic SEO best practices with brand voice.
+
+Scenario:
+1. Create a bank with a delta-mode mental model ("editorial-preferences").
+2. Ingest an SEO best practices document -> trigger mental model refresh.
+3. Ingest a brand voice document -> trigger mental model refresh (delta).
+4. Verify the delta fuses both documents organically.
+
+Requires: HINDSIGHT_RUN_GEMINI_EVALS=1 + a Gemini/OpenAI API key.
+"""
+
+import os
+import uuid
+from collections import Counter
+
+import pytest
+
+from hindsight_api import MemoryEngine, RequestContext
+
+# ---------------------------------------------------------------------------
+# Gate
+# ---------------------------------------------------------------------------
+_GEMINI_KEY = os.getenv("HINDSIGHT_GEMINI_API_KEY") or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")
+_OPENAI_KEY = os.getenv("OPENAI_API_KEY")
+_RUN = os.getenv("HINDSIGHT_RUN_GEMINI_EVALS") == "1" and (bool(_GEMINI_KEY) or bool(_OPENAI_KEY))
+pytestmark = pytest.mark.skipif(not _RUN, reason="Set HINDSIGHT_RUN_GEMINI_EVALS=1 + LLM API key")
+
+# ---------------------------------------------------------------------------
+# Test documents — short but representative
+# ---------------------------------------------------------------------------
+
+SEO_BEST_PRACTICES = """\
+# SEO Content Best Practices
+
+## Content Structure
+- Use clear H1/H2/H3 heading hierarchy for every article.
+- Keep paragraphs under 3 sentences for scannability.
+- Use bullet points and numbered lists to break up dense information.
+
+## Tone and Voice
+- Write in a professional, authoritative tone.
+- Use industry-standard SEO terminology (e.g., "SERP", "CTR", "backlink").
+- Address the reader in second person ("you").
+
+## Keyword Strategy
+- Place primary keyword in H1, first paragraph, and meta description.
+- Target keyword density of 1-2% for primary terms.
+- Include long-tail question keywords in H2/H3 subheadings.
+
+## Technical Requirements
+- Meta titles: 50-60 characters, primary keyword first.
+- Meta descriptions: 150-160 characters, include CTA.
+- Internal links: minimum 3 per article.
+- Image alt text: descriptive, keyword-rich where natural.
+
+## E-E-A-T Compliance
+- Include author bios with credentials.
+- Cite authoritative sources.
+- Update content quarterly to maintain freshness.
+"""
+
+BRAND_VOICE = """\
+# Plot Brand Voice Guide
+
+## Who We Are
+Plot is a finance app for freelancers. We handle invoicing, expense tracking,
+and tax prep for people whose income is irregular.
+
+## Voice Principles
+- We talk like a smart friend who knows about money — not a bank, not a guru.
+- Clarity always wins. If a 12-year-old can't understand it, rewrite it.
+- We never lecture or moralize about financial decisions.
+
+## Tone by Context
+- Marketing: Confident, slightly wry. Example: "Built for income that doesn't show up on the same day every month."
+- Support: Direct, human, accountable. Example: "That's our bug, not yours. We're fixing it now."
+- Product UI: Quiet, precise. Example: "Income from Stripe — Mar 14."
+- Errors: Calm, specific. Example: "We couldn't sync your bank. Try reconnecting."
+
+## Writing Rules
+- Always use contractions (it's, we're, you'll).
+- Use Oxford comma.
+- Always say "you", never "users" or "customers".
+- Avoid jargon: never say "leverage", "empower", "solution", "holistic", "game-changing".
+- No puns. Wit is fine — wordplay and wry asides, not dad jokes.
+
+## What We Sound Like
+- YES: "Here's what we found." / NO: "We are pleased to present our findings."
+- YES: "Looks like this payment is late." / NO: "ALERT: Payment overdue! Action required!"
+"""
+
+
+class TestDeltaEditorialFusion:
+    """Real-LLM test verifying delta mode correctly fuses two documents."""
+
+    async def test_delta_fuses_seo_and_brand_voice(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+    ):
+        bank_id = f"test-editorial-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        try:
+            mm = await memory.create_mental_model(
+                bank_id=bank_id,
+                name="Editorial Preferences",
+                source_query=(
+                    "What are the editorial preferences and content guidelines? "
+                    "Include tone, voice, formatting rules, and vocabulary rules."
+                ),
+                content="",
+                trigger={
+                    "mode": "delta",
+                    "refresh_after_consolidation": False,
+                    "fact_types": ["observation"],
+                    "exclude_mental_models": True,
+                },
+                request_context=request_context,
+            )
+            mm_id = mm["id"]
+
+            # Phase 1: Ingest SEO best practices
+            await memory.retain_async(
+                bank_id=bank_id, content=SEO_BEST_PRACTICES,
+                document_id="seo-best-practices", request_context=request_context,
+            )
+            mm_after_seo = await memory.refresh_mental_model(
+                bank_id=bank_id, mental_model_id=mm_id, request_context=request_context,
+            )
+            seo_content = mm_after_seo["content"]
+            assert len(seo_content) > 100, f"First refresh produced too little content: {len(seo_content)} chars"
+
+            # Phase 2: Ingest brand voice -> delta refresh
+            await memory.retain_async(
+                bank_id=bank_id, content=BRAND_VOICE,
+                document_id="brand-voice", request_context=request_context,
+            )
+            mm_after_brand = await memory.refresh_mental_model(
+                bank_id=bank_id, mental_model_id=mm_id, request_context=request_context,
+            )
+            fused = mm_after_brand["content"]
+            rr = mm_after_brand.get("reflect_response") or {}
+            fused_lower = fused.lower()
+
+            # -- Verify fusion quality --
+
+            # Brand voice concepts present (LLM may paraphrase, check synonyms)
+            for concept, signals in {
+                "contractions": ["contraction", "it's", "we're", "you'll"],
+                "oxford comma": ["oxford comma"],
+                "vocabulary rules": ["jargon", "leverage", "empower", "forbidden"],
+            }.items():
+                assert any(s in fused_lower for s in signals), (
+                    f"Brand voice concept '{concept}' missing (looked for {signals}).\n"
+                    f"Fused content:\n{fused[:500]}"
+                )
+
+            # SEO concepts still present (not wiped by delta)
+            for concept, signals in {
+                "keywords": ["keyword"],
+                "structure": ["heading", "h1", "h2", "structure"],
+                "seo": ["meta", "e-e-a-t", "seo", "search"],
+            }.items():
+                assert any(s in fused_lower for s in signals), (
+                    f"SEO concept '{concept}' missing (looked for {signals}).\n"
+                    f"Fused content:\n{fused[:500]}"
+                )
+
+            # Brand voice overrides generic tone
+            assert any(t in fused_lower for t in ["friend", "wry", "plot", "witty"]), (
+                f"Brand-specific tone missing from fused content.\nFused:\n{fused[:500]}"
+            )
+
+            # No duplicate paragraphs
+            lines = [
+                ln.strip() for ln in fused.split("\n")
+                if ln.strip() and not ln.strip().startswith("#")
+            ]
+            dupes = {line: cnt for line, cnt in Counter(lines).items() if cnt > 1}
+            assert not dupes, (
+                "Duplicate paragraphs:\n" +
+                "\n".join(f"  [{c}x] {t[:80]}" for t, c in dupes.items())
+            )
+
+            # based_on accumulates from both docs
+            obs_count = len(rr.get("based_on", {}).get("observation", []))
+            assert obs_count > 5, f"Expected observations from both docs, got {obs_count}"
+
+        finally:
+            await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_delta_retain.py

@@ -9,6 +9,14 @@ import pytest
 
 from hindsight_api import RequestContext
 from hindsight_api.engine.memory_engine import Budget
+from hindsight_api.extensions import (
+    OperationValidatorExtension,
+    RecallContext,
+    ReflectContext,
+    RetainContext,
+    RetainResult,
+    ValidationResult,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -17,6 +25,31 @@ def _ts():
     return datetime.now(timezone.utc).timestamp()
 
 
+class _RetainResultCapture(OperationValidatorExtension):
+    """Minimal OperationValidator that records each RetainResult it receives.
+
+    Used by tests to assert on fields the engine sets on RetainResult (e.g.
+    processed_content_tokens), without having to scrape logs or internals.
+    The pre-operation validators must be implemented to satisfy the
+    abstract base class, but they always accept.
+    """
+
+    def __init__(self) -> None:
+        self.results: list[RetainResult] = []
+
+    async def validate_retain(self, ctx: RetainContext) -> ValidationResult:
+        return ValidationResult.accept()
+
+    async def validate_recall(self, ctx: RecallContext) -> ValidationResult:
+        return ValidationResult.accept()
+
+    async def validate_reflect(self, ctx: ReflectContext) -> ValidationResult:
+        return ValidationResult.accept()
+
+    async def on_retain_complete(self, result: RetainResult) -> None:
+        self.results.append(result)
+
+
 # ============================================================
 # Core Delta Retain Tests
 # ============================================================
@@ -840,3 +873,185 @@ async def test_delta_retain_recall_with_chunks(memory, request_context):
 
     finally:
         await memory.delete_bank(bank_id, request_context=request_context)
+
+
+# ============================================================
+# processed_content_tokens on RetainResult
+# ============================================================
+#
+# These tests verify the signal the engine exposes via
+# RetainResult.processed_content_tokens for the post-retain hook. That
+# field lets a metering/billing extension tell the difference between:
+#   * a retain that went through the full extraction pipeline (None),
+#   * a retain whose chunks all matched prior content (0),
+#   * a retain where only some chunks were new/changed (N>0, the
+#     content+context tokens of the chunks that were actually processed).
+
+
+def test_merge_processed_content_tokens_helper():
+    """Unit check on the None-propagating aggregator used by the engine."""
+    from hindsight_api.engine.retain.orchestrator import (
+        _merge_processed_content_tokens,
+    )
+
+    assert _merge_processed_content_tokens(0, 0) == 0
+    assert _merge_processed_content_tokens(5, 7) == 12
+    # None "wins" in either slot — once any sub-result bypassed dedup, the
+    # aggregate is None so callers bill full content.
+    assert _merge_processed_content_tokens(None, 10) is None
+    assert _merge_processed_content_tokens(10, None) is None
+    assert _merge_processed_content_tokens(None, None) is None
+
+
+@pytest.mark.asyncio
+async def test_processed_content_tokens_first_retain_is_none(memory, request_context):
+    """
+    First retain to a new document goes through the full (non-delta) path,
+    so processed_content_tokens should be None — the caller has no dedup
+    signal and should bill the full submitted content.
+    """
+    bank_id = f"test_pct_first_{_ts()}"
+    document_id = "new-doc"
+    capture = _RetainResultCapture()
+    memory._operation_validator = capture
+
+    try:
+        await memory.retain_async(
+            bank_id=bank_id,
+            content="Alice works at Google.",
+            context="test",
+            document_id=document_id,
+            request_context=request_context,
+        )
+        assert len(capture.results) == 1
+        assert capture.results[0].processed_content_tokens is None, (
+            "First retain (full path) should report processed_content_tokens=None"
+        )
+    finally:
+        memory._operation_validator = None
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_processed_content_tokens_unchanged_resubmit_is_zero(memory, request_context):
+    """
+    Re-retaining identical content to the same document_id should hit the
+    'no chunks changed' path and report processed_content_tokens=0.
+    """
+    bank_id = f"test_pct_unchanged_{_ts()}"
+    document_id = "conversation-001"
+    capture = _RetainResultCapture()
+    memory._operation_validator = capture
+    content = "Alice works at Google. Bob works at Microsoft."
+
+    try:
+        await memory.retain_async(
+            bank_id=bank_id,
+            content=content,
+            context="team info",
+            document_id=document_id,
+            request_context=request_context,
+        )
+        # Identical resubmit.
+        await memory.retain_async(
+            bank_id=bank_id,
+            content=content,
+            context="team info",
+            document_id=document_id,
+            request_context=request_context,
+        )
+        assert len(capture.results) == 2
+        assert capture.results[0].processed_content_tokens is None
+        assert capture.results[1].processed_content_tokens == 0, (
+            "Unchanged resubmit should report zero processed content tokens"
+        )
+    finally:
+        memory._operation_validator = None
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_processed_content_tokens_appended_reports_delta(memory, request_context):
+    """
+    Appending new content to an existing document should surface a
+    non-zero processed_content_tokens that is less than the full
+    submitted content tokens — only the new/changed chunks are counted.
+    """
+    bank_id = f"test_pct_appended_{_ts()}"
+    document_id = "growing-doc"
+    capture = _RetainResultCapture()
+    memory._operation_validator = capture
+
+    v1 = "Alice works at Google."
+    # Make v2 large enough that the delta diff classifies some chunks as
+    # unchanged (shared prefix) and some as new (the appended tail). The
+    # chunker splits on ``retain_chunk_size`` (default 3000), so we pad
+    # each part with a comfortable margin of filler text to force a chunk
+    # boundary between them.
+    filler = " The project budget is fine. " * 400  # ~12 KB
+    v2 = v1 + filler + " Bob works at Microsoft."
+
+    try:
+        await memory.retain_async(
+            bank_id=bank_id,
+            content=v1,
+            context="profile",
+            document_id=document_id,
+            request_context=request_context,
+        )
+        await memory.retain_async(
+            bank_id=bank_id,
+            content=v2,
+            context="profile",
+            document_id=document_id,
+            request_context=request_context,
+        )
+        assert len(capture.results) == 2
+
+        # Second retain should either:
+        #   * Be on the delta path with a positive partial count strictly
+        #     less than the full submission (the common case), OR
+        #   * Fall back to full retain if the chunker decided nothing
+        #     matched (in which case we report None and bill full).
+        # Both are correct signals for the billing extension; the test
+        # just asserts they're shaped sanely.
+        from hindsight_api.engine.memory_engine import count_tokens
+
+        submitted_tokens = count_tokens(v2) + count_tokens("profile")
+        second = capture.results[1].processed_content_tokens
+        if second is None:
+            # Fell back to full retain — acceptable signal.
+            return
+        assert second > 0, "Partial-delta retain should report a positive token count"
+        assert second < submitted_tokens, (
+            "Partial-delta retain should report fewer processed tokens "
+            "than the full submitted payload"
+        )
+    finally:
+        memory._operation_validator = None
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_processed_content_tokens_without_document_id_is_none(memory, request_context):
+    """
+    A retain without a document_id can't participate in per-document
+    dedup, so the engine should report processed_content_tokens=None
+    and let the caller bill the full submitted payload.
+    """
+    bank_id = f"test_pct_no_doc_{_ts()}"
+    capture = _RetainResultCapture()
+    memory._operation_validator = capture
+
+    try:
+        await memory.retain_async(
+            bank_id=bank_id,
+            content="A one-off observation with no document_id.",
+            context="test",
+            request_context=request_context,
+        )
+        assert len(capture.results) == 1
+        assert capture.results[0].processed_content_tokens is None
+    finally:
+        memory._operation_validator = None
+        await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_delta_retain_duplicates.py

@@ -0,0 +1,404 @@
+"""
+Tests for delta retain chunk ordering and duplicate prevention.
+
+Verifies that:
+1. Chunks are stored with deterministic indices (not task completion order)
+2. Delta retain can correctly identify unchanged chunks on subsequent upserts
+3. Repeated upserts of same content don't produce duplicate memory units
+4. Concurrent retains on the same document produce clean final state (no duplicates)
+"""
+
+import asyncio
+import logging
+import os
+from datetime import datetime, timezone
+
+import pytest
+import pytest_asyncio
+
+from hindsight_api import RequestContext
+from hindsight_api.engine.task_backend import SyncTaskBackend
+
+logger = logging.getLogger(__name__)
+
+
+def _ts():
+    return datetime.now(timezone.utc).timestamp()
+
+
+@pytest.mark.asyncio
+async def test_repeated_upsert_chunks_not_scrambled(memory, request_context):
+    """
+    Verify that chunks are stored with correct indices matching the
+    deterministic chunking order, not task completion order.
+
+    This is critical for delta retain: if chunk indices don't match the
+    deterministic order, delta will think all chunks changed on every
+    upsert and fall back to full re-processing.
+    """
+    bank_id = f"test_chunk_order_{_ts()}"
+    document_id = "chunk-order-doc"
+
+    try:
+        # Create content that produces multiple distinct chunks
+        chunk1_text = "Alice works at Google on Search. " * 100  # ~3300 chars
+        chunk2_text = "Bob works at Microsoft on Azure. " * 100  # ~3400 chars
+        content = chunk1_text + chunk2_text
+
+        assert len(content) > 6000, "Should produce at least 2 chunks"
+
+        await memory.retain_async(
+            bank_id=bank_id,
+            content=content,
+            context="team info",
+            document_id=document_id,
+            request_context=request_context,
+        )
+
+        # Load chunks from DB and verify order matches deterministic chunking
+        from hindsight_api.engine.retain import chunk_storage, fact_extraction
+
+        pool = await memory._get_pool()
+
+        # Get the chunk texts from DB
+        async with pool.acquire() as conn:
+            chunk_rows = await conn.fetch(
+                "SELECT chunk_index, chunk_text, content_hash FROM chunks WHERE bank_id = $1 AND document_id = $2 ORDER BY chunk_index",
+                bank_id,
+                document_id,
+            )
+
+        # Compute expected chunks deterministically (default chunk_size is 3000)
+        chunk_size = 3000
+        expected_chunks = fact_extraction.chunk_text(content, max_chars=chunk_size)
+
+        logger.info(f"Expected {len(expected_chunks)} chunks, got {len(chunk_rows)} in DB")
+
+        # Verify each chunk at its index has the correct content hash
+        for i, expected_text in enumerate(expected_chunks):
+            expected_hash = chunk_storage.compute_chunk_hash(expected_text)
+            matching_rows = [r for r in chunk_rows if r["chunk_index"] == i]
+            assert len(matching_rows) == 1, f"Expected exactly 1 chunk at index {i}, got {len(matching_rows)}"
+            actual_hash = matching_rows[0]["content_hash"]
+            assert actual_hash == expected_hash, (
+                f"Chunk at index {i} has wrong content hash. "
+                f"Expected hash of first 50 chars: {repr(expected_text[:50])}, "
+                f"got hash of: {repr(matching_rows[0]['chunk_text'][:50])}"
+            )
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_delta_detects_unchanged_after_first_retain(memory, request_context):
+    """
+    After first retain stores chunks with correct indices, a second retain
+    with identical content should use the delta path and detect all chunks
+    as unchanged (no re-processing).
+    """
+    bank_id = f"test_delta_unchanged_{_ts()}"
+    document_id = "delta-unchanged-doc"
+
+    try:
+        # Multi-chunk content with distinct sections
+        chunk1_text = "Alice works at Google on Search. " * 100
+        chunk2_text = "Bob works at Microsoft on Azure. " * 100
+        content = chunk1_text + chunk2_text
+
+        # First retain
+        v1_units = await memory.retain_async(
+            bank_id=bank_id,
+            content=content,
+            context="team info",
+            document_id=document_id,
+            request_context=request_context,
+        )
+        assert len(v1_units) > 0
+
+        pool = await memory._get_pool()
+        async with pool.acquire() as conn:
+            v1_count = await conn.fetchval(
+                "SELECT count(*) FROM memory_units WHERE bank_id = $1 AND document_id = $2",
+                bank_id,
+                document_id,
+            )
+
+        # Second retain — same content, should be detected as unchanged by delta
+        v2_units = await memory.retain_async(
+            bank_id=bank_id,
+            content=content,
+            context="team info",
+            document_id=document_id,
+            request_context=request_context,
+        )
+
+        # Delta should detect all unchanged → return empty (no new units)
+        assert v2_units == [], f"Delta with unchanged content should return empty, got {len(v2_units)} units"
+
+        # Memory unit count should not change
+        async with pool.acquire() as conn:
+            v2_count = await conn.fetchval(
+                "SELECT count(*) FROM memory_units WHERE bank_id = $1 AND document_id = $2",
+                bank_id,
+                document_id,
+            )
+        assert v2_count == v1_count, (
+            f"Memory unit count changed on same-content upsert: {v1_count} -> {v2_count}"
+        )
+
+        # Third retain — verify stability
+        v3_units = await memory.retain_async(
+            bank_id=bank_id,
+            content=content,
+            context="team info",
+            document_id=document_id,
+            request_context=request_context,
+        )
+        assert v3_units == [], "Third retain should also detect unchanged"
+
+        async with pool.acquire() as conn:
+            v3_count = await conn.fetchval(
+                "SELECT count(*) FROM memory_units WHERE bank_id = $1 AND document_id = $2",
+                bank_id,
+                document_id,
+            )
+        assert v3_count == v1_count, (
+            f"Memory unit count changed on third upsert: {v1_count} -> {v3_count}"
+        )
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_stale_request_skipped_when_newer_retain_completed(memory, request_context):
+    """
+    When two retains race on the same document, the one that started earlier
+    (stale) should be skipped if the newer one already completed.
+
+    Simulates: Request B (newer content) completes while Request A (older content)
+    was waiting for the advisory lock. When A finally acquires the lock, it sees
+    the document was updated after its start_time and skips.
+    """
+    bank_id = f"test_stale_skip_{_ts()}"
+    document_id = "stale-skip-doc"
+
+    try:
+        # First: establish the document with initial content
+        newer_content = "Alice works at Google. Bob works at Microsoft. Charlie works at Apple."
+        await memory.retain_async(
+            bank_id=bank_id,
+            content=newer_content,
+            context="team",
+            document_id=document_id,
+            request_context=request_context,
+        )
+
+        pool = await memory._get_pool()
+        async with pool.acquire() as conn:
+            after_newer_count = await conn.fetchval(
+                "SELECT count(*) FROM memory_units WHERE bank_id = $1 AND document_id = $2",
+                bank_id,
+                document_id,
+            )
+        assert after_newer_count > 0, "Should have facts from newer content"
+
+        # Simulate the race condition by pushing the document's updated_at into
+        # the future. This makes any new retain appear "stale" (its start_time
+        # is before updated_at), as if another request already completed.
+        async with pool.acquire() as conn:
+            await conn.execute(
+                "UPDATE documents SET updated_at = NOW() + INTERVAL '10 seconds' WHERE id = $1 AND bank_id = $2",
+                document_id,
+                bank_id,
+            )
+
+        # Now try to retain with older/different content. The stale-request check
+        # should detect that updated_at > start_time and skip this request.
+        older_content = "Alice works at Google."
+        result = await memory.retain_async(
+            bank_id=bank_id,
+            content=older_content,
+            context="team",
+            document_id=document_id,
+            request_context=request_context,
+        )
+
+        # The stale request should have been skipped (empty result)
+        assert result == [], f"Stale request should return empty, got {result}"
+
+        # Memory units should be unchanged (newer content preserved)
+        async with pool.acquire() as conn:
+            final_count = await conn.fetchval(
+                "SELECT count(*) FROM memory_units WHERE bank_id = $1 AND document_id = $2",
+                bank_id,
+                document_id,
+            )
+        assert final_count == after_newer_count, (
+            f"Stale request should not change memory units: {after_newer_count} -> {final_count}"
+        )
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+# ============================================================
+# Concurrent Retain Stress Test
+# ============================================================
+
+
+@pytest_asyncio.fixture(scope="function")
+async def memory_no_llm(pg0_db_url, embeddings, cross_encoder, query_analyzer):
+    """
+    MemoryEngine with provider=none (chunks mode, no LLM needed).
+    Each chunk is stored verbatim as a single memory unit — fast and deterministic.
+    """
+    from hindsight_api.engine.memory_engine import MemoryEngine
+
+    mem = MemoryEngine(
+        db_url=pg0_db_url,
+        memory_llm_provider="none",
+        memory_llm_api_key="",
+        memory_llm_model="none",
+        embeddings=embeddings,
+        cross_encoder=cross_encoder,
+        query_analyzer=query_analyzer,
+        pool_min_size=2,
+        pool_max_size=10,
+        run_migrations=False,
+        task_backend=SyncTaskBackend(),
+        skip_llm_verification=True,
+    )
+    await mem.initialize()
+    yield mem
+    await mem.close()
+
+
+@pytest.mark.asyncio
+async def test_concurrent_upserts_no_duplicates(memory_no_llm, request_context):
+    """
+    Stress test: N concurrent retains of the same document with different content.
+
+    Each version has distinct content so we can verify the final state is exactly
+    one version's data — no duplicates, no mixed data from different versions.
+
+    With provider=none (chunks mode), each chunk becomes a verbatim memory unit,
+    so we can inspect exactly which chunks survived.
+
+    The test verifies:
+    - Exactly one version's document row survives (by content_hash)
+    - All memory units belong to a single version (no cross-version mixing)
+    - No duplicate memory units exist
+    - Chunk count matches what the winning version should have
+    """
+    bank_id = f"test_concurrent_{_ts()}"
+    document_id = "concurrent-doc"
+    num_concurrent = 20
+
+    try:
+        # Each version has unique, identifiable content.
+        # Make content large enough for multiple chunks (~3000 chars per chunk).
+        versions = []
+        for v in range(num_concurrent):
+            # Each version's chunks will contain "VERSION_XX" markers so we can
+            # identify which version's data survived in the final state.
+            content = f"VERSION_{v:02d} " + f"Person_{v} works at Company_{v}. " * 200
+            versions.append(content)
+
+        # Fire all retains concurrently
+        async def _retain_version(version_content: str) -> None:
+            await memory_no_llm.retain_async(
+                bank_id=bank_id,
+                content=version_content,
+                document_id=document_id,
+                request_context=request_context,
+            )
+
+        results = await asyncio.gather(
+            *[_retain_version(v) for v in versions],
+            return_exceptions=True,
+        )
+
+        # Some may have been aborted (pipeline_aborted) — that's expected.
+        # Check for unexpected errors.
+        errors = [r for r in results if isinstance(r, Exception)]
+        for err in errors:
+            logger.warning(f"Concurrent retain error (may be expected): {err}")
+
+        # --- Verify final state ---
+        pool = await memory_no_llm._get_pool()
+
+        # 1. Exactly one document row should exist
+        async with pool.acquire() as conn:
+            doc_rows = await conn.fetch(
+                "SELECT id, content_hash FROM documents WHERE id = $1 AND bank_id = $2",
+                document_id,
+                bank_id,
+            )
+        assert len(doc_rows) == 1, f"Expected 1 document row, got {len(doc_rows)}"
+        winning_hash = doc_rows[0]["content_hash"]
+
+        # Find which version won by matching content_hash
+        import hashlib
+
+        from hindsight_api.engine.retain.fact_extraction import _sanitize_text
+
+        winning_version = None
+        for v, content in enumerate(versions):
+            sanitized = _sanitize_text(content) or ""
+            h = hashlib.sha256(sanitized.encode()).hexdigest()
+            if h == winning_hash:
+                winning_version = v
+                break
+        assert winning_version is not None, "Could not identify winning version from content_hash"
+        logger.info(f"Winning version: {winning_version} (out of {num_concurrent} concurrent retains)")
+
+        # 2. All memory units should belong to the winning version
+        async with pool.acquire() as conn:
+            units = await conn.fetch(
+                "SELECT text, chunk_id, id::text as unit_id FROM memory_units WHERE bank_id = $1 AND document_id = $2",
+                bank_id,
+                document_id,
+            )
+        unit_texts = [r["text"] for r in units]
+        assert len(unit_texts) > 0, "Should have at least 1 memory unit"
+
+        # In chunks mode, each memory unit text IS the chunk text.
+        # Every unit should contain the winning version's unique person name.
+        # We check for "Person_N" rather than "VERSION_N" because the text
+        # splitter may cut mid-text, so later chunks might not start with the prefix.
+        winning_person = f"Person_{winning_version}"
+        wrong_version_units = [
+            (r["text"], r["chunk_id"], r["unit_id"])
+            for r in units
+            if winning_person not in r["text"]
+        ]
+        assert not wrong_version_units, (
+            f"Found {len(wrong_version_units)} memory units NOT from winning version "
+            f"{winning_version} (expected '{winning_person}' in every unit). "
+            f"Details: {[(t[:60], cid, uid) for t, cid, uid in wrong_version_units]}"
+        )
+
+        # 3. No duplicate memory units
+        from collections import Counter
+
+        text_counts = Counter(unit_texts)
+        duplicates = {text[:80]: count for text, count in text_counts.items() if count > 1}
+        assert not duplicates, f"Found duplicate memory units: {duplicates}"
+
+        # 4. Chunk count matches expected
+        from hindsight_api.engine.retain.fact_extraction import chunk_text
+
+        expected_chunks = chunk_text(versions[winning_version], max_chars=3000)
+        assert len(unit_texts) == len(expected_chunks), (
+            f"Expected {len(expected_chunks)} chunks for winning version, got {len(unit_texts)} memory units"
+        )
+
+        logger.info(
+            f"Concurrent test passed: version {winning_version} won with "
+            f"{len(unit_texts)} memory units, no duplicates"
+        )
+
+    finally:
+        await memory_no_llm.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_embeddings_openai_batch_size.py

@@ -0,0 +1,150 @@
+"""
+Tests for HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE config wiring.
+
+Regression test for issue #1142: `OpenAIEmbeddings` hardcoded `batch_size=100` is
+incompatible with OpenAI-compatible providers that enforce stricter per-request
+limits (e.g. DashScope / Aliyun Tongyi cap at 10). Users must be able to override
+the batch size via env var so `encode()` splits into smaller chunks.
+"""
+
+import os
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def setup_test_env():
+    """Save/restore env vars touched by these tests."""
+    from hindsight_api.config import clear_config_cache
+
+    env_vars_to_save = [
+        "HINDSIGHT_API_EMBEDDINGS_PROVIDER",
+        "HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY",
+        "HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL",
+        "HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE",
+        "HINDSIGHT_API_EMBEDDINGS_OPENROUTER_API_KEY",
+        "HINDSIGHT_API_LLM_API_KEY",
+        "HINDSIGHT_API_LLM_PROVIDER",
+    ]
+
+    original_values = {key: os.environ.get(key) for key in env_vars_to_save}
+
+    clear_config_cache()
+
+    yield
+
+    for key, original_value in original_values.items():
+        if original_value is None:
+            os.environ.pop(key, None)
+        else:
+            os.environ[key] = original_value
+
+    clear_config_cache()
+
+
+def test_default_openai_batch_size_is_100():
+    """Default batch size is 100 when env var unset (preserves legacy behavior)."""
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ.pop("HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE", None)
+
+    config = HindsightConfig.from_env()
+    assert config.embeddings_openai_batch_size == 100
+
+
+def test_openai_batch_size_env_var_is_read():
+    """HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE overrides the default."""
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"] = "10"
+
+    config = HindsightConfig.from_env()
+    assert config.embeddings_openai_batch_size == 10
+
+
+def test_openai_embeddings_provider_uses_configured_batch_size():
+    """create_embeddings_from_env() propagates config to OpenAIEmbeddings for 'openai' provider."""
+    from hindsight_api.engine.embeddings import OpenAIEmbeddings, create_embeddings_from_env
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_PROVIDER"] = "openai"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY"] = "sk-test"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"] = "10"
+
+    embeddings = create_embeddings_from_env()
+    assert isinstance(embeddings, OpenAIEmbeddings)
+    assert embeddings.batch_size == 10
+
+
+def test_openrouter_provider_uses_configured_batch_size():
+    """'openrouter' provider also honors the shared batch-size config (both paths use OpenAIEmbeddings)."""
+    from hindsight_api.engine.embeddings import OpenAIEmbeddings, create_embeddings_from_env
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_PROVIDER"] = "openrouter"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENROUTER_API_KEY"] = "sk-or-test"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"] = "8"
+
+    embeddings = create_embeddings_from_env()
+    assert isinstance(embeddings, OpenAIEmbeddings)
+    assert embeddings.batch_size == 8
+
+
+def test_zero_batch_size_is_rejected():
+    """Zero would cause `range(0, N, 0)` to crash at runtime — fail fast at config load."""
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"] = "0"
+
+    with pytest.raises(ValueError, match="must be >= 1"):
+        HindsightConfig.from_env()
+
+
+def test_negative_batch_size_is_rejected():
+    """Negative values would silently skip batching — reject at config load."""
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"] = "-5"
+
+    with pytest.raises(ValueError, match="must be >= 1"):
+        HindsightConfig.from_env()
+
+
+def test_non_numeric_batch_size_is_rejected():
+    """Non-integer strings are rejected with a clear error pointing at the env var name."""
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"] = "not-a-number"
+
+    with pytest.raises(ValueError, match="HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"):
+        HindsightConfig.from_env()
+
+
+def test_openai_encode_splits_on_configured_batch_size(monkeypatch):
+    """encode() sends multiple upstream requests when len(texts) > batch_size."""
+    from types import SimpleNamespace
+
+    from hindsight_api.engine.embeddings import OpenAIEmbeddings
+
+    emb = OpenAIEmbeddings(api_key="sk-test", model="text-embedding-3-small", batch_size=10)
+
+    calls: list[int] = []
+
+    def fake_create(*, model, input):
+        calls.append(len(input))
+        return SimpleNamespace(data=[SimpleNamespace(index=i, embedding=[0.0] * 1536) for i in range(len(input))])
+
+    emb._client = SimpleNamespace(embeddings=SimpleNamespace(create=fake_create))
+    emb._dimension = 1536
+
+    vectors = emb.encode(["x"] * 25)
+
+    assert len(vectors) == 25
+    assert calls == [10, 10, 5], (
+        f"Expected upstream calls of size 10, 10, 5 when batch_size=10 and 25 inputs, got {calls}"
+    )

hindsight-api-slim/tests/test_file_retain.py

@@ -543,6 +543,196 @@ async def test_async_file_retain_serializes_datetime_timestamp(memory_no_llm_ver
     assert row["timestamp"] == "2024-01-15T10:30:00+00:00"
 
 
+@pytest.mark.asyncio
+async def test_file_retain_maps_timestamp_to_event_date(memory_no_llm_verify, sample_txt_content):
+    """Regression (PR #1092): file retain must translate 'timestamp' -> 'event_date'.
+
+    The retain orchestrator only reads 'event_date' from each content dict.
+    _handle_file_convert_retain previously forwarded 'timestamp' unchanged, so every
+    file-retained memory silently defaulted to utcnow() and the 'unset' sentinel
+    was a no-op. This test intercepts the inner batch_retain task the handler
+    submits and asserts the key mapping is correct for all three inputs:
+    explicit ISO timestamp, 'unset' sentinel, and omitted (None).
+    """
+    from hindsight_api.engine.parsers.base import FileParser
+    from hindsight_api.models import RequestContext
+
+    memory = memory_no_llm_verify
+
+    class NoopParser(FileParser):
+        async def convert(self, file_data: bytes, filename: str) -> str:
+            return file_data.decode("utf-8")
+
+        def supports(self, filename: str, content_type: str | None = None) -> bool:
+            return filename.endswith(".txt")
+
+        def name(self) -> str:
+            return "event_date_regression_parser"
+
+    memory._parser_registry.register(NoopParser())
+
+    class MockFile:
+        def __init__(self, content, filename, content_type):
+            self.content = content
+            self.filename = filename
+            self.content_type = content_type
+
+        async def read(self):
+            return self.content
+
+    # Capture the inner batch_retain submission from _handle_file_convert_retain so we
+    # can inspect its content dict without running the (LLM-dependent) retain pipeline.
+    original_submit = memory._task_backend.submit_task
+    captured: list[dict] = []
+
+    async def capturing_submit(task_dict):
+        if task_dict.get("type") == "batch_retain":
+            captured.append(task_dict)
+            return
+        await original_submit(task_dict)
+
+    memory._task_backend.submit_task = capturing_submit
+    try:
+        context = RequestContext(internal=True)
+
+        async def run_case(label: str, timestamp_value) -> dict:
+            bank_id = f"test_file_event_date_{label}_{datetime.now(timezone.utc).timestamp()}"
+            await memory.get_bank_profile(bank_id, request_context=context)
+
+            captured.clear()
+            await memory.submit_async_file_retain(
+                bank_id=bank_id,
+                file_items=[
+                    {
+                        "file": MockFile(sample_txt_content, f"{label}.txt", "text/plain"),
+                        "document_id": f"doc_{label}",
+                        "context": "regression test",
+                        "metadata": {},
+                        "tags": [],
+                        "timestamp": timestamp_value,
+                        "parser": ["event_date_regression_parser"],
+                    }
+                ],
+                document_tags=None,
+                request_context=context,
+            )
+            assert len(captured) == 1, f"{label}: expected exactly one batch_retain submission"
+            contents = captured[0]["contents"]
+            assert len(contents) == 1
+            return contents[0]
+
+        # Explicit ISO timestamp -> event_date must equal that string.
+        content = await run_case("explicit", "2024-01-15T10:30:00+00:00")
+        assert "timestamp" not in content, "raw 'timestamp' must not leak into retain content"
+        assert content["event_date"] == "2024-01-15T10:30:00+00:00"
+
+        # 'unset' sentinel -> event_date must be explicit None (orchestrator stores NULL).
+        content = await run_case("unset", "unset")
+        assert "timestamp" not in content
+        assert "event_date" in content, "'unset' must produce an explicit event_date=None"
+        assert content["event_date"] is None
+
+        # Omitted timestamp -> event_date key must be absent (orchestrator defaults to utcnow).
+        content = await run_case("missing", None)
+        assert "timestamp" not in content
+        assert "event_date" not in content
+    finally:
+        memory._task_backend.submit_task = original_submit
+
+
+@pytest.mark.asyncio
+async def test_file_retain_forwards_all_content_fields(memory_no_llm_verify, sample_txt_content):
+    """Regression: _handle_file_convert_retain must forward every FileRetainMetadata
+    field to the inner batch_retain task without renaming or dropping it.
+
+    Covers document_id, context, metadata, tags (per-content), plus strategy
+    and document_tags (per-request). The timestamp -> event_date mapping has
+    its own test above. Existing file retain tests only assert HTTP 200 or
+    inspect the outer file_convert_retain task_payload; none verify what
+    arrives at the retain pipeline. If any of these fields were silently
+    dropped or mis-keyed -- the same failure mode as #1092 for timestamp --
+    those tests would still pass.
+    """
+    from hindsight_api.engine.parsers.base import FileParser
+    from hindsight_api.models import RequestContext
+
+    memory = memory_no_llm_verify
+
+    class NoopParser(FileParser):
+        async def convert(self, file_data: bytes, filename: str) -> str:
+            return file_data.decode("utf-8")
+
+        def supports(self, filename: str, content_type: str | None = None) -> bool:
+            return filename.endswith(".txt")
+
+        def name(self) -> str:
+            return "all_fields_regression_parser"
+
+    memory._parser_registry.register(NoopParser())
+
+    class MockFile:
+        def __init__(self, content, filename, content_type):
+            self.content = content
+            self.filename = filename
+            self.content_type = content_type
+
+        async def read(self):
+            return self.content
+
+    original_submit = memory._task_backend.submit_task
+    captured: list[dict] = []
+
+    async def capturing_submit(task_dict):
+        if task_dict.get("type") == "batch_retain":
+            captured.append(task_dict)
+            return
+        await original_submit(task_dict)
+
+    memory._task_backend.submit_task = capturing_submit
+    try:
+        request_context = RequestContext(internal=True)
+        bank_id = f"test_file_all_fields_{datetime.now(timezone.utc).timestamp()}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        await memory.submit_async_file_retain(
+            bank_id=bank_id,
+            file_items=[
+                {
+                    "file": MockFile(sample_txt_content, "doc.txt", "text/plain"),
+                    "document_id": "my_doc_id",
+                    "context": "meeting notes from Alice",
+                    "metadata": {"author": "Alice", "year": "2024"},
+                    "tags": ["report", "q1"],
+                    "timestamp": None,
+                    "parser": ["all_fields_regression_parser"],
+                    "strategy": "my_strategy",
+                }
+            ],
+            document_tags=["batch_tag"],
+            request_context=request_context,
+        )
+
+        assert len(captured) == 1, "expected exactly one batch_retain submission"
+        payload = captured[0]
+        assert payload["type"] == "batch_retain"
+
+        # Per-request fields (live on the outer task payload, not per-content).
+        assert payload.get("strategy") == "my_strategy", "strategy must be forwarded at request level"
+        assert payload.get("document_tags") == ["batch_tag"], "document_tags must be forwarded at request level"
+
+        # Per-content fields.
+        assert len(payload["contents"]) == 1
+        content = payload["contents"][0]
+        assert content["document_id"] == "my_doc_id"
+        assert content["context"] == "meeting notes from Alice"
+        assert content["metadata"] == {"author": "Alice", "year": "2024"}
+        assert content["tags"] == ["report", "q1"]
+        # content is the converted markdown (raw bytes decoded by NoopParser).
+        assert content["content"] == sample_txt_content.decode("utf-8")
+    finally:
+        memory._task_backend.submit_task = original_submit
+
+
 @pytest.mark.asyncio
 async def test_file_conversion_failure_sets_status_to_failed(memory_no_llm_verify, sample_txt_content):
     """Test that when file conversion fails, the operation status is set to 'failed' not 'completed'."""

hindsight-api-slim/tests/test_hierarchical_config.py

@@ -98,7 +98,7 @@ async def test_hierarchical_fields_categorization():
     assert "retain_chunk_batch_size" in configurable
 
     # Verify count is correct
-    assert len(configurable) == 22
+    assert len(configurable) == 35
 
     # Verify credential fields (NEVER exposed)
     assert "llm_api_key" in credentials
@@ -458,7 +458,7 @@ async def test_config_get_bank_config_no_static_or_credential_fields_leak(memory
             assert field in config, f"Expected configurable field '{field}' missing from config"
 
         # Should have a small number of configurable fields (not hundreds)
-        assert len(config) < 25, f"Too many fields returned: {len(config)}"
+        assert len(config) < 50, f"Too many fields returned: {len(config)}"
 
     finally:
         await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_http_api_integration.py

@@ -286,6 +286,34 @@ async def test_full_api_workflow(api_client, test_bank_id):
         )
         assert response.status_code == 410  # Deprecated endpoint
 
+    # Entity co-occurrence graph — shape is stable even when there are no
+    # co-occurrences; every edge must reference two nodes that are also present.
+    response = await api_client.get(f"/v1/default/banks/{test_bank_id}/entities/graph")
+    assert response.status_code == 200
+    entity_graph = response.json()
+    assert set(entity_graph.keys()) >= {"nodes", "edges", "total_entities", "total_edges", "limit"}
+    assert entity_graph["limit"] == 1000
+    assert len(entity_graph["nodes"]) == entity_graph["total_entities"]
+    assert len(entity_graph["edges"]) == entity_graph["total_edges"]
+    node_ids = {n["data"]["id"] for n in entity_graph["nodes"]}
+    for edge in entity_graph["edges"]:
+        assert edge["data"]["source"] in node_ids
+        assert edge["data"]["target"] in node_ids
+        assert edge["data"]["linkType"] == "cooccurrence"
+        assert edge["data"]["weight"] >= 1
+
+    # min_count filter — raising the threshold can only shrink the edge set.
+    response = await api_client.get(
+        f"/v1/default/banks/{test_bank_id}/entities/graph?min_count=1000000"
+    )
+    assert response.status_code == 200
+    filtered_graph = response.json()
+    assert filtered_graph["total_edges"] == 0
+
+    # "graph" must route to the graph endpoint, not be parsed as an entity_id.
+    # Regression guard in case someone reorders the FastAPI route registration.
+    assert entity_graph["total_entities"] >= 0
+
     # ================================================================
     # 9. List All Banks (should include our test bank)
     # ================================================================

hindsight-api-slim/tests/test_load_large_batch.py

@@ -25,6 +25,8 @@ from hindsight_api.engine.llm_wrapper import TokenUsage
 
 logger = logging.getLogger(__name__)
 
+pytestmark = pytest.mark.xdist_group("load_batch_tests")
+
 
 def generate_content(char_count: int) -> str:
     """Generate realistic content of approximately char_count characters."""
@@ -117,9 +119,18 @@ class TestLargeBatchRetain:
         except Exception:
             pass
 
+    @pytest.fixture
+    def disable_observations(self):
+        from hindsight_api.config import _get_raw_config
+        config = _get_raw_config()
+        original = config.enable_observations
+        config.enable_observations = False
+        yield
+        config.enable_observations = original
+
     @pytest.mark.asyncio
     @pytest.mark.timeout(300)  # 5 minute timeout
-    async def test_large_batch_500k_chars_20_items(self, memory_with_mock_llm, request_context):
+    async def test_large_batch_500k_chars_20_items(self, memory_with_mock_llm, request_context, disable_observations):
         """
         Test retaining a batch of 20 content items totaling ~500k chars.
 
@@ -283,7 +294,7 @@ class TestLargeBatchRetain:
 
     @pytest.mark.asyncio
     @pytest.mark.timeout(60)
-    async def test_db_connection_pool_under_load(self, memory_with_mock_llm, request_context):
+    async def test_db_connection_pool_under_load(self, memory_with_mock_llm, request_context, disable_observations):
         """
         Test that DB connection pool handles concurrent operations.
 

hindsight-api-slim/tests/test_mcp_tools.py

@@ -177,6 +177,10 @@ def mock_memory():
     memory.get_bank_stats = AsyncMock(return_value={"nodes": 100, "links": 50})
     memory.update_bank = AsyncMock(return_value={"id": "test-bank", "name": "Updated"})
     memory.delete_bank = AsyncMock(return_value={"deleted_memories": 10, "deleted_entities": 5})
+
+    # Config resolver (used by update_bank MCP tool for config fields)
+    memory._config_resolver = MagicMock()
+    memory._config_resolver.update_bank_config = AsyncMock()
     memory.list_banks = AsyncMock(return_value=[])
 
     return memory
@@ -1265,9 +1269,14 @@ class TestTagsAndBankTools:
     async def test_update_bank(self, mock_memory):
         mcp = _make_mcp_server(mock_memory, {"update_bank"}, include_bank_id=True)
         result = await _tools(mcp)["update_bank"].fn(name="New Name", mission="New Mission")
+        # name is updated via engine
         call_kwargs = mock_memory.update_bank.call_args.kwargs
         assert call_kwargs["name"] == "New Name"
-        assert call_kwargs["mission"] == "New Mission"
+        # mission is routed to config resolver as reflect_mission
+        config_call = mock_memory._config_resolver.update_bank_config.call_args
+        assert config_call.args[1] == {"reflect_mission": "New Mission"}
+        # bank_id is the first positional arg
+        assert config_call.args[0] == "test-bank"
 
     async def test_delete_bank(self, mock_memory):
         mcp = _make_mcp_server(mock_memory, {"delete_bank"}, include_bank_id=True)
@@ -1396,6 +1405,120 @@ class TestUpdateBankVariants:
         result = await _tools(mcp)["update_bank"].fn(name="X")
         assert "error" in result
 
+    async def test_update_bank_config_updates_dict(self, mock_memory):
+        """config_updates dict is passed directly to config resolver."""
+        mcp = _make_mcp_server(mock_memory, {"update_bank"}, include_bank_id=True)
+        await _tools(mcp)["update_bank"].fn(config_updates={"reflect_mission": "Guide reflect output"})
+        config_call = mock_memory._config_resolver.update_bank_config.call_args
+        assert config_call.args[1] == {"reflect_mission": "Guide reflect output"}
+        # name should NOT be updated when not provided
+        mock_memory.update_bank.assert_not_called()
+
+    async def test_update_bank_mission_maps_to_reflect_mission(self, mock_memory):
+        """Deprecated mission param is mapped to reflect_mission in config."""
+        mcp = _make_mcp_server(mock_memory, {"update_bank"}, include_bank_id=True)
+        await _tools(mcp)["update_bank"].fn(mission="My mission")
+        config_call = mock_memory._config_resolver.update_bank_config.call_args
+        assert config_call.args[1] == {"reflect_mission": "My mission"}
+
+    async def test_update_bank_config_reflect_mission_takes_precedence_over_mission(self, mock_memory):
+        """When both mission and config_updates.reflect_mission are provided, config wins."""
+        mcp = _make_mcp_server(mock_memory, {"update_bank"}, include_bank_id=True)
+        await _tools(mcp)["update_bank"].fn(
+            mission="old", config_updates={"reflect_mission": "new"}
+        )
+        config_call = mock_memory._config_resolver.update_bank_config.call_args
+        assert config_call.args[1]["reflect_mission"] == "new"
+
+    async def test_update_bank_multiple_config_fields(self, mock_memory):
+        """Multiple config fields can be set in a single config_updates dict."""
+        mcp = _make_mcp_server(mock_memory, {"update_bank"}, include_bank_id=True)
+        await _tools(mcp)["update_bank"].fn(config_updates={
+            "retain_mission": "Extract technical decisions",
+            "disposition_skepticism": 5,
+            "disposition_literalism": 1,
+            "disposition_empathy": 4,
+            "enable_observations": True,
+            "observations_mission": "Focus on preferences",
+            "retain_extraction_mode": "custom",
+            "retain_custom_instructions": "Extract only action items",
+            "retain_chunk_size": 2000,
+        })
+        config_call = mock_memory._config_resolver.update_bank_config.call_args
+        updates = config_call.args[1]
+        assert updates["retain_mission"] == "Extract technical decisions"
+        assert updates["disposition_skepticism"] == 5
+        assert updates["disposition_literalism"] == 1
+        assert updates["disposition_empathy"] == 4
+        assert updates["enable_observations"] is True
+        assert updates["observations_mission"] == "Focus on preferences"
+        assert updates["retain_extraction_mode"] == "custom"
+        assert updates["retain_custom_instructions"] == "Extract only action items"
+        assert updates["retain_chunk_size"] == 2000
+
+    async def test_update_bank_name_and_config_together(self, mock_memory):
+        """name goes to engine, config_updates goes to config resolver."""
+        mcp = _make_mcp_server(mock_memory, {"update_bank"}, include_bank_id=True)
+        await _tools(mcp)["update_bank"].fn(
+            name="My Bank",
+            config_updates={"reflect_mission": "Reflect guide", "retain_mission": "Retain guide"},
+        )
+        assert mock_memory.update_bank.call_args.kwargs["name"] == "My Bank"
+        updates = mock_memory._config_resolver.update_bank_config.call_args.args[1]
+        assert updates["reflect_mission"] == "Reflect guide"
+        assert updates["retain_mission"] == "Retain guide"
+
+    async def test_update_bank_no_config_call_when_only_name(self, mock_memory):
+        """When only name is provided, config resolver should not be called."""
+        mcp = _make_mcp_server(mock_memory, {"update_bank"}, include_bank_id=True)
+        await _tools(mcp)["update_bank"].fn(name="Just Name")
+        mock_memory.update_bank.assert_called_once()
+        mock_memory._config_resolver.update_bank_config.assert_not_called()
+
+    async def test_update_bank_config_updates_single_bank(self, mock_memory):
+        """config_updates works in single-bank mode too."""
+        mcp = _make_mcp_server(mock_memory, {"update_bank"}, include_bank_id=False)
+        result = await _tools(mcp)["update_bank"].fn(
+            config_updates={"retain_mission": "Extract everything", "disposition_empathy": 5}
+        )
+        assert isinstance(result, dict)
+        config_call = mock_memory._config_resolver.update_bank_config.call_args
+        updates = config_call.args[1]
+        assert updates["retain_mission"] == "Extract everything"
+        assert updates["disposition_empathy"] == 5
+
+    async def test_update_bank_with_bank_id_override(self, mock_memory):
+        """bank_id override routes config update to the correct bank."""
+        mcp = _make_mcp_server(mock_memory, {"update_bank"}, include_bank_id=True)
+        await _tools(mcp)["update_bank"].fn(
+            config_updates={"reflect_mission": "Test"}, bank_id="other-bank"
+        )
+        config_call = mock_memory._config_resolver.update_bank_config.call_args
+        assert config_call.args[0] == "other-bank"
+
+    async def test_update_bank_config_resolver_validation_error(self, mock_memory):
+        """ValueError from config resolver (e.g. invalid field) is returned as error."""
+        mock_memory._config_resolver.update_bank_config.side_effect = ValueError(
+            "Cannot override static (server-level) fields: ['database_url']"
+        )
+        mcp = _make_mcp_server(mock_memory, {"update_bank"}, include_bank_id=True)
+        result = await _tools(mcp)["update_bank"].fn(config_updates={"database_url": "bad"})
+        assert "error" in result
+        assert "static" in result
+
+    async def test_update_bank_any_configurable_field(self, mock_memory):
+        """Any field in _CONFIGURABLE_FIELDS is accepted (future-proof)."""
+        mcp = _make_mcp_server(mock_memory, {"update_bank"}, include_bank_id=True)
+        await _tools(mcp)["update_bank"].fn(config_updates={
+            "recall_budget_fixed_low": 100,
+            "consolidation_llm_batch_size": 8,
+            "entity_labels": ["PERSON", "ORG"],
+        })
+        updates = mock_memory._config_resolver.update_bank_config.call_args.args[1]
+        assert updates["recall_budget_fixed_low"] == 100
+        assert updates["consolidation_llm_batch_size"] == 8
+        assert updates["entity_labels"] == ["PERSON", "ORG"]
+
     async def test_get_bank_stats_engine_error(self, mock_memory):
         mock_memory.get_bank_stats.side_effect = RuntimeError("DB error")
         mcp = _make_mcp_server(mock_memory, {"get_bank_stats"}, include_bank_id=True)

hindsight-api-slim/tests/test_mental_model_delta.py

@@ -0,0 +1,906 @@
+"""Tests for delta-mode mental model refresh.
+
+Delta mode performs a surgical update on the existing mental model content:
+- Unchanged sections are preserved byte-for-byte.
+- Stale content is removed.
+- New content from observations/facts is added, preferably by extending existing sections.
+
+Fallback rules:
+- If the mental model has no existing content, delta falls back to a full regeneration.
+- If the source_query has changed since the last refresh, delta falls back to a full regeneration.
+
+This file contains two kinds of tests:
+
+1. TestDeltaRefreshPlumbing: fast, deterministic tests that monkey-patch reflect_async
+   and the LLM call to verify branching logic (fallback conditions, provenance tracking).
+
+2. TestDeltaRefreshGeminiEval: real-LLM behavioral evals against Gemini. These are
+   gated on HINDSIGHT_RUN_GEMINI_EVALS=1 (plus a Gemini API key) because they cost
+   money/time and require network access. They verify the actual quality of delta
+   updates — format preservation, surgical edits, observation-grounding.
+"""
+
+import os
+import uuid
+from typing import Any
+
+import pytest
+
+from hindsight_api import MemoryEngine, RequestContext
+from hindsight_api.engine.llm_wrapper import LLMConfig
+from hindsight_api.engine.response_models import ReflectResult
+
+
+def _canned_reflect_result(text: str, facts: list[dict] | None = None) -> ReflectResult:
+    """Build a minimal ReflectResult for monkey-patching reflect_async."""
+    return ReflectResult.model_validate(
+        {
+            "text": text,
+            "based_on": {
+                "observation": facts or [],
+                "world": [],
+                "experience": [],
+                "mental-models": [],
+                "directives": [],
+            },
+        }
+    )
+
+
+@pytest.fixture
+def patch_reflect(monkeypatch):
+    """Helper that patches memory.reflect_async to return a canned result and records the call.
+
+    Usage:
+        calls = patch_reflect(memory, text="hello", facts=[...])
+        await memory.refresh_mental_model(...)
+        assert len(calls) == 1
+    """
+
+    def _install(memory: MemoryEngine, *, text: str, facts: list[dict] | None = None):
+        calls: list[dict] = []
+
+        async def fake_reflect_async(**kwargs):
+            calls.append(kwargs)
+            return _canned_reflect_result(text, facts)
+
+        monkeypatch.setattr(memory, "reflect_async", fake_reflect_async)
+        return calls
+
+    return _install
+
+
+@pytest.fixture
+def patch_llm_call(monkeypatch):
+    """Patch the reflect LLM config's ``.call()`` used for the structured delta call.
+
+    The structured-delta path passes ``response_format=DeltaOperationList``, so the
+    LLM returns a Pydantic instance.  Each invocation of ``patch_llm_call`` installs
+    a single canned response, in any of these shapes:
+
+    - ``DeltaOperationList`` instance → returned as-is
+    - ``[]`` (empty list) → no operations (this is the no-change case)
+    - ``[{"op": "...", ...}, ...]`` → wrapped into ``{"operations": [...]}``
+    - ``{"operations": [...]}`` → validated directly
+    """
+    from hindsight_api.engine.reflect.delta_ops import DeltaOperationList
+
+    def _to_op_list(resp: Any) -> DeltaOperationList:
+        if isinstance(resp, DeltaOperationList):
+            return resp
+        if isinstance(resp, dict):
+            if "operations" in resp:
+                return DeltaOperationList.model_validate(resp)
+            # Treat a bare op dict as a one-op list for ergonomics.
+            return DeltaOperationList.model_validate({"operations": [resp]})
+        if isinstance(resp, list):
+            return DeltaOperationList.model_validate({"operations": resp})
+        if isinstance(resp, str):
+            # Tests that expect *no* call ever still install a sentinel; treat as no-op.
+            return DeltaOperationList()
+        raise TypeError(f"unsupported canned LLM response: {type(resp)!r}")
+
+    def _install(memory: MemoryEngine, *, returns):
+        calls: list[dict] = []
+        canned = _to_op_list(returns)
+
+        async def fake_call(*, messages, **kwargs):
+            calls.append({"messages": messages, **kwargs})
+            return canned
+
+        monkeypatch.setattr(memory._reflect_llm_config, "call", fake_call)
+        return calls
+
+    return _install
+
+
+class TestDeltaRefreshPlumbing:
+    """Deterministic tests that verify the branching/plumbing of delta-mode refresh."""
+
+    async def test_full_mode_does_not_call_delta_merge(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        patch_llm_call,
+    ):
+        """When trigger.mode='full', no second LLM call for delta merge occurs."""
+        bank_id = f"test-delta-full-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content="# Team\n\nOriginal content.",
+            trigger={"mode": "full"},
+            request_context=request_context,
+        )
+
+        patch_reflect(memory, text="# Team\n\nRegenerated from scratch.")
+        llm_calls = patch_llm_call(memory, returns="should-not-be-called")
+
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        assert refreshed is not None
+        assert refreshed["content"] == "# Team\n\nRegenerated from scratch."
+        assert len(llm_calls) == 0, "Delta merge LLM call must not happen in full mode"
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_delta_mode_empty_content_falls_back_to_full(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        patch_llm_call,
+    ):
+        """When the mental model has no existing content there is nothing to anchor
+        a surgical edit on, so delta falls back to full regeneration. The user's
+        candidate from reflect_async is used verbatim.
+        """
+        bank_id = f"test-delta-empty-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content="",  # no existing content
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+
+        patch_reflect(memory, text="# Team\n\nFull fresh synthesis.")
+        llm_calls = patch_llm_call(memory, returns=[])
+
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        assert refreshed["content"] == "# Team\n\nFull fresh synthesis."
+        assert len(llm_calls) == 0  # delta path skipped entirely
+        rr = refreshed.get("reflect_response") or {}
+        assert rr.get("delta_applied") is not True
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_delta_mode_source_query_change_falls_back_to_full(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        patch_llm_call,
+    ):
+        """If source_query changes after a refresh, the next delta run must do a full rewrite."""
+        bank_id = f"test-delta-query-change-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content="# Team\n\nBaseline.",
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+
+        # First refresh: establishes last_refreshed_source_query.
+        patch_reflect(memory, text="# Team\n\nFirst pass.")
+        patch_llm_call(memory, returns="unused-first")
+        await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        # Now change the source_query — a genuine topic shift.
+        await memory.update_mental_model(
+            bank_id=bank_id,
+            mental_model_id=mm["id"],
+            source_query="Tell me about customers instead",
+            request_context=request_context,
+        )
+
+        # Second refresh under the new query must do a FULL rewrite, not a delta merge.
+        patch_reflect(memory, text="# Customers\n\nBrand new topic.")
+        llm_calls = patch_llm_call(memory, returns="should-not-be-called")
+
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        assert refreshed["content"] == "# Customers\n\nBrand new topic."
+        assert len(llm_calls) == 0, "Source-query change must bypass the delta merge"
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_delta_mode_applies_ops_when_query_stable(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        patch_llm_call,
+    ):
+        """When content exists and source_query is stable, the delta LLM produces ops
+        that are applied against the parsed structured doc. The unchanged section
+        renders byte-identical, the new fact lands in a new block.
+        """
+        bank_id = f"test-delta-apply-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        existing = (
+            "# Team\n"
+            "\n"
+            "Alice is the lead.\n"
+            "\n"
+            "## Members\n"
+            "\n"
+            "- Alice — lead\n"
+        )
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content=existing,
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+
+        # First refresh: empty op list → structured doc unchanged → markdown is the
+        # render of the parsed existing content. This also seeds the tracking column.
+        patch_reflect(memory, text="ignored — full mode candidate")
+        patch_llm_call(memory, returns=[])  # zero ops
+        await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        # Second refresh: a new fact arrives; LLM returns one append_block op.
+        candidate = "# Team\n\nAlice is the lead. Bob joined as junior engineer."
+        patch_reflect(
+            memory,
+            text=candidate,
+            facts=[
+                {
+                    "id": "obs-bob",
+                    "text": "Bob joined the team as junior engineer",
+                    "type": "observation",
+                    "context": None,
+                }
+            ],
+        )
+        ops = [
+            {
+                "op": "append_block",
+                "section_id": "members",
+                "block": {
+                    "type": "bullet_list",
+                    "items": ["Bob — junior engineer"],
+                },
+            }
+        ]
+        llm_calls = patch_llm_call(memory, returns=ops)
+
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        assert len(llm_calls) == 1, "Structured-delta LLM call must fire exactly once"
+        system_msg = llm_calls[0]["messages"][0]["content"]
+        user_msg = llm_calls[0]["messages"][1]["content"]
+        # Prompt must include the structured doc + supporting facts + the system prompt.
+        assert "integrating" in system_msg.lower()
+        assert "operations" in system_msg.lower()
+        assert "obs-bob" in user_msg
+        assert "Bob joined" in user_msg
+        # The structured JSON of the current doc must include the section id "members".
+        assert '"id": "members"' in user_msg
+
+        # New content includes the new bullet.
+        assert "Bob — junior engineer" in refreshed["content"]
+        # Unchanged section ("Alice is the lead.") still present.
+        assert "Alice is the lead." in refreshed["content"]
+        rr = refreshed.get("reflect_response") or {}
+        assert rr.get("delta_applied") is True
+        applied = rr.get("delta_operations_applied") or []
+        assert len(applied) == 1
+        assert applied[0]["op"] == "append_block"
+        assert applied[0]["section_id"] == "members"
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_delta_zero_ops_keeps_existing_content_byte_identical(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        patch_llm_call,
+    ):
+        """Zero operations from the LLM must mean zero changes in the rendered output.
+
+        This is the structural guarantee: any sections/blocks not mentioned by an
+        op come through byte-identical. A no-op refresh therefore re-renders the
+        same structured doc — which (after the first refresh has parsed and
+        re-rendered it) is byte-stable.
+        """
+        bank_id = f"test-delta-noop-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        existing = (
+            "# Team\n"
+            "\n"
+            "Alice is the lead.\n"
+            "\n"
+            "## Members\n"
+            "\n"
+            "- Alice\n"
+        )
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content=existing,
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+        # First refresh: parses + renders existing into structured form. The output
+        # may not match `existing` byte-for-byte (whitespace normalised by renderer).
+        patch_reflect(memory, text="ignored — full mode candidate")
+        patch_llm_call(memory, returns=[])
+        first = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+        normalised = first["content"]
+
+        # Second refresh: zero ops again → same bytes as first refresh.
+        # Must include at least one fact so the no-new-facts short-circuit doesn't fire.
+        patch_reflect(
+            memory,
+            text="something completely different from existing",
+            facts=[{"id": "obs-1", "text": "irrelevant", "type": "observation", "context": None}],
+        )
+        patch_llm_call(memory, returns=[])
+        second = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+        assert second["content"] == normalised
+        rr = second.get("reflect_response") or {}
+        assert rr.get("delta_applied") is True  # delta path ran; produced no changes
+        assert rr.get("delta_operations_applied") == []
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_delta_llm_failure_falls_back_to_candidate(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        monkeypatch,
+    ):
+        """When the structured-delta LLM call raises, refresh falls back to the
+        candidate markdown so the user still sees a fresh synthesis instead of
+        an opaque failure.
+        """
+        bank_id = f"test-delta-llm-fail-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content="# Team\n\nExisting.\n",
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+        # Seed tracking column with a successful zero-op refresh.
+        patch_reflect(memory, text="ignored")
+
+        async def ok_call(*, messages, **kwargs):
+            from hindsight_api.engine.reflect.delta_ops import DeltaOperationList
+
+            return DeltaOperationList()
+
+        monkeypatch.setattr(memory._reflect_llm_config, "call", ok_call)
+        await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        # Now the second refresh: LLM raises. Refresh must not crash; it should
+        # store the candidate markdown.
+        candidate = "# Team\n\nFallback candidate from reflect_async.\n"
+        patch_reflect(
+            memory,
+            text=candidate,
+            facts=[{"id": "obs-new", "text": "some new fact", "type": "observation", "context": None}],
+        )
+
+        async def boom(*, messages, **kwargs):
+            raise RuntimeError("simulated provider 500")
+
+        monkeypatch.setattr(memory._reflect_llm_config, "call", boom)
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        assert "Fallback candidate" in refreshed["content"]
+        rr = refreshed.get("reflect_response") or {}
+        assert rr.get("delta_applied") is False
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_empty_reflect_answer_preserves_existing_content(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        patch_llm_call,
+        monkeypatch,
+    ):
+        """Regression: when the reflect agent returns an empty answer (small models
+        sometimes hit this after exhausting tool-call retries), the refresh must
+        NOT overwrite the existing content with an empty string.
+
+        Previously this destroyed the working document on every transient upstream
+        failure, and the next refresh saw current_content == "" and skipped the
+        delta path entirely — a snowball that emptied valuable mental models.
+
+        The scenario covered here is the realistic failure path: the structured
+        delta call also fails (because the empty supporting facts produce empty
+        / invalid JSON) so the fallback path kicks in. Without the guard, the
+        fallback would write "" to the DB; with it, the existing content stays.
+        """
+        bank_id = f"test-empty-reflect-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        existing = (
+            "# Team\n"
+            "\n"
+            "Alice is the lead.\n"
+            "\n"
+            "## Members\n"
+            "\n"
+            "- Alice\n"
+        )
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content=existing,
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+
+        # Reflect returns "" — this is the upstream failure mode.
+        # Must include at least one fact so the no-new-facts short-circuit doesn't fire.
+        patch_reflect(
+            memory,
+            text="",
+            facts=[{"id": "obs-new", "text": "some fact", "type": "observation", "context": None}],
+        )
+
+        # Delta call also fails (mirrors the real groq behaviour where empty
+        # supporting facts often produce empty / invalid JSON). Refresh then
+        # falls back to the empty candidate, which the guard rejects.
+        async def boom(*, messages, **kwargs):
+            raise RuntimeError("simulated empty/invalid JSON from provider")
+
+        monkeypatch.setattr(memory._reflect_llm_config, "call", boom)
+
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        # Existing content preserved exactly.
+        assert refreshed["content"] == existing, (
+            "Empty reflect answer overwrote existing content — guard regressed"
+        )
+        rr = refreshed.get("reflect_response") or {}
+        assert rr.get("refresh_skipped") == "empty_candidate"
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+# ---------------------------------------------------------------------------
+# Real-Gemini evaluation tests
+# ---------------------------------------------------------------------------
+
+_GEMINI_API_KEY = (
+    os.getenv("HINDSIGHT_GEMINI_API_KEY")
+    or os.getenv("GEMINI_API_KEY")
+    or os.getenv("GOOGLE_API_KEY")
+)
+_OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+_RUN_LLM_EVAL = os.getenv("HINDSIGHT_RUN_GEMINI_EVALS") == "1" and (
+    bool(_GEMINI_API_KEY) or bool(_OPENAI_API_KEY)
+)
+
+
+pytestmark_gemini = pytest.mark.skipif(
+    not _RUN_LLM_EVAL,
+    reason=(
+        "Real-LLM delta evals are gated. Set HINDSIGHT_RUN_GEMINI_EVALS=1 and provide "
+        "GEMINI_API_KEY (preferred) or OPENAI_API_KEY to run."
+    ),
+)
+
+
+@pytest.fixture
+async def gemini_memory(memory_no_llm_verify: MemoryEngine):
+    """MemoryEngine wired to a real LLM for reflect + structured delta.
+
+    Prefers Gemini (the original target) but falls back to OpenAI when the
+    Gemini key is unavailable — the structured-delta architecture works
+    against either, and waiting on a single provider's key would block
+    iteration. The chosen model is logged so test failures are unambiguous
+    about which provider produced them.
+    """
+    if _GEMINI_API_KEY:
+        provider = "gemini"
+        model = os.getenv("HINDSIGHT_GEMINI_EVAL_MODEL", "gemini-2.0-flash")
+        cfg = LLMConfig(provider=provider, api_key=_GEMINI_API_KEY, base_url="", model=model)
+    else:
+        provider = "openai"
+        model = os.getenv("HINDSIGHT_OPENAI_EVAL_MODEL", "gpt-4o-mini")
+        cfg = LLMConfig(provider=provider, api_key=_OPENAI_API_KEY or "", base_url="", model=model)
+    print(f"\n[delta-eval] using provider={provider} model={model}")
+    memory_no_llm_verify._reflect_llm_config = cfg
+    memory_no_llm_verify._llm_config = cfg
+    memory_no_llm_verify._retain_llm_config = cfg
+    memory_no_llm_verify._consolidation_llm_config = cfg
+    yield memory_no_llm_verify
+
+
+_NEWS_FEED_SKILL_MARKDOWN = """## Purpose
+
+Generate a concise, top-N personalized AI/ML news brief in response to user-triggered requests such as "ai news", "top 5 this week", or "what matters for builders today".
+
+## Scope
+
+- **In scope**: collecting, filtering, and summarizing AI/ML articles from user-preferred RSS feeds, applying user preferences stored in the AI News Feed Preferences mental model, and delivering the brief to the user.
+- **Out of scope**: non-AI news, detailed article content, legal or privacy reviews beyond user preferences, and posting the brief to external platforms without explicit user approval.
+
+## Rules
+
+- **Always**:
+  1. Use the AI News Feed Preferences mental model to retrieve user preferences; do not embed preferences in the skill file.
+  2. Do not post the brief to any platform unless the user explicitly approves.
+  3. Do not persist preferences locally; rely solely on the mental model.
+  4. Refresh the feed after consolidation if the trigger-refresh-after-consolidation flag is true.
+- **Prefer**:
+  1. Provide a concise summary (about 2-3 sentences per article) for the top-N articles.
+  2. Default to the top-5 articles unless the user specifies otherwise.
+  3. Order articles chronologically or by relevance as per user preference.
+  4. Highlight any user-specified topics or tags if present.
+
+## Procedure
+
+1. **Trigger detection** — identify a request containing keywords like "ai news", "top N", or "what matters".
+2. **Preference retrieval** — call memory recall for the AI News Feed Preferences mental model to obtain RSS feed URLs and any filtering criteria.
+3. **Feed consolidation** — fetch all feeds, de-duplicate entries, and apply any user-specified filters.
+4. **Article selection** — choose the top-N articles based on date or user preference; if trigger-refresh-after-consolidation is true, re-fetch feeds before selection.
+5. **Summarization** — generate a brief summary for each article, keeping it short and to the point.
+6. **Approval check** — if the brief is to be posted externally, verify explicit user approval; otherwise, deliver it directly to the user.
+7. **Memory retention** — store any new learnings or preferences observed during the task using memory retain.
+
+## Inputs and Context
+
+- **Source feeds**: user-specified RSS URLs stored in the mental model (e.g., https://aiagentmemory.org/index.xml).
+- **Time window**: the latest update from each feed; typically the last 7 days for weekly briefs.
+- **User preferences**: stored in the AI News Feed Preferences mental model; may include topics, tags, or language.
+
+## Output Shape
+
+- **Structure**: list of articles with title, publication date, source, and a 2-sentence summary.
+- **Format**: plain text or markdown (as requested by the user).
+- **Length**: concise — approximately 2-3 sentences per article; total brief about 200-300 words for top-5.
+- **Voice/Tone**: neutral, informative, and concise; use bullet points for clarity.
+
+## Stop Conditions
+
+- If the mental model cannot be retrieved, refuse or request clarification.
+- If the user has not provided any RSS feed URLs, ask for a preferred source.
+- If the brief is requested for posting and explicit approval is missing, refuse.
+- If the user explicitly requests to remove a skill or stop the briefing, comply immediately.
+
+## Open Questions
+
+- Desired brief length or word count?
+- Preferred summary style (bullet vs paragraph).
+- Whether the user wants to include non-AI but AI-related topics.
+- Frequency or schedule for automated briefs (if any).
+- Specific user-defined tags or topics to highlight.
+"""
+
+
+@pytestmark_gemini
+class TestDeltaRefreshGeminiEval:
+    """Real-LLM evals for the structured-delta refresh path.
+
+    The structural guarantee these tests verify: sections and blocks not
+    targeted by an LLM-emitted operation are byte-identical between the
+    pre-refresh and post-refresh markdown render. This is what the
+    structured-ops architecture buys us — the LLM cannot drift on text it
+    never re-emits.
+
+    Real Gemini is used (not a mock) because the failure mode we're guarding
+    against is precisely "the LLM doesn't reliably do what the prompt says,
+    even at temperature 0". Mocked output would prove the wiring works but
+    not that the contract holds against an actual model.
+    """
+
+    async def _seed(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        bank_id: str,
+        existing_markdown: str,
+        memories: list[str],
+    ) -> dict[str, Any]:
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Skill Doc",
+            source_query="Document the news-feed skill: purpose, rules, procedure, stop conditions.",
+            content=existing_markdown,
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+        await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[{"content": m} for m in memories],
+            request_context=request_context,
+        )
+        await memory.wait_for_background_tasks()
+        # First refresh: parses existing into structured form. With well-aligned
+        # memories the LLM should emit zero ops, so the structured doc is just
+        # the parsed existing content. The rendered markdown is canonicalised.
+        first = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+        return {"mm": mm, "first": first}
+
+    async def test_no_change_when_observations_agree_with_existing(
+        self, gemini_memory: MemoryEngine, request_context: RequestContext
+    ):
+        """When observations only restate the existing doc, a second delta
+        refresh produces output byte-identical to the first refresh's output.
+
+        The first refresh canonicalises whitespace via the parser+renderer; we
+        compare the *second* refresh against the *first* (not against the raw
+        seed markdown), which is the actual repeat-refresh behaviour users
+        will see in production.
+        """
+        bank_id = f"eval-delta-noop-{uuid.uuid4().hex[:8]}"
+        seeded = await self._seed(
+            gemini_memory,
+            request_context,
+            bank_id,
+            existing_markdown=_NEWS_FEED_SKILL_MARKDOWN,
+            memories=[
+                "The news-feed skill produces a concise top-N AI/ML news brief.",
+                "Default brief size is top 5 unless the user specifies otherwise.",
+                "Source feed: https://aiagentmemory.org/index.xml.",
+                "The skill must not post externally without explicit approval.",
+            ],
+        )
+        first_content = seeded["first"]["content"]
+
+        second = await gemini_memory.refresh_mental_model(
+            bank_id=bank_id,
+            mental_model_id=seeded["mm"]["id"],
+            request_context=request_context,
+        )
+        second_content = second["content"]
+
+        # Byte-identical render across refreshes when no new fact has arrived.
+        assert second_content == first_content, (
+            "Repeat delta refresh changed bytes when no new facts arrived.\n"
+            f"--- diff sample (first 300 chars different) ---\n"
+            f"first:  {first_content[:300]!r}\n"
+            f"second: {second_content[:300]!r}"
+        )
+        rr = second.get("reflect_response") or {}
+        # The LLM may emit zero ops (best case) or non-effective ops (still no
+        # change to render); both are acceptable so long as the bytes match.
+        assert rr.get("delta_applied") is True
+
+        await gemini_memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_new_observation_is_merged_surgically(
+        self, gemini_memory: MemoryEngine, request_context: RequestContext
+    ):
+        """A new fact arrives; only the section relevant to it should change.
+
+        Asserts the architectural guarantee at the section level: every
+        section that the LLM did NOT name in an operation must render exactly
+        the same bytes after the refresh as before. The new fact itself must
+        appear somewhere in the output.
+        """
+        from hindsight_api.engine.reflect.structured_doc import (
+            StructuredDocument,
+            render_section,
+        )
+
+        bank_id = f"eval-delta-add-{uuid.uuid4().hex[:8]}"
+        seeded = await self._seed(
+            gemini_memory,
+            request_context,
+            bank_id,
+            existing_markdown=_NEWS_FEED_SKILL_MARKDOWN,
+            memories=[
+                "The news-feed skill produces a concise top-N AI/ML news brief.",
+                "Default brief size is top 5.",
+                "Source feed: https://aiagentmemory.org/index.xml.",
+            ],
+        )
+        first_content = seeded["first"]["content"]
+        first_struct = StructuredDocument.model_validate(
+            seeded["first"]["reflect_response"]["delta_operations_applied"]
+            and seeded["first"].get("structured_content")
+            or {"version": 1, "sections": []}
+        )
+        # The first refresh's structured snapshot is what the second refresh
+        # will operate on. Re-fetch via get_mental_model would also work.
+        # For preservation comparison we re-parse first_content.
+        from hindsight_api.engine.reflect.structured_doc import parse_markdown
+
+        before = parse_markdown(first_content)
+
+        # Introduce a brand-new fact that fits into "Inputs and Context" or
+        # similar — but the model may pick any reasonable section.
+        await gemini_memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {
+                    "content": (
+                        "The default time window for the news brief is the last 7 days, "
+                        "matching the weekly cadence preferred by the user."
+                    )
+                },
+            ],
+            request_context=request_context,
+        )
+        await gemini_memory.wait_for_background_tasks()
+
+        refreshed = await gemini_memory.refresh_mental_model(
+            bank_id=bank_id,
+            mental_model_id=seeded["mm"]["id"],
+            request_context=request_context,
+        )
+        content = refreshed["content"]
+        rr = refreshed.get("reflect_response") or {}
+        applied_ops = rr.get("delta_operations_applied") or []
+        touched_section_ids = {op.get("section_id") for op in applied_ops if op.get("section_id")}
+
+        # The fact must show up.
+        assert "7 days" in content or "seven days" in content.lower(), (
+            f"New fact about 7-day window missing from delta output: {content!r}"
+        )
+
+        # Every untouched section must render byte-identical to its pre-refresh form.
+        after = parse_markdown(content)
+        before_by_id = {s.id: s for s in before.sections}
+        for section in after.sections:
+            if section.id in touched_section_ids:
+                continue
+            orig = before_by_id.get(section.id)
+            if orig is None:
+                continue  # newly added section, no preservation contract
+            assert render_section(orig) == render_section(section), (
+                f"Untouched section {section.id!r} drifted between refreshes — the "
+                f"structured-ops architecture's preservation guarantee was violated.\n"
+                f"BEFORE:\n{render_section(orig)!r}\n"
+                f"AFTER:\n{render_section(section)!r}"
+            )
+
+        assert rr.get("delta_applied") is True
+
+        await gemini_memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_no_change_repeated_three_times_stays_byte_stable(
+        self, gemini_memory: MemoryEngine, request_context: RequestContext
+    ):
+        """Three consecutive no-change refreshes must produce three identical
+        markdown outputs. This is the regression test for the original
+        complaint where prose-merge delta drifted content across versions even
+        when no observation changed.
+        """
+        bank_id = f"eval-delta-stable-{uuid.uuid4().hex[:8]}"
+        seeded = await self._seed(
+            gemini_memory,
+            request_context,
+            bank_id,
+            existing_markdown=_NEWS_FEED_SKILL_MARKDOWN,
+            memories=[
+                "The news-feed skill produces a top-N AI brief on demand.",
+                "It must not post without explicit user approval.",
+            ],
+        )
+        c1 = seeded["first"]["content"]
+        r2 = await gemini_memory.refresh_mental_model(
+            bank_id=bank_id,
+            mental_model_id=seeded["mm"]["id"],
+            request_context=request_context,
+        )
+        r3 = await gemini_memory.refresh_mental_model(
+            bank_id=bank_id,
+            mental_model_id=seeded["mm"]["id"],
+            request_context=request_context,
+        )
+        assert r2["content"] == c1, "second refresh drifted vs first"
+        assert r3["content"] == c1, "third refresh drifted vs first"
+
+        await gemini_memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_source_query_change_forces_full_rewrite(
+        self, gemini_memory: MemoryEngine, request_context: RequestContext
+    ):
+        """Changing source_query must bypass delta and produce a full regeneration."""
+        bank_id = f"eval-delta-query-change-{uuid.uuid4().hex[:8]}"
+        await gemini_memory.get_bank_profile(bank_id, request_context=request_context)
+
+        mm = await gemini_memory.create_mental_model(
+            bank_id=bank_id,
+            name="Subject",
+            source_query="Summarize the team and how it operates.",
+            content="# Team Overview\n\nAlice leads the team.\n",
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+
+        await gemini_memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {"content": "Alice leads the team."},
+                {"content": "The product is a memory system for AI agents."},
+                {"content": "Customers include small SaaS startups and enterprise pilots."},
+            ],
+            request_context=request_context,
+        )
+        await gemini_memory.wait_for_background_tasks()
+
+        # First refresh seeds tracking column under the team query.
+        await gemini_memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        # Change the topic entirely.
+        await gemini_memory.update_mental_model(
+            bank_id=bank_id,
+            mental_model_id=mm["id"],
+            source_query="Summarize our customers and what we sell them.",
+            request_context=request_context,
+        )
+
+        refreshed = await gemini_memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+        content = refreshed["content"].lower()
+        # Content should now be about customers/product, not (only) about Alice leading the team.
+        assert "customer" in content or "product" in content, (
+            f"Full rewrite should cover the new topic, got: {refreshed['content']!r}"
+        )
+        # delta_applied should be absent/False because we took the full path.
+        assert (refreshed.get("reflect_response") or {}).get("delta_applied") is not True
+
+        await gemini_memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_mental_models.py

@@ -8,7 +8,8 @@ import uuid
 
 import pytest
 
-from hindsight_api.engine.memory_engine import MemoryEngine
+from hindsight_api.engine.memory_engine import MemoryEngine, fq_table
+from hindsight_api.engine.retain import embedding_utils
 
 
 @pytest.fixture
@@ -688,6 +689,51 @@ class TestMentalModelHistory:
         assert len(history) == 1
         assert history[0]["previous_content"] == "Original content"
         assert "changed_at" in history[0]
+        assert "previous_reflect_response" in history[0]
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_history_snapshots_previous_reflect_response(
+        self, memory: MemoryEngine, request_context
+    ):
+        """Each history entry snapshots the reflect_response that produced previous_content."""
+        bank_id = f"test-mm-history-reflect-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Test Model",
+            source_query="What is the test?",
+            content="v1",
+            request_context=request_context,
+        )
+
+        rr_v1 = {"text": "v1", "based_on": {"observation": [{"id": "o1", "text": "obs1"}]}, "mental_models": []}
+        await memory.update_mental_model(
+            bank_id=bank_id,
+            mental_model_id=mm["id"],
+            content="v2",
+            reflect_response=rr_v1,
+            request_context=request_context,
+        )
+
+        rr_v2 = {"text": "v2", "based_on": {"observation": [{"id": "o2", "text": "obs2"}]}, "mental_models": []}
+        await memory.update_mental_model(
+            bank_id=bank_id,
+            mental_model_id=mm["id"],
+            content="v3",
+            reflect_response=rr_v2,
+            request_context=request_context,
+        )
+
+        history = await memory.get_mental_model_history(bank_id, mm["id"], request_context=request_context)
+        assert len(history) == 2
+        # Most recent first: replacing v2 snapshotted rr_v1 (the reflect that produced v2).
+        assert history[0]["previous_content"] == "v2"
+        assert history[0]["previous_reflect_response"] == rr_v1
+        # The first update replaced v1, which had no reflect_response stored yet.
+        assert history[1]["previous_content"] == "v1"
+        assert history[1]["previous_reflect_response"] is None
 
         await memory.delete_bank(bank_id, request_context=request_context)
 
@@ -763,6 +809,222 @@ class TestMentalModelHistory:
         await memory.delete_bank(bank_id, request_context=request_context)
 
 
+class TestMentalModelStaleness:
+    """Tests for compute_mental_model_is_stale scope semantics.
+
+    Memories are inserted directly into ``memory_units`` so the scenarios don't
+    depend on the LLM fact-extraction pipeline.
+    """
+
+    @staticmethod
+    async def _insert_memory(
+        memory: MemoryEngine,
+        bank_id: str,
+        *,
+        tags: list[str] | None = None,
+        fact_type: str = "experience",
+    ) -> str:
+        from datetime import datetime, timezone
+
+        pool = await memory._get_pool()
+        mem_id = str(uuid.uuid4())
+        now = datetime.now(timezone.utc)
+        async with pool.acquire() as conn:
+            await conn.execute(
+                f"""
+                INSERT INTO {fq_table("memory_units")}
+                    (id, bank_id, text, event_date, fact_type, tags, created_at)
+                VALUES ($1, $2, $3, $4, $5, $6::varchar[], $4)
+                """,
+                mem_id,
+                bank_id,
+                "test memory",
+                now,
+                fact_type,
+                tags if tags is not None else [],
+            )
+        return mem_id
+
+    async def test_fresh_mental_model_is_not_stale(self, memory: MemoryEngine, request_context):
+        bank_id = f"test-mm-stale-fresh-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id, name="MM", source_query="q", content="c", request_context=request_context
+        )
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is False
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_untagged_mm_stale_on_any_new_memory(
+        self, memory: MemoryEngine, request_context
+    ):
+        bank_id = f"test-mm-stale-untagged-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id, name="MM", source_query="q", content="c", request_context=request_context
+        )
+        await self._insert_memory(memory, bank_id, tags=["something"])
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is True
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_tagged_mm_ignores_out_of_scope_memory(
+        self, memory: MemoryEngine, request_context
+    ):
+        bank_id = f"test-mm-stale-oos-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="MM",
+            source_query="q",
+            content="c",
+            tags=["user_a"],
+            request_context=request_context,
+        )
+        # Memory tagged with unrelated tag → not in scope, MM should not be stale
+        await self._insert_memory(memory, bank_id, tags=["user_b"])
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is False
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_tagged_mm_stale_on_overlapping_memory(
+        self, memory: MemoryEngine, request_context
+    ):
+        bank_id = f"test-mm-stale-overlap-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="MM",
+            source_query="q",
+            content="c",
+            tags=["user_a"],
+            request_context=request_context,
+        )
+        await self._insert_memory(memory, bank_id, tags=["user_a", "extra"])
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is True
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_tags_match_all_strict_requires_all_tags(
+        self, memory: MemoryEngine, request_context
+    ):
+        """tags_match='all_strict' → memory must contain ALL MM tags (and be tagged)."""
+        bank_id = f"test-mm-stale-all-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="MM",
+            source_query="q",
+            content="c",
+            tags=["user_a", "proj_x"],
+            trigger={"refresh_after_consolidation": False, "tags_match": "all_strict"},
+            request_context=request_context,
+        )
+        # Memory only has one of the tags → does NOT match all_strict
+        await self._insert_memory(memory, bank_id, tags=["user_a"])
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is False, "all_strict must require ALL MM tags"
+
+        # Now add a memory with both tags → matches
+        await self._insert_memory(memory, bank_id, tags=["user_a", "proj_x"])
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is True
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_tags_match_any_strict_excludes_untagged(
+        self, memory: MemoryEngine, request_context
+    ):
+        """tags_match='any_strict' → untagged memory does NOT keep MM in scope."""
+        bank_id = f"test-mm-stale-anystrict-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="MM",
+            source_query="q",
+            content="c",
+            tags=["user_a"],
+            trigger={"refresh_after_consolidation": False, "tags_match": "any_strict"},
+            request_context=request_context,
+        )
+        await self._insert_memory(memory, bank_id, tags=None)
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is False
+
+        await self._insert_memory(memory, bank_id, tags=["user_a"])
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is True
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_fact_type_filter_narrows_scope(
+        self, memory: MemoryEngine, request_context
+    ):
+        bank_id = f"test-mm-stale-fact-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="MM",
+            source_query="q",
+            content="c",
+            trigger={"refresh_after_consolidation": False, "fact_types": ["world"]},
+            request_context=request_context,
+        )
+        # Out-of-scope fact_type → not stale
+        await self._insert_memory(memory, bank_id, fact_type="experience")
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is False
+
+        # Matching fact_type → stale
+        await self._insert_memory(memory, bank_id, fact_type="world")
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is True
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_tool_search_mental_models_returns_is_stale_per_mm(
+        self, memory: MemoryEngine, request_context
+    ):
+        """Regression: tool_search_mental_models must compute is_stale per-MM via scope,
+        not via a bank-wide pending_consolidation short-circuit."""
+        from hindsight_api.engine.reflect.tools import tool_search_mental_models
+
+        bank_id = f"test-mm-stale-tool-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        fresh = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="fresh MM",
+            source_query="q",
+            content="fresh",
+            tags=["user_b"],
+            request_context=request_context,
+        )
+        stale = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="stale MM",
+            source_query="q",
+            content="stale",
+            tags=["user_a"],
+            request_context=request_context,
+        )
+        # Memory only in user_a's scope → only `stale` MM should be flagged.
+        await self._insert_memory(memory, bank_id, tags=["user_a"])
+
+        pool = await memory._get_pool()
+        async with pool.acquire() as conn:
+            embedding = (
+                await embedding_utils.generate_embeddings_batch(memory.embeddings, ["q"])
+            )[0]
+            result = await tool_search_mental_models(
+                memory, conn, bank_id, "q", embedding, max_results=10
+            )
+        by_id = {m["id"]: m for m in result["mental_models"]}
+        assert by_id[fresh["id"]]["is_stale"] is False
+        assert by_id[stale["id"]]["is_stale"] is True
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
 class TestMentalModelRefreshTagSecurity:
     """Test that mental model refresh respects tag-based security boundaries."""
 
@@ -1253,6 +1515,145 @@ class TestMentalModelTriggerTagsConfig:
         await memory.delete_bank(bank_id, request_context=request_context)
 
 
+class TestMentalModelRefreshMaxTokens:
+    """Verify that refresh_mental_model honors the per-model max_tokens column.
+
+    These tests mock the engine's collaborators so we can assert the exact kwargs
+    passed to reflect_async without spinning up a DB or LLM. The bug being guarded
+    against: the per-model ``max_tokens`` column was ignored during refresh, so
+    reflect_async fell back to its default (4096) and the generated content could
+    exceed the user-configured limit when there were many facts to synthesize.
+    """
+
+    async def test_refresh_passes_stored_max_tokens_to_reflect(self, request_context):
+        from unittest.mock import AsyncMock
+
+        from hindsight_api.engine.memory_engine import MemoryEngine
+        from hindsight_api.engine.response_models import ReflectResult
+
+        custom_max_tokens = 777
+        mental_model = {
+            "id": "mm-1",
+            "bank_id": "bank-1",
+            "name": "Capped Model",
+            "source_query": "Summarize the facts",
+            "content": "initial",
+            "tags": None,
+            "max_tokens": custom_max_tokens,
+            "trigger": {"refresh_after_consolidation": False},
+        }
+
+        engine = MemoryEngine.__new__(MemoryEngine)
+        engine._authenticate_tenant = AsyncMock(return_value=None)  # type: ignore[method-assign]
+        engine.get_mental_model = AsyncMock(return_value=mental_model)  # type: ignore[method-assign]
+        engine.reflect_async = AsyncMock(  # type: ignore[method-assign]
+            return_value=ReflectResult(text="stub synthesis", based_on={})
+        )
+        engine.update_mental_model = AsyncMock(return_value=mental_model)  # type: ignore[method-assign]
+
+        await engine.refresh_mental_model(
+            bank_id="bank-1",
+            mental_model_id="mm-1",
+            request_context=request_context,
+        )
+
+        assert engine.reflect_async.await_count == 1
+        kwargs = engine.reflect_async.await_args.kwargs
+        assert kwargs.get("max_tokens") == custom_max_tokens, (
+            f"refresh_mental_model should forward the stored max_tokens ({custom_max_tokens}) "
+            f"to reflect_async, but got max_tokens={kwargs.get('max_tokens')!r}"
+        )
+
+    async def test_refresh_content_respects_max_tokens(self, memory: MemoryEngine, request_context):
+        """End-to-end: refreshed content must stay within the model's max_tokens cap.
+
+        We seed the bank with enough varied facts that an unconstrained synthesis
+        would happily produce a long answer, then refresh a mental model with a
+        small max_tokens and assert the resulting content is actually within the
+        cap (with a small tolerance for cross-tokenizer drift, since the LLM may
+        not use cl100k_base).
+        """
+        from hindsight_api.engine.memory_engine import count_tokens
+
+        bank_id = f"test-refresh-cap-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        # Seed enough content that an uncapped reflect would produce a long answer.
+        await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {"content": (
+                    "Alice is the staff frontend engineer. She owns the design system, "
+                    "leads accessibility reviews, mentors three junior engineers, and runs "
+                    "the weekly UI guild meeting every Thursday at 2pm Pacific."
+                )},
+                {"content": (
+                    "Bob is the backend tech lead. He owns the payments service, the "
+                    "billing reconciliation pipeline, and the on-call rotation for the "
+                    "platform team. He is the primary reviewer for any database migration."
+                )},
+                {"content": (
+                    "Carol manages the data platform. Her team operates the warehouse, "
+                    "the streaming ingestion layer, and the metrics pipeline that feeds "
+                    "the executive dashboards refreshed every fifteen minutes."
+                )},
+                {"content": (
+                    "The team holds a company-wide demo every other Friday. Engineering "
+                    "presents shipped work, design walks through prototypes, and product "
+                    "shares roadmap updates for the upcoming quarter."
+                )},
+                {"content": (
+                    "Dan is the security lead. He runs the quarterly threat-modeling "
+                    "exercises, owns the incident response runbook, and coordinates the "
+                    "annual external penetration test with the vendor."
+                )},
+                {"content": (
+                    "Erin runs developer experience. She maintains the local-dev tooling, "
+                    "the CI pipelines, the release automation, and the internal "
+                    "documentation portal that everyone uses to onboard new hires."
+                )},
+            ],
+            request_context=request_context,
+        )
+        await memory.wait_for_background_tasks()
+
+        cap = 200
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Summary (capped)",
+            source_query="Give me a complete overview of every team member, what they own, and the recurring meetings.",
+            content="initial",
+            max_tokens=cap,
+            request_context=request_context,
+        )
+
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id,
+            mental_model_id=mm["id"],
+            request_context=request_context,
+        )
+
+        assert refreshed is not None
+        content = refreshed["content"]
+        assert content, "refresh produced empty content"
+
+        # The provider enforces the cap exactly in its own tokenizer, but our
+        # local count uses tiktoken (cl100k_base) which can disagree with
+        # provider tokenizers (Gemini's SentencePiece in particular tends to run
+        # ~30% higher for English prose). We use a generous tolerance — the test
+        # is guarding against the regression where the cap was ignored entirely
+        # and content grew toward reflect_async's default of 4096 tokens.
+        observed_tokens = count_tokens(content)
+        tolerance = 1.5
+        assert observed_tokens <= cap * tolerance, (
+            f"refreshed content exceeds max_tokens cap: "
+            f"observed≈{observed_tokens} tokens, cap={cap} (tolerance x{tolerance}). "
+            f"content={content!r}"
+        )
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
 class TestMentalModelTriggerSchema:
     """Unit tests for MentalModelTrigger schema validation (no DB needed)."""
 

hindsight-api-slim/tests/test_observation_invalidation.py

@@ -8,6 +8,7 @@ These tests verify that:
    resets the target memory itself for re-consolidation
 4. delete_bank(fact_type=...) also cleans up affected observations
 """
+
 import uuid
 from unittest.mock import AsyncMock, patch
 
@@ -20,6 +21,7 @@ from hindsight_api.engine.memory_engine import MemoryEngine
 # Helpers
 # ---------------------------------------------------------------------------
 
+
 async def _insert_memory(conn, bank_id: str, text: str, fact_type: str = "experience") -> uuid.UUID:
     """Insert a memory unit directly, bypassing LLM retain pipeline."""
     mem_id = uuid.uuid4()
@@ -36,9 +38,7 @@ async def _insert_memory(conn, bank_id: str, text: str, fact_type: str = "experi
     return mem_id
 
 
-async def _insert_observation(
-    conn, bank_id: str, text: str, source_memory_ids: list[uuid.UUID]
-) -> uuid.UUID:
+async def _insert_observation(conn, bank_id: str, text: str, source_memory_ids: list[uuid.UUID]) -> uuid.UUID:
     """Insert an observation unit directly."""
     obs_id = uuid.uuid4()
     await conn.execute(
@@ -79,8 +79,8 @@ async def _ensure_bank(memory: MemoryEngine, bank_id: str, request_context: Requ
 # Tests: delete_memory_unit
 # ---------------------------------------------------------------------------
 
-class TestDeleteMemoryUnitObservationCleanup:
 
+class TestDeleteMemoryUnitObservationCleanup:
     @pytest.mark.asyncio
     async def test_deleting_source_memory_removes_observation(
         self, memory: MemoryEngine, request_context: RequestContext
@@ -207,12 +207,10 @@ class TestDeleteMemoryUnitObservationCleanup:
 # Tests: delete_document
 # ---------------------------------------------------------------------------
 
-class TestDeleteDocumentObservationCleanup:
 
+class TestDeleteDocumentObservationCleanup:
     @pytest.mark.asyncio
-    async def test_deleting_document_removes_observations(
-        self, memory: MemoryEngine, request_context: RequestContext
-    ):
+    async def test_deleting_document_removes_observations(self, memory: MemoryEngine, request_context: RequestContext):
         """Deleting a document removes observations derived from its memory units."""
         bank_id = f"test-invalidate-doc-{uuid.uuid4().hex[:8]}"
         await _ensure_bank(memory, bank_id, request_context)
@@ -248,9 +246,7 @@ class TestDeleteDocumentObservationCleanup:
             m3 = await _insert_memory(conn, bank_id, "Alice is an avid outdoor person.")
 
             # Observation referencing both doc memories and the standalone memory
-            obs_id = await _insert_observation(
-                conn, bank_id, "Alice enjoys outdoor activities.", [m1, m2, m3]
-            )
+            obs_id = await _insert_observation(conn, bank_id, "Alice enjoys outdoor activities.", [m1, m2, m3])
 
         # Patch out consolidation so it doesn't re-set consolidated_at before we can check it
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
@@ -267,12 +263,117 @@ class TestDeleteDocumentObservationCleanup:
         await memory.delete_bank(bank_id, request_context=request_context)
 
 
+# ---------------------------------------------------------------------------
+# Tests: document upsert via retain pipeline (regression for orphan observations)
+# ---------------------------------------------------------------------------
+
+class TestDocumentUpsertObservationCleanup:
+    """Regression: re-ingesting a document via the retain pipeline must clean up
+    observations derived from the outgoing memory_units, the same way the
+    explicit ``MemoryEngine.delete_document`` API does.
+
+    Before the fix, ``fact_storage.handle_document_tracking`` deleted the
+    document via FK cascade — removing the source memory_units silently — but
+    never invalidated the dependent observations. They became orphans whose
+    ``source_memory_ids`` arrays pointed at IDs that no longer existed in
+    ``memory_units``.
+    """
+
+    @pytest.mark.asyncio
+    async def test_upsert_document_removes_observations_from_outgoing_memories(
+        self, memory: MemoryEngine, request_context: RequestContext
+    ):
+        from hindsight_api.engine.retain.fact_storage import handle_document_tracking
+
+        bank_id = f"test-upsert-obs-cleanup-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(memory, bank_id, request_context)
+
+        pool = await memory._get_pool()
+        doc_id = str(uuid.uuid4())
+
+        # Pre-populate: one document, two source memories under it, one
+        # standalone memory not in the document, and an observation that joins
+        # all three. After the upsert, the two doc memories should be gone
+        # (cascade) AND the observation should be invalidated (the bug we're
+        # fixing). The standalone memory should be reset for re-consolidation.
+        async with pool.acquire() as conn:
+            await conn.execute(
+                """
+                INSERT INTO documents (id, bank_id, original_text, content_hash, created_at, updated_at)
+                VALUES ($1, $2, 'old version', 'hash-old', NOW(), NOW())
+                """,
+                doc_id,
+                bank_id,
+            )
+            doc_mem_a = uuid.uuid4()
+            doc_mem_b = uuid.uuid4()
+            for mem_id, text in [(doc_mem_a, "Old fact A."), (doc_mem_b, "Old fact B.")]:
+                await conn.execute(
+                    """
+                    INSERT INTO memory_units (id, bank_id, text, fact_type, event_date, document_id,
+                                              created_at, updated_at, consolidated_at)
+                    VALUES ($1, $2, $3, 'experience', NOW(), $4, NOW(), NOW(), NOW())
+                    """,
+                    mem_id,
+                    bank_id,
+                    text,
+                    doc_id,
+                )
+            standalone_mem = await _insert_memory(conn, bank_id, "Standalone fact C.")
+            obs_id = await _insert_observation(
+                conn,
+                bank_id,
+                "Aggregated observation joining doc + standalone facts.",
+                [doc_mem_a, doc_mem_b, standalone_mem],
+            )
+
+        # Trigger the upsert path directly. ``handle_document_tracking`` is
+        # what the retain orchestrator calls on every document re-ingest.
+        async with pool.acquire() as conn:
+            async with conn.transaction():
+                await handle_document_tracking(
+                    conn,
+                    bank_id=bank_id,
+                    document_id=doc_id,
+                    combined_content="new version replacing old facts",
+                    is_first_batch=True,
+                    retain_params=None,
+                    document_tags=None,
+                )
+
+        async with pool.acquire() as conn:
+            obs_ids = await _get_observation_ids(conn, bank_id)
+            assert str(obs_id) not in obs_ids, (
+                "Observation derived from the outgoing memory_units should have been "
+                "deleted during the upsert (regression: orphan observations were "
+                "previously left behind because handle_document_tracking didn't call "
+                "delete_stale_observations_for_memories)"
+            )
+
+            # The standalone memory survives (different document_id) and should
+            # be reset for re-consolidation since one of its observations was
+            # invalidated by the upsert.
+            consolidated_at = await _get_consolidated_at(conn, standalone_mem)
+            assert consolidated_at is None, (
+                "Surviving co-source memory should be reset for re-consolidation"
+            )
+
+            # The two doc-scoped memories are gone via FK cascade.
+            doc_mem_count = await conn.fetchval(
+                "SELECT COUNT(*) FROM memory_units WHERE id = ANY($1::uuid[])",
+                [doc_mem_a, doc_mem_b],
+            )
+            assert doc_mem_count == 0
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
 # ---------------------------------------------------------------------------
 # Tests: delete_bank with fact_type filter
 # ---------------------------------------------------------------------------
 
-class TestDeleteBankByTypeObservationCleanup:
 
+class TestDeleteBankByTypeObservationCleanup:
     @pytest.mark.asyncio
     async def test_clearing_experience_memories_removes_affected_observations(
         self, memory: MemoryEngine, request_context: RequestContext
@@ -285,9 +386,7 @@ class TestDeleteBankByTypeObservationCleanup:
         async with pool.acquire() as conn:
             exp1 = await _insert_memory(conn, bank_id, "Alice went hiking last week.", "experience")
             world1 = await _insert_memory(conn, bank_id, "Alice is a hiker.", "world")
-            obs_id = await _insert_observation(
-                conn, bank_id, "Alice is a regular hiker.", [exp1, world1]
-            )
+            obs_id = await _insert_observation(conn, bank_id, "Alice is a regular hiker.", [exp1, world1])
 
         # Patch out consolidation so it doesn't re-set consolidated_at before we can check it
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
@@ -330,8 +429,8 @@ class TestDeleteBankByTypeObservationCleanup:
 # Tests: clear_observations_for_memory
 # ---------------------------------------------------------------------------
 
-class TestClearObservationsForMemory:
 
+class TestClearObservationsForMemory:
     @pytest.mark.asyncio
     async def test_clears_observations_and_resets_all_source_memories(
         self, memory: MemoryEngine, request_context: RequestContext
@@ -348,9 +447,7 @@ class TestClearObservationsForMemory:
 
         # Patch out consolidation so it doesn't re-set consolidated_at before we can check it
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            result = await memory.clear_observations_for_memory(
-                bank_id, str(m1), request_context=request_context
-            )
+            result = await memory.clear_observations_for_memory(bank_id, str(m1), request_context=request_context)
 
         assert result["deleted_count"] == 1
 
@@ -365,9 +462,7 @@ class TestClearObservationsForMemory:
         await memory.delete_bank(bank_id, request_context=request_context)
 
     @pytest.mark.asyncio
-    async def test_no_observations_returns_zero(
-        self, memory: MemoryEngine, request_context: RequestContext
-    ):
+    async def test_no_observations_returns_zero(self, memory: MemoryEngine, request_context: RequestContext):
         """Returns 0 when the memory has no associated observations."""
         bank_id = f"test-clear-obs-noop-{uuid.uuid4().hex[:8]}"
         await _ensure_bank(memory, bank_id, request_context)
@@ -376,9 +471,7 @@ class TestClearObservationsForMemory:
         async with pool.acquire() as conn:
             m1 = await _insert_memory(conn, bank_id, "Alice loves hiking.")
 
-        result = await memory.clear_observations_for_memory(
-            bank_id, str(m1), request_context=request_context
-        )
+        result = await memory.clear_observations_for_memory(bank_id, str(m1), request_context=request_context)
 
         assert result["deleted_count"] == 0
 
@@ -405,9 +498,7 @@ class TestClearObservationsForMemory:
             obs1_id = await _insert_observation(conn, bank_id, "Alice is an avid hiker.", [m1, m2])
             obs2_id = await _insert_observation(conn, bank_id, "Alice is a mountaineer.", [m3])
 
-        result = await memory.clear_observations_for_memory(
-            bank_id, str(m1), request_context=request_context
-        )
+        result = await memory.clear_observations_for_memory(bank_id, str(m1), request_context=request_context)
 
         assert result["deleted_count"] == 1
 
@@ -439,9 +530,7 @@ class TestClearObservationsForMemory:
 
         # Patch out consolidation so it doesn't re-set consolidated_at before we can check it
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            result = await memory.clear_observations_for_memory(
-                bank_id, str(m1), request_context=request_context
-            )
+            result = await memory.clear_observations_for_memory(bank_id, str(m1), request_context=request_context)
 
         assert result["deleted_count"] == 2
 
@@ -493,11 +582,8 @@ async def _insert_document_with_memories(
 
 
 class TestUpdateDocumentTagsObservationCleanup:
-
     @pytest.mark.asyncio
-    async def test_update_tags_returns_updated_document(
-        self, memory: MemoryEngine, request_context: RequestContext
-    ):
+    async def test_update_tags_returns_updated_document(self, memory: MemoryEngine, request_context: RequestContext):
         """update_document returns the updated document with new tags."""
         bank_id = f"test-tag-update-basic-{uuid.uuid4().hex[:8]}"
         await _ensure_bank(memory, bank_id, request_context)
@@ -507,9 +593,7 @@ class TestUpdateDocumentTagsObservationCleanup:
             doc_id = f"doc-{uuid.uuid4().hex[:8]}"
             await _insert_document_with_memories(conn, bank_id, doc_id, [("Alice loves hiking.", "experience")])
 
-        result = await memory.update_document(
-            doc_id, bank_id, tags=["new-tag"], request_context=request_context
-        )
+        result = await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
 
         assert result is True
 
@@ -523,18 +607,14 @@ class TestUpdateDocumentTagsObservationCleanup:
         bank_id = f"test-tag-update-missing-{uuid.uuid4().hex[:8]}"
         await _ensure_bank(memory, bank_id, request_context)
 
-        result = await memory.update_document(
-            "nonexistent-doc", bank_id, tags=["tag"], request_context=request_context
-        )
+        result = await memory.update_document("nonexistent-doc", bank_id, tags=["tag"], request_context=request_context)
 
         assert result is False
 
         await memory.delete_bank(bank_id, request_context=request_context)
 
     @pytest.mark.asyncio
-    async def test_update_tags_propagates_to_memory_units(
-        self, memory: MemoryEngine, request_context: RequestContext
-    ):
+    async def test_update_tags_propagates_to_memory_units(self, memory: MemoryEngine, request_context: RequestContext):
         """Changing document tags also updates all associated memory unit tags."""
         bank_id = f"test-tag-update-propagate-{uuid.uuid4().hex[:8]}"
         await _ensure_bank(memory, bank_id, request_context)
@@ -547,23 +627,17 @@ class TestUpdateDocumentTagsObservationCleanup:
             )
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
 
         async with pool.acquire() as conn:
             for mem_id in mem_ids:
-                tags = await conn.fetchval(
-                    "SELECT tags FROM memory_units WHERE id = $1", mem_id
-                )
+                tags = await conn.fetchval("SELECT tags FROM memory_units WHERE id = $1", mem_id)
                 assert list(tags) == ["new-tag"], f"Memory unit {mem_id} should have updated tags"
 
         await memory.delete_bank(bank_id, request_context=request_context)
 
     @pytest.mark.asyncio
-    async def test_update_tags_invalidates_observations(
-        self, memory: MemoryEngine, request_context: RequestContext
-    ):
+    async def test_update_tags_invalidates_observations(self, memory: MemoryEngine, request_context: RequestContext):
         """Observations referencing the document's memory units are deleted on tag change."""
         bank_id = f"test-tag-update-obs-{uuid.uuid4().hex[:8]}"
         await _ensure_bank(memory, bank_id, request_context)
@@ -577,9 +651,7 @@ class TestUpdateDocumentTagsObservationCleanup:
             obs_id = await _insert_observation(conn, bank_id, "Alice is a hiker.", mem_ids)
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
 
         async with pool.acquire() as conn:
             obs_ids = await _get_observation_ids(conn, bank_id)
@@ -607,9 +679,7 @@ class TestUpdateDocumentTagsObservationCleanup:
             assert await _get_consolidated_at(conn, mem_ids[0]) is not None
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
 
         async with pool.acquire() as conn:
             consolidated_at = await _get_consolidated_at(conn, mem_ids[0])
@@ -634,9 +704,7 @@ class TestUpdateDocumentTagsObservationCleanup:
             await _insert_observation(conn, bank_id, "Alice is a hiker.", mem_ids)
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()) as mock_consolidate:
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
             mock_consolidate.assert_awaited_once()
 
         await memory.delete_bank(bank_id, request_context=request_context)
@@ -652,15 +720,11 @@ class TestUpdateDocumentTagsObservationCleanup:
         pool = await memory._get_pool()
         async with pool.acquire() as conn:
             doc_id = f"doc-{uuid.uuid4().hex[:8]}"
-            await _insert_document_with_memories(
-                conn, bank_id, doc_id, [("Alice loves hiking.", "experience")]
-            )
+            await _insert_document_with_memories(conn, bank_id, doc_id, [("Alice loves hiking.", "experience")])
             # No observations inserted
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()) as mock_consolidate:
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
             mock_consolidate.assert_not_awaited()
 
         await memory.delete_bank(bank_id, request_context=request_context)
@@ -689,9 +753,7 @@ class TestUpdateDocumentTagsObservationCleanup:
             assert await _get_consolidated_at(conn, other_mem) is not None
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
 
         async with pool.acquire() as conn:
             obs_ids = await _get_observation_ids(conn, bank_id)
@@ -719,17 +781,128 @@ class TestUpdateDocumentTagsObservationCleanup:
             )
             # Unrelated memory not in the document
             unrelated = await _insert_memory(conn, bank_id, "Bob likes cycling.")
-            unrelated_obs_id = await _insert_observation(
-                conn, bank_id, "Bob is a cyclist.", [unrelated]
-            )
+            unrelated_obs_id = await _insert_observation(conn, bank_id, "Bob is a cyclist.", [unrelated])
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
 
         async with pool.acquire() as conn:
             obs_ids = await _get_observation_ids(conn, bank_id)
             assert str(unrelated_obs_id) in obs_ids, "Unrelated observation should remain untouched"
 
         await memory.delete_bank(bank_id, request_context=request_context)
+
+
+# ---------------------------------------------------------------------------
+# Tests: consolidation-vs-delete race — filtering stale source_memory_ids
+# ---------------------------------------------------------------------------
+
+
+class TestConsolidationSourceMemoryFiltering:
+    """
+    When a source memory is deleted concurrently with consolidation, the
+    observation must not be written referencing the dead uuid. We exercise
+    the guard by calling the consolidator helpers directly with a deleted
+    source id in the input list.
+    """
+
+    @pytest.mark.asyncio
+    async def test_create_observation_filters_deleted_source_memories(
+        self, memory: MemoryEngine, request_context: RequestContext
+    ):
+        from hindsight_api.engine.consolidation.consolidator import _create_observation_directly
+
+        bank_id = f"test-race-create-filter-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(memory, bank_id, request_context)
+
+        pool = await memory._get_pool()
+        async with pool.acquire() as conn:
+            live = await _insert_memory(conn, bank_id, "Alice loves hiking.")
+            dead = uuid.uuid4()  # never existed — stands in for a concurrently deleted source
+
+            result = await _create_observation_directly(
+                conn=conn,
+                memory_engine=memory,
+                bank_id=bank_id,
+                source_memory_ids=[live, dead],
+                observation_text="Alice enjoys hiking regularly.",
+            )
+
+            assert result["action"] == "created"
+            stored = await conn.fetchval(
+                "SELECT source_memory_ids FROM memory_units WHERE id = $1",
+                uuid.UUID(result["observation_id"]),
+            )
+            stored_set = {str(s) for s in stored}
+            assert str(live) in stored_set
+            assert str(dead) not in stored_set, "Deleted source must not appear in stored observation"
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    @pytest.mark.asyncio
+    async def test_create_observation_skipped_when_all_sources_deleted(
+        self, memory: MemoryEngine, request_context: RequestContext
+    ):
+        from hindsight_api.engine.consolidation.consolidator import _create_observation_directly
+
+        bank_id = f"test-race-create-skip-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(memory, bank_id, request_context)
+
+        pool = await memory._get_pool()
+        async with pool.acquire() as conn:
+            result = await _create_observation_directly(
+                conn=conn,
+                memory_engine=memory,
+                bank_id=bank_id,
+                source_memory_ids=[uuid.uuid4(), uuid.uuid4()],
+                observation_text="All sources gone.",
+            )
+
+            assert result["action"] == "skipped"
+            assert result["reason"] == "sources_deleted"
+
+            obs_ids = await _get_observation_ids(conn, bank_id)
+            assert obs_ids == [], "No observation row should exist"
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    @pytest.mark.asyncio
+    async def test_update_observation_skipped_when_all_new_sources_deleted(
+        self, memory: MemoryEngine, request_context: RequestContext
+    ):
+        from hindsight_api.engine.consolidation.consolidator import _execute_update_action
+        from hindsight_api.engine.response_models import MemoryFact
+
+        bank_id = f"test-race-update-skip-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(memory, bank_id, request_context)
+
+        pool = await memory._get_pool()
+        async with pool.acquire() as conn:
+            original_source = await _insert_memory(conn, bank_id, "Alice hikes.")
+            obs_id = await _insert_observation(conn, bank_id, "Alice is a hiker.", [original_source])
+            original_text = "Alice is a hiker."
+
+            observation_model = MemoryFact(
+                id=str(obs_id),
+                text=original_text,
+                fact_type="observation",
+                source_fact_ids=[str(original_source)],
+                tags=[],
+            )
+
+            await _execute_update_action(
+                conn=conn,
+                memory_engine=memory,
+                bank_id=bank_id,
+                source_memory_ids=[uuid.uuid4(), uuid.uuid4()],  # all dead
+                observation_id=str(obs_id),
+                new_text="This update must not land.",
+                observations=[observation_model],
+            )
+
+            row = await conn.fetchrow("SELECT text, source_memory_ids FROM memory_units WHERE id = $1", obs_id)
+            assert row["text"] == original_text, "Observation text must not change"
+            stored_sources = {str(s) for s in row["source_memory_ids"]}
+            assert stored_sources == {str(original_source)}, "Dead sources must not be appended"
+
+        await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_per_operation_llm_config.py

@@ -276,6 +276,51 @@ class TestReflectUsesReflectLLMConfig:
         # Verify it's different from the retain config
         assert engine._reflect_llm_config.model != engine._retain_llm_config.model
 
+    @pytest.mark.asyncio
+    async def test_reflect_allowed_when_default_llm_none_but_reflect_configured(self, monkeypatch):
+        """A disabled default LLM should not block a separately configured reflect LLM."""
+        from types import SimpleNamespace
+        from unittest.mock import AsyncMock
+
+        from hindsight_api import MemoryEngine
+        from hindsight_api.engine.reflect.models import ReflectAgentResult
+        from hindsight_api.models import RequestContext
+
+        engine = MemoryEngine(
+            memory_llm_provider="none",
+            memory_llm_model="none",
+            reflect_llm_provider="mock",
+            reflect_llm_model="reflect-specific-model",
+            skip_llm_verification=True,
+            lazy_reranker=True,
+        )
+
+        engine._authenticate_tenant = AsyncMock()  # type: ignore[method-assign]
+        engine.get_bank_profile = AsyncMock(return_value={"name": "Test", "mission": ""})  # type: ignore[method-assign]
+        engine.get_bank_stats = AsyncMock(return_value=SimpleNamespace(last_consolidated_at=None, pending_consolidation=0))  # type: ignore[method-assign]
+        engine.list_directives = AsyncMock(return_value=[])  # type: ignore[method-assign]
+        engine._get_pool = AsyncMock(return_value=SimpleNamespace())  # type: ignore[method-assign]
+        engine._config_resolver = SimpleNamespace(
+            resolve_full_config=AsyncMock(return_value=SimpleNamespace(llm_gemini_safety_settings=None)),
+            get_bank_config=AsyncMock(return_value={}),
+        )
+
+        async def fake_run_reflect_agent(**kwargs):
+            assert kwargs["llm_config"].provider == "mock"
+            return ReflectAgentResult(text="reflect works")
+
+        monkeypatch.setattr("hindsight_api.engine.memory_engine.run_reflect_agent", fake_run_reflect_agent)
+
+        result = await engine.reflect_async(
+            bank_id="bank-1",
+            query="test",
+            request_context=RequestContext(),
+            exclude_mental_models=True,
+            fact_types=["observation"],
+        )
+
+        assert result.text == "reflect works"
+
 
 class TestRetryAndBackoffConfiguration:
     """Test retry and backoff configuration options."""

hindsight-api-slim/tests/test_recall_budget_config.py

@@ -0,0 +1,254 @@
+"""
+Tests for the configurable recall-budget mapping (Budget enum -> thinking_budget int).
+
+Two functions are supported:
+- "fixed": returns the recall_budget_fixed_<level> integer directly (legacy default).
+- "adaptive": returns round(max_tokens * recall_budget_adaptive_<level>),
+              clamped to [recall_budget_min, recall_budget_max].
+
+Both the function selector and the per-level numbers are hierarchical config
+fields (global env -> tenant -> bank), so they can be overridden per bank.
+"""
+
+import dataclasses
+
+import pytest
+
+from hindsight_api.config import (
+    DEFAULT_RECALL_BUDGET_ADAPTIVE_HIGH,
+    DEFAULT_RECALL_BUDGET_ADAPTIVE_LOW,
+    DEFAULT_RECALL_BUDGET_ADAPTIVE_MID,
+    DEFAULT_RECALL_BUDGET_FIXED_HIGH,
+    DEFAULT_RECALL_BUDGET_FIXED_LOW,
+    DEFAULT_RECALL_BUDGET_FIXED_MID,
+    DEFAULT_RECALL_BUDGET_MAX,
+    DEFAULT_RECALL_BUDGET_MIN,
+    DEFAULT_RECALL_BUDGET_FUNCTION,
+    ENV_RECALL_BUDGET_ADAPTIVE_LOW,
+    ENV_RECALL_BUDGET_ADAPTIVE_MID,
+    ENV_RECALL_BUDGET_FIXED_HIGH,
+    ENV_RECALL_BUDGET_FIXED_LOW,
+    ENV_RECALL_BUDGET_FIXED_MID,
+    ENV_RECALL_BUDGET_MAX,
+    ENV_RECALL_BUDGET_MIN,
+    ENV_RECALL_BUDGET_FUNCTION,
+    RECALL_BUDGET_FUNCTIONS,
+    HindsightConfig,
+)
+from hindsight_api.config_resolver import _validate_recall_budget_updates
+from hindsight_api.engine.memory_engine import Budget, _resolve_thinking_budget
+
+
+_BUDGET_FIELD_NAMES = (
+    "recall_budget_function",
+    "recall_budget_fixed_low",
+    "recall_budget_fixed_mid",
+    "recall_budget_fixed_high",
+    "recall_budget_adaptive_low",
+    "recall_budget_adaptive_mid",
+    "recall_budget_adaptive_high",
+    "recall_budget_min",
+    "recall_budget_max",
+)
+
+
+class TestBudgetConfigFields:
+    def test_fields_exist_on_dataclass(self):
+        names = {f.name for f in dataclasses.fields(HindsightConfig)}
+        for field_name in _BUDGET_FIELD_NAMES:
+            assert field_name in names, f"Missing dataclass field: {field_name}"
+
+    def test_fields_are_configurable(self):
+        configurable = HindsightConfig.get_configurable_fields()
+        for field_name in _BUDGET_FIELD_NAMES:
+            assert field_name in configurable, f"Field not in _CONFIGURABLE_FIELDS: {field_name}"
+
+    def test_default_function_is_fixed_for_backwards_compat(self):
+        # The whole point of function="fixed" being default is to preserve legacy behavior.
+        assert DEFAULT_RECALL_BUDGET_FUNCTION == "fixed"
+        assert "fixed" in RECALL_BUDGET_FUNCTIONS
+        assert "adaptive" in RECALL_BUDGET_FUNCTIONS
+
+    def test_default_fixed_values_match_legacy_hardcoded_mapping(self):
+        # These are the values that used to live in the hardcoded budget_mapping dict.
+        assert DEFAULT_RECALL_BUDGET_FIXED_LOW == 100
+        assert DEFAULT_RECALL_BUDGET_FIXED_MID == 300
+        assert DEFAULT_RECALL_BUDGET_FIXED_HIGH == 1000
+
+    def test_default_adaptive_clamps_are_sane(self):
+        assert DEFAULT_RECALL_BUDGET_MIN >= 1
+        assert DEFAULT_RECALL_BUDGET_MAX > DEFAULT_RECALL_BUDGET_MIN
+
+    def test_env_var_constants(self):
+        assert ENV_RECALL_BUDGET_FUNCTION == "HINDSIGHT_API_RECALL_BUDGET_FUNCTION"
+        assert ENV_RECALL_BUDGET_FIXED_LOW == "HINDSIGHT_API_RECALL_BUDGET_FIXED_LOW"
+        assert ENV_RECALL_BUDGET_FIXED_MID == "HINDSIGHT_API_RECALL_BUDGET_FIXED_MID"
+        assert ENV_RECALL_BUDGET_FIXED_HIGH == "HINDSIGHT_API_RECALL_BUDGET_FIXED_HIGH"
+        assert ENV_RECALL_BUDGET_ADAPTIVE_LOW == "HINDSIGHT_API_RECALL_BUDGET_ADAPTIVE_LOW"
+        assert ENV_RECALL_BUDGET_ADAPTIVE_MID == "HINDSIGHT_API_RECALL_BUDGET_ADAPTIVE_MID"
+        assert ENV_RECALL_BUDGET_MIN == "HINDSIGHT_API_RECALL_BUDGET_MIN"
+        assert ENV_RECALL_BUDGET_MAX == "HINDSIGHT_API_RECALL_BUDGET_MAX"
+
+    def test_from_env_reads_overrides(self, monkeypatch):
+        monkeypatch.setenv(ENV_RECALL_BUDGET_FUNCTION, "adaptive")
+        monkeypatch.setenv(ENV_RECALL_BUDGET_FIXED_MID, "777")
+        monkeypatch.setenv(ENV_RECALL_BUDGET_ADAPTIVE_MID, "0.5")
+        monkeypatch.setenv(ENV_RECALL_BUDGET_MIN, "5")
+        monkeypatch.setenv(ENV_RECALL_BUDGET_MAX, "9999")
+
+        config = HindsightConfig.from_env()
+        assert config.recall_budget_function == "adaptive"
+        assert config.recall_budget_fixed_mid == 777
+        assert config.recall_budget_adaptive_mid == 0.5
+        assert config.recall_budget_min == 5
+        assert config.recall_budget_max == 9999
+
+    def test_from_env_invalid_function_falls_back_to_default(self, monkeypatch):
+        # Defensive parsing: an invalid env value logs a warning and falls back.
+        monkeypatch.setenv(ENV_RECALL_BUDGET_FUNCTION, "garbage")
+        config = HindsightConfig.from_env()
+        assert config.recall_budget_function == DEFAULT_RECALL_BUDGET_FUNCTION
+
+
+class TestResolveThinkingBudgetFixedFunction:
+    @pytest.fixture
+    def fixed_config(self):
+        return {
+            "recall_budget_function": "fixed",
+            "recall_budget_fixed_low": 100,
+            "recall_budget_fixed_mid": 300,
+            "recall_budget_fixed_high": 1000,
+            "recall_budget_adaptive_low": 0.025,
+            "recall_budget_adaptive_mid": 0.075,
+            "recall_budget_adaptive_high": 0.25,
+            "recall_budget_min": 20,
+            "recall_budget_max": 2000,
+        }
+
+    def test_low_mid_high_match_fixed_values(self, fixed_config):
+        assert _resolve_thinking_budget(fixed_config, Budget.LOW, 4096) == 100
+        assert _resolve_thinking_budget(fixed_config, Budget.MID, 4096) == 300
+        assert _resolve_thinking_budget(fixed_config, Budget.HIGH, 4096) == 1000
+
+    def test_none_budget_defaults_to_mid(self, fixed_config):
+        assert _resolve_thinking_budget(fixed_config, None, 4096) == 300
+
+    def test_max_tokens_does_not_affect_fixed_function(self, fixed_config):
+        # Whole point of "fixed": result is independent of max_tokens.
+        assert _resolve_thinking_budget(fixed_config, Budget.MID, 1) == 300
+        assert _resolve_thinking_budget(fixed_config, Budget.MID, 1_000_000) == 300
+
+    def test_per_bank_overrides_take_effect(self, fixed_config):
+        fixed_config["recall_budget_fixed_mid"] = 42
+        assert _resolve_thinking_budget(fixed_config, Budget.MID, 4096) == 42
+
+
+class TestResolveThinkingBudgetAdaptiveFunction:
+    @pytest.fixture
+    def adaptive_config(self):
+        return {
+            "recall_budget_function": "adaptive",
+            "recall_budget_fixed_low": 100,
+            "recall_budget_fixed_mid": 300,
+            "recall_budget_fixed_high": 1000,
+            "recall_budget_adaptive_low": 0.025,
+            "recall_budget_adaptive_mid": 0.075,
+            "recall_budget_adaptive_high": 0.25,
+            "recall_budget_min": 20,
+            "recall_budget_max": 2000,
+        }
+
+    def test_scales_with_max_tokens(self, adaptive_config):
+        # 4096 * 0.075 = 307.2 -> 307
+        assert _resolve_thinking_budget(adaptive_config, Budget.MID, 4096) == 307
+        # 8192 * 0.075 = 614.4 -> 614
+        assert _resolve_thinking_budget(adaptive_config, Budget.MID, 8192) == 614
+
+    def test_clamps_to_floor_when_max_tokens_tiny(self, adaptive_config):
+        # 100 * 0.025 = 2.5 -> 2 -> clamped to floor 20
+        assert _resolve_thinking_budget(adaptive_config, Budget.LOW, 100) == 20
+
+    def test_clamps_to_ceiling_when_max_tokens_huge(self, adaptive_config):
+        # 100_000 * 0.25 = 25_000 -> clamped to ceiling 2000
+        assert _resolve_thinking_budget(adaptive_config, Budget.HIGH, 100_000) == 2000
+
+    def test_none_budget_defaults_to_mid(self, adaptive_config):
+        assert _resolve_thinking_budget(adaptive_config, None, 4096) == 307
+
+    def test_custom_clamps_per_bank(self, adaptive_config):
+        adaptive_config["recall_budget_min"] = 500
+        adaptive_config["recall_budget_max"] = 600
+        # 4096 * 0.075 = 307 -> below floor 500
+        assert _resolve_thinking_budget(adaptive_config, Budget.MID, 4096) == 500
+        # 4096 * 0.25 = 1024 -> above ceiling 600
+        assert _resolve_thinking_budget(adaptive_config, Budget.HIGH, 4096) == 600
+
+
+class TestResolveThinkingBudgetFallbacks:
+    def test_empty_config_uses_legacy_defaults(self):
+        # Resilience: missing keys should not crash; fallback to legacy mapping.
+        assert _resolve_thinking_budget({}, Budget.LOW, 4096) == 100
+        assert _resolve_thinking_budget({}, Budget.MID, 4096) == 300
+        assert _resolve_thinking_budget({}, Budget.HIGH, 4096) == 1000
+
+    def test_unknown_function_falls_back_to_fixed(self):
+        # Defensive: if some bad config slipped past validation, behave like "fixed".
+        assert _resolve_thinking_budget({"recall_budget_function": "garbage"}, Budget.MID, 4096) == 300
+
+
+class TestValidateRecallBudgetUpdates:
+    def test_no_op_passes(self):
+        _validate_recall_budget_updates({})
+        _validate_recall_budget_updates({"unrelated_field": 123})
+
+    def test_valid_function_values(self):
+        _validate_recall_budget_updates({"recall_budget_function": "fixed"})
+        _validate_recall_budget_updates({"recall_budget_function": "adaptive"})
+
+    def test_invalid_function_raises(self):
+        with pytest.raises(ValueError, match="recall_budget_function"):
+            _validate_recall_budget_updates({"recall_budget_function": "wrong"})
+        with pytest.raises(ValueError, match="recall_budget_function"):
+            _validate_recall_budget_updates({"recall_budget_function": 123})
+
+    def test_fixed_must_be_positive_integer(self):
+        for key in ("recall_budget_fixed_low", "recall_budget_fixed_mid", "recall_budget_fixed_high"):
+            _validate_recall_budget_updates({key: 1})
+            _validate_recall_budget_updates({key: 100_000})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: 0})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: -5})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: 1.5})  # float not allowed for fixed
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: True})  # bool sneaks past int check
+
+    def test_adaptive_must_be_positive_number(self):
+        for key in ("recall_budget_adaptive_low", "recall_budget_adaptive_mid", "recall_budget_adaptive_high"):
+            _validate_recall_budget_updates({key: 0.001})
+            _validate_recall_budget_updates({key: 1.0})
+            _validate_recall_budget_updates({key: 5})  # int is acceptable as a number
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: 0})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: -0.1})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: True})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: "0.5"})
+
+    def test_min_must_be_le_max_when_both_set(self):
+        _validate_recall_budget_updates({"recall_budget_min": 10, "recall_budget_max": 1000})
+        _validate_recall_budget_updates({"recall_budget_min": 100, "recall_budget_max": 100})
+        with pytest.raises(ValueError, match="recall_budget_min"):
+            _validate_recall_budget_updates({"recall_budget_min": 5000, "recall_budget_max": 100})
+
+    def test_min_max_must_be_positive_integers(self):
+        for key in ("recall_budget_min", "recall_budget_max"):
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: 0})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: -1})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: 1.5})

hindsight-api-slim/tests/test_recall_config.py

@@ -0,0 +1,231 @@
+"""
+Tests for the internal recall configuration knobs used during mental model
+refresh: recall_include_chunks, recall_max_tokens, recall_chunks_max_tokens.
+
+These are exposed both as hierarchical config fields (env → tenant → bank)
+and as overrides on a mental model's `trigger` JSONB field.
+"""
+
+import dataclasses
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from hindsight_api.engine.reflect.tools import tool_recall
+from hindsight_api.engine.response_models import RecallResult as RecallResultModel
+from hindsight_api.models import RequestContext
+
+
+def _make_mock_engine():
+    engine = MagicMock()
+    engine.recall_async = AsyncMock(return_value=RecallResultModel(results=[], entities={}, chunks={}))
+    return engine
+
+
+@pytest.fixture
+def mock_request_context():
+    # internal=True bypasses the tenant extension, letting these unit tests
+    # exercise engine methods without standing up auth.
+    return RequestContext(internal=True)
+
+
+class TestToolRecallIncludeChunks:
+    """tool_recall must honor the include_chunks parameter (was hardcoded True)."""
+
+    @pytest.mark.asyncio
+    async def test_default_includes_chunks(self, mock_request_context):
+        engine = _make_mock_engine()
+
+        await tool_recall(engine, "bank-1", "q", mock_request_context)
+
+        kwargs = engine.recall_async.call_args.kwargs
+        assert kwargs["include_chunks"] is True
+
+    @pytest.mark.asyncio
+    async def test_include_chunks_false_propagates(self, mock_request_context):
+        engine = _make_mock_engine()
+
+        await tool_recall(engine, "bank-1", "q", mock_request_context, include_chunks=False)
+
+        kwargs = engine.recall_async.call_args.kwargs
+        assert kwargs["include_chunks"] is False
+
+    @pytest.mark.asyncio
+    async def test_max_chunk_tokens_propagates(self, mock_request_context):
+        engine = _make_mock_engine()
+
+        await tool_recall(
+            engine, "bank-1", "q", mock_request_context, max_chunk_tokens=2500, max_tokens=512
+        )
+
+        kwargs = engine.recall_async.call_args.kwargs
+        assert kwargs["max_chunk_tokens"] == 2500
+        assert kwargs["max_tokens"] == 512
+
+
+class TestRecallConfigFields:
+    """Hierarchical config fields for internal recall."""
+
+    def test_fields_exist_on_dataclass(self):
+        from hindsight_api.config import HindsightConfig
+
+        names = {f.name for f in dataclasses.fields(HindsightConfig)}
+        assert "recall_include_chunks" in names
+        assert "recall_max_tokens" in names
+        assert "recall_chunks_max_tokens" in names
+
+    def test_fields_are_configurable(self):
+        from hindsight_api.config import HindsightConfig
+
+        configurable = HindsightConfig.get_configurable_fields()
+        assert "recall_include_chunks" in configurable
+        assert "recall_max_tokens" in configurable
+        assert "recall_chunks_max_tokens" in configurable
+
+    def test_default_values(self):
+        from hindsight_api.config import (
+            DEFAULT_RECALL_CHUNKS_MAX_TOKENS,
+            DEFAULT_RECALL_INCLUDE_CHUNKS,
+            DEFAULT_RECALL_MAX_TOKENS,
+        )
+
+        assert DEFAULT_RECALL_INCLUDE_CHUNKS is True
+        assert DEFAULT_RECALL_MAX_TOKENS == 2048
+        assert DEFAULT_RECALL_CHUNKS_MAX_TOKENS == 1000
+
+    def test_env_var_constants(self):
+        from hindsight_api.config import (
+            ENV_RECALL_CHUNKS_MAX_TOKENS,
+            ENV_RECALL_INCLUDE_CHUNKS,
+            ENV_RECALL_MAX_TOKENS,
+        )
+
+        assert ENV_RECALL_INCLUDE_CHUNKS == "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS"
+        assert ENV_RECALL_MAX_TOKENS == "HINDSIGHT_API_RECALL_MAX_TOKENS"
+        assert ENV_RECALL_CHUNKS_MAX_TOKENS == "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS"
+
+    @patch.dict(
+        "os.environ",
+        {
+            "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS": "false",
+            "HINDSIGHT_API_RECALL_MAX_TOKENS": "777",
+            "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS": "333",
+        },
+    )
+    def test_from_env_reads_overrides(self):
+        from hindsight_api.config import HindsightConfig
+
+        config = HindsightConfig.from_env()
+        assert config.recall_include_chunks is False
+        assert config.recall_max_tokens == 777
+        assert config.recall_chunks_max_tokens == 333
+
+
+class TestMentalModelTriggerRecallFields:
+    """MentalModelTrigger Pydantic model accepts the new override fields."""
+
+    def test_trigger_accepts_new_fields(self):
+        from hindsight_api.api.http import MentalModelTrigger
+
+        trigger = MentalModelTrigger(
+            include_chunks=False,
+            recall_max_tokens=512,
+            recall_chunks_max_tokens=0,
+        )
+        assert trigger.include_chunks is False
+        assert trigger.recall_max_tokens == 512
+        assert trigger.recall_chunks_max_tokens == 0
+
+    def test_trigger_defaults_are_none(self):
+        from hindsight_api.api.http import MentalModelTrigger
+
+        trigger = MentalModelTrigger()
+        assert trigger.include_chunks is None
+        assert trigger.recall_max_tokens is None
+        assert trigger.recall_chunks_max_tokens is None
+
+
+class TestRefreshTriggerWiring:
+    """Verify mental-model refresh forwards trigger overrides into reflect_async kwargs."""
+
+    @pytest.mark.asyncio
+    async def test_trigger_overrides_passed_to_reflect_async(self, mock_request_context):
+        from hindsight_api.engine.memory_engine import MemoryEngine
+        from hindsight_api.engine.response_models import ReflectResult
+
+        engine = MemoryEngine.__new__(MemoryEngine)
+
+        async def fake_get_mental_model(bank_id, mental_model_id, request_context):
+            return {
+                "id": mental_model_id,
+                "source_query": "What do we know?",
+                "tags": [],
+                "trigger": {
+                    "include_chunks": False,
+                    "recall_max_tokens": 512,
+                    "recall_chunks_max_tokens": 0,
+                    "fact_types": ["world"],
+                },
+            }
+
+        captured = {}
+
+        async def fake_reflect_async(**kwargs):
+            captured.update(kwargs)
+            return ReflectResult(text="ok", based_on={})
+
+        async def fake_update_mental_model(*args, **kwargs):
+            return None
+
+        engine.get_mental_model = fake_get_mental_model
+        engine.reflect_async = fake_reflect_async
+        engine.update_mental_model = fake_update_mental_model
+        engine._operation_validator = None
+        engine._tenant_extension = None
+
+        await engine.refresh_mental_model(
+            bank_id="bank-1",
+            mental_model_id="mm-1",
+            request_context=mock_request_context,
+        )
+
+        assert captured["recall_include_chunks"] is False
+        assert captured["recall_max_tokens_override"] == 512
+        assert captured["recall_chunks_max_tokens_override"] == 0
+        assert captured["fact_types"] == ["world"]
+
+    @pytest.mark.asyncio
+    async def test_missing_trigger_fields_pass_none(self, mock_request_context):
+        from hindsight_api.engine.memory_engine import MemoryEngine
+        from hindsight_api.engine.response_models import ReflectResult
+
+        engine = MemoryEngine.__new__(MemoryEngine)
+
+        async def fake_get_mental_model(bank_id, mental_model_id, request_context):
+            return {"id": mental_model_id, "source_query": "q", "tags": [], "trigger": {}}
+
+        captured = {}
+
+        async def fake_reflect_async(**kwargs):
+            captured.update(kwargs)
+            return ReflectResult(text="ok", based_on={})
+
+        async def fake_update_mental_model(*args, **kwargs):
+            return None
+
+        engine.get_mental_model = fake_get_mental_model
+        engine.reflect_async = fake_reflect_async
+        engine.update_mental_model = fake_update_mental_model
+        engine._operation_validator = None
+        engine._tenant_extension = None
+
+        await engine.refresh_mental_model(
+            bank_id="bank-1",
+            mental_model_id="mm-1",
+            request_context=mock_request_context,
+        )
+
+        # When trigger fields are absent, None is forwarded so reflect_async falls back to bank/global config.
+        assert captured["recall_include_chunks"] is None
+        assert captured["recall_max_tokens_override"] is None
+        assert captured["recall_chunks_max_tokens_override"] is None

hindsight-api-slim/tests/test_recall_time_range.py

@@ -0,0 +1,214 @@
+"""Tests for created_after / created_before time-range filtering in recall.
+
+Inserts memory_units with known timestamps directly via SQL, then verifies
+that recall_async respects the time bounds — never returning memories
+outside the requested range.
+
+No LLM required — uses mock provider.
+"""
+
+import uuid
+from datetime import datetime, timezone
+
+import pytest
+import pytest_asyncio
+
+from hindsight_api import MemoryEngine, RequestContext
+from hindsight_api.engine.retain import embedding_utils
+
+# Three points in time, each 1 hour apart
+T1 = datetime(2026, 1, 1, 10, 0, 0, tzinfo=timezone.utc)
+T2 = datetime(2026, 1, 1, 11, 0, 0, tzinfo=timezone.utc)
+T3 = datetime(2026, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
+
+# Stable UUIDs for the three facts (deterministic for assertion readability)
+ID_OLD = "00000000-0000-0000-0000-000000000001"
+ID_MID = "00000000-0000-0000-0000-000000000002"
+ID_NEW = "00000000-0000-0000-0000-000000000003"
+
+RC = RequestContext(tenant_id="default")
+
+
+async def _insert_fact(
+    conn,
+    *,
+    fact_id: str,
+    text: str,
+    bank_id: str,
+    embedding_str: str,
+    created_at: datetime,
+    updated_at: datetime | None = None,
+    fact_type: str = "world",
+) -> None:
+    """Insert a memory_unit with a specific created_at/updated_at timestamp."""
+    updated = updated_at or created_at
+    await conn.execute(
+        """
+        INSERT INTO memory_units (id, bank_id, text, fact_type, embedding, created_at, updated_at)
+        VALUES ($1, $2, $3, $4, $5::vector, $6, $7)
+        """,
+        fact_id,
+        bank_id,
+        text,
+        fact_type,
+        embedding_str,
+        created_at,
+        updated,
+    )
+
+
+@pytest_asyncio.fixture
+async def seeded_memory(memory_no_llm_verify: MemoryEngine):
+    """Insert three facts at T1, T2, T3 and return the engine."""
+    engine = memory_no_llm_verify
+    bank_id = f"test-time-range-{uuid.uuid4().hex[:8]}"
+
+    await engine.get_bank_profile(bank_id, request_context=RC)
+
+    # Generate real embeddings so semantic retrieval works
+    embeddings = await embedding_utils.generate_embeddings_batch(
+        engine.embeddings,
+        ["the cat sat on the mat", "dogs are loyal animals", "birds can fly in the sky"],
+    )
+
+    def _to_str(emb: list[float]) -> str:
+        return "[" + ",".join(str(v) for v in emb) + "]"
+
+    pool = await engine._get_pool()
+    async with pool.acquire() as conn:
+        await _insert_fact(
+            conn, fact_id=ID_OLD, text="the cat sat on the mat",
+            bank_id=bank_id, embedding_str=_to_str(embeddings[0]),
+            created_at=T1, updated_at=T1,
+        )
+        await _insert_fact(
+            conn, fact_id=ID_MID, text="dogs are loyal animals",
+            bank_id=bank_id, embedding_str=_to_str(embeddings[1]),
+            created_at=T2, updated_at=T2,
+        )
+        await _insert_fact(
+            conn, fact_id=ID_NEW, text="birds can fly in the sky",
+            bank_id=bank_id, embedding_str=_to_str(embeddings[2]),
+            created_at=T3, updated_at=T3,
+        )
+
+    yield engine, bank_id
+
+    await engine.delete_bank(bank_id, request_context=RC)
+
+
+def _result_ids(result) -> set[str]:
+    return {str(r.id) for r in result.results}
+
+
+class TestRecallTimeRange:
+    """Verify created_after / created_before filtering at the recall level."""
+
+    async def test_no_filter_returns_all(self, seeded_memory):
+        engine, bank_id = seeded_memory
+        result = await engine.recall_async(
+            bank_id=bank_id, query="animals and nature",
+            request_context=RC, max_tokens=10000,
+        )
+        ids = _result_ids(result)
+        assert ID_OLD in ids
+        assert ID_MID in ids
+        assert ID_NEW in ids
+
+    async def test_created_after_excludes_old(self, seeded_memory):
+        """created_after=T1 excludes fact-old (updated_at == T1, not > T1)."""
+        engine, bank_id = seeded_memory
+        result = await engine.recall_async(
+            bank_id=bank_id, query="animals and nature",
+            request_context=RC, max_tokens=10000,
+            created_after=T1,
+        )
+        ids = _result_ids(result)
+        assert ID_OLD not in ids, "fact-old (updated_at=T1) must be excluded by created_after=T1"
+        assert ID_MID in ids
+        assert ID_NEW in ids
+
+    async def test_created_after_excludes_old_and_mid(self, seeded_memory):
+        """created_after=T2 returns only fact-new."""
+        engine, bank_id = seeded_memory
+        result = await engine.recall_async(
+            bank_id=bank_id, query="animals and nature",
+            request_context=RC, max_tokens=10000,
+            created_after=T2,
+        )
+        ids = _result_ids(result)
+        assert ID_OLD not in ids
+        assert ID_MID not in ids, "fact-mid (updated_at=T2) must be excluded by created_after=T2"
+        assert ID_NEW in ids
+
+    async def test_created_before_excludes_new(self, seeded_memory):
+        """created_before=T3 excludes fact-new (updated_at == T3, not < T3)."""
+        engine, bank_id = seeded_memory
+        result = await engine.recall_async(
+            bank_id=bank_id, query="animals and nature",
+            request_context=RC, max_tokens=10000,
+            created_before=T3,
+        )
+        ids = _result_ids(result)
+        assert ID_OLD in ids
+        assert ID_MID in ids
+        assert ID_NEW not in ids, "fact-new (updated_at=T3) must be excluded by created_before=T3"
+
+    async def test_created_before_excludes_mid_and_new(self, seeded_memory):
+        """created_before=T2 returns only fact-old."""
+        engine, bank_id = seeded_memory
+        result = await engine.recall_async(
+            bank_id=bank_id, query="animals and nature",
+            request_context=RC, max_tokens=10000,
+            created_before=T2,
+        )
+        ids = _result_ids(result)
+        assert ID_OLD in ids
+        assert ID_MID not in ids
+        assert ID_NEW not in ids
+
+    async def test_range_both_bounds(self, seeded_memory):
+        """created_after=T1, created_before=T3 returns only fact-mid."""
+        engine, bank_id = seeded_memory
+        result = await engine.recall_async(
+            bank_id=bank_id, query="animals and nature",
+            request_context=RC, max_tokens=10000,
+            created_after=T1, created_before=T3,
+        )
+        ids = _result_ids(result)
+        assert ID_OLD not in ids
+        assert ID_MID in ids, "fact-mid (T2) must be in range (T1, T3)"
+        assert ID_NEW not in ids
+
+    async def test_empty_range_returns_nothing(self, seeded_memory):
+        """A range after all facts returns empty results."""
+        engine, bank_id = seeded_memory
+        result = await engine.recall_async(
+            bank_id=bank_id, query="animals and nature",
+            request_context=RC, max_tokens=10000,
+            created_after=T3,
+        )
+        assert len(result.results) == 0, f"Expected no results after T3, got: {_result_ids(result)}"
+
+    async def test_updated_at_catches_consolidation_updates(self, seeded_memory):
+        """A fact created at T1 but updated at T3 appears with created_after=T2."""
+        engine, bank_id = seeded_memory
+
+        # Simulate consolidation updating fact-old
+        pool = await engine._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute(
+                "UPDATE memory_units SET updated_at = $1 WHERE id = $2",
+                T3, ID_OLD,
+            )
+
+        result = await engine.recall_async(
+            bank_id=bank_id, query="animals and nature",
+            request_context=RC, max_tokens=10000,
+            created_after=T2,
+        )
+        ids = _result_ids(result)
+        assert ID_OLD in ids, (
+            "fact-old created at T1, updated at T3 — created_after=T2 must find it via updated_at"
+        )
+        assert ID_NEW in ids

hindsight-api-slim/tests/test_reflect_agent.py

@@ -396,6 +396,88 @@ class TestReflectAgentMocked:
         # Verify recall was actually called (normalization worked)
         mock_functions["recall_fn"].assert_called_once()
 
+    @pytest.mark.asyncio
+    async def test_short_circuit_answer_is_capped_by_max_tokens(self, mock_llm, mock_functions):
+        """When the LLM short-circuits (returns text without calling a tool) and the text
+        exceeds max_tokens, the agent must rewrite it through a capped call so the final
+        user-visible answer respects the configured limit.
+        """
+        # Build a long response that's well over the cap in cl100k_base tokens.
+        long_answer = " ".join(
+            [
+                "This is a detailed paragraph about the team, their roles, and their recurring meetings."
+            ]
+            * 80
+        )
+        # The short-circuit path: tool_calls empty, content populated.
+        mock_llm.call_with_tools.return_value = LLMToolCallResult(
+            tool_calls=[],
+            content=long_answer,
+            finish_reason="stop",
+            input_tokens=10,
+            output_tokens=500,
+        )
+        mock_llm.call = AsyncMock(
+            return_value=(
+                "Short rewritten answer.",
+                TokenUsage(input_tokens=50, output_tokens=10, total_tokens=60),
+            )
+        )
+
+        cap = 50
+        result = await run_reflect_agent(
+            llm_config=mock_llm,
+            bank_id="test-bank",
+            query="test query",
+            bank_profile={"name": "Test", "mission": "Testing"},
+            max_tokens=cap,
+            **mock_functions,
+        )
+
+        # The rewrite call must have been made, and it must carry the cap.
+        assert mock_llm.call.await_count == 1, (
+            f"expected exactly one capped rewrite call, got {mock_llm.call.await_count}"
+        )
+        rewrite_kwargs = mock_llm.call.await_args.kwargs
+        assert rewrite_kwargs.get("max_completion_tokens") == cap, (
+            f"rewrite call should use max_completion_tokens={cap}, "
+            f"got {rewrite_kwargs.get('max_completion_tokens')}"
+        )
+
+        # The final answer is the rewritten text, not the oversized original.
+        assert result.text == "Short rewritten answer."
+
+        # The trace records the rewrite step so we can see it was invoked.
+        assert any(entry.scope == "final_rewrite" for entry in result.llm_trace), (
+            f"llm_trace should include a final_rewrite entry, got {result.llm_trace}"
+        )
+
+    @pytest.mark.asyncio
+    async def test_short_circuit_answer_under_cap_is_not_rewritten(self, mock_llm, mock_functions):
+        """If the short-circuit answer already fits within max_tokens, no extra rewrite
+        call should happen — we don't want to pay for a second LLM call in the common case.
+        """
+        short_answer = "Small answer that already fits."
+        mock_llm.call_with_tools.return_value = LLMToolCallResult(
+            tool_calls=[],
+            content=short_answer,
+            finish_reason="stop",
+            input_tokens=10,
+            output_tokens=8,
+        )
+
+        result = await run_reflect_agent(
+            llm_config=mock_llm,
+            bank_id="test-bank",
+            query="test query",
+            bank_profile={"name": "Test", "mission": "Testing"},
+            max_tokens=200,
+            **mock_functions,
+        )
+
+        assert result.text == short_answer
+        mock_llm.call.assert_not_called()
+
     @pytest.mark.asyncio
     async def test_max_iterations_reached(self, mock_llm, mock_functions):
         """Test that agent stops after max iterations even with errors."""
@@ -573,6 +655,52 @@ class TestContextOverflowBehavior:
         mock_llm.call.assert_called_once()
 
 
+class TestDirectiveLeakageOnEmptyBank:
+    """Test that directives don't leak into the answer when the bank has no data.
+
+    Uses a real LLM to verify the behaviour end-to-end.
+    """
+
+    @pytest.mark.asyncio
+    async def test_directive_not_echoed_on_empty_bank(self, memory, request_context):
+        """When a bank has a directive but zero memories, reflect must NOT
+        parrot the directive text back as its answer.
+        """
+        import uuid
+
+        directive_text = (
+            "When making SEO or content decisions, prefer observed performance data "
+            "over industry best practices. Always check the Content Performance page "
+            "before recommending a format or approach."
+        )
+
+        bank_id = f"test-directive-leak-{uuid.uuid4().hex[:8]}"
+        try:
+            # Ensure bank exists (auto-creates it), but retain nothing.
+            await memory.get_bank_profile(bank_id, request_context=request_context)
+
+            await memory.create_directive(
+                bank_id=bank_id,
+                name="SEO Directive",
+                content=directive_text,
+                request_context=request_context,
+            )
+
+            result = await memory.reflect_async(
+                bank_id=bank_id,
+                query="What content strategy should we use?",
+                request_context=request_context,
+            )
+
+            # The directive content must NOT leak into the answer.
+            assert directive_text not in result.text, (
+                f"Directive content leaked into the answer verbatim. "
+                f"Got: {result.text!r}"
+            )
+        finally:
+            await memory.delete_bank(bank_id, request_context=request_context)
+
+
 class TestContextOverflowIntegration:
     """Integration test: real LLM with a very small max_context_tokens.
 

hindsight-api-slim/tests/test_retain.py

@@ -1,6 +1,7 @@
 """
 Test retain function and chunk storage.
 """
+import asyncio
 import logging
 from datetime import datetime, timedelta, timezone
 
@@ -378,6 +379,7 @@ async def test_mentioned_at_vs_occurred(memory, request_context):
             content="Alice graduated from MIT in March 2020.",
             context="education history",
             event_date=conversation_date,  # When this conversation happened
+            fact_type_override="world",
             request_context=request_context,
         )
 
@@ -1120,6 +1122,57 @@ async def test_document_upsert_behavior(memory, request_context):
         await memory.delete_bank(bank_id, request_context=request_context)
 
 
+@pytest.mark.asyncio
+async def test_document_upsert_preserves_created_at(memory, request_context):
+    """Re-ingesting a document keeps the original created_at; updated_at advances."""
+    bank_id = f"test_upsert_ts_{datetime.now(timezone.utc).timestamp()}"
+    document_id = "timestamp_doc"
+
+    try:
+        await memory.retain_async(
+            bank_id=bank_id,
+            content="Initial content about the project.",
+            document_id=document_id,
+            request_context=request_context,
+        )
+
+        async with memory._pool.acquire() as conn:
+            v1_row = await conn.fetchrow(
+                "SELECT created_at, updated_at FROM documents WHERE id = $1 AND bank_id = $2",
+                document_id,
+                bank_id,
+            )
+        assert v1_row is not None
+        v1_created = v1_row["created_at"]
+        v1_updated = v1_row["updated_at"]
+
+        # Small delay so updated_at can advance visibly
+        await asyncio.sleep(1.1)
+
+        await memory.retain_async(
+            bank_id=bank_id,
+            content="Updated content about the project, with more detail.",
+            document_id=document_id,
+            request_context=request_context,
+        )
+
+        async with memory._pool.acquire() as conn:
+            v2_row = await conn.fetchrow(
+                "SELECT created_at, updated_at FROM documents WHERE id = $1 AND bank_id = $2",
+                document_id,
+                bank_id,
+            )
+        assert v2_row is not None
+        assert v2_row["created_at"] == v1_created, (
+            f"created_at should be preserved across upsert (was {v1_created}, now {v2_row['created_at']})"
+        )
+        assert v2_row["updated_at"] > v1_updated, (
+            f"updated_at should advance on upsert (was {v1_updated}, now {v2_row['updated_at']})"
+        )
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
 # ============================================================
 # Chunk Storage Advanced Tests
 # ============================================================
@@ -1149,6 +1202,7 @@ async def test_chunk_fact_mapping(memory, request_context):
             content=content,
             context="technical documentation",
             document_id=document_id,
+            fact_type_override="world",
             request_context=request_context,
         )
 

hindsight-api-slim/tests/test_retain_orchestrator_mapping.py

@@ -0,0 +1,117 @@
+"""Unit tests for retain orchestrator mapping and embeddings length guarantee.
+
+Regression coverage for issue #1037: a silent length mismatch between the
+extracted facts and the generated embeddings caused
+`_map_results_to_contents` to raise IndexError during batch_retain.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime
+from unittest.mock import MagicMock
+
+import pytest
+
+from hindsight_api.engine.retain import embedding_utils
+from hindsight_api.engine.retain.orchestrator import _map_results_to_contents
+from hindsight_api.engine.retain.types import ProcessedFact, RetainContent
+
+
+def _make_processed_fact(content_index: int, text: str = "fact") -> ProcessedFact:
+    return ProcessedFact(
+        fact_text=text,
+        fact_type="world",
+        embedding=[0.0, 0.0, 0.0],
+        occurred_start=None,
+        occurred_end=None,
+        mentioned_at=datetime(2026, 1, 1),
+        context="",
+        metadata={},
+        content_index=content_index,
+    )
+
+
+def _make_content(text: str = "x") -> RetainContent:
+    return RetainContent(content=text)
+
+
+class TestMapResultsToContents:
+    def test_groups_unit_ids_by_content_index(self):
+        contents = [_make_content("a"), _make_content("b"), _make_content("c")]
+        processed = [
+            _make_processed_fact(0, "a1"),
+            _make_processed_fact(0, "a2"),
+            _make_processed_fact(2, "c1"),
+        ]
+        unit_ids = ["u-a1", "u-a2", "u-c1"]
+
+        result = _map_results_to_contents(contents, processed, unit_ids)
+
+        assert result == [["u-a1", "u-a2"], [], ["u-c1"]]
+
+    def test_handles_out_of_range_content_index(self):
+        contents = [_make_content("a"), _make_content("b")]
+        processed = [
+            _make_processed_fact(-1, "f1"),
+            _make_processed_fact(99, "f2"),
+        ]
+        unit_ids = ["u1", "u2"]
+
+        result = _map_results_to_contents(contents, processed, unit_ids)
+
+        assert result == [["u1"], ["u2"]]
+
+    def test_empty_inputs(self):
+        assert _map_results_to_contents([], [], []) == []
+
+    def test_length_mismatch_raises(self):
+        # Regression for #1037: previously the function silently overran unit_ids.
+        contents = [_make_content("a")]
+        processed = [_make_processed_fact(0), _make_processed_fact(0)]
+        unit_ids = ["u1"]  # one fewer than processed_facts
+
+        with pytest.raises(ValueError, match="length mismatch"):
+            _map_results_to_contents(contents, processed, unit_ids)
+
+    def test_unit_ids_assigned_by_processed_fact_position(self):
+        # Even if processed_facts are interleaved across contents, each unit_id
+        # must follow its corresponding processed_fact (positional alignment).
+        contents = [_make_content("a"), _make_content("b")]
+        processed = [
+            _make_processed_fact(1, "b1"),
+            _make_processed_fact(0, "a1"),
+            _make_processed_fact(1, "b2"),
+        ]
+        unit_ids = ["u-b1", "u-a1", "u-b2"]
+
+        result = _map_results_to_contents(contents, processed, unit_ids)
+
+        assert result == [["u-a1"], ["u-b1", "u-b2"]]
+
+
+class TestEmbeddingsBatchLengthGuarantee:
+    def test_raises_when_backend_returns_fewer_embeddings(self):
+        # Regression for #1037: backends that silently truncate must not pass
+        # through — `zip(extracted_facts, embeddings)` would otherwise drop
+        # facts and break unit_id alignment downstream.
+        backend = MagicMock()
+        backend.encode.return_value = [[0.1, 0.2]]  # only 1 vector for 3 inputs
+
+        with pytest.raises(RuntimeError, match="returned 1 vectors for 3 input texts"):
+            asyncio.run(embedding_utils.generate_embeddings_batch(backend, ["a", "b", "c"]))
+
+    def test_raises_when_backend_returns_more_embeddings(self):
+        backend = MagicMock()
+        backend.encode.return_value = [[0.1], [0.2], [0.3]]
+
+        with pytest.raises(RuntimeError, match="returned 3 vectors for 2 input texts"):
+            asyncio.run(embedding_utils.generate_embeddings_batch(backend, ["a", "b"]))
+
+    def test_passes_through_aligned_embeddings(self):
+        backend = MagicMock()
+        backend.encode.return_value = [[0.1], [0.2]]
+
+        result = asyncio.run(embedding_utils.generate_embeddings_batch(backend, ["a", "b"]))
+
+        assert result == [[0.1], [0.2]]

hindsight-api-slim/tests/test_structured_doc.py

@@ -0,0 +1,438 @@
+"""Unit tests for the structured document schema, renderer, parser, and
+delta-operation applicator.
+
+These tests are pure-Python (no DB, no LLM) and run fast. They guard the
+mechanical guarantees that the structured-delta architecture relies on:
+
+- Deterministic rendering (same input → same bytes).
+- Round-trip parse → render is stable for canonical markdown.
+- Section IDs are stable slugs and survive disambiguation.
+- Operations target sections/blocks by id/index and never silently corrupt
+  the document; invalid ops are dropped, not applied half-way.
+- Sections and blocks not mentioned by any op come through byte-identical.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from hindsight_api.engine.reflect.delta_ops import (
+    AddSectionOp,
+    AppendBlockOp,
+    DeltaOperationList,
+    InsertBlockOp,
+    RemoveBlockOp,
+    RemoveSectionOp,
+    RenameSectionOp,
+    ReplaceBlockOp,
+    ReplaceSectionBlocksOp,
+    apply_operations,
+)
+from hindsight_api.engine.reflect.structured_doc import (
+    BulletListBlock,
+    CodeBlock,
+    OrderedListBlock,
+    ParagraphBlock,
+    Section,
+    StructuredDocument,
+    make_unique_id,
+    parse_markdown,
+    render_block,
+    render_document,
+    render_section,
+    slugify_heading,
+)
+
+# Helpers --------------------------------------------------------------------
+
+
+def _team_overview_doc() -> StructuredDocument:
+    return StructuredDocument(
+        sections=[
+            Section(
+                id="team-overview",
+                heading="Team Overview",
+                level=1,
+                blocks=[ParagraphBlock(text="Quick summary of the engineering team.")],
+            ),
+            Section(
+                id="members",
+                heading="Members",
+                level=2,
+                blocks=[
+                    BulletListBlock(
+                        items=[
+                            "**Alice** — team lead, owns planning.",
+                            "**Bob** — senior engineer, mentors juniors.",
+                        ]
+                    )
+                ],
+            ),
+            Section(
+                id="cadence",
+                heading="Cadence",
+                level=2,
+                blocks=[ParagraphBlock(text="Standups happen daily at 9am.")],
+            ),
+        ]
+    )
+
+
+# Slug ----------------------------------------------------------------------
+
+
+class TestSlugify:
+    def test_basic(self):
+        assert slugify_heading("Purpose") == "purpose"
+
+    def test_multi_word(self):
+        assert slugify_heading("Stop Conditions") == "stop-conditions"
+
+    def test_punctuation_collapses(self):
+        assert slugify_heading("Inputs / Context !") == "inputs-context"
+
+    def test_unicode_falls_back(self):
+        # Non-ASCII chars are stripped; if nothing remains, slug becomes "section".
+        assert slugify_heading("???") == "section"
+
+    def test_make_unique_id_no_collision(self):
+        assert make_unique_id("rules", set()) == "rules"
+
+    def test_make_unique_id_collision(self):
+        assert make_unique_id("rules", {"rules"}) == "rules-2"
+        assert make_unique_id("rules", {"rules", "rules-2"}) == "rules-3"
+
+
+# Renderer ------------------------------------------------------------------
+
+
+class TestRenderer:
+    def test_paragraph(self):
+        assert render_block(ParagraphBlock(text="hello world")) == "hello world"
+
+    def test_bullet_list(self):
+        block = BulletListBlock(items=["one", "two"])
+        assert render_block(block) == "- one\n- two"
+
+    def test_ordered_list_uses_sequential_numbering(self):
+        block = OrderedListBlock(items=["one", "two", "three"])
+        assert render_block(block) == "1. one\n2. two\n3. three"
+
+    def test_code_block_with_language(self):
+        block = CodeBlock(language="json", text='{"a": 1}')
+        assert render_block(block) == '```json\n{"a": 1}\n```'
+
+    def test_code_block_no_language(self):
+        block = CodeBlock(text="raw text")
+        assert render_block(block) == "```\nraw text\n```"
+
+    def test_section_heading_level(self):
+        section = Section(
+            id="purpose", heading="Purpose", level=3, blocks=[ParagraphBlock(text="hi")]
+        )
+        assert render_section(section).startswith("### Purpose\n\nhi")
+
+    def test_document_round_trip_is_stable(self):
+        doc = _team_overview_doc()
+        rendered = render_document(doc)
+        # Re-rendering must produce the same bytes.
+        assert render_document(doc) == rendered
+        # Headings, members, cadence all present.
+        assert "# Team Overview" in rendered
+        assert "## Members" in rendered
+        assert "## Cadence" in rendered
+        assert "- **Alice**" in rendered
+        assert "Standups happen daily at 9am" in rendered
+        # Sections separated by exactly one blank line, document ends with newline.
+        assert rendered.endswith("\n")
+        assert "\n\n\n" not in rendered
+
+    def test_empty_document_renders_empty(self):
+        assert render_document(StructuredDocument()) == ""
+
+
+# Parser --------------------------------------------------------------------
+
+
+class TestParser:
+    def test_simple_document(self):
+        markdown = (
+            "# Team Overview\n"
+            "\n"
+            "Quick summary.\n"
+            "\n"
+            "## Members\n"
+            "\n"
+            "- Alice\n"
+            "- Bob\n"
+            "\n"
+            "## Cadence\n"
+            "\n"
+            "Standups daily.\n"
+        )
+        doc = parse_markdown(markdown)
+        assert [s.id for s in doc.sections] == ["team-overview", "members", "cadence"]
+        assert [s.level for s in doc.sections] == [1, 2, 2]
+        assert isinstance(doc.sections[0].blocks[0], ParagraphBlock)
+        assert isinstance(doc.sections[1].blocks[0], BulletListBlock)
+        assert doc.sections[1].blocks[0].items == ["Alice", "Bob"]
+
+    def test_horizontal_rule_treated_as_blank(self):
+        markdown = "## Rules\n\n- one\n\n---\n\n## Stop\n\nstop here.\n"
+        doc = parse_markdown(markdown)
+        assert [s.id for s in doc.sections] == ["rules", "stop"]
+        # Horizontal rule must NOT become a paragraph.
+        assert all(
+            not (isinstance(b, ParagraphBlock) and "---" in b.text)
+            for s in doc.sections
+            for b in s.blocks
+        )
+
+    def test_ordered_list(self):
+        markdown = "## Steps\n\n1. one\n2. two\n3. three\n"
+        doc = parse_markdown(markdown)
+        block = doc.sections[0].blocks[0]
+        assert isinstance(block, OrderedListBlock)
+        assert block.items == ["one", "two", "three"]
+
+    def test_code_block(self):
+        markdown = '## Example\n\n```json\n{"a": 1}\n```\n'
+        doc = parse_markdown(markdown)
+        block = doc.sections[0].blocks[0]
+        assert isinstance(block, CodeBlock)
+        assert block.language == "json"
+        assert block.text == '{"a": 1}'
+
+    def test_implicit_overview_when_content_before_first_heading(self):
+        markdown = "preamble paragraph.\n\n## Members\n\n- Alice\n"
+        doc = parse_markdown(markdown)
+        assert doc.sections[0].id == "overview"
+        assert isinstance(doc.sections[0].blocks[0], ParagraphBlock)
+        assert doc.sections[1].id == "members"
+
+    def test_duplicate_headings_get_unique_ids(self):
+        markdown = "## Notes\n\nfirst.\n\n## Notes\n\nsecond.\n"
+        doc = parse_markdown(markdown)
+        assert [s.id for s in doc.sections] == ["notes", "notes-2"]
+
+    def test_round_trip_via_render(self):
+        original = _team_overview_doc()
+        markdown = render_document(original)
+        roundtripped = parse_markdown(markdown)
+        # Re-render must match the original render exactly.
+        assert render_document(roundtripped) == markdown
+
+
+# Operation applicator ------------------------------------------------------
+
+
+class TestApplyOperations:
+    def test_zero_ops_returns_identical_document(self):
+        doc = _team_overview_doc()
+        result = apply_operations(doc, [])
+        assert result.document.model_dump() == doc.model_dump()
+        assert render_document(result.document) == render_document(doc)
+        assert result.applied == []
+        assert result.changed is False
+
+    def test_unknown_section_op_is_skipped(self):
+        doc = _team_overview_doc()
+        op = AppendBlockOp(section_id="does-not-exist", block=ParagraphBlock(text="x"))
+        result = apply_operations(doc, [op])
+        assert result.applied == []
+        assert len(result.skipped) == 1
+        assert "unknown section_id" in result.skipped[0]["reason"]
+        # Document unchanged.
+        assert render_document(result.document) == render_document(doc)
+
+    def test_append_block_to_existing_section(self):
+        doc = _team_overview_doc()
+        op = AppendBlockOp(
+            section_id="members",
+            block=BulletListBlock(items=["**Carol** — junior engineer."]),
+        )
+        result = apply_operations(doc, [op])
+        members = result.document.section_by_id("members")
+        assert members is not None
+        assert len(members.blocks) == 2  # original list + new bullet block
+        # Other sections byte-identical
+        original = doc.model_dump()
+        new = result.document.model_dump()
+        assert new["sections"][0] == original["sections"][0]  # team-overview
+        assert new["sections"][2] == original["sections"][2]  # cadence
+
+    def test_insert_block_at_index(self):
+        doc = _team_overview_doc()
+        op = InsertBlockOp(
+            section_id="members",
+            index=0,
+            block=ParagraphBlock(text="Roster as of 2026:"),
+        )
+        result = apply_operations(doc, [op])
+        members = result.document.section_by_id("members")
+        assert isinstance(members.blocks[0], ParagraphBlock)
+        assert members.blocks[0].text.startswith("Roster")
+
+    def test_insert_block_out_of_range_skipped(self):
+        doc = _team_overview_doc()
+        op = InsertBlockOp(
+            section_id="members", index=99, block=ParagraphBlock(text="x")
+        )
+        result = apply_operations(doc, [op])
+        assert result.applied == []
+        assert "index out of range" in result.skipped[0]["reason"]
+
+    def test_replace_block(self):
+        doc = _team_overview_doc()
+        op = ReplaceBlockOp(
+            section_id="cadence",
+            index=0,
+            block=ParagraphBlock(text="Standups happen daily at 10am."),
+        )
+        result = apply_operations(doc, [op])
+        cadence = result.document.section_by_id("cadence")
+        assert isinstance(cadence.blocks[0], ParagraphBlock)
+        assert cadence.blocks[0].text.endswith("10am.")
+
+    def test_remove_block(self):
+        doc = _team_overview_doc()
+        op = RemoveBlockOp(section_id="members", index=0)
+        result = apply_operations(doc, [op])
+        members = result.document.section_by_id("members")
+        assert members.blocks == []
+
+    def test_add_section_at_end(self):
+        doc = _team_overview_doc()
+        op = AddSectionOp(
+            heading="Open Questions",
+            blocks=[ParagraphBlock(text="None right now.")],
+        )
+        result = apply_operations(doc, [op])
+        assert result.document.sections[-1].id == "open-questions"
+        assert result.document.sections[-1].heading == "Open Questions"
+
+    def test_add_section_after_existing(self):
+        doc = _team_overview_doc()
+        op = AddSectionOp(
+            heading="Charter",
+            after_section_id="team-overview",
+            blocks=[ParagraphBlock(text="Mission statement.")],
+        )
+        result = apply_operations(doc, [op])
+        ids = [s.id for s in result.document.sections]
+        assert ids == ["team-overview", "charter", "members", "cadence"]
+
+    def test_add_section_after_unknown_skipped(self):
+        doc = _team_overview_doc()
+        op = AddSectionOp(
+            heading="Charter",
+            after_section_id="nope",
+            blocks=[],
+        )
+        result = apply_operations(doc, [op])
+        assert result.applied == []
+        assert "unknown after_section_id" in result.skipped[0]["reason"]
+
+    def test_add_section_with_id_collision_disambiguates(self):
+        doc = _team_overview_doc()
+        op = AddSectionOp(heading="Members", blocks=[])
+        result = apply_operations(doc, [op])
+        # Two sections with heading "Members": the new one gets "members-2".
+        ids = [s.id for s in result.document.sections]
+        assert "members" in ids
+        assert "members-2" in ids
+
+    def test_remove_section(self):
+        doc = _team_overview_doc()
+        op = RemoveSectionOp(section_id="cadence")
+        result = apply_operations(doc, [op])
+        assert [s.id for s in result.document.sections] == ["team-overview", "members"]
+
+    def test_replace_section_blocks_preserves_id_and_heading(self):
+        doc = _team_overview_doc()
+        op = ReplaceSectionBlocksOp(
+            section_id="members",
+            blocks=[ParagraphBlock(text="See the org chart.")],
+        )
+        result = apply_operations(doc, [op])
+        members = result.document.section_by_id("members")
+        assert members.heading == "Members"
+        assert members.id == "members"
+        assert len(members.blocks) == 1
+        assert isinstance(members.blocks[0], ParagraphBlock)
+
+    def test_rename_section_keeps_id(self):
+        doc = _team_overview_doc()
+        op = RenameSectionOp(section_id="cadence", new_heading="Operating Cadence")
+        result = apply_operations(doc, [op])
+        section = result.document.section_by_id("cadence")
+        assert section.heading == "Operating Cadence"
+        # ID stable so future ops still resolve.
+        assert section.id == "cadence"
+
+    def test_unmodified_sections_byte_identical_in_render(self):
+        """The structural guarantee: sections not touched by any op render
+        identically character-for-character.
+        """
+        doc = _team_overview_doc()
+        op = AppendBlockOp(
+            section_id="members",
+            block=ParagraphBlock(text="New: Carol joined as junior engineer."),
+        )
+        result = apply_operations(doc, [op])
+        before_overview = render_section(doc.section_by_id("team-overview"))
+        after_overview = render_section(
+            result.document.section_by_id("team-overview")
+        )
+        before_cadence = render_section(doc.section_by_id("cadence"))
+        after_cadence = render_section(
+            result.document.section_by_id("cadence")
+        )
+        assert before_overview == after_overview
+        assert before_cadence == after_cadence
+
+
+class TestDeltaOperationListSchema:
+    """Sanity-check that the discriminated-union schema serialises as the LLM
+    will see it: each op has a literal ``op`` string that picks the variant.
+    """
+
+    def test_round_trip_via_json(self):
+        ops = DeltaOperationList(
+            operations=[
+                AppendBlockOp(
+                    section_id="members",
+                    block=ParagraphBlock(text="hi"),
+                ),
+                AddSectionOp(
+                    heading="Open Questions",
+                    after_section_id="cadence",
+                    blocks=[ParagraphBlock(text="None.")],
+                ),
+                RemoveSectionOp(section_id="charter"),
+            ]
+        )
+        payload = ops.model_dump_json()
+        roundtripped = DeltaOperationList.model_validate_json(payload)
+        assert len(roundtripped.operations) == 3
+
+    def test_invalid_op_field_rejected(self):
+        with pytest.raises(Exception):  # pydantic ValidationError
+            DeltaOperationList.model_validate(
+                {"operations": [{"op": "not_a_real_op", "section_id": "x"}]}
+            )
+
+    def test_extra_field_rejected(self):
+        with pytest.raises(Exception):
+            DeltaOperationList.model_validate(
+                {
+                    "operations": [
+                        {
+                            "op": "append_block",
+                            "section_id": "members",
+                            "block": {"type": "paragraph", "text": "hi"},
+                            "extra_field": "no",
+                        }
+                    ]
+                }
+            )

hindsight-api-slim/tests/test_tei_cross_encoder.py

@@ -549,6 +549,29 @@ class TestRemoteTEICrossEncoderConfig:
 
         clear_config_cache()  # Clear cache after test
 
+    def test_create_from_env_with_custom_timeout(self):
+        """Test that HINDSIGHT_API_RERANKER_TEI_HTTP_TIMEOUT is respected."""
+        import os
+
+        from hindsight_api.config import clear_config_cache
+        from hindsight_api.engine.cross_encoder import create_cross_encoder_from_env
+
+        with patch.dict(
+            os.environ,
+            {
+                "HINDSIGHT_API_RERANKER_PROVIDER": "tei",
+                "HINDSIGHT_API_RERANKER_TEI_URL": "http://test:9000",
+                "HINDSIGHT_API_RERANKER_TEI_HTTP_TIMEOUT": "120.0",
+            },
+        ):
+            clear_config_cache()
+            encoder = create_cross_encoder_from_env()
+
+            assert isinstance(encoder, RemoteTEICrossEncoder)
+            assert encoder.timeout == 120.0
+
+        clear_config_cache()
+
 
 # ============================================================================
 # TEI Reranker Performance Benchmark Tests

hindsight-api-slim/tests/test_verify_connection_soft_fail.py

@@ -0,0 +1,157 @@
+"""
+Tests that LLM connection verification failures don't crash server startup.
+
+When the LLM provider is unavailable (e.g. 429 quota exhaustion), the server
+should log a warning and continue booting rather than crash-looping.
+See: https://github.com/vectorize-io/hindsight/issues/1147
+"""
+
+import logging
+from unittest.mock import AsyncMock, patch
+
+import pytest
+import pytest_asyncio
+
+from hindsight_api import MemoryEngine
+from hindsight_api.engine.task_backend import SyncTaskBackend
+
+
+@pytest_asyncio.fixture(scope="function")
+async def engine_with_failing_llm(pg0_db_url, embeddings, cross_encoder, query_analyzer):
+    """Create a MemoryEngine whose LLM verify_connection raises."""
+    mem = MemoryEngine(
+        db_url=pg0_db_url,
+        memory_llm_provider="mock",
+        memory_llm_api_key="",
+        memory_llm_model="mock",
+        embeddings=embeddings,
+        cross_encoder=cross_encoder,
+        query_analyzer=query_analyzer,
+        pool_min_size=1,
+        pool_max_size=5,
+        run_migrations=False,
+        task_backend=SyncTaskBackend(),
+        skip_llm_verification=False,  # Enable verification — we want to test the soft-fail path
+    )
+    yield mem
+    try:
+        if mem._pool and not mem._pool._closing:
+            await mem.close()
+    except Exception:
+        pass
+
+
+@pytest.mark.asyncio
+async def test_initialize_succeeds_when_verify_connection_fails(
+    engine_with_failing_llm,
+    caplog,
+):
+    """Server should start even if LLM verify_connection raises (e.g. 429)."""
+    engine = engine_with_failing_llm
+
+    # Patch the mock provider's verify_connection to simulate a 429 error
+    with patch.object(
+        engine._llm_config._provider_impl,
+        "verify_connection",
+        new_callable=AsyncMock,
+        side_effect=RuntimeError("429 RESOURCE_EXHAUSTED: Quota exceeded"),
+    ):
+        with caplog.at_level(logging.WARNING):
+            # Should NOT raise — the server boots despite the LLM being unavailable
+            await engine.initialize()
+
+    # Verify the warning was logged
+    assert any("LLM connection verification failed" in record.message for record in caplog.records)
+    assert any("429 RESOURCE_EXHAUSTED" in record.message for record in caplog.records)
+
+
+@pytest.mark.asyncio
+async def test_initialize_logs_warning_per_failing_config(
+    pg0_db_url,
+    embeddings,
+    cross_encoder,
+    query_analyzer,
+    caplog,
+):
+    """Each distinct LLM config that fails verification gets its own warning."""
+    engine = MemoryEngine(
+        db_url=pg0_db_url,
+        memory_llm_provider="mock",
+        memory_llm_api_key="",
+        memory_llm_model="default-model",
+        retain_llm_provider="mock",
+        retain_llm_model="retain-model",  # Different model → separate verification
+        embeddings=embeddings,
+        cross_encoder=cross_encoder,
+        query_analyzer=query_analyzer,
+        pool_min_size=1,
+        pool_max_size=5,
+        run_migrations=False,
+        task_backend=SyncTaskBackend(),
+        skip_llm_verification=False,
+    )
+
+    # Both default and retain verify_connection will raise
+    with (
+        patch.object(
+            engine._llm_config._provider_impl,
+            "verify_connection",
+            new_callable=AsyncMock,
+            side_effect=RuntimeError("429 quota exceeded for default"),
+        ),
+        patch.object(
+            engine._retain_llm_config._provider_impl,
+            "verify_connection",
+            new_callable=AsyncMock,
+            side_effect=RuntimeError("429 quota exceeded for retain"),
+        ),
+    ):
+        with caplog.at_level(logging.WARNING):
+            await engine.initialize()
+
+    warning_messages = [r.message for r in caplog.records if "LLM connection verification failed" in r.message]
+    assert len(warning_messages) == 2
+    assert any("'default'" in msg for msg in warning_messages)
+    assert any("'retain'" in msg for msg in warning_messages)
+
+    try:
+        if engine._pool and not engine._pool._closing:
+            await engine.close()
+    except Exception:
+        pass
+
+
+@pytest.mark.asyncio
+async def test_initialize_succeeds_when_verify_connection_succeeds(
+    pg0_db_url,
+    embeddings,
+    cross_encoder,
+    query_analyzer,
+    caplog,
+):
+    """Verify the happy path still works — no warnings when verification passes."""
+    engine = MemoryEngine(
+        db_url=pg0_db_url,
+        memory_llm_provider="mock",
+        memory_llm_api_key="",
+        memory_llm_model="mock",
+        embeddings=embeddings,
+        cross_encoder=cross_encoder,
+        query_analyzer=query_analyzer,
+        pool_min_size=1,
+        pool_max_size=5,
+        run_migrations=False,
+        task_backend=SyncTaskBackend(),
+        skip_llm_verification=False,
+    )
+
+    with caplog.at_level(logging.WARNING):
+        await engine.initialize()
+
+    assert not any("LLM connection verification failed" in r.message for r in caplog.records)
+
+    try:
+        if engine._pool and not engine._pool._closing:
+            await engine.close()
+    except Exception:
+        pass

hindsight-api/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hindsight-api"
-version = "0.5.1"
+version = "0.5.4"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"

hindsight-cli/.openapi-coverage.toml

@@ -21,7 +21,13 @@
 # Operation-level skips
 # ---------------------------------------------------------------------------
 [skip]
-# (empty — every operation is currently wired)
+# UI-only endpoint powering the control-plane stats chart.
+# Zero-filled bucket arrays don't map to a useful CLI command.
+get_memories_timeseries = "UI-only endpoint for the control plane stats chart"
+
+# UI-only endpoint powering the control-plane entity constellation view.
+# Returns nodes/edges in cytoscape shape; not a useful CLI command.
+get_entity_graph = "UI-only endpoint for the control plane entity constellation"
 
 # ---------------------------------------------------------------------------
 # Per-operation parameter skips

hindsight-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hindsight-cli"
-version = "0.5.1"
+version = "0.5.4"
 edition = "2021"
 authors = ["Hindsight Team"]
 description = "A beautiful CLI for Hindsight - semantic memory system"

hindsight-cli/src/api.rs

@@ -513,7 +513,7 @@ impl ApiClient {
         self.runtime.block_on(async {
             let response = self
                 .client
-                .list_memories(bank_id, limit, offset, q, type_filter, None)
+                .list_memories(bank_id, None, limit, offset, q, type_filter, None)
                 .await?;
             Ok(response.into_inner())
         })
@@ -735,7 +735,7 @@ impl ApiClient {
         self.runtime.block_on(async {
             let response = self
                 .client
-                .get_operation_status(bank_id, operation_id, None)
+                .get_operation_status(bank_id, operation_id, None, None)
                 .await?;
             Ok(response.into_inner())
         })

hindsight-cli/src/commands/mental_model.rs

@@ -118,12 +118,16 @@ pub fn create(
     // behaviour is preserved otherwise.
     let trigger = if trigger_refresh_after_consolidation {
         Some(types::MentalModelTriggerInput {
+            mode: types::Mode::Full,
             refresh_after_consolidation: true,
             exclude_mental_models: false,
             exclude_mental_model_ids: None,
             fact_types: None,
             tag_groups: None,
             tags_match: None,
+            include_chunks: None,
+            recall_max_tokens: None,
+            recall_chunks_max_tokens: None,
         })
     } else {
         None
@@ -192,12 +196,16 @@ pub fn update(
     // Only build a trigger override when the user actually passed the flag;
     // sending None leaves the existing trigger config untouched on the server.
     let trigger = trigger_refresh_after_consolidation.map(|refresh| types::MentalModelTriggerInput {
+        mode: types::Mode::Full,
         refresh_after_consolidation: refresh,
         exclude_mental_models: false,
         exclude_mental_model_ids: None,
         fact_types: None,
         tag_groups: None,
         tags_match: None,
+        include_chunks: None,
+        recall_max_tokens: None,
+        recall_chunks_max_tokens: None,
     });
 
     let request = types::UpdateMentalModelRequest {

hindsight-cli/src/config.rs

@@ -7,6 +7,8 @@ use std::path::PathBuf;
 const DEFAULT_API_URL: &str = "http://localhost:8888";
 const CONFIG_FILE_NAME: &str = "config";
 const CONFIG_DIR_NAME: &str = ".hindsight";
+const PROFILE_DIR_NAME: &str = "cli-profiles";
+const PROFILE_ENV_VAR: &str = "HINDSIGHT_PROFILE";
 
 #[derive(Debug)]
 pub struct Config {
@@ -18,6 +20,7 @@ pub struct Config {
 #[derive(Debug, Clone, PartialEq)]
 pub enum ConfigSource {
     LocalFile,
+    Profile(String),
     Environment,
     Default,
 }
@@ -26,6 +29,7 @@ impl std::fmt::Display for ConfigSource {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             ConfigSource::LocalFile => write!(f, "config file"),
+            ConfigSource::Profile(name) => write!(f, "profile '{}'", name),
             ConfigSource::Environment => write!(f, "environment variable"),
             ConfigSource::Default => write!(f, "default"),
         }
@@ -33,27 +37,43 @@ impl std::fmt::Display for ConfigSource {
 }
 
 impl Config {
-    /// Load configuration with the following priority:
-    /// 1. Environment variable (HINDSIGHT_API_URL, HINDSIGHT_API_KEY) - highest priority, for overrides
-    /// 2. Local config file (~/.hindsight/config.toml)
-    /// 3. Default (http://localhost:8888)
+    /// Load configuration with no explicit profile. See [`Self::load_with_profile`].
     pub fn load() -> Result<Self> {
-        // Load API key from environment (highest priority)
+        Self::load_with_profile(None)
+    }
+
+    /// Load configuration with the following priority:
+    /// 1. Environment variable (HINDSIGHT_API_URL/HINDSIGHT_API_KEY) - highest priority
+    /// 2. Named profile (from `profile_name` arg, else `$HINDSIGHT_PROFILE`)
+    ///    at `~/.hindsight/cli-profiles/<name>.toml`
+    /// 3. Local config file (`~/.hindsight/config`)
+    /// 4. Default (http://localhost:8888)
+    pub fn load_with_profile(profile_name: Option<&str>) -> Result<Self> {
         let env_api_key = env::var("HINDSIGHT_API_KEY").ok();
 
-        // 1. Environment variable takes highest priority (for overrides)
+        // 1. Environment variable takes highest priority
         if let Ok(api_url) = env::var("HINDSIGHT_API_URL") {
             return Self::validate_and_create(api_url, env_api_key, ConfigSource::Environment);
         }
 
-        // 2. Try local config file
+        // 2. Named profile (explicit flag takes precedence over env var)
+        let resolved_profile: Option<String> = profile_name
+            .map(|s| s.to_string())
+            .or_else(|| env::var(PROFILE_ENV_VAR).ok().filter(|s| !s.is_empty()));
+
+        if let Some(name) = resolved_profile {
+            let (api_url, file_api_key) = Self::load_profile(&name)?;
+            let api_key = env_api_key.or(file_api_key);
+            return Self::validate_and_create(api_url, api_key, ConfigSource::Profile(name));
+        }
+
+        // 3. Local config file
         if let Some((api_url, file_api_key)) = Self::load_from_file()? {
-            // Environment api_key takes precedence over file api_key
             let api_key = env_api_key.or(file_api_key);
             return Self::validate_and_create(api_url, api_key, ConfigSource::LocalFile);
         }
 
-        // 3. Fall back to default
+        // 4. Fall back to default
         Self::validate_and_create(DEFAULT_API_URL.to_string(), env_api_key, ConfigSource::Default)
     }
 
@@ -151,6 +171,173 @@ impl Config {
     pub fn api_url(&self) -> &str {
         &self.api_url
     }
+
+    // ---------- profile support ----------
+
+    pub fn profile_dir() -> Option<PathBuf> {
+        Self::config_dir().map(|dir| dir.join(PROFILE_DIR_NAME))
+    }
+
+    pub fn profile_file_path(name: &str) -> Option<PathBuf> {
+        Self::profile_dir().map(|dir| dir.join(format!("{}.toml", name)))
+    }
+
+    /// Load a named profile. Returns (api_url, api_key) or an error if the profile
+    /// file is missing / malformed.
+    pub fn load_profile(name: &str) -> Result<(String, Option<String>)> {
+        validate_profile_name(name)?;
+        let dir = Self::profile_dir()
+            .ok_or_else(|| anyhow::anyhow!("Could not determine home directory"))?;
+        load_profile_from_dir(&dir, name)
+    }
+
+    /// Save a named profile to `~/.hindsight/cli-profiles/<name>.toml`.
+    /// Sets file permissions to 0600 on Unix to protect the API key.
+    pub fn save_profile(name: &str, api_url: &str, api_key: Option<&str>) -> Result<PathBuf> {
+        validate_profile_name(name)?;
+        let dir = Self::profile_dir()
+            .ok_or_else(|| anyhow::anyhow!("Could not determine home directory"))?;
+        save_profile_to_dir(&dir, name, api_url, api_key)
+    }
+
+    /// List profile names (without `.toml` extension), sorted alphabetically.
+    pub fn list_profiles() -> Result<Vec<String>> {
+        let dir = match Self::profile_dir() {
+            Some(d) => d,
+            None => return Ok(vec![]),
+        };
+        list_profiles_in_dir(&dir)
+    }
+
+    /// Delete a named profile. Returns Ok(path) on success. Errors if the profile
+    /// does not exist.
+    pub fn delete_profile(name: &str) -> Result<PathBuf> {
+        validate_profile_name(name)?;
+        let path = Self::profile_file_path(name)
+            .ok_or_else(|| anyhow::anyhow!("Could not determine home directory"))?;
+        if !path.exists() {
+            anyhow::bail!("profile '{}' not found at {}", name, path.display());
+        }
+        fs::remove_file(&path)
+            .with_context(|| format!("Failed to delete profile file: {}", path.display()))?;
+        Ok(path)
+    }
+}
+
+fn load_profile_from_dir(dir: &std::path::Path, name: &str) -> Result<(String, Option<String>)> {
+    let path = dir.join(format!("{}.toml", name));
+    if !path.exists() {
+        anyhow::bail!(
+            "profile '{}' not found at {}; create with: hindsight profile create {} --api-url <url>",
+            name,
+            path.display(),
+            name
+        );
+    }
+
+    let content = fs::read_to_string(&path)
+        .with_context(|| format!("Failed to read profile file: {}", path.display()))?;
+
+    let mut api_url: Option<String> = None;
+    let mut api_key: Option<String> = None;
+
+    for line in content.lines() {
+        if let Some(v) = parse_config_value(line, "api_url") {
+            api_url = Some(v);
+        } else if let Some(v) = parse_config_value(line, "api_key") {
+            api_key = Some(v);
+        }
+    }
+
+    let api_url = api_url.ok_or_else(|| {
+        anyhow::anyhow!(
+            "profile '{}' at {} is missing required 'api_url' field",
+            name,
+            path.display()
+        )
+    })?;
+
+    Ok((api_url, api_key))
+}
+
+fn save_profile_to_dir(
+    dir: &std::path::Path,
+    name: &str,
+    api_url: &str,
+    api_key: Option<&str>,
+) -> Result<PathBuf> {
+    if !api_url.starts_with("http://") && !api_url.starts_with("https://") {
+        anyhow::bail!(
+            "Invalid API URL: {}. Must start with http:// or https://",
+            api_url
+        );
+    }
+
+    if !dir.exists() {
+        fs::create_dir_all(dir)
+            .with_context(|| format!("Failed to create profile directory: {}", dir.display()))?;
+    }
+
+    let path = dir.join(format!("{}.toml", name));
+    let mut content = format!("api_url = \"{}\"\n", api_url);
+    if let Some(key) = api_key {
+        content.push_str(&format!("api_key = \"{}\"\n", key));
+    }
+
+    fs::write(&path, content)
+        .with_context(|| format!("Failed to write profile file: {}", path.display()))?;
+
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        let perms = fs::Permissions::from_mode(0o600);
+        fs::set_permissions(&path, perms).with_context(|| {
+            format!("Failed to set permissions on profile file: {}", path.display())
+        })?;
+    }
+
+    Ok(path)
+}
+
+fn list_profiles_in_dir(dir: &std::path::Path) -> Result<Vec<String>> {
+    if !dir.exists() {
+        return Ok(vec![]);
+    }
+    let mut names: Vec<String> = fs::read_dir(dir)
+        .with_context(|| format!("Failed to read profile directory: {}", dir.display()))?
+        .filter_map(|entry| entry.ok())
+        .filter_map(|entry| {
+            let path = entry.path();
+            if path.extension().and_then(|s| s.to_str()) != Some("toml") {
+                return None;
+            }
+            path.file_stem()
+                .and_then(|s| s.to_str())
+                .map(|s| s.to_string())
+        })
+        .collect();
+    names.sort();
+    Ok(names)
+}
+
+/// Reject empty, path-like, or hidden profile names so they can't escape the
+/// profile directory.
+fn validate_profile_name(name: &str) -> Result<()> {
+    if name.is_empty() {
+        anyhow::bail!("profile name cannot be empty");
+    }
+    if name.starts_with('.')
+        || name.contains('/')
+        || name.contains('\\')
+        || name.contains("..")
+        || name.contains(char::is_whitespace)
+    {
+        anyhow::bail!(
+            "invalid profile name '{}': must not contain path separators, whitespace, or start with '.'",
+            name
+        );
+    }
+    Ok(())
 }
 
 /// Prompt user for API URL interactively
@@ -197,6 +384,104 @@ mod tests {
         assert_eq!(format!("{}", ConfigSource::LocalFile), "config file");
         assert_eq!(format!("{}", ConfigSource::Environment), "environment variable");
         assert_eq!(format!("{}", ConfigSource::Default), "default");
+        assert_eq!(
+            format!("{}", ConfigSource::Profile("prod".to_string())),
+            "profile 'prod'"
+        );
+    }
+
+    fn tempdir() -> PathBuf {
+        let pid = std::process::id();
+        let nanos = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_nanos();
+        let counter = std::sync::atomic::AtomicU64::new(0);
+        let n = counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        let dir = std::env::temp_dir().join(format!("hindsight-cli-test-{}-{}-{}", pid, nanos, n));
+        std::fs::create_dir_all(&dir).unwrap();
+        dir
+    }
+
+    #[test]
+    fn test_validate_profile_name_ok() {
+        assert!(validate_profile_name("prod").is_ok());
+        assert!(validate_profile_name("staging-1").is_ok());
+        assert!(validate_profile_name("openclaw-plugin").is_ok());
+        assert!(validate_profile_name("a_b_c").is_ok());
+    }
+
+    #[test]
+    fn test_validate_profile_name_rejects_unsafe() {
+        assert!(validate_profile_name("").is_err());
+        assert!(validate_profile_name(".hidden").is_err());
+        assert!(validate_profile_name("a/b").is_err());
+        assert!(validate_profile_name("a\\b").is_err());
+        assert!(validate_profile_name("..").is_err());
+        assert!(validate_profile_name("foo bar").is_err());
+    }
+
+    #[test]
+    fn test_save_and_load_profile_roundtrip() {
+        let dir = tempdir();
+        let path = save_profile_to_dir(&dir, "prod", "https://api.example.com", Some("hsk_abc"))
+            .unwrap();
+        assert!(path.exists());
+
+        let (url, key) = load_profile_from_dir(&dir, "prod").unwrap();
+        assert_eq!(url, "https://api.example.com");
+        assert_eq!(key.as_deref(), Some("hsk_abc"));
+
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::PermissionsExt;
+            let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777;
+            assert_eq!(mode, 0o600);
+        }
+    }
+
+    #[test]
+    fn test_save_profile_rejects_invalid_url() {
+        let dir = tempdir();
+        let err = save_profile_to_dir(&dir, "foo", "localhost:8888", None).unwrap_err();
+        assert!(err.to_string().contains("Invalid API URL"));
+    }
+
+    #[test]
+    fn test_load_profile_missing_returns_helpful_error() {
+        let dir = tempdir();
+        let err = load_profile_from_dir(&dir, "nope").unwrap_err().to_string();
+        assert!(err.contains("profile 'nope' not found"));
+        assert!(err.contains("hindsight profile create nope"));
+    }
+
+    #[test]
+    fn test_load_profile_missing_api_url_fails() {
+        let dir = tempdir();
+        let path = dir.join("broken.toml");
+        std::fs::write(&path, "api_key = \"x\"\n").unwrap();
+        let err = load_profile_from_dir(&dir, "broken").unwrap_err().to_string();
+        assert!(err.contains("missing required 'api_url'"));
+    }
+
+    #[test]
+    fn test_list_profiles_returns_sorted_names() {
+        let dir = tempdir();
+        save_profile_to_dir(&dir, "prod", "https://prod.example.com", None).unwrap();
+        save_profile_to_dir(&dir, "dev", "https://dev.example.com", None).unwrap();
+        save_profile_to_dir(&dir, "staging", "https://staging.example.com", None).unwrap();
+        // Non-toml files should be ignored.
+        std::fs::write(dir.join("README"), "hi").unwrap();
+
+        let names = list_profiles_in_dir(&dir).unwrap();
+        assert_eq!(names, vec!["dev", "prod", "staging"]);
+    }
+
+    #[test]
+    fn test_list_profiles_missing_dir_is_empty() {
+        let dir = tempdir().join("nonexistent");
+        let names = list_profiles_in_dir(&dir).unwrap();
+        assert!(names.is_empty());
     }
 
     #[test]

hindsight-cli/src/main.rs

@@ -45,6 +45,12 @@ struct Cli {
     #[arg(short = 'v', long, global = true)]
     verbose: bool,
 
+    /// Named profile to load from ~/.hindsight/cli-profiles/<name>.toml
+    /// (env var HINDSIGHT_PROFILE is used if this flag is omitted).
+    /// Environment variables (HINDSIGHT_API_URL / HINDSIGHT_API_KEY) still override profile values.
+    #[arg(short = 'p', long, global = true, env = "HINDSIGHT_PROFILE")]
+    profile: Option<String>,
+
     #[command(subcommand)]
     command: Commands,
 }
@@ -129,7 +135,7 @@ enum Commands {
 
     /// Configure the CLI (API URL, API key, etc.)
     #[command(
-        after_help = "Configuration priority:\n  1. Environment variables (HINDSIGHT_API_URL, HINDSIGHT_API_KEY) - highest priority\n  2. Config file (~/.hindsight/config)\n  3. Default (http://localhost:8888)"
+        after_help = "Configuration priority:\n  1. Environment variables (HINDSIGHT_API_URL, HINDSIGHT_API_KEY) - highest priority\n  2. Named profile (-p / HINDSIGHT_PROFILE, see 'hindsight profile')\n  3. Config file (~/.hindsight/config)\n  4. Default (http://localhost:8888)"
     )]
     Configure {
         /// API URL to connect to (interactive prompt if not provided)
@@ -139,6 +145,40 @@ enum Commands {
         #[arg(long)]
         api_key: Option<String>,
     },
+
+    /// Manage named connection profiles (~/.hindsight/cli-profiles/<name>.toml)
+    #[command(subcommand)]
+    Profile(ProfileCommands),
+}
+
+#[derive(Subcommand)]
+enum ProfileCommands {
+    /// Create or overwrite a profile
+    Create {
+        /// Profile name (used with -p/--profile or $HINDSIGHT_PROFILE)
+        name: String,
+        /// API URL (required)
+        #[arg(long)]
+        api_url: String,
+        /// API key (optional; stored in profile file with 0600 permissions)
+        #[arg(long)]
+        api_key: Option<String>,
+    },
+    /// List all known profiles
+    List,
+    /// Show the contents of a profile
+    Show {
+        /// Profile name
+        name: String,
+    },
+    /// Delete a profile
+    Delete {
+        /// Profile name
+        name: String,
+        /// Skip confirmation prompt
+        #[arg(short = 'y', long)]
+        yes: bool,
+    },
 }
 
 #[derive(Subcommand)]
@@ -1091,7 +1131,8 @@ enum DirectiveCommands {
 }
 
 fn main() {
-    if let Err(_) = run() {
+    if let Err(e) = run() {
+        ui::print_error(&format!("{:#}", e));
         std::process::exit(1);
     }
 }
@@ -1101,19 +1142,25 @@ fn run() -> Result<()> {
 
     let output_format: OutputFormat = cli.output.into();
     let verbose = cli.verbose;
+    let profile = cli.profile.clone();
 
     // Handle configure command before loading full config (it doesn't need API client)
     if let Commands::Configure { api_url, api_key } = cli.command {
         return handle_configure(api_url, api_key, output_format);
     }
 
+    // Handle profile management commands — no API client required.
+    if let Commands::Profile(cmd) = cli.command {
+        return handle_profile(cmd, output_format);
+    }
+
     // Handle ui command - needs config but not API client
     if let Commands::Ui = cli.command {
-        return handle_ui(output_format);
+        return handle_ui(profile.as_deref(), output_format);
     }
 
     // Load configuration
-    let config = Config::from_env().unwrap_or_else(|e| {
+    let config = Config::load_with_profile(profile.as_deref()).unwrap_or_else(|e| {
         ui::print_error(&format!("Configuration error: {}", e));
         errors::print_config_help();
         std::process::exit(1);
@@ -1130,6 +1177,7 @@ fn run() -> Result<()> {
     // Execute command and handle errors
     let result: Result<()> = match cli.command {
         Commands::Configure { .. } => unreachable!(), // Handled above
+        Commands::Profile(_) => unreachable!(),       // Handled above
         Commands::Ui => unreachable!(),               // Handled above
         Commands::Explore => commands::explore::run(&client),
 
@@ -1875,11 +1923,11 @@ fn handle_configure(
     Ok(())
 }
 
-fn handle_ui(output_format: OutputFormat) -> Result<()> {
+fn handle_ui(profile: Option<&str>, output_format: OutputFormat) -> Result<()> {
     use std::process::Command;
 
     // Load configuration to get the API URL
-    let config = Config::load().unwrap_or_else(|e| {
+    let config = Config::load_with_profile(profile).unwrap_or_else(|e| {
         ui::print_error(&format!("Configuration error: {}", e));
         errors::print_config_help();
         std::process::exit(1);
@@ -1921,3 +1969,118 @@ fn handle_ui(output_format: OutputFormat) -> Result<()> {
 
     Ok(())
 }
+
+fn mask_api_key(key: &str) -> String {
+    if key.len() > 8 {
+        format!("{}...{}", &key[..4], &key[key.len() - 4..])
+    } else {
+        "****".to_string()
+    }
+}
+
+fn handle_profile(cmd: ProfileCommands, output_format: OutputFormat) -> Result<()> {
+    match cmd {
+        ProfileCommands::Create {
+            name,
+            api_url,
+            api_key,
+        } => {
+            let path = Config::save_profile(&name, &api_url, api_key.as_deref())?;
+            if output_format == OutputFormat::Pretty {
+                ui::print_success(&format!("Profile '{}' saved to {}", name, path.display()));
+                println!();
+                println!("  API URL: {}", api_url);
+                if let Some(ref key) = api_key {
+                    println!("  API Key: {}", mask_api_key(key));
+                }
+                println!();
+                println!(
+                    "Use with: hindsight -p {} <command>  (or export HINDSIGHT_PROFILE={})",
+                    name, name
+                );
+            } else {
+                let result = serde_json::json!({
+                    "name": name,
+                    "api_url": api_url,
+                    "api_key_set": api_key.is_some(),
+                    "path": path.display().to_string(),
+                });
+                output::print_output(&result, output_format)?;
+            }
+            Ok(())
+        }
+        ProfileCommands::List => {
+            let names = Config::list_profiles()?;
+            if output_format == OutputFormat::Pretty {
+                if names.is_empty() {
+                    ui::print_info("No profiles found.");
+                    println!();
+                    println!("Create one with: hindsight profile create <name> --api-url <url>");
+                } else {
+                    ui::print_info("Profiles:");
+                    for name in &names {
+                        println!("  • {}", name);
+                    }
+                }
+            } else {
+                output::print_output(&serde_json::json!({ "profiles": names }), output_format)?;
+            }
+            Ok(())
+        }
+        ProfileCommands::Show { name } => {
+            let (api_url, api_key) = Config::load_profile(&name)?;
+            let path = Config::profile_file_path(&name)
+                .map(|p| p.display().to_string())
+                .unwrap_or_default();
+            if output_format == OutputFormat::Pretty {
+                ui::print_info(&format!("Profile '{}'", name));
+                println!();
+                println!("  Path:    {}", path);
+                println!("  API URL: {}", api_url);
+                if let Some(ref key) = api_key {
+                    println!("  API Key: {}", mask_api_key(key));
+                }
+            } else {
+                let result = serde_json::json!({
+                    "name": name,
+                    "path": path,
+                    "api_url": api_url,
+                    "api_key_set": api_key.is_some(),
+                });
+                output::print_output(&result, output_format)?;
+            }
+            Ok(())
+        }
+        ProfileCommands::Delete { name, yes } => {
+            let path = Config::profile_file_path(&name)
+                .ok_or_else(|| anyhow::anyhow!("Could not determine home directory"))?;
+            if !path.exists() {
+                anyhow::bail!("profile '{}' not found at {}", name, path.display());
+            }
+            if !yes && output_format == OutputFormat::Pretty {
+                print!("Delete profile '{}' at {}? [y/N]: ", name, path.display());
+                std::io::Write::flush(&mut std::io::stdout())?;
+                let mut input = String::new();
+                std::io::stdin().read_line(&mut input)?;
+                if !matches!(input.trim().to_lowercase().as_str(), "y" | "yes") {
+                    ui::print_info("Aborted.");
+                    return Ok(());
+                }
+            }
+            let deleted = Config::delete_profile(&name)?;
+            if output_format == OutputFormat::Pretty {
+                ui::print_success(&format!("Deleted profile '{}' ({})", name, deleted.display()));
+            } else {
+                output::print_output(
+                    &serde_json::json!({
+                        "name": name,
+                        "path": deleted.display().to_string(),
+                        "deleted": true,
+                    }),
+                    output_format,
+                )?;
+            }
+            Ok(())
+        }
+    }
+}

hindsight-cli/tests/cli_profile.rs

@@ -0,0 +1,321 @@
+//! End-to-end tests for the `hindsight profile` CRUD subcommands and the
+//! global `-p/--profile` flag.
+//!
+//! These tests do not require a running Hindsight API server: they exercise
+//! the binary against a temporary HOME directory and assert on the profile
+//! files it reads/writes. The only time a command is expected to contact the
+//! API is the `-p` precedence test, which uses `hindsight version` pointed at
+//! a deliberately-unreachable URL so we can assert that the profile value
+//! (not the default `http://localhost:8888`) was picked up from the error
+//! message.
+//!
+//! These integration tests are Unix-only because they rely on overriding
+//! `$HOME` to redirect `dirs::home_dir()` at a tempdir. On Windows
+//! `dirs::home_dir()` resolves via `FOLDERID_Profile` (the Win32 shell API)
+//! and ignores the env var, so running these tests there would pollute the
+//! real user profile directory. The Windows runtime path is still exercised
+//! by the `config::tests::*` unit tests, which drive the path-based
+//! `save_profile_to_dir` / `load_profile_from_dir` helpers directly.
+#![cfg(unix)]
+
+use std::path::PathBuf;
+use std::process::{Command, Output};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+static COUNTER: AtomicU64 = AtomicU64::new(0);
+
+fn unique_tempdir(tag: &str) -> PathBuf {
+    let pid = std::process::id();
+    let nanos = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_nanos();
+    let n = COUNTER.fetch_add(1, Ordering::SeqCst);
+    let dir = std::env::temp_dir().join(format!("hindsight-profile-test-{}-{}-{}-{}", tag, pid, nanos, n));
+    std::fs::create_dir_all(&dir).unwrap();
+    dir
+}
+
+fn hindsight_binary() -> String {
+    std::env::var("CARGO_BIN_EXE_hindsight").unwrap_or_else(|_| {
+        let debug = "./target/debug/hindsight";
+        let release = "./target/release/hindsight";
+        if std::path::Path::new(debug).exists() {
+            debug.to_string()
+        } else if std::path::Path::new(release).exists() {
+            release.to_string()
+        } else {
+            "hindsight".to_string()
+        }
+    })
+}
+
+fn run_with_home(home: &std::path::Path, args: &[&str]) -> Output {
+    Command::new(hindsight_binary())
+        .env("HOME", home)
+        // Unset anything that would bypass the profile/config-file resolution
+        // we're trying to exercise here.
+        .env_remove("HINDSIGHT_API_URL")
+        .env_remove("HINDSIGHT_API_KEY")
+        .env_remove("HINDSIGHT_PROFILE")
+        .args(args)
+        .output()
+        .expect("failed to spawn hindsight binary")
+}
+
+fn assert_success(out: &Output) {
+    if !out.status.success() {
+        panic!(
+            "command failed: status={:?}\n--- stdout ---\n{}\n--- stderr ---\n{}",
+            out.status,
+            String::from_utf8_lossy(&out.stdout),
+            String::from_utf8_lossy(&out.stderr),
+        );
+    }
+}
+
+fn stdout(out: &Output) -> String {
+    String::from_utf8_lossy(&out.stdout).into_owned()
+}
+
+fn stderr(out: &Output) -> String {
+    String::from_utf8_lossy(&out.stderr).into_owned()
+}
+
+#[test]
+fn profile_create_writes_toml_and_json_output() {
+    let home = unique_tempdir("create");
+    let out = run_with_home(
+        &home,
+        &[
+            "--output",
+            "json",
+            "profile",
+            "create",
+            "prod",
+            "--api-url",
+            "https://api.example.com",
+            "--api-key",
+            "hsk_abcdef1234",
+        ],
+    );
+    assert_success(&out);
+
+    let payload: serde_json::Value =
+        serde_json::from_str(&stdout(&out)).expect("expected JSON output");
+    assert_eq!(payload["name"], "prod");
+    assert_eq!(payload["api_url"], "https://api.example.com");
+    assert_eq!(payload["api_key_set"], true);
+
+    let path = home.join(".hindsight/cli-profiles/prod.toml");
+    assert!(path.exists(), "profile file was not created at {}", path.display());
+    let body = std::fs::read_to_string(&path).unwrap();
+    assert!(body.contains("api_url = \"https://api.example.com\""));
+    assert!(body.contains("api_key = \"hsk_abcdef1234\""));
+
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777;
+        assert_eq!(mode, 0o600, "profile file should be mode 0600");
+    }
+}
+
+#[test]
+fn profile_create_rejects_invalid_api_url() {
+    let home = unique_tempdir("invalid-url");
+    let out = run_with_home(
+        &home,
+        &["profile", "create", "foo", "--api-url", "localhost:8888"],
+    );
+    assert!(!out.status.success());
+    assert!(stderr(&out).contains("Invalid API URL"));
+}
+
+#[test]
+fn profile_create_rejects_unsafe_names() {
+    let home = unique_tempdir("unsafe-name");
+    for bad in &["..", ".hidden", "a/b", "a b"] {
+        let out = run_with_home(
+            &home,
+            &["profile", "create", bad, "--api-url", "https://example.com"],
+        );
+        assert!(
+            !out.status.success(),
+            "expected failure for profile name {:?}",
+            bad
+        );
+    }
+}
+
+#[test]
+fn profile_list_returns_sorted_names() {
+    let home = unique_tempdir("list");
+    for name in &["prod", "dev", "staging"] {
+        let out = run_with_home(
+            &home,
+            &[
+                "profile",
+                "create",
+                name,
+                "--api-url",
+                &format!("https://{}.example.com", name),
+            ],
+        );
+        assert_success(&out);
+    }
+
+    let out = run_with_home(&home, &["--output", "json", "profile", "list"]);
+    assert_success(&out);
+    let payload: serde_json::Value = serde_json::from_str(&stdout(&out)).unwrap();
+    let names: Vec<&str> = payload["profiles"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|v| v.as_str().unwrap())
+        .collect();
+    assert_eq!(names, vec!["dev", "prod", "staging"]);
+}
+
+#[test]
+fn profile_list_empty_when_no_profiles() {
+    let home = unique_tempdir("list-empty");
+    let out = run_with_home(&home, &["--output", "json", "profile", "list"]);
+    assert_success(&out);
+    let payload: serde_json::Value = serde_json::from_str(&stdout(&out)).unwrap();
+    assert!(payload["profiles"].as_array().unwrap().is_empty());
+}
+
+#[test]
+fn profile_show_returns_stored_values() {
+    let home = unique_tempdir("show");
+    assert_success(&run_with_home(
+        &home,
+        &[
+            "profile",
+            "create",
+            "prod",
+            "--api-url",
+            "https://api.example.com",
+            "--api-key",
+            "hsk_xyz",
+        ],
+    ));
+
+    let out = run_with_home(&home, &["--output", "json", "profile", "show", "prod"]);
+    assert_success(&out);
+    let payload: serde_json::Value = serde_json::from_str(&stdout(&out)).unwrap();
+    assert_eq!(payload["name"], "prod");
+    assert_eq!(payload["api_url"], "https://api.example.com");
+    assert_eq!(payload["api_key_set"], true);
+}
+
+#[test]
+fn profile_show_missing_profile_errors_with_hint() {
+    let home = unique_tempdir("show-missing");
+    let out = run_with_home(&home, &["profile", "show", "nope"]);
+    assert!(!out.status.success());
+    let err = stderr(&out) + &stdout(&out);
+    assert!(err.contains("profile 'nope' not found"));
+    assert!(err.contains("hindsight profile create nope"));
+}
+
+#[test]
+fn profile_delete_removes_file() {
+    let home = unique_tempdir("delete");
+    assert_success(&run_with_home(
+        &home,
+        &[
+            "profile",
+            "create",
+            "prod",
+            "--api-url",
+            "https://api.example.com",
+        ],
+    ));
+    let path = home.join(".hindsight/cli-profiles/prod.toml");
+    assert!(path.exists());
+
+    let out = run_with_home(&home, &["profile", "delete", "prod", "-y"]);
+    assert_success(&out);
+    assert!(!path.exists(), "profile file should have been removed");
+}
+
+#[test]
+fn profile_delete_missing_errors() {
+    let home = unique_tempdir("delete-missing");
+    let out = run_with_home(&home, &["profile", "delete", "nope", "-y"]);
+    assert!(!out.status.success());
+}
+
+#[test]
+fn p_flag_overrides_config_file_api_url() {
+    // Build a HOME that contains BOTH a legacy ~/.hindsight/config pointing
+    // at URL_A and a named profile pointing at URL_B. Running `hindsight -p`
+    // should pick the profile URL, not the config-file URL — we verify by
+    // looking at the connection-error message (no API server needed).
+    let home = unique_tempdir("precedence");
+    let hindsight_dir = home.join(".hindsight");
+    std::fs::create_dir_all(&hindsight_dir).unwrap();
+    std::fs::write(
+        hindsight_dir.join("config"),
+        "api_url = \"http://127.0.0.1:9/from-config\"\n",
+    )
+    .unwrap();
+
+    assert_success(&run_with_home(
+        &home,
+        &[
+            "profile",
+            "create",
+            "prod",
+            "--api-url",
+            "http://127.0.0.1:9/from-profile",
+        ],
+    ));
+
+    // `version` will fail to connect (port 9 is "discard"), but the error
+    // message echoes the API URL actually used.
+    let out = run_with_home(&home, &["-p", "prod", "version"]);
+    assert!(!out.status.success());
+    let err = stderr(&out) + &stdout(&out);
+    assert!(
+        err.contains("from-profile"),
+        "expected profile URL in output, got:\n{}",
+        err
+    );
+    assert!(
+        !err.contains("from-config"),
+        "config-file URL should not have been used:\n{}",
+        err
+    );
+}
+
+#[test]
+fn hindsight_profile_env_var_is_honored() {
+    let home = unique_tempdir("env-var");
+    assert_success(&run_with_home(
+        &home,
+        &[
+            "profile",
+            "create",
+            "staging",
+            "--api-url",
+            "http://127.0.0.1:9/from-env",
+        ],
+    ));
+
+    // Re-run without `-p` but with HINDSIGHT_PROFILE set.
+    let out = Command::new(hindsight_binary())
+        .env("HOME", &home)
+        .env_remove("HINDSIGHT_API_URL")
+        .env_remove("HINDSIGHT_API_KEY")
+        .env("HINDSIGHT_PROFILE", "staging")
+        .args(["version"])
+        .output()
+        .unwrap();
+    assert!(!out.status.success());
+    let err = String::from_utf8_lossy(&out.stderr).into_owned()
+        + &String::from_utf8_lossy(&out.stdout);
+    assert!(err.contains("from-env"), "expected profile URL in output:\n{}", err);
+}

hindsight-clients/go/api/openapi.yaml

@@ -7,7 +7,7 @@ info:
     name: Apache 2.0
     url: https://www.apache.org/licenses/LICENSE-2.0.html
   title: Hindsight HTTP API
-  version: 0.5.1
+  version: 0.5.4
 servers:
 - url: /
 paths:
@@ -166,6 +166,14 @@ paths:
           nullable: true
           type: string
         style: form
+      - explode: true
+        in: query
+        name: consolidation_state
+        required: false
+        schema:
+          nullable: true
+          type: string
+        style: form
       - explode: true
         in: query
         name: limit
@@ -464,6 +472,53 @@ paths:
       summary: Get statistics for memory bank
       tags:
       - Banks
+  /v1/default/banks/{bank_id}/stats/memories-timeseries:
+    get:
+      description: "Memories ingested over a period, bucketed by time and broken down\
+        \ by fact type."
+      operationId: get_memories_timeseries
+      parameters:
+      - explode: false
+        in: path
+        name: bank_id
+        required: true
+        schema:
+          title: Bank Id
+          type: string
+        style: simple
+      - explode: true
+        in: query
+        name: period
+        required: false
+        schema:
+          default: 7d
+          title: Period
+          type: string
+        style: form
+      - explode: false
+        in: header
+        name: authorization
+        required: false
+        schema:
+          nullable: true
+          type: string
+        style: simple
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/MemoriesTimeseriesResponse'
+          description: Successful Response
+        "422":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      summary: Memory ingestion time-series
+      tags:
+      - Banks
   /v1/default/banks/{bank_id}/entities:
     get:
       description: "List all entities (people, organizations, etc.) known by the bank,\
@@ -524,6 +579,66 @@ paths:
       summary: List entities
       tags:
       - Entities
+  /v1/default/banks/{bank_id}/entities/graph:
+    get:
+      description: Return a graph of entities (nodes) and their co-occurrences (edges)
+        for visualization.
+      operationId: get_entity_graph
+      parameters:
+      - explode: false
+        in: path
+        name: bank_id
+        required: true
+        schema:
+          title: Bank Id
+          type: string
+        style: simple
+      - description: Maximum number of co-occurrence edges to return
+        explode: true
+        in: query
+        name: limit
+        required: false
+        schema:
+          default: 1000
+          description: Maximum number of co-occurrence edges to return
+          title: Limit
+          type: integer
+        style: form
+      - description: Minimum cooccurrence_count to include an edge
+        explode: true
+        in: query
+        name: min_count
+        required: false
+        schema:
+          default: 1
+          description: Minimum cooccurrence_count to include an edge
+          title: Min Count
+          type: integer
+        style: form
+      - explode: false
+        in: header
+        name: authorization
+        required: false
+        schema:
+          nullable: true
+          type: string
+        style: simple
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/EntityGraphResponse'
+          description: Successful Response
+        "422":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      summary: Get entity co-occurrence graph
+      tags:
+      - Entities
   /v1/default/banks/{bank_id}/entities/{entity_id}:
     get:
       description: Get detailed information about an entity including observations
@@ -1781,6 +1896,19 @@ paths:
           title: Operation Id
           type: string
         style: simple
+      - description: Include the raw task payload (submission params) in the response.
+          May be large.
+        explode: true
+        in: query
+        name: include_payload
+        required: false
+        schema:
+          default: false
+          description: Include the raw task payload (submission params) in the response.
+            May be large.
+          title: Include Payload
+          type: boolean
+        style: form
       - explode: false
         in: header
         name: authorization
@@ -3439,6 +3567,7 @@ components:
       description: Response model for bank statistics endpoint.
       example:
         bank_id: user123
+        failed_consolidation: 0
         failed_operations: 0
         last_consolidated_at: 2024-01-15T10:30:00Z
         links_breakdown:
@@ -3500,6 +3629,12 @@ components:
         failed_operations:
           title: Failed Operations
           type: integer
+        operations_by_status:
+          additionalProperties:
+            type: integer
+          description: "Async operations grouped by status (pending, in_progress,\
+            \ completed, failed, cancelled)."
+          title: Operations By Status
         last_consolidated_at:
           nullable: true
           type: string
@@ -3508,6 +3643,12 @@ components:
           description: Number of memories not yet processed into observations
           title: Pending Consolidation
           type: integer
+        failed_consolidation:
+          default: 0
+          description: Number of source memories (world/experience) whose consolidation
+            permanently failed and can be retried via the consolidation recovery endpoint.
+          title: Failed Consolidation
+          type: integer
         total_observations:
           default: 0
           description: Total number of observations
@@ -3576,6 +3717,66 @@ components:
         entities_allow_free_form:
           nullable: true
           type: boolean
+        retain_default_strategy:
+          nullable: true
+          type: string
+        retain_strategies:
+          additionalProperties: {}
+          nullable: true
+        retain_chunk_batch_size:
+          nullable: true
+          type: integer
+        mcp_enabled_tools:
+          items:
+            type: string
+          nullable: true
+          type: array
+        consolidation_llm_batch_size:
+          nullable: true
+          type: integer
+        consolidation_source_facts_max_tokens:
+          nullable: true
+          type: integer
+        consolidation_source_facts_max_tokens_per_observation:
+          nullable: true
+          type: integer
+        max_observations_per_scope:
+          nullable: true
+          type: integer
+        reflect_source_facts_max_tokens:
+          nullable: true
+          type: integer
+        llm_gemini_safety_settings:
+          items: {}
+          nullable: true
+          type: array
+        recall_budget_function:
+          nullable: true
+          type: string
+        recall_budget_fixed_low:
+          nullable: true
+          type: integer
+        recall_budget_fixed_mid:
+          nullable: true
+          type: integer
+        recall_budget_fixed_high:
+          nullable: true
+          type: integer
+        recall_budget_adaptive_low:
+          nullable: true
+          type: number
+        recall_budget_adaptive_mid:
+          nullable: true
+          type: number
+        recall_budget_adaptive_high:
+          nullable: true
+          type: number
+        recall_budget_min:
+          nullable: true
+          type: integer
+        recall_budget_max:
+          nullable: true
+          type: integer
       title: BankTemplateConfig
     BankTemplateDirective:
       description: |-
@@ -4381,6 +4582,58 @@ components:
       - mention_count
       - observations
       title: EntityDetailResponse
+    EntityGraphResponse:
+      description: Response model for entity co-occurrence graph endpoint.
+      example:
+        edges:
+        - data:
+            color: '#ffd700'
+            id: uuid-1-uuid-2
+            lastCooccurred: 2024-02-01T14:00:00Z
+            lineStyle: solid
+            linkType: cooccurrence
+            source: uuid-1
+            target: uuid-2
+            weight: 5
+        limit: 1000
+        nodes:
+        - data:
+            color: '#42a5f5'
+            id: uuid-1
+            label: Alice
+            mentionCount: 12
+        - data:
+            color: '#42a5f5'
+            id: uuid-2
+            label: Google
+            mentionCount: 8
+        total_edges: 1
+        total_entities: 2
+      properties:
+        nodes:
+          items:
+            additionalProperties: {}
+          type: array
+        edges:
+          items:
+            additionalProperties: {}
+          type: array
+        total_entities:
+          title: Total Entities
+          type: integer
+        total_edges:
+          title: Total Edges
+          type: integer
+        limit:
+          title: Limit
+          type: integer
+      required:
+      - edges
+      - limit
+      - nodes
+      - total_edges
+      - total_entities
+      title: EntityGraphResponse
     EntityIncludeOptions:
       description: Options for including entity observations in recall results.
       properties:
@@ -4736,6 +4989,44 @@ components:
       - offset
       - total
       title: ListTagsResponse
+    MemoriesTimeseriesResponse:
+      description: Time-series of memory ingestion bucketed by time and fact type.
+      example:
+        period: period
+        trunc: trunc
+        bank_id: bank_id
+        buckets:
+        - world: 0
+          observation: 1
+          time: time
+          experience: 6
+        - world: 0
+          observation: 1
+          time: time
+          experience: 6
+      properties:
+        bank_id:
+          title: Bank Id
+          type: string
+        period:
+          description: "One of: 1h, 12h, 1d, 7d, 30d, 90d."
+          title: Period
+          type: string
+        trunc:
+          description: "Bucket granularity: minute, hour, day."
+          title: Trunc
+          type: string
+        buckets:
+          description: "Per-bucket counts, always returned fully padded for the requested\
+            \ period."
+          items:
+            $ref: '#/components/schemas/MemoryTimeseriesBucket'
+          type: array
+      required:
+      - bank_id
+      - period
+      - trunc
+      title: MemoriesTimeseriesResponse
     MemoryItem:
       description: Single memory item for retain.
       example:
@@ -4793,6 +5084,36 @@ components:
       required:
       - content
       title: MemoryItem
+    MemoryTimeseriesBucket:
+      description: One bucket in the memory ingestion time-series.
+      example:
+        world: 0
+        observation: 1
+        time: time
+        experience: 6
+      properties:
+        time:
+          description: Bucket start timestamp in ISO-8601 (UTC).
+          title: Time
+          type: string
+        world:
+          default: 0
+          description: World-fact memories ingested in this bucket.
+          title: World
+          type: integer
+        experience:
+          default: 0
+          description: Experience memories ingested in this bucket.
+          title: Experience
+          type: integer
+        observation:
+          default: 0
+          description: Observations recorded in this bucket.
+          title: Observation
+          type: integer
+      required:
+      - time
+      title: MemoryTimeseriesBucket
     MentalModelListResponse:
       description: Response model for listing mental models.
       example:
@@ -4806,7 +5127,9 @@ components:
           created_at: created_at
           id: id
           trigger:
+            mode: full
             refresh_after_consolidation: false
+            recall_chunks_max_tokens: 1
             tag_groups:
             - match: any_strict
               tags:
@@ -4822,9 +5145,12 @@ components:
             exclude_mental_model_ids:
             - exclude_mental_model_ids
             - exclude_mental_model_ids
+            include_chunks: true
             tags_match: any
             exclude_mental_models: false
+            recall_max_tokens: 6
           last_refreshed_at: last_refreshed_at
+          is_stale: true
           content: content
           tags:
           - tags
@@ -4838,7 +5164,9 @@ components:
           created_at: created_at
           id: id
           trigger:
+            mode: full
             refresh_after_consolidation: false
+            recall_chunks_max_tokens: 1
             tag_groups:
             - match: any_strict
               tags:
@@ -4854,9 +5182,12 @@ components:
             exclude_mental_model_ids:
             - exclude_mental_model_ids
             - exclude_mental_model_ids
+            include_chunks: true
             tags_match: any
             exclude_mental_models: false
+            recall_max_tokens: 6
           last_refreshed_at: last_refreshed_at
+          is_stale: true
           content: content
           tags:
           - tags
@@ -4881,7 +5212,9 @@ components:
         created_at: created_at
         id: id
         trigger:
+          mode: full
           refresh_after_consolidation: false
+          recall_chunks_max_tokens: 1
           tag_groups:
           - match: any_strict
             tags:
@@ -4897,9 +5230,12 @@ components:
           exclude_mental_model_ids:
           - exclude_mental_model_ids
           - exclude_mental_model_ids
+          include_chunks: true
           tags_match: any
           exclude_mental_models: false
+          recall_max_tokens: 6
         last_refreshed_at: last_refreshed_at
+        is_stale: true
         content: content
         tags:
         - tags
@@ -4939,6 +5275,9 @@ components:
         reflect_response:
           additionalProperties: {}
           nullable: true
+        is_stale:
+          nullable: true
+          type: boolean
       required:
       - bank_id
       - id
@@ -4947,6 +5286,19 @@ components:
     MentalModelTrigger-Input:
       description: Trigger settings for a mental model.
       properties:
+        mode:
+          default: full
+          description: "Refresh mode. 'full' (default) regenerates the mental model\
+            \ content from scratch on each refresh. 'delta' performs surgical edits\
+            \ against the existing content: unchanged sections are preserved byte-for-byte,\
+            \ stale content is removed, new content is added. If the mental model\
+            \ has no existing content, or if the source_query has changed since the\
+            \ last refresh, delta mode falls back to a full regeneration automatically."
+          enum:
+          - full
+          - delta
+          title: Mode
+          type: string
         refresh_after_consolidation:
           default: false
           description: "If true, refresh this mental model after observations consolidation\
@@ -4986,11 +5338,22 @@ components:
             $ref: '#/components/schemas/MentalModelTrigger_Input_tag_groups_inner'
           nullable: true
           type: array
+        include_chunks:
+          nullable: true
+          type: boolean
+        recall_max_tokens:
+          nullable: true
+          type: integer
+        recall_chunks_max_tokens:
+          nullable: true
+          type: integer
       title: MentalModelTrigger
     MentalModelTrigger-Output:
       description: Trigger settings for a mental model.
       example:
+        mode: full
         refresh_after_consolidation: false
+        recall_chunks_max_tokens: 1
         tag_groups:
         - match: any_strict
           tags:
@@ -5006,9 +5369,24 @@ components:
         exclude_mental_model_ids:
         - exclude_mental_model_ids
         - exclude_mental_model_ids
+        include_chunks: true
         tags_match: any
         exclude_mental_models: false
+        recall_max_tokens: 6
       properties:
+        mode:
+          default: full
+          description: "Refresh mode. 'full' (default) regenerates the mental model\
+            \ content from scratch on each refresh. 'delta' performs surgical edits\
+            \ against the existing content: unchanged sections are preserved byte-for-byte,\
+            \ stale content is removed, new content is added. If the mental model\
+            \ has no existing content, or if the source_query has changed since the\
+            \ last refresh, delta mode falls back to a full regeneration automatically."
+          enum:
+          - full
+          - delta
+          title: Mode
+          type: string
         refresh_after_consolidation:
           default: false
           description: "If true, refresh this mental model after observations consolidation\
@@ -5048,6 +5426,15 @@ components:
             $ref: '#/components/schemas/MentalModelTrigger_Output_tag_groups_inner'
           nullable: true
           type: array
+        include_chunks:
+          nullable: true
+          type: boolean
+        recall_max_tokens:
+          nullable: true
+          type: integer
+        recall_chunks_max_tokens:
+          nullable: true
+          type: integer
       title: MentalModelTrigger
     OperationResponse:
       description: Response model for a single async operation.
@@ -5055,6 +5442,7 @@ components:
         created_at: 2024-01-15T10:30:00Z
         id: 550e8400-e29b-41d4-a716-446655440000
         items_count: 5
+        retry_count: 0
         status: pending
         task_type: retain
       properties:
@@ -5079,6 +5467,12 @@ components:
         error_message:
           nullable: true
           type: string
+        retry_count:
+          nullable: true
+          type: integer
+        next_retry_at:
+          nullable: true
+          type: string
       required:
       - created_at
       - error_message
@@ -5123,6 +5517,12 @@ components:
         error_message:
           nullable: true
           type: string
+        retry_count:
+          nullable: true
+          type: integer
+        next_retry_at:
+          nullable: true
+          type: string
         result_metadata:
           additionalProperties: {}
           nullable: true
@@ -5131,6 +5531,9 @@ components:
             $ref: '#/components/schemas/ChildOperationStatus'
           nullable: true
           type: array
+        task_payload:
+          additionalProperties: {}
+          nullable: true
       required:
       - operation_id
       - status

hindsight-clients/go/api_audit.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.4
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.