feat(agent-plugin): add portable Hindsight plugin for the Agent Plugins standard

Add a vendor-neutral Hindsight plugin conforming to Vercel's Agent Plugins
1.0.0 standard (plugin.json + mcp.json + skills/SKILL.md), so one artifact
gives long-term memory to any compatible client (Codex, Cursor, GitHub
Copilot, Kiro, VS Code) instead of a per-IDE integration. The plugin is a
thin transport wrapper over Hindsight's existing MCP server (retain / recall
/ reflect); a bundled skill teaches the agent when to use it.

Wiring:
- CI: test-agent-plugin-integration job runs the manifest validator, gated on
  hindsight-integrations/agent-plugin/** changes.
- Docs: integrations.json gallery entry + docs-integrations/agent-plugin.md.
- Release: agent-plugin added to release-integration.sh and the changelog
  generator; both learn to read a root-level plugin.json and link the
  changelog to the source tree (git-distributed bundle, no registry package).

.env.example

@@ -35,6 +35,25 @@ HINDSIGHT_API_LLM_BASE_URL=https://api.openai.com/v1
 # Disable it only for those backends; consolidation still enforces the cap.
 # HINDSIGHT_API_LLM_SUPPORTS_MAX_ITEMS=true
 
+# Pin a conversation to one backend prompt cache (OpenAI-compatible providers only).
+# Server-side prompt caches are per backend server, so the same conversation has to
+# reach the same one to hit. Values: auto (default), xai_conv_id (sends xAI's
+# x-grok-conv-id header), openai_prompt_cache_key (sends OpenAI's prompt_cache_key
+# field), none (sends nothing). "auto" picks from the base URL host and is an
+# allowlist: x.ai / grok.com get the header, native OpenAI / openai.com / Azure
+# OpenAI get the field, and every other backend gets nothing. Per-operation
+# overrides take precedence. Set to none to disable entirely.
+# HINDSIGHT_API_LLM_CACHE_AFFINITY=auto
+# HINDSIGHT_API_RETAIN_LLM_CACHE_AFFINITY=none
+# HINDSIGHT_API_REFLECT_LLM_CACHE_AFFINITY=xai_conv_id
+# HINDSIGHT_API_CONSOLIDATION_LLM_CACHE_AFFINITY=none
+
+# Ask litellm/litellmrouter/bedrock for structured output via a forced tool call
+# instead of response_format. Enable it for backends that reject response_format
+# outright -- e.g. Bedrock Claude in ap-southeast-2 ("Extra inputs are not permitted");
+# the same model in us-east-1 accepts response_format and needs nothing here.
+# HINDSIGHT_API_LLM_STRUCTURED_OUTPUT_FORCED_TOOL=false
+
 # Diagnostic: on any LLM 4xx, log the exact assembled request ([LLM_4XX_DUMP]) --
 # serialized request config (message bodies stripped) + capped per-message previews.
 # For debugging otherwise-unreproducible rejected calls. Off by default.

.github/star-history/data.json

@@ -1000,6 +1000,22 @@
     {
       "date": "2026-08-07",
       "stars": 19183
+    },
+    {
+      "date": "2026-08-08",
+      "stars": 19271
+    },
+    {
+      "date": "2026-08-09",
+      "stars": 19334
+    },
+    {
+      "date": "2026-08-10",
+      "stars": 19414
+    },
+    {
+      "date": "2026-08-11",
+      "stars": 19506
     }
   ]
 }

.github/workflows/test.yml

@@ -25,7 +25,7 @@ jobs:
       cli: ${{ steps.filter.outputs.cli }}
       docker: ${{ steps.filter.outputs.docker }}
       helm: ${{ steps.filter.outputs.helm }}
-      docs: ${{ steps.filter.outputs.docs }}
+      doc-examples: ${{ steps.filter.outputs.doc-examples }}
       embed: ${{ steps.filter.outputs.embed }}
       all-npm: ${{ steps.filter.outputs.all-npm }}
       hindsight-all: ${{ steps.filter.outputs.hindsight-all }}
@@ -43,6 +43,7 @@ jobs:
       integrations-continue: ${{ steps.filter.outputs.integrations-continue }}
       integrations-cursor-cli: ${{ steps.filter.outputs.integrations-cursor-cli }}
       integrations-zcode: ${{ steps.filter.outputs.integrations-zcode }}
+      integrations-agent-plugin: ${{ steps.filter.outputs.integrations-agent-plugin }}
       integrations-copilot-cli: ${{ steps.filter.outputs.integrations-copilot-cli }}
       integrations-crewai: ${{ steps.filter.outputs.integrations-crewai }}
       integrations-litellm: ${{ steps.filter.outputs.integrations-litellm }}
@@ -117,12 +118,17 @@ jobs:
             - 'docker/**'
           helm:
             - 'helm/**'
-          docs:
-            - 'hindsight-docs/**'
-            - '*.md'
-            # Integration changes can add/rename integrations, which the docs
-            # build's integrations check validates against integrations.json.
-            - 'hindsight-integrations/**'
+          # The RUNNABLE samples only. This replaces a broad `docs` filter that also
+          # matched 'hindsight-docs/**', '*.md' and 'hindsight-integrations/**' — the
+          # last of those so an integration rename would be caught by the docs build's
+          # integrations check, except that build (build-docs) has no `if:` and runs
+          # unconditionally anyway. test-doc-examples was the filter's only consumer,
+          # and it executes every sample against a live LLM-backed server, so every
+          # integration and prose-only PR paid for four provider-credentialed runs that
+          # none of those files can affect.
+          doc-examples:
+            - 'hindsight-docs/examples/**'
+            - 'scripts/test-doc-examples.sh'
           embed:
             - 'hindsight-embed/**'
           all-npm:
@@ -189,6 +195,8 @@ jobs:
             - 'hindsight-integrations/zed/**'
           integrations-zcode:
             - 'hindsight-integrations/zcode/**'
+          integrations-agent-plugin:
+            - 'hindsight-integrations/agent-plugin/**'
           integrations-n8n:
             - 'hindsight-integrations/n8n/**'
           integrations-zapier:
@@ -791,6 +799,29 @@ jobs:
       working-directory: ./hindsight-integrations/zcode
       run: uv run pytest tests -v
 
+  test-agent-plugin-integration:
+    needs: [detect-changes]
+    if: >-
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-agent-plugin == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version-file: ".python-version"
+
+    - name: Validate Agent Plugin manifests
+      working-directory: ./hindsight-integrations/agent-plugin
+      run: python3 validate.py
+
   build-ai-sdk-integration:
     needs: [detect-changes]
     if: >-
@@ -4528,6 +4559,69 @@ jobs:
           fi
         done || true
 
+  # Compatibility gate against hermes-agent's *main* branch.
+  #
+  # `hermes memory setup` installs `hindsight-all` into Hermes' own venv to run
+  # memory in local_embedded mode, and Hermes exact-pins every direct dependency
+  # (`==X.Y.Z`) as a deliberate supply-chain policy — they will not loosen a pin
+  # for us. So any version range Hindsight declares that excludes one of their
+  # pins makes the two impossible to co-install for every Hermes user on
+  # embedded memory. That was #3251: our `cryptography>=48.0.1` / `pillow>=12.3.0`
+  # against their `==46.0.7` / `==12.2.0`, which left `pip check` permanently
+  # broken. Both sides bump on their own schedule, so this needs a standing gate
+  # rather than a one-off fix; tracking main surfaces the next collision while
+  # it is still cheap to fix on either side.
+  #
+  # Deliberately not gated on has_secrets — the resolution, wiring, runtime and
+  # daemon-boot checks need no credentials, so this runs on fork PRs too. Only
+  # retain/recall need an LLM and the script skips them when no key is present.
+  test-hermes-compat:
+    needs: [detect-changes]
+    if: >-
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.core == 'true' ||
+      needs.detect-changes.outputs.embed == 'true' ||
+      needs.detect-changes.outputs.hindsight-all == 'true' ||
+      needs.detect-changes.outputs.clients-python == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    # hindsight-all pulls the local-ml extra, so the embedded daemon can load
+    # sentence-transformers models. Cache them like the test-embed job does.
+    - name: Cache HuggingFace models
+      uses: actions/cache@v5
+      with:
+        path: ~/.cache/huggingface
+        key: ${{ runner.os }}-huggingface-hermes-compat-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-huggingface-hermes-compat-
+          ${{ runner.os }}-huggingface-
+
+    - name: Run Hermes compatibility test
+      run: ./scripts/test-hermes-compat.sh
+
+    - name: Collect embedded daemon logs on failure
+      if: failure()
+      run: |
+        for f in ~/.hindsight/profiles/hermes-ci*.log ~/.hindsight/profiles/hermes-ci*.stderr.log; do
+          if [ -f "$f" ]; then
+            echo "=== $f ==="
+            cat "$f"
+          fi
+        done || true
+
   test-hindsight-all:
     needs: [detect-changes]
     if: >-
@@ -4629,7 +4723,7 @@ jobs:
       needs.detect-changes.outputs.clients-python == 'true' ||
       needs.detect-changes.outputs.clients-go == 'true' ||
       needs.detect-changes.outputs.cli == 'true' ||
-      needs.detect-changes.outputs.docs == 'true' ||
+      needs.detect-changes.outputs.doc-examples == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -5033,13 +5127,26 @@ jobs:
           exit 0
         fi
 
-        echo "Checking OpenAPI compatibility against base branch: $BASE_BRANCH"
+        # Compare against the point this branch was cut from, NOT the live tip of
+        # the base branch. Using the tip reports every endpoint main has gained
+        # since the branch was cut as "removed by this PR" — a false positive that
+        # fails PRs which touch no spec at all, and whose only cure is an unrelated
+        # rebase. The merge-base answers the question the check actually asks:
+        # did *this branch* remove something?
+        MERGE_BASE="$(git merge-base "origin/$BASE_BRANCH" HEAD)"
 
-        # Extract the old OpenAPI spec from base branch
-        git show "origin/$BASE_BRANCH:hindsight-docs/static/openapi.json" > /tmp/old-openapi.json
+        if [ -z "$MERGE_BASE" ]; then
+          echo "⚠️  Warning: Could not determine merge-base with $BASE_BRANCH. Skipping compatibility check."
+          exit 0
+        fi
+
+        echo "Checking OpenAPI compatibility against $BASE_BRANCH merge-base: $MERGE_BASE"
+
+        # Extract the old OpenAPI spec from the merge-base
+        git show "$MERGE_BASE:hindsight-docs/static/openapi.json" > /tmp/old-openapi.json
 
         if [ ! -s /tmp/old-openapi.json ]; then
-          echo "⚠️  Warning: Could not find OpenAPI spec in base branch. Skipping compatibility check."
+          echo "⚠️  Warning: Could not find OpenAPI spec at the merge-base. Skipping compatibility check."
           exit 0
         fi
 
@@ -5081,6 +5188,42 @@ jobs:
         cd hindsight-dev
         uv run cli-coverage-check
 
+  # hindsight-dev/tests had no job of its own, so nothing ran it: the benchmark
+  # harness was only exercised by the nightly Performance Tests workflow, where a
+  # plain construction bug in the answer/judge LLM config surfaced as a red
+  # benchmark hours later instead of on the PR that introduced it.
+  test-dev:
+    needs: [detect-changes]
+    if: >-
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.core == 'true' ||
+      needs.detect-changes.outputs.dev == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version-file: ".python-version"
+
+    - name: Install hindsight-dev dependencies
+      run: |
+        cd hindsight-dev && uv sync --frozen --extra test --index-strategy unsafe-best-match
+
+    - name: Run hindsight-dev tests
+      working-directory: hindsight-dev
+      run: uv run pytest tests/ -v
+
   # Report CI status back to the PR for pull_request_review events.
   # GitHub does not automatically link pull_request_review check runs to the PR,
   # so we create a commit status on the PR head SHA and post a comment.
@@ -5100,6 +5243,7 @@ jobs:
       - test-codex-integration
       - test-cursor-cli-integration
       - test-zcode-integration
+      - test-agent-plugin-integration
       - build-ai-sdk-integration
       - test-ai-sdk-integration-deno
       - test-opencode-integration
@@ -5155,6 +5299,7 @@ jobs:
       - test-embed
       - test-embed-windows
       - verify-embed-control-center-bundle
+      - test-hermes-compat
       - test-hindsight-all
       - test-hindsight-agent-sdk
       - test-claude-agent-sdk-integration

docker/docker-compose/pgroonga/Dockerfile

@@ -3,21 +3,11 @@
 # pgroonga is a multilingual full-text search extension built on Groonga.
 # It works out of the box for CJK (Chinese, Japanese, Korean) and other
 # non-whitespace-segmented languages via the TokenBigram tokenizer.
-FROM groonga/pgroonga:latest-debian-pg17
+FROM groonga/pgroonga:4.0.8-debian-17
 
 # Install pgvector on top of the pgroonga base image (which already provides
-# pgroonga and the Groonga library).
+# pgroonga, the Groonga library, and the PostgreSQL PGDG package repository).
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential \
-    git \
-    postgresql-server-dev-17 \
+    postgresql-17-pgvector=0.8.6-1.pgdg13+1 \
+    && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
-
-RUN cd /tmp && \
-    git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git && \
-    cd pgvector && \
-    make && \
-    make install
-
-RUN rm -rf /tmp/pgvector && \
-    apt-get purge -y --auto-remove build-essential git postgresql-server-dev-17

helm/hindsight/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: hindsight
 description: Hindsight helm chart
 type: application
-version: 0.8.6
-appVersion: "0.8.6"
+version: 0.9.0
+appVersion: "0.9.0"
 keywords:
   - ai
   - memory

helm/hindsight/values.yaml

@@ -36,16 +36,22 @@ api:
       cpu: 500m
       memory: 1Gi
 
-  # Liveness and readiness probes
+  # Liveness and readiness probes.
+  # Liveness uses /health/live, which performs no database access: a slow or
+  # unreachable database must gate traffic (readiness), never restart pods.
+  # Needs an image from this chart's appVersion or newer — older ones serve
+  # /health only, and would fail this probe with a 404.
   livenessProbe:
     httpGet:
-      path: /health
+      path: /health/live
       port: 8888
     initialDelaySeconds: 30
     periodSeconds: 10
     timeoutSeconds: 5
     failureThreshold: 3
 
+  # Readiness checks the database, so a pod that cannot reach it is pulled out
+  # of the Service and put back once the database recovers.
   readinessProbe:
     httpGet:
       path: /health
@@ -131,10 +137,15 @@ worker:
       cpu: 500m
       memory: 1Gi
 
-  # Liveness and readiness probes
+  # Liveness and readiness probes.
+  # Liveness uses /health/live, which performs no database access. Restarting a
+  # worker whose database is merely slow requeues its claimed operations with
+  # retry_count incremented, so DB checks must stay out of liveness.
+  # Needs an image from this chart's appVersion or newer — older ones serve
+  # /health only, and would fail this probe with a 404.
   livenessProbe:
     httpGet:
-      path: /health
+      path: /health/live
       port: 8889
     initialDelaySeconds: 30
     periodSeconds: 10

hindsight-all-npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vectorize-io/hindsight-all",
-  "version": "0.8.6",
+  "version": "0.9.0",
   "description": "Node.js programmatic lifecycle manager for Hindsight — embeds a local hindsight daemon in a Node application. Pair with @vectorize-io/hindsight-client for memory operations.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

hindsight-all-slim/pyproject.toml

@@ -4,15 +4,15 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hindsight-all-slim"
-version = "0.8.6"
+version = "0.9.0"
 description = "Hindsight: Agent Memory That Works Like Human Memory - Slim All-in-One Bundle"
 license = "MIT"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "hindsight-api-slim==0.8.6",
+    "hindsight-api-slim==0.9.0",
     "hindsight-client>=0.0.7",
-    "hindsight-embed>=0.1.0",
+    "hindsight-embed==0.9.0",
 ]
 
 [tool.uv.sources]

hindsight-all/pyproject.toml

@@ -4,15 +4,15 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-all"
-version = "0.8.6"
+version = "0.9.0"
 description = "Hindsight: Agent Memory That Works Like Human Memory - All-in-One Bundle"
 license = "MIT"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "hindsight-api-slim[all]==0.8.6",
+    "hindsight-api-slim[all]==0.9.0",
     "hindsight-client>=0.0.7",
-    "hindsight-embed>=0.1.0",
+    "hindsight-embed==0.9.0",
 ]
 
 [tool.uv.sources]
@@ -22,7 +22,7 @@ hindsight-embed = { workspace = true }
 
 [project.optional-dependencies]
 local-llm = [
-    "hindsight-api-slim[local-llm]==0.8.6",
+    "hindsight-api-slim[local-llm]==0.9.0",
 ]
 test = [
     "pytest>=7.0.0",

hindsight-api-slim/hindsight_api/__init__.py

@@ -53,4 +53,4 @@ __all__ = [
     "RemoteTEICrossEncoder",
     "LLMConfig",
 ]
-__version__ = "0.8.6"
+__version__ = "0.9.0"

hindsight-api-slim/hindsight_api/_text_search.py

@@ -0,0 +1,23 @@
+"""Text-search SQL shapes shared by index DDL and the queries that must hit it.
+
+A PostgreSQL expression index is only selectable when the query repeats the
+indexed expression verbatim, so the DDL (``migrations.py`` and the Alembic
+versions) and the read arms (``engine/sql/postgresql.py``) cannot be allowed to
+drift. Both sides call the helpers here — same idea as
+``_pg_search.pg_search_bm25_columns``.
+"""
+
+
+def mental_models_text_document(alias: str | None = None) -> str:
+    """The ``mental_models`` full-text document: model/page name + content.
+
+    Mirrors the generating expression of the native tsvector column created by
+    the ``n9i0j1k2l3m4`` (learnings / pinned_reflections) migration, so every
+    backend indexes and queries the exact same document. ``content`` is NOT NULL,
+    hence the deliberate lack of a ``COALESCE`` around it.
+
+    ``alias`` qualifies the columns for queries that join the table (``mm``);
+    leave it unset for DDL, where the expression is already table-scoped.
+    """
+    prefix = f"{alias}." if alias else ""
+    return f"(COALESCE({prefix}name, '') || ' ' || {prefix}content)"

hindsight-api-slim/hindsight_api/admin/cli.py

@@ -847,7 +847,8 @@ def import_bank_command(
         f"Imported bank '{result.bank_id}': {result.documents_imported} doc(s), "
         f"{result.facts_imported} fact(s), {result.observations_imported} observation(s), "
         f"{result.mental_models_imported} mental model(s), "
-        f"{result.mental_model_history_imported} mm-history row(s), {result.directives_imported} directive(s), "
+        f"{result.mental_model_history_imported} mm-history row(s), "
+        f"{result.knowledge_pages_imported} knowledge page(s), {result.directives_imported} directive(s), "
         f"{result.webhooks_imported} webhook(s), {result.history_rows_imported} history row(s)"
     )
 

hindsight-api-slim/hindsight_api/alembic/versions/c1e7a9d3f5b2_drop_entity_schema_from_memory_links.py

@@ -1,268 +0,0 @@
-"""Drop the deprecated entity schema from ``memory_links``.
-
-Entity edges are no longer materialized in ``memory_links``. Retain stores
-memory-to-entity associations in ``unit_entities``, and both read paths derive
-entity edges on demand from that table — the /graph endpoint builds them from
-shared ``unit_entities`` rows, and recall expands via the ``unit_entities``
-self-join. Migration ``e9b2c7d1f3a4`` deleted the materialized entity rows and
-current writers only ever pass ``entity_id = NULL``, so the entity-specific
-schema on ``memory_links`` is now dead weight:
-
-- the ``entity_id`` column and its FK to ``entities``
-- ``link_type = 'entity'`` in the table CHECK constraint
-- the entity index (``idx_memory_links_entity`` on PG, ``idx_ml_entity`` on Oracle)
-- the ``entity_id`` term in the function-based ``idx_memory_links_unique``
-
-Once entity edges are gone, every remaining row (temporal, semantic, and the
-causal types) has a single meaningful identity — ``(from_unit_id, to_unit_id,
-link_type)`` — so the unique index collapses to those three columns and
-preserves the existing effective uniqueness semantics.
-
-Production ``memory_links`` tables and the entity index can be very large, so
-this migration is written to avoid long exclusive locks: the residual delete is
-chunked with per-batch commits, every index is swapped with ``CONCURRENTLY``,
-and the new CHECK is added ``NOT VALID`` then validated separately so writers are
-never blocked on a full-table scan.
-
-Downgrade restores the former schema *shape* (column, FK, index, CHECK, and the
-expression unique index) but cannot reconstruct the historical entity rows that
-``e9b2c7d1f3a4`` already deleted — new retains do not produce them either, so the
-restored entity index would simply stay empty.
-
-Revision ID: c1e7a9d3f5b2
-Revises: e4a7c1b9d2f6
-Create Date: 2026-08-04
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-from hindsight_api.alembic._dialect import run_for_dialect
-
-revision: str = "c1e7a9d3f5b2"
-down_revision: str | Sequence[str] | None = "e4a7c1b9d2f6"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-_NIL_ENTITY_UUID = "00000000-0000-0000-0000-000000000000"
-_LINK_TYPES_WITHOUT_ENTITY = "'temporal', 'semantic', 'causes', 'caused_by', 'enables', 'prevents'"
-_LINK_TYPES_WITH_ENTITY = "'temporal', 'semantic', 'entity', 'causes', 'caused_by', 'enables', 'prevents'"
-
-
-def _pg_schema_prefix() -> str:
-    """Schema-qualifier for raw SQL on PG (multi-tenant search_path)."""
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def _pg_upgrade() -> None:
-    schema = _pg_schema_prefix()
-
-    # CONCURRENTLY index swaps and the DO block's per-batch COMMIT both require
-    # running outside Alembic's migration transaction — an autocommit_block
-    # commits it and switches the connection to autocommit for the duration.
-    with op.get_context().autocommit_block():
-        # 1. Defensively delete any residual entity rows. e9b2c7d1f3a4 already
-        #    did this, but a bank that predates its deployment can still carry
-        #    them, and they must be gone before the three-column unique index
-        #    (which no longer distinguishes entity rows) and the entity-free
-        #    CHECK are created. Chunked with per-batch commits so a very large
-        #    table drains in bounded transactions instead of one long-locking
-        #    delete.
-        op.execute(
-            f"""
-            DO $$
-            DECLARE
-                deleted INTEGER;
-            BEGIN
-                LOOP
-                    DELETE FROM {schema}memory_links
-                    WHERE ctid IN (
-                        SELECT ctid FROM {schema}memory_links
-                        WHERE link_type = 'entity'
-                        LIMIT 50000
-                    );
-                    GET DIAGNOSTICS deleted = ROW_COUNT;
-                    EXIT WHEN deleted = 0;
-                    COMMIT;
-                END LOOP;
-            END$$;
-            """
-        )
-
-        # 2. Drop the entity indexes. idx_memory_links_entity_covering was
-        #    already removed by e1b2c3d4f5a6; dropped again defensively.
-        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_entity")
-        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_entity_covering")
-
-        # 3. Replace the expression unique index with the three-column form.
-        #    Build the new one first (under a temporary name) so uniqueness is
-        #    never unprotected, then drop the old expression index and rename.
-        #    Non-entity rows already collapse to (from, to, link_type) under the
-        #    old COALESCE(entity_id, nil) expression, so this cannot find a
-        #    duplicate once the entity rows are gone.
-        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_unique_new")
-        op.execute(
-            f"CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_unique_new "
-            f"ON {schema}memory_links (from_unit_id, to_unit_id, link_type)"
-        )
-        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_unique")
-        op.execute(f"ALTER INDEX IF EXISTS {schema}idx_memory_links_unique_new RENAME TO idx_memory_links_unique")
-
-        # 4. Drop the FK and the now-unreferenced column. Both are metadata-only
-        #    on PostgreSQL (DROP COLUMN marks the attribute dropped, no rewrite).
-        op.execute(f"ALTER TABLE {schema}memory_links DROP CONSTRAINT IF EXISTS fk_memory_links_entity_id_entities")
-        op.execute(f"ALTER TABLE {schema}memory_links DROP COLUMN IF EXISTS entity_id")
-
-        # 5. Recreate the link_type CHECK without 'entity'. Added NOT VALID then
-        #    validated separately: the ADD takes a brief lock without scanning,
-        #    and VALIDATE takes only SHARE UPDATE EXCLUSIVE, so concurrent reads
-        #    and writes are never blocked on the scan.
-        op.execute(f"ALTER TABLE {schema}memory_links DROP CONSTRAINT IF EXISTS memory_links_link_type_check")
-        op.execute(
-            f"ALTER TABLE {schema}memory_links ADD CONSTRAINT memory_links_link_type_check "
-            f"CHECK (link_type IN ({_LINK_TYPES_WITHOUT_ENTITY})) NOT VALID"
-        )
-        op.execute(f"ALTER TABLE {schema}memory_links VALIDATE CONSTRAINT memory_links_link_type_check")
-
-
-def _pg_downgrade() -> None:
-    schema = _pg_schema_prefix()
-
-    with op.get_context().autocommit_block():
-        # Restore the column and FK (nullable, as in the initial schema).
-        op.execute(f"ALTER TABLE {schema}memory_links ADD COLUMN IF NOT EXISTS entity_id uuid")
-        op.execute(f"ALTER TABLE {schema}memory_links DROP CONSTRAINT IF EXISTS fk_memory_links_entity_id_entities")
-        op.execute(
-            f"ALTER TABLE {schema}memory_links ADD CONSTRAINT fk_memory_links_entity_id_entities "
-            f"FOREIGN KEY (entity_id) REFERENCES {schema}entities (id) ON DELETE CASCADE"
-        )
-
-        # Restore the entity partial index.
-        op.execute(
-            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_entity "
-            f"ON {schema}memory_links (entity_id) WHERE entity_id IS NOT NULL"
-        )
-
-        # Restore the expression unique index (entity_id back in the key).
-        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_unique_old")
-        op.execute(
-            f"CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_unique_old "
-            f"ON {schema}memory_links (from_unit_id, to_unit_id, link_type, "
-            f"COALESCE(entity_id, '{_NIL_ENTITY_UUID}'::uuid))"
-        )
-        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_unique")
-        op.execute(f"ALTER INDEX IF EXISTS {schema}idx_memory_links_unique_old RENAME TO idx_memory_links_unique")
-
-        # Restore 'entity' as a permitted link_type.
-        op.execute(f"ALTER TABLE {schema}memory_links DROP CONSTRAINT IF EXISTS memory_links_link_type_check")
-        op.execute(
-            f"ALTER TABLE {schema}memory_links ADD CONSTRAINT memory_links_link_type_check "
-            f"CHECK (link_type IN ({_LINK_TYPES_WITH_ENTITY})) NOT VALID"
-        )
-        op.execute(f"ALTER TABLE {schema}memory_links VALIDATE CONSTRAINT memory_links_link_type_check")
-
-
-def _oracle_exec_ignoring(sql: str, *ignore_codes: int) -> None:
-    """Run a DDL statement, swallowing the given ORA-NNNNN codes for idempotency.
-
-    Oracle has no ``IF EXISTS``/``IF NOT EXISTS`` for most objects; the standard
-    pattern (see e4a7c1b9d2f6) is EXECUTE IMMEDIATE inside a PL/SQL block that
-    re-raises anything but the expected "already absent"/"already present" code.
-    """
-    conditions = " AND ".join(f"SQLCODE != {code}" for code in ignore_codes)
-    escaped = sql.replace("'", "''")
-    op.execute(
-        f"""
-        BEGIN
-            EXECUTE IMMEDIATE '{escaped}';
-        EXCEPTION WHEN OTHERS THEN
-            IF {conditions} THEN RAISE; END IF;
-        END;
-        """
-    )
-
-
-def _oracle_upgrade() -> None:
-    # 1. Defensively drain residual entity rows in bounded, committed chunks so a
-    #    large table doesn't delete under one long-held lock / huge undo segment.
-    op.execute(
-        """
-        BEGIN
-            LOOP
-                DELETE FROM memory_links WHERE link_type = 'entity' AND ROWNUM <= 50000;
-                EXIT WHEN SQL%ROWCOUNT = 0;
-                COMMIT;
-            END LOOP;
-        END;
-        """
-    )
-
-    # 2. Build the three-column unique index under a temporary name before
-    #    dropping the old expression index, then rename. Oracle DDL implicitly
-    #    commits, so ordering it this way keeps duplicate protection continuous
-    #    instead of leaving a gap between drop and create. ONLINE avoids blocking
-    #    concurrent DML during the (potentially large) index builds/drops.
-    #    ORA-00955: name already in use; ORA-01418: index does not exist.
-    _oracle_exec_ignoring("DROP INDEX idx_memory_links_unique_new", -1418)
-    _oracle_exec_ignoring(
-        "CREATE UNIQUE INDEX idx_memory_links_unique_new ON memory_links (from_unit_id, to_unit_id, link_type) ONLINE",
-        -955,
-    )
-    _oracle_exec_ignoring("DROP INDEX idx_memory_links_unique", -1418)
-    _oracle_exec_ignoring("ALTER INDEX idx_memory_links_unique_new RENAME TO idx_memory_links_unique", -1418)
-
-    # 3. Drop the entity index. ORA-01418: index does not exist.
-    _oracle_exec_ignoring("DROP INDEX idx_ml_entity ONLINE", -1418)
-
-    # 4. Drop the entity FK, then the column. ORA-02443: constraint does not
-    #    exist; ORA-00904: column does not exist.
-    _oracle_exec_ignoring("ALTER TABLE memory_links DROP CONSTRAINT fk_ml_entity", -2443)
-    _oracle_exec_ignoring("ALTER TABLE memory_links DROP COLUMN entity_id", -904)
-
-    # 5. Recreate the link_type CHECK without 'entity'. ORA-02443: constraint
-    #    does not exist; ORA-02264: name already used by an existing constraint.
-    _oracle_exec_ignoring("ALTER TABLE memory_links DROP CONSTRAINT chk_ml_link_type", -2443)
-    _oracle_exec_ignoring(
-        f"ALTER TABLE memory_links ADD CONSTRAINT chk_ml_link_type CHECK (link_type IN ({_LINK_TYPES_WITHOUT_ENTITY}))",
-        -2264,
-    )
-
-
-def _oracle_downgrade() -> None:
-    # Restore the column, FK, entity index, expression unique index, and the
-    # 'entity'-permitting CHECK. ORA-01430: column already exists; ORA-00955:
-    # name already in use; ORA-01418: index does not exist; ORA-02443/-2264:
-    # constraint absent / name in use.
-    _oracle_exec_ignoring("ALTER TABLE memory_links ADD (entity_id RAW(16))", -1430)
-    _oracle_exec_ignoring(
-        "ALTER TABLE memory_links ADD CONSTRAINT fk_ml_entity "
-        "FOREIGN KEY (entity_id) REFERENCES entities (id) ON DELETE CASCADE",
-        -2264,
-    )
-    _oracle_exec_ignoring("CREATE INDEX idx_ml_entity ON memory_links (entity_id)", -955)
-
-    _oracle_exec_ignoring("DROP INDEX idx_memory_links_unique_old", -1418)
-    _oracle_exec_ignoring(
-        "CREATE UNIQUE INDEX idx_memory_links_unique_old ON memory_links ("
-        "from_unit_id, to_unit_id, link_type, "
-        "NVL(entity_id, HEXTORAW('00000000000000000000000000000000'))) ONLINE",
-        -955,
-    )
-    _oracle_exec_ignoring("DROP INDEX idx_memory_links_unique", -1418)
-    _oracle_exec_ignoring("ALTER INDEX idx_memory_links_unique_old RENAME TO idx_memory_links_unique", -1418)
-
-    _oracle_exec_ignoring("ALTER TABLE memory_links DROP CONSTRAINT chk_ml_link_type", -2443)
-    _oracle_exec_ignoring(
-        f"ALTER TABLE memory_links ADD CONSTRAINT chk_ml_link_type CHECK (link_type IN ({_LINK_TYPES_WITH_ENTITY}))",
-        -2264,
-    )
-
-
-def upgrade() -> None:
-    run_for_dialect(pg=_pg_upgrade, oracle=_oracle_upgrade)
-
-
-def downgrade() -> None:
-    run_for_dialect(pg=_pg_downgrade, oracle=_oracle_downgrade)

hindsight-api-slim/hindsight_api/alembic/versions/f2a6d8c4b1e9_drop_stale_global_memory_units_vector_index.py

@@ -1,7 +1,7 @@
 """Repair: drop the stale global memory_units vector index on per-bank backends.
 
 Revision ID: f2a6d8c4b1e9
-Revises: c1e7a9d3f5b2
+Revises: e4a7c1b9d2f6
 Create Date: 2026-08-06
 
 Migration d5e6f7a8b9c0 dropped the global ``idx_memory_units_embedding`` for
@@ -33,7 +33,7 @@ from alembic import context, op
 from hindsight_api.alembic._dialect import run_for_dialect
 
 revision: str = "f2a6d8c4b1e9"
-down_revision: str | Sequence[str] | None = "c1e7a9d3f5b2"
+down_revision: str | Sequence[str] | None = "e4a7c1b9d2f6"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 

hindsight-api-slim/hindsight_api/api/http.py

@@ -17,6 +17,7 @@ from typing import Any, Literal, TypeVar
 
 from fastapi import Depends, FastAPI, File, Form, Header, HTTPException, Query, Request, UploadFile
 from fastapi.middleware.gzip import GZipMiddleware
+from fastapi.responses import JSONResponse
 
 from hindsight_api.api import page_markdown
 from hindsight_api.api.disconnect import ClientDisconnectCancellationMiddleware, get_scope_cancellation_token
@@ -185,6 +186,7 @@ from hindsight_api.engine.response_models import (
 from hindsight_api.engine.search.tags import TagGroup, TagsMatch
 from hindsight_api.engine.structured_output import validate_response_schema
 from hindsight_api.extensions import HttpExtension, OperationValidationError, load_extension
+from hindsight_api.liveness import LivenessResponse, liveness_response
 from hindsight_api.metrics import (
     create_metrics_collector,
     get_metrics_collector,
@@ -1892,6 +1894,19 @@ class DocumentImportSubmitResponse(BaseModel):
     status: str = "pending"
 
 
+class DocumentExportSubmitResponse(BaseModel):
+    """Response for the async document-export endpoint (202).
+
+    The export runs in the background; poll the operations endpoint for status.
+    On completion the operation's ``result_metadata`` carries ``download_url``
+    (fetch the ZIP from GET /v1/default/files/download/{key}), ``storage_key``,
+    ``byte_size``, and ``filename``.
+    """
+
+    operation_id: str
+    status: str = "pending"
+
+
 class DeleteResponse(BaseModel):
     """Response model for delete operations."""
 
@@ -2646,6 +2661,24 @@ class BankTemplateConfig(BaseModel):
         description="Persist raw source text (documents.original_text / chunks.chunk_text). "
         "Set false to keep only derived facts.",
     )
+    enable_auto_consolidation: bool | None = Field(
+        default=None, description="Automatically consolidate observations after retain"
+    )
+    consolidation_max_memories_per_round: int | None = Field(
+        default=None, description="Max memory units fed into a single consolidation round"
+    )
+    consolidation_llm_parallelism: int | None = Field(
+        default=None, description="Number of consolidation LLM batches processed concurrently"
+    )
+    recall_include_chunks: bool | None = Field(default=None, description="Include raw chunks in recall results")
+    recall_max_tokens: int | None = Field(default=None, description="Max tokens of results returned by recall")
+    recall_chunks_max_tokens: int | None = Field(
+        default=None, description="Max tokens of raw chunks returned by recall (when recall_include_chunks is set)"
+    )
+    memory_defense: dict | None = Field(
+        default=None,
+        description="Memory Defense policy for this bank (validated against the DefensePolicy schema on write)",
+    )
 
     def get_config_updates(self) -> dict[str, Any]:
         """Return only the fields that were explicitly set (non-None)."""
@@ -4004,17 +4037,24 @@ def _register_routes(app: FastAPI):
     # Global exception handler for authentication errors
     @app.exception_handler(AuthenticationError)
     async def authentication_error_handler(request, exc: AuthenticationError):
-        from fastapi.responses import JSONResponse
-
         return JSONResponse(
             status_code=401,
             content={"detail": str(exc)},
         )
 
+    async def _readiness_response() -> JSONResponse:
+        """Shared body of /health and /health/ready: 200 if healthy, 503 if not."""
+        health = await app.state.memory.health_check()
+        status_code = 200 if health.get("status") == "healthy" else 503
+        return JSONResponse(content=health, status_code=status_code)
+
     @app.get(
         "/health",
         summary="Health check endpoint",
-        description="Checks the health of the API and database connection",
+        description="Readiness check: verifies the API can reach the database. "
+        "Alias of /health/ready. Use /health/live for liveness probes — this one "
+        "fails whenever the database is unreachable, which must gate traffic, not "
+        "restart the process.",
         tags=["Monitoring"],
     )
     async def health_endpoint():
@@ -4023,11 +4063,43 @@ def _register_routes(app: FastAPI):
 
         Returns 200 if healthy, 503 if unhealthy.
         """
-        from fastapi.responses import JSONResponse
+        return await _readiness_response()
 
-        health = await app.state.memory.health_check()
-        status_code = 200 if health.get("status") == "healthy" else 503
-        return JSONResponse(content=health, status_code=status_code)
+    @app.get(
+        "/health/ready",
+        summary="Readiness probe",
+        description="Returns 200 when the API can serve traffic (database reachable), "
+        "503 otherwise. Identical to /health, which stays supported as its alias.",
+        tags=["Monitoring"],
+        operation_id="get_readiness",
+    )
+    async def readiness_endpoint():
+        """
+        Readiness probe that verifies database connectivity.
+
+        Returns 200 if ready, 503 if not. A 503 should remove this pod from the
+        Service; it must not restart it.
+        """
+        return await _readiness_response()
+
+    @app.get(
+        "/health/live",
+        response_model=LivenessResponse,
+        summary="Liveness probe",
+        description="Returns 200 whenever the process can serve a request. Performs no "
+        "database access, so a slow or unreachable database never restarts the pod. "
+        "Point livenessProbe here and readinessProbe at /health.",
+        tags=["Monitoring"],
+        operation_id="get_liveness",
+    )
+    async def liveness_endpoint() -> LivenessResponse:
+        """
+        Liveness probe: in-process only, never touches the database.
+
+        Answering at all is the check — Hindsight serves requests and task work on
+        one event loop, so a wedged loop cannot respond within the probe timeout.
+        """
+        return liveness_response()
 
     @app.get(
         "/version",
@@ -4391,6 +4463,8 @@ def _register_routes(app: FastAPI):
             return data
         except OperationValidationError as e:
             raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
         except (AuthenticationError, HTTPException):
             raise
         except Exception as e:
@@ -5012,6 +5086,8 @@ def _register_routes(app: FastAPI):
             )
         except OperationValidationError as e:
             raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
         except (AuthenticationError, HTTPException):
             raise
         except Exception as e:
@@ -5573,39 +5649,26 @@ def _register_routes(app: FastAPI):
     ):
         """Export a bank's knowledge base as a flat markdown bundle."""
         try:
-            nodes = await app.state.memory.list_knowledge_nodes(bank_id=bank_id, request_context=request_context)
+            export = await app.state.memory.export_knowledge_base(bank_id=bank_id, request_context=request_context)
             files = [
-                KnowledgePageBundleFile(path=page_markdown.INDEX_FILENAME, content=page_markdown.render_index(nodes))
-            ]
-            for node in nodes:
-                if node.get("kind") != "page":
-                    continue
-                page = await app.state.memory.get_knowledge_page(
-                    bank_id=bank_id, page_id=node["id"], request_context=request_context
+                KnowledgePageBundleFile(
+                    path=page_markdown.INDEX_FILENAME, content=page_markdown.render_index(export.nodes)
                 )
-                if page is None:
-                    continue
+            ]
+            for page in export.pages:
                 files.append(
                     KnowledgePageBundleFile(
-                        path=page_markdown.page_filename(node["id"]), content=page_markdown.render_document(page)
+                        path=page_markdown.page_filename(page.node_id),
+                        content=page_markdown.render_document(page.page),
                     )
                 )
-                if node.get("mental_model_id"):
-                    history = (
-                        await app.state.memory.get_mental_model_history(
-                            bank_id=bank_id,
-                            mental_model_id=node["mental_model_id"],
-                            request_context=request_context,
+                if page.history:
+                    files.append(
+                        KnowledgePageBundleFile(
+                            path=page_markdown.log_filename(page.node_id),
+                            content=page_markdown.render_log(page.page, page.history),
                         )
-                        or []
                     )
-                    if history:
-                        files.append(
-                            KnowledgePageBundleFile(
-                                path=page_markdown.log_filename(node["id"]),
-                                content=page_markdown.render_log(page, history),
-                            )
-                        )
             return KnowledgePageBundleResponse(files=files)
         except OperationValidationError as e:
             raise HTTPException(status_code=e.status_code, detail=e.reason)
@@ -6982,26 +7045,56 @@ def _register_routes(app: FastAPI):
         # greedy GET /documents/{document_id:path} route, which would otherwise
         # capture "export"/"import" as a document id.
         "/v1/default/banks/{bank_id}/document-transfer",
-        summary="Export documents",
-        description="Export documents (extracted facts, entity names, causal links, chunks) from a bank as a "
-        "transfer ZIP archive. Embeddings and database ids are not included — importing re-embeds with the target "
-        "bank's model and re-resolves entities. Consolidated observations are excluded unless include_observations=true. "
-        "Pass document_id query params to export specific documents, or omit to export the whole bank.",
+        summary="Export documents (removed — use POST .../document-transfer/export)",
+        description="**Removed.** The synchronous whole-bank export loaded the entire bank into memory and "
+        "held a database connection for the full request, which could exhaust memory and take down the shared "
+        "API on large banks. Use the asynchronous POST /v1/default/banks/{bank_id}/document-transfer/export "
+        "instead: it returns an operation_id, runs the export in the background, and exposes a download URL on "
+        "completion.",
+        operation_id="export_documents_sync_removed",
+        tags=["Document Transfer"],
+        deprecated=True,
+    )
+    async def api_export_documents_removed(
+        bank_id: str,
+        request_context: RequestContext = Depends(get_request_context),
+    ):
+        """Removed synchronous export — always 410, pointing at the async endpoint."""
+        raise HTTPException(
+            status_code=410,
+            detail=(
+                "Synchronous document export has been removed because it could take down the shared API on "
+                f"large banks. Submit an async export via POST /v1/default/banks/{bank_id}/document-transfer/export, "
+                f"poll GET /v1/default/banks/{bank_id}/operations/{{operation_id}}, then download the archive from "
+                "the download_url in the operation's result_metadata."
+            ),
+        )
+
+    @app.post(
+        "/v1/default/banks/{bank_id}/document-transfer/export",
+        response_model=DocumentExportSubmitResponse,
+        status_code=202,
+        summary="Export documents (async)",
+        description="Submit an async export of a bank's documents (extracted facts, entity names, causal links, "
+        "chunks) as a transfer ZIP archive. Embeddings and database ids are not included — importing re-embeds "
+        "with the target bank's model and re-resolves entities. Runs as a background operation to avoid pinning "
+        "the API on large banks. Returns an operation_id; poll "
+        "GET /v1/default/banks/{bank_id}/operations/{operation_id}. On completion the operation's result_metadata "
+        "carries download_url (fetch the ZIP from GET /v1/default/files/download/{key}), storage_key, byte_size, "
+        "and filename. Pass document_id query params to export specific documents, or omit to export the whole "
+        "bank; include_observations=true also carries consolidated observations (whole-bank export only).",
         operation_id="export_documents",
         tags=["Document Transfer"],
-        responses={200: {"content": {"application/zip": {}}, "description": "Transfer archive"}},
     )
     async def api_export_documents(
         bank_id: str,
         document_id: list[str] | None = Query(default=None, description="Document id(s) to export; omit for all"),
         include_observations: bool = Query(
-            default=False, description="Also export consolidated observations (restored on import)"
+            default=False, description="Also export consolidated observations (restored on import; whole-bank only)"
         ),
         request_context: RequestContext = Depends(get_request_context),
     ):
-        """Export documents from a bank into a transfer ZIP archive."""
-        from fastapi.responses import Response
-
+        """Submit an async document-export operation for a bank."""
         try:
             if not get_config().enable_document_export_api:
                 raise HTTPException(
@@ -7016,7 +7109,7 @@ def _register_routes(app: FastAPI):
                 raise HTTPException(status_code=404, detail=f"Bank '{bank_id}' not found")
 
             try:
-                archive = await app.state.memory.export_documents_async(
+                submission = await app.state.memory.submit_export_documents_async(
                     bank_id,
                     request_context,
                     list(document_id) if document_id else None,
@@ -7025,11 +7118,7 @@ def _register_routes(app: FastAPI):
             except ValueError as e:
                 # e.g. include_observations combined with a document_id subset.
                 raise HTTPException(status_code=400, detail=str(e))
-            return Response(
-                content=archive,
-                media_type="application/zip",
-                headers={"Content-Disposition": f'attachment; filename="{bank_id}-documents.zip"'},
-            )
+            return DocumentExportSubmitResponse(operation_id=submission["operation_id"])
         except OperationValidationError as e:
             raise HTTPException(status_code=e.status_code, detail=e.reason)
         except (AuthenticationError, HTTPException):
@@ -7037,7 +7126,9 @@ def _register_routes(app: FastAPI):
         except Exception as e:
             import traceback
 
-            logger.error(f"Error in GET /v1/default/banks/{bank_id}/document-transfer: {traceback.format_exc()}")
+            logger.error(
+                f"Error in POST /v1/default/banks/{bank_id}/document-transfer/export: {traceback.format_exc()}"
+            )
             raise HTTPException(status_code=500, detail=str(e))
 
     @app.post(
@@ -7091,6 +7182,60 @@ def _register_routes(app: FastAPI):
             logger.error(f"Error in POST /v1/default/banks/{bank_id}/document-transfer: {traceback.format_exc()}")
             raise HTTPException(status_code=500, detail=str(e))
 
+    @app.get(
+        "/v1/default/files/download/{key:path}",
+        summary="Download a stored file (async export archive)",
+        description="Stream a file previously written to file storage — currently the transfer ZIP produced by "
+        "an async document export. The key comes from the export operation's result_metadata (storage_key / "
+        "download_url). Access is authorized against the bank the key belongs to.",
+        operation_id="download_file",
+        tags=["Document Transfer"],
+        responses={200: {"content": {"application/zip": {}}, "description": "Stored file"}},
+    )
+    async def api_download_file(
+        key: str,
+        request_context: RequestContext = Depends(get_request_context),
+    ):
+        """Download a bank-scoped stored file (export archive) by storage key."""
+        from fastapi.responses import Response
+
+        try:
+            if not get_config().enable_document_export_api:
+                raise HTTPException(
+                    status_code=404,
+                    detail="Document export API is disabled. "
+                    "Set HINDSIGHT_API_ENABLE_DOCUMENT_EXPORT_API=true to enable.",
+                )
+            # Only bank-scoped keys are downloadable. Parse the bank id out of the
+            # "banks/{bank_id}/..." key (request validation, so it belongs here); the
+            # engine method then authorizes the caller against that bank and retrieves
+            # the file, so a caller can't fetch another tenant's or bank's archive
+            # (IDOR guard). The unguessable uuid in the key is defence in depth, not
+            # the access control.
+            parts = key.split("/")
+            if ".." in parts or len(parts) < 2 or parts[0] != "banks" or not parts[1]:
+                raise HTTPException(status_code=404, detail="File not found")
+            bank_id = parts[1]
+
+            data = await app.state.memory.retrieve_bank_file(bank_id, key, request_context)
+            if data is None:
+                raise HTTPException(status_code=404, detail="File not found")
+
+            return Response(
+                content=data,
+                media_type="application/zip",
+                headers={"Content-Disposition": f'attachment; filename="{bank_id}-documents.zip"'},
+            )
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            logger.error(f"Error in GET /v1/default/files/download/{key}: {traceback.format_exc()}")
+            raise HTTPException(status_code=500, detail=str(e))
+
     @app.get(
         "/v1/bank-template-schema",
         summary="Get bank template JSON Schema",

hindsight-api-slim/hindsight_api/config.py

@@ -161,6 +161,12 @@ ENV_LLM_BEDROCK_SERVICE_TIER = "HINDSIGHT_API_LLM_BEDROCK_SERVICE_TIER"
 ENV_LLM_GEMINI_SERVICE_TIER = "HINDSIGHT_API_LLM_GEMINI_SERVICE_TIER"
 ENV_LLM_EXTRA_BODY = "HINDSIGHT_API_LLM_EXTRA_BODY"
 ENV_LLM_DEFAULT_HEADERS = "HINDSIGHT_API_LLM_DEFAULT_HEADERS"
+# Backend prompt-cache pinning for the OpenAI-compatible providers and Fireworks. Server-side
+# prompt caches are per backend server, so the same conversation has to reach the same
+# one: "xai_conv_id" sends xAI's documented x-grok-conv-id header, and
+# "openai_prompt_cache_key" sends OpenAI's prompt_cache_key field. See
+# engine/cache_affinity.py. Per-operation variants override the global one.
+ENV_LLM_CACHE_AFFINITY = "HINDSIGHT_API_LLM_CACHE_AFFINITY"
 # Grammar-enforced structured output. The global flag applies to every internal
 # LLM call; the per-operation variants override it for a single operation, so an
 # operator can enable strict schema where it fixes malformed/truncated JSON
@@ -171,6 +177,11 @@ ENV_LLM_STRICT_SCHEMA_RETAIN = "HINDSIGHT_API_LLM_STRICT_SCHEMA_RETAIN"
 ENV_LLM_STRICT_SCHEMA_REFLECT = "HINDSIGHT_API_LLM_STRICT_SCHEMA_REFLECT"
 ENV_LLM_STRICT_SCHEMA_CONSOLIDATION = "HINDSIGHT_API_LLM_STRICT_SCHEMA_CONSOLIDATION"
 ENV_LLM_SUPPORTS_MAX_ITEMS = "HINDSIGHT_API_LLM_SUPPORTS_MAX_ITEMS"
+# Route structured output through a forced function tool instead of the
+# OpenAI-style ``response_format`` on the LiteLLM-backed providers (``litellm``,
+# ``litellmrouter``, ``bedrock``). Off by default; see
+# DEFAULT_LLM_STRUCTURED_OUTPUT_FORCED_TOOL.
+ENV_LLM_STRUCTURED_OUTPUT_FORCED_TOOL = "HINDSIGHT_API_LLM_STRUCTURED_OUTPUT_FORCED_TOOL"
 ENV_LLM_SEND_BANK_AS_USER = "HINDSIGHT_API_LLM_SEND_BANK_AS_USER"
 ENV_LLM_OLLAMA_NUM_CTX = "HINDSIGHT_API_LLM_OLLAMA_NUM_CTX"
 
@@ -220,6 +231,14 @@ DEFAULT_LLM_EXTRA_BODY = None  # None = no extra body params; JSON dict merged i
 DEFAULT_LLM_DEFAULT_HEADERS = (
     None  # None = no extra headers; JSON dict passed as default_headers to provider SDK clients
 )
+# "auto" is safe as a default because it is an allowlist, not a best-effort probe:
+# it emits a hint only for hosts documented to accept one (x.ai / grok.com get the
+# header, native OpenAI / openai.com / Azure OpenAI get the field) and resolves to
+# "none" for every other backend, so vLLM, ollama, groq, openrouter and any custom
+# OpenAI-compatible endpoint keep receiving byte-identical requests. Measured on a
+# live xAI backend: 29% of a shared prefix cached without the header vs 99% with it,
+# so defaulting to "none" silently costs most deployments the benefit.
+DEFAULT_LLM_CACHE_AFFINITY = "auto"
 
 
 def parse_gemini_service_tier(value: str | None) -> str | None:
@@ -316,6 +335,7 @@ ENV_RETAIN_LLM_TIMEOUT = "HINDSIGHT_API_RETAIN_LLM_TIMEOUT"
 ENV_RETAIN_LLM_LITELLMROUTER_CONFIG = "HINDSIGHT_API_RETAIN_LLM_LITELLMROUTER_CONFIG"
 ENV_RETAIN_LLM_REASONING_EFFORT = "HINDSIGHT_API_RETAIN_LLM_REASONING_EFFORT"
 ENV_RETAIN_LLM_EXTRA_BODY = "HINDSIGHT_API_RETAIN_LLM_EXTRA_BODY"
+ENV_RETAIN_LLM_CACHE_AFFINITY = "HINDSIGHT_API_RETAIN_LLM_CACHE_AFFINITY"
 
 # Fireworks AI batch inference. Fireworks' batch API is a proprietary
 # account-scoped dataset/job REST API on a control-plane host, distinct from the
@@ -340,6 +360,7 @@ ENV_REFLECT_LLM_TIMEOUT = "HINDSIGHT_API_REFLECT_LLM_TIMEOUT"
 ENV_REFLECT_LLM_LITELLMROUTER_CONFIG = "HINDSIGHT_API_REFLECT_LLM_LITELLMROUTER_CONFIG"
 ENV_REFLECT_LLM_REASONING_EFFORT = "HINDSIGHT_API_REFLECT_LLM_REASONING_EFFORT"
 ENV_REFLECT_LLM_EXTRA_BODY = "HINDSIGHT_API_REFLECT_LLM_EXTRA_BODY"
+ENV_REFLECT_LLM_CACHE_AFFINITY = "HINDSIGHT_API_REFLECT_LLM_CACHE_AFFINITY"
 
 ENV_CONSOLIDATION_LLM_PROVIDER = "HINDSIGHT_API_CONSOLIDATION_LLM_PROVIDER"
 ENV_CONSOLIDATION_LLM_API_KEY = "HINDSIGHT_API_CONSOLIDATION_LLM_API_KEY"
@@ -353,6 +374,7 @@ ENV_CONSOLIDATION_LLM_TIMEOUT = "HINDSIGHT_API_CONSOLIDATION_LLM_TIMEOUT"
 ENV_CONSOLIDATION_LLM_LITELLMROUTER_CONFIG = "HINDSIGHT_API_CONSOLIDATION_LLM_LITELLMROUTER_CONFIG"
 ENV_CONSOLIDATION_LLM_REASONING_EFFORT = "HINDSIGHT_API_CONSOLIDATION_LLM_REASONING_EFFORT"
 ENV_CONSOLIDATION_LLM_EXTRA_BODY = "HINDSIGHT_API_CONSOLIDATION_LLM_EXTRA_BODY"
+ENV_CONSOLIDATION_LLM_CACHE_AFFINITY = "HINDSIGHT_API_CONSOLIDATION_LLM_CACHE_AFFINITY"
 
 ENV_EMBEDDINGS_PROVIDER = "HINDSIGHT_API_EMBEDDINGS_PROVIDER"
 ENV_EMBEDDINGS_LOCAL_MODEL = "HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL"
@@ -720,6 +742,7 @@ WORKER_SLOT_TYPE_DEFAULTS: dict[str, int] = {
     "refresh_mental_model": 0,
     "graph_maintenance": 0,
     "import_documents": 0,
+    "export_documents": 0,
 }
 
 
@@ -874,6 +897,7 @@ PROVIDER_DEFAULT_MODELS = {
     "requesty": "openai/gpt-4o-mini",
     "fireworks": "accounts/fireworks/models/llama-v3p1-8b-instruct",
     "nous": "deepseek/deepseek-v4-flash",
+    "xai-oauth": "grok-4.5",
 }
 DEFAULT_LLM_MODEL = "gpt-4o-mini"  # Fallback if provider not in table
 # Built-in llama.cpp defaults
@@ -892,6 +916,17 @@ DEFAULT_LLAMACPP_EXTRA_ARGS = None  # Space-separated extra CLI args for llama.c
 DEFAULT_LLM_STRICT_SCHEMA = False
 DEFAULT_LLM_SUPPORTS_MAX_ITEMS = True
 
+# True = ask LiteLLM-backed providers for structured output via a single forced
+# function tool (the response schema becomes the tool's parameters) instead of
+# the OpenAI-style ``response_format``. Needed where the backend rejects the
+# response_format route outright — notably Bedrock Claude, whose Converse layer
+# refuses the translated ``outputConfig`` ("Extra inputs are not permitted") while
+# accepting the identical schema as a tool (issue #3300). Verified region-dependent:
+# ap-southeast-2 / au.* rejects it, us-east-1 / us.* accepts it, so this is opt-in
+# rather than keyed off the provider. Default False keeps ``response_format``, which
+# every other LiteLLM backend handles natively.
+DEFAULT_LLM_STRUCTURED_OUTPUT_FORCED_TOOL = False
+
 DEFAULT_LLM_MAX_CONCURRENT = 32
 DEFAULT_LLM_MAX_RETRIES = 3  # Max retry attempts for LLM API calls
 DEFAULT_LLM_INITIAL_BACKOFF = 1.0  # Initial backoff in seconds for retry exponential backoff
@@ -1679,6 +1714,7 @@ class LLMMemberConfig:
     default_headers: dict | None
     bedrock_service_tier: str | None
     gemini_service_tier: str | None
+    cache_affinity: str | None = None
     vertexai_project_id: str | None = None
     vertexai_region: str | None = None
     vertexai_service_account_key: str | None = None
@@ -1772,6 +1808,7 @@ def _parse_llm_members(prefix: str) -> list[LLMMemberConfig]:
                 reasoning_effort=os.getenv(base + "REASONING_EFFORT") or None,
                 extra_body=json.loads(os.getenv(base + "EXTRA_BODY", "null")),
                 default_headers=json.loads(os.getenv(base + "DEFAULT_HEADERS", "null")),
+                cache_affinity=os.getenv(base + "CACHE_AFFINITY") or None,
                 bedrock_service_tier=os.getenv(base + "BEDROCK_SERVICE_TIER") or None,
                 gemini_service_tier=(
                     parse_gemini_service_tier(gemini_service_tier) if provider.lower() == "gemini" else None
@@ -2079,6 +2116,12 @@ class HindsightConfig:
     llm_default_headers: (
         dict | None
     )  # Custom headers passed as default_headers to provider SDK clients (e.g. {"X-Component-Id": "hindsight"} for proxies / request tracing)
+    # Backend prompt-cache pinning for the OpenAI-compatible providers and Fireworks:
+    # "none" (default),
+    # "xai_conv_id", "openai_prompt_cache_key" or "auto". Static (server-level) like the
+    # two fields above -- it is a transport detail of the configured endpoint, not a
+    # per-bank behaviour. See ENV_LLM_CACHE_AFFINITY and engine/cache_affinity.py.
+    llm_cache_affinity: str | None
     llm_strict_schema: bool  # Grammar-enforce structured output via the provider's strongest schema mode (see DEFAULT_LLM_STRICT_SCHEMA)
     # Per-operation strict-schema overrides. Resolved from the per-operation env
     # var, falling back to llm_strict_schema's global env var. See
@@ -2090,6 +2133,10 @@ class HindsightConfig:
         default=DEFAULT_LLM_SUPPORTS_MAX_ITEMS,
         kw_only=True,
     )  # Whether structured-output schemas accept JSON Schema maxItems
+    llm_structured_output_forced_tool: bool = field(
+        default=DEFAULT_LLM_STRUCTURED_OUTPUT_FORCED_TOOL,
+        kw_only=True,
+    )  # LiteLLM-backed providers: structured output via a forced tool call, not response_format
     # Tags outbound OpenAI-compatible LLM + embedding calls with `user=<bank_id>` for
     # per-bank cost attribution. Downstream cost gateways (OpenRouter usage accounting,
     # LiteLLM, Helicone) key attribution on the OpenAI `user` field. Opt-in; never
@@ -2150,6 +2197,7 @@ class HindsightConfig:
     retain_llm_litellmrouter_config: dict | None
     retain_llm_reasoning_effort: str | None
     retain_llm_extra_body: dict | None
+    retain_llm_cache_affinity: str | None
 
     # Fireworks AI batch inference (static, server-level)
     fireworks_account_id: str | None
@@ -2168,6 +2216,7 @@ class HindsightConfig:
     reflect_llm_litellmrouter_config: dict | None
     reflect_llm_reasoning_effort: str | None
     reflect_llm_extra_body: dict | None
+    reflect_llm_cache_affinity: str | None
 
     consolidation_llm_provider: str | None
     consolidation_llm_api_key: str | None
@@ -2181,6 +2230,7 @@ class HindsightConfig:
     consolidation_llm_litellmrouter_config: dict | None
     consolidation_llm_reasoning_effort: str | None
     consolidation_llm_extra_body: dict | None
+    consolidation_llm_cache_affinity: str | None
 
     # Embeddings
     embeddings_provider: str
@@ -3016,6 +3066,7 @@ class HindsightConfig:
             ),
             llm_extra_body=json.loads(os.getenv(ENV_LLM_EXTRA_BODY, "null")),
             llm_default_headers=json.loads(os.getenv(ENV_LLM_DEFAULT_HEADERS, "null")),
+            llm_cache_affinity=os.getenv(ENV_LLM_CACHE_AFFINITY, DEFAULT_LLM_CACHE_AFFINITY) or None,
             llm_strict_schema=os.getenv(ENV_LLM_STRICT_SCHEMA, str(DEFAULT_LLM_STRICT_SCHEMA)).lower() in ("true", "1"),
             llm_strict_schema_retain=_resolve_operation_strict_schema(ENV_LLM_STRICT_SCHEMA_RETAIN),
             llm_strict_schema_reflect=_resolve_operation_strict_schema(ENV_LLM_STRICT_SCHEMA_REFLECT),
@@ -3024,6 +3075,10 @@ class HindsightConfig:
                 ENV_LLM_SUPPORTS_MAX_ITEMS,
                 DEFAULT_LLM_SUPPORTS_MAX_ITEMS,
             ),
+            llm_structured_output_forced_tool=_parse_boolean_env(
+                ENV_LLM_STRUCTURED_OUTPUT_FORCED_TOOL,
+                DEFAULT_LLM_STRUCTURED_OUTPUT_FORCED_TOOL,
+            ),
             llm_send_bank_as_user=os.getenv(ENV_LLM_SEND_BANK_AS_USER, str(DEFAULT_LLM_SEND_BANK_AS_USER)).lower()
             in ("true", "1"),
             llm_ollama_num_ctx=_parse_optional_positive_int(
@@ -3095,6 +3150,7 @@ class HindsightConfig:
             retain_llm_litellmrouter_config=_parse_llm_router_config(ENV_RETAIN_LLM_LITELLMROUTER_CONFIG),
             retain_llm_reasoning_effort=os.getenv(ENV_RETAIN_LLM_REASONING_EFFORT) or None,
             retain_llm_extra_body=json.loads(os.getenv(ENV_RETAIN_LLM_EXTRA_BODY, "null")),
+            retain_llm_cache_affinity=os.getenv(ENV_RETAIN_LLM_CACHE_AFFINITY) or None,
             reflect_llm_provider=os.getenv(ENV_REFLECT_LLM_PROVIDER) or None,
             reflect_llm_api_key=os.getenv(ENV_REFLECT_LLM_API_KEY) or None,
             reflect_llm_model=os.getenv(ENV_REFLECT_LLM_MODEL)
@@ -3122,6 +3178,7 @@ class HindsightConfig:
             reflect_llm_litellmrouter_config=_parse_llm_router_config(ENV_REFLECT_LLM_LITELLMROUTER_CONFIG),
             reflect_llm_reasoning_effort=os.getenv(ENV_REFLECT_LLM_REASONING_EFFORT) or None,
             reflect_llm_extra_body=json.loads(os.getenv(ENV_REFLECT_LLM_EXTRA_BODY, "null")),
+            reflect_llm_cache_affinity=os.getenv(ENV_REFLECT_LLM_CACHE_AFFINITY) or None,
             consolidation_llm_provider=os.getenv(ENV_CONSOLIDATION_LLM_PROVIDER) or None,
             consolidation_llm_api_key=os.getenv(ENV_CONSOLIDATION_LLM_API_KEY) or None,
             consolidation_llm_model=os.getenv(ENV_CONSOLIDATION_LLM_MODEL)
@@ -3149,6 +3206,7 @@ class HindsightConfig:
             consolidation_llm_litellmrouter_config=_parse_llm_router_config(ENV_CONSOLIDATION_LLM_LITELLMROUTER_CONFIG),
             consolidation_llm_reasoning_effort=os.getenv(ENV_CONSOLIDATION_LLM_REASONING_EFFORT) or None,
             consolidation_llm_extra_body=json.loads(os.getenv(ENV_CONSOLIDATION_LLM_EXTRA_BODY, "null")),
+            consolidation_llm_cache_affinity=os.getenv(ENV_CONSOLIDATION_LLM_CACHE_AFFINITY) or None,
             # Multi-LLM chains (indexed members + routing strategy)
             llm_members=_parse_llm_members(""),
             llm_strategy=_parse_llm_strategy(os.getenv(ENV_LLM_STRATEGY)),

hindsight-api-slim/hindsight_api/config_resolver.py

@@ -11,8 +11,10 @@ multiple API servers.
 import asyncio
 import json
 import logging
-from dataclasses import asdict, replace
-from typing import TYPE_CHECKING, Any
+from dataclasses import asdict, fields, replace
+from functools import lru_cache
+from types import UnionType
+from typing import TYPE_CHECKING, Any, Union, get_args, get_origin
 
 from hindsight_api.config import (
     RECALL_BUDGET_FUNCTIONS,
@@ -295,7 +297,8 @@ class ConfigResolver:
 
                     # Only return active overrides for configurable fields. JSON null is a tombstone
                     # for "Server Default" in the bank-config UI and should not override defaults.
-                    return {k: v for k, v in normalized.items() if k in self._configurable_fields and v is not None}
+                    active = {k: v for k, v in normalized.items() if k in self._configurable_fields and v is not None}
+                    return _coerce_stored_bank_overrides(bank_id, active)
         except Exception as e:
             logger.error(f"Failed to load bank config for {bank_id}: {e}")
 
@@ -335,7 +338,7 @@ class ConfigResolver:
                         k: v for k, v in normalized.items() if k in self._configurable_fields and v is not None
                     }
                     if overrides:
-                        result[row["bank_id"]] = overrides
+                        result[row["bank_id"]] = _coerce_stored_bank_overrides(row["bank_id"], overrides)
         except Exception as e:
             logger.error(f"Failed to bulk-load bank configs: {e}")
         return result
@@ -419,6 +422,11 @@ class ConfigResolver:
                 logger.warning(f"Failed to check permissions for bank {bank_id}: {e}")
                 # Continue without permission check (fail open for backward compatibility)
 
+        # Validate every value against its declared field type before the
+        # field-specific checks below, so a wrong-shaped value is reported as such
+        # instead of tripping a structural validator with a confusing message.
+        _validate_config_value_types(normalized_updates)
+
         # Validate entity_labels structure
         if "entity_labels" in normalized_updates and normalized_updates["entity_labels"] is not None:
             from .engine.retain.entity_labels import parse_entity_labels
@@ -435,6 +443,16 @@ class ConfigResolver:
                 raise ValueError(
                     "Strategy names must not be empty strings. Remove entries with empty names before saving."
                 )
+            # A strategy's overrides are applied with dataclasses.replace() at retain
+            # time, so a wrong-shaped value there wedges the bank exactly as a
+            # top-level one would. Same contract, same door.
+            for strategy_name, strategy_overrides in normalized_updates["retain_strategies"].items():
+                if not isinstance(strategy_overrides, dict):
+                    raise ValueError(f"Invalid retain strategy {strategy_name!r}: must be an object")
+                try:
+                    _validate_config_value_types(normalize_config_dict(strategy_overrides))
+                except ValueError as e:
+                    raise ValueError(f"Invalid retain strategy {strategy_name!r}: {e}") from e
 
         # Validate recall budget fields
         _validate_recall_budget_updates(normalized_updates)
@@ -530,6 +548,147 @@ class ConfigResolver:
         logger.info(f"Reset bank config for {bank_id} to defaults")
 
 
+# Fields whose accepted input shape is deliberately wider than the dataclass
+# annotation, because a dedicated structural validator normalizes them later.
+_WIDENED_FIELD_TYPES: dict[str, tuple[type, ...]] = {
+    # parse_entity_labels() accepts both the bare list of label groups and the
+    # {"attributes": [...]} envelope, though the field is annotated `list | None`.
+    "entity_labels": (list, dict),
+}
+
+
+def _runtime_types(declared: Any) -> tuple[type, ...]:
+    """Runtime-checkable base classes for a dataclass field annotation.
+
+    Unwraps unions (``str | None``) and generic aliases (``list[str]`` -> ``list``);
+    ``None`` is dropped because callers handle the tombstone separately. Returns an
+    empty tuple for anything not reducible to concrete classes, which the callers
+    read as "no type contract to enforce".
+    """
+    if declared is type(None):
+        return ()
+    origin = get_origin(declared)
+    if origin in (Union, UnionType):
+        return tuple(t for arg in get_args(declared) for t in _runtime_types(arg))
+    if origin is not None:
+        return (origin,) if isinstance(origin, type) else ()
+    return (declared,) if isinstance(declared, type) else ()
+
+
+@lru_cache(maxsize=1)
+def _configurable_field_types() -> dict[str, tuple[type, ...]]:
+    """Map each configurable field to the value types it accepts."""
+    configurable = HindsightConfig.get_configurable_fields()
+    field_types: dict[str, tuple[type, ...]] = {}
+    for field in fields(HindsightConfig):
+        if field.name not in configurable:
+            continue
+        allowed = _WIDENED_FIELD_TYPES.get(field.name) or _runtime_types(field.type)
+        if allowed:
+            field_types[field.name] = allowed
+    return field_types
+
+
+def _value_matches_type(value: Any, allowed: tuple[type, ...]) -> bool:
+    """Whether ``value`` satisfies a field's declared type contract."""
+    if isinstance(value, bool):
+        # bool is an int subclass; it must not slip into a numeric field.
+        return bool in allowed
+    if isinstance(value, int) and float in allowed:
+        # JSON draws no int/float distinction: 1 is a valid ratio.
+        return True
+    return isinstance(value, allowed)
+
+
+# Field types are reported to API clients, so name them the way the JSON payload
+# reads rather than by their Python class.
+_TYPE_DESCRIPTIONS: dict[type, str] = {
+    bool: "a boolean",
+    int: "an integer",
+    float: "a number",
+    str: "a string",
+    list: "a list",
+    dict: "an object",
+}
+
+
+def _describe_types(allowed: tuple[type, ...]) -> str:
+    return " or ".join(dict.fromkeys(_TYPE_DESCRIPTIONS.get(t, t.__name__) for t in allowed))
+
+
+def _validate_config_value_types(updates: dict[str, Any]) -> None:
+    """Reject values whose type contradicts the declared HindsightConfig type.
+
+    Without this, the bank-config API happily stores e.g. a JSON object in
+    ``observations_mission``; the write succeeds and the bank then fails every
+    consolidation with ``expected string or bytes-like object, got 'dict'`` from
+    deep inside prompt assembly (issue #3218). Reject at the door instead, naming
+    the field and the expected type.
+    """
+    field_types = _configurable_field_types()
+    for key, value in updates.items():
+        allowed = field_types.get(key)
+        # None is the "clear this override" tombstone; unknown keys are rejected
+        # elsewhere as non-configurable.
+        if allowed is None or value is None:
+            continue
+        if not _value_matches_type(value, allowed):
+            raise ValueError(f"{key} must be {_describe_types(allowed)}, got {type(value).__name__}")
+
+
+def _coerce_stored_bank_overrides(bank_id: str, overrides: dict[str, Any], where: str = "") -> dict[str, Any]:
+    """Make stored bank overrides safe to consume, tolerating pre-validation shapes.
+
+    ``_validate_config_value_types`` rejects bad types at write time, but banks
+    configured before that landed can still hold e.g. a JSON object in a
+    string-typed field. Every consumer that treats such a value as text blows up
+    identically on every run (``escape_for_prompt`` -> ``re.sub`` ->
+    "expected string or bytes-like object, got 'dict'"), so the bank's
+    consolidation never recovers on its own (issue #3218).
+
+    String fields are JSON-encoded, which preserves the author's intent — the
+    structure still reaches the prompt, as text. Anything else is dropped so the
+    bank falls back to the tenant/global value rather than wedging.
+
+    ``where`` labels the location in warnings; it is set when recursing into a
+    retain strategy, whose overrides reach the same fields via ``apply_strategy``.
+    """
+    field_types = _configurable_field_types()
+    coerced: dict[str, Any] = {}
+    for key, value in overrides.items():
+        allowed = field_types.get(key)
+        # None passes through: the caller has already dropped top-level tombstones,
+        # and inside a retain strategy a null is a deliberate override to None.
+        if allowed is None or value is None or _value_matches_type(value, allowed):
+            coerced[key] = value
+            continue
+        if str in allowed:
+            coerced[key] = json.dumps(value, ensure_ascii=False)
+            logger.warning(
+                f"Bank {bank_id} config field '{key}'{where} holds a {type(value).__name__} but is a string field; "
+                f"using its JSON encoding. Re-save this field as a string to silence this warning."
+            )
+        else:
+            logger.warning(
+                f"Bank {bank_id} config field '{key}'{where} holds a {type(value).__name__} but must be "
+                f"{_describe_types(allowed)}; ignoring the override and falling back to the server default."
+            )
+
+    # Strategy overrides are spliced onto the resolved config by apply_strategy(),
+    # so a bad value nested there wedges the bank just as a top-level one does.
+    strategies = coerced.get("retain_strategies")
+    if isinstance(strategies, dict):
+        coerced["retain_strategies"] = {
+            name: (
+                _coerce_stored_bank_overrides(bank_id, strategy, where=f" in retain strategy {name!r}")
+                if isinstance(strategy, dict)
+                else strategy
+            )
+            for name, strategy in strategies.items()
+        }
+    return coerced
+
+
 _RECALL_BUDGET_FIXED_KEYS = (
     "recall_budget_fixed_low",
     "recall_budget_fixed_mid",

hindsight-api-slim/hindsight_api/engine/cache_affinity.py

@@ -0,0 +1,185 @@
+"""Server-side prompt-cache affinity hints for OpenAI-compatible providers.
+
+Prompt caching only pays off when the same conversation reaches the same backend
+cache, and providers expose different mechanisms for that:
+
+- xAI stores prompt-cache entries **per backend server** and routes requests
+  carrying the same ``x-grok-conv-id`` to one server (docs.x.ai, "Maximizing
+  Cache Hits"). Without it, consecutive calls of one agentic loop can each land
+  on a cache-cold replica.
+- OpenAI accepts a ``prompt_cache_key`` request field that improves its own
+  cache routing.
+
+Hindsight already does provider-specific cache work for its first-class
+providers (``anthropic_llm`` sets ``cache_control`` breakpoints; ``gemini_llm``
+runs an explicit ``CachedContent`` manager). This module is the equivalent for
+the OpenAI-compatible family — ``OpenAICompatibleLLM`` and its ``fireworks``
+and ``nous`` subclasses — which sent no affinity hint at all.
+
+Default ``auto`` per member (``cache_affinity``). ``auto`` is an allowlist, not a
+best-effort probe: it emits a hint only for hosts documented to accept one and
+resolves to ``none`` for everything else, so an unknown OpenAI-compatible backend
+never receives an unfamiliar field. Every helper here is fail-open — when no id
+can be derived the request goes out byte-identical to before. Set ``none`` to
+disable entirely.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from enum import StrEnum
+from typing import Any
+from urllib.parse import urlparse
+
+logger = logging.getLogger(__name__)
+
+# xAI's documented cache-pinning header, and OpenAI's cache-routing field.
+XAI_CONV_ID_HEADER = "x-grok-conv-id"
+OPENAI_PROMPT_CACHE_KEY_PARAM = "prompt_cache_key"
+
+# Hosts (exact or parent domain) whose backends implement the xAI header.
+_XAI_DOMAINS = ("x.ai", "grok.com")
+# Hosts (exact or parent domain) that accept OpenAI's prompt_cache_key field.
+_OPENAI_DOMAINS = ("openai.com", "openai.azure.com")
+
+
+class CacheAffinityMode(StrEnum):
+    """How (and whether) to pin a request to a backend prompt cache."""
+
+    NONE = "none"
+    XAI_CONV_ID = "xai_conv_id"
+    OPENAI_PROMPT_CACHE_KEY = "openai_prompt_cache_key"
+    AUTO = "auto"
+
+
+def parse_cache_affinity(value: str | None) -> CacheAffinityMode:
+    """Validate a configured cache-affinity mode, defaulting to ``none``.
+
+    Raises ``ValueError`` on an unrecognized value so a typo fails loudly at
+    provider construction rather than silently disabling the feature — the whole
+    point of the setting is that its effect is invisible in the response.
+    """
+    if not value:
+        return CacheAffinityMode.NONE
+    try:
+        return CacheAffinityMode(value.strip().lower())
+    except ValueError as e:
+        valid = ", ".join(mode.value for mode in CacheAffinityMode)
+        raise ValueError(f"Invalid cache_affinity {value!r}. Must be one of: {valid}.") from e
+
+
+def _host_matches(hostname: str, domain: str) -> bool:
+    """True when ``hostname`` is ``domain`` itself or a subdomain of it.
+
+    Parsed-host suffix matching, never a substring test: a bare
+    ``"x.ai" in base_url`` also matches ``vertex.ai`` and
+    ``https://x.ai.evil.example``. The in-tree Azure check
+    (``".openai.azure.com" in self.base_url``) gets away with a substring only
+    because its needle is long and dotted; ``x.ai`` is four characters.
+    """
+    return hostname == domain or hostname.endswith(f".{domain}")
+
+
+def resolve_cache_affinity(mode: CacheAffinityMode, provider: str, base_url: str | None) -> CacheAffinityMode:
+    """Resolve ``auto`` to a concrete mode from the provider and base-URL host.
+
+    Non-``auto`` modes are returned unchanged. ``auto`` resolves to
+    ``xai_conv_id`` for an x.ai / grok.com host, ``openai_prompt_cache_key`` for
+    native OpenAI (no base URL) or an openai.com / Azure OpenAI host, and
+    ``none`` for everything else — an unknown backend gets no unfamiliar field.
+
+    The xAI check is host-only and deliberately provider-independent: the
+    documented setup for an xAI endpoint is ``provider=openai`` plus an x.ai base
+    URL, exactly like Azure OpenAI, so keying on the provider name would miss it.
+    """
+    if mode is not CacheAffinityMode.AUTO:
+        return mode
+
+    hostname = (urlparse(base_url).hostname or "") if base_url else ""
+    if hostname and any(_host_matches(hostname, domain) for domain in _XAI_DOMAINS):
+        return CacheAffinityMode.XAI_CONV_ID
+    if provider.lower() == "openai":
+        if not hostname:
+            return CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY
+        if any(_host_matches(hostname, domain) for domain in _OPENAI_DOMAINS):
+            return CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY
+    return CacheAffinityMode.NONE
+
+
+def _first_message_fingerprint(messages: Any) -> str | None:
+    """Hash the first message into a 32-hex id, or None if the shape is wrong.
+
+    Used only when no trace context is bound (direct provider use, tests). The
+    first message is the system prompt, so the id is stable as the message list
+    grows through an agent loop — which is the property cache pinning needs —
+    while differing across conversations whose first messages differ.
+
+    Shape-checked rather than truthiness-checked: a bare string ``messages``
+    would index to its first character and mint an id from garbage. Anything
+    unexpected returns None and the request goes out with no affinity hint.
+    """
+    if not isinstance(messages, list) or not messages or not isinstance(messages[0], dict):
+        return None
+    try:
+        canonical = json.dumps(messages[0], sort_keys=True, ensure_ascii=False, default=str)
+    except (TypeError, ValueError):
+        logger.debug("Cache affinity: first message not serializable; sending no hint", exc_info=True)
+        return None
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:32]
+
+
+def cache_affinity_id(messages: Any) -> str | None:
+    """Return the affinity id for the in-flight call, or None to send nothing.
+
+    Primary source is the operation's ``trace_id`` — one uuid per
+    retain/reflect/consolidation run, generated in ``LLMProvider.with_config``
+    and bound around every underlying provider call, so every LLM call of one
+    run shares it. That is engine identity rather than payload hashing: it stays
+    constant across a run even when the first message changes mid-run.
+
+    The value is always 32 lowercase hex characters, including for the trace_id
+    path (hashed rather than passed through) so the wire format is uniform and
+    carries no uuid semantics.
+    """
+    from .llm_trace import current_trace_context
+
+    trace_ctx = current_trace_context()
+    if trace_ctx is not None and trace_ctx.trace_id:
+        return hashlib.sha256(str(trace_ctx.trace_id).encode("utf-8")).hexdigest()[:32]
+    return _first_message_fingerprint(messages)
+
+
+def apply_cache_affinity(request: dict[str, Any], mode: CacheAffinityMode) -> None:
+    """Add this request's cache-affinity hint to ``request`` in place.
+
+    ``mode`` must already be resolved (see :func:`resolve_cache_affinity`);
+    ``none`` — and an unresolved ``auto`` — add nothing.
+
+    User-wins semantics throughout, matching the file's ``setdefault`` precedent
+    in ``_apply_provider_extra_body_defaults``: an ``x-grok-conv-id`` the caller
+    already placed in ``extra_headers`` is kept, and a ``prompt_cache_key`` in
+    the operator's configured ``extra_body`` (the escape hatch for a backend
+    that wants its own value) suppresses ours entirely.
+
+    Never raises: when no id can be derived the request is left byte-identical
+    to a pre-affinity one.
+    """
+    affinity_id = cache_affinity_id(request.get("messages"))
+    if affinity_id is None:
+        return
+
+    if mode is CacheAffinityMode.XAI_CONV_ID:
+        extra_headers = request.setdefault("extra_headers", {})
+        extra_headers.setdefault(XAI_CONV_ID_HEADER, affinity_id)
+    elif mode is CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY:
+        # prompt_cache_key is a first-class named parameter on
+        # chat.completions.create() in the resolved openai SDK, so it goes at the
+        # top level rather than through extra_body. An operator value in
+        # extra_body would still reach the same wire field, so honour it and
+        # send nothing rather than sending both.
+        extra_body = request.get("extra_body")
+        if isinstance(extra_body, dict) and OPENAI_PROMPT_CACHE_KEY_PARAM in extra_body:
+            return
+        request.setdefault(OPENAI_PROMPT_CACHE_KEY_PARAM, affinity_id)

hindsight-api-slim/hindsight_api/engine/consolidation/consolidator.py

@@ -303,7 +303,7 @@ async def _dedup_reconcile_create(
             live_source_ids = await _filter_live_source_memories(conn, bank_id, create_source_ids)
             if not live_source_ids:
                 return None
-            if store.writes_memory_rows_in_sql:
+            if store.writes_memory_rows_in_sql_for(bank_id):
                 # Oracle-safe: _native_search_vector_update emits the to_tsvector clause only for a
                 # native PG tsvector column, "" otherwise (see #3021 — the raw ::regconfig cast
                 # breaks Oracle). RETURNING-gate on the twin's probe-time text so a concurrent
@@ -385,7 +385,7 @@ async def _dedup_reconcile_update(
     store = get_memories()
     async with acquire_with_retry(pool) as conn:
         async with conn.transaction():
-            if store.writes_memory_rows_in_sql:
+            if store.writes_memory_rows_in_sql_for(bank_id):
                 # Snapshot the updated row's sources with a PLAIN read (no FOR UPDATE). Lock order
                 # must be sources-before-observation: _filter_live_source_memories below takes
                 # FOR SHARE on the SOURCE rows first, then the fold UPDATE locks the observation
@@ -583,7 +583,7 @@ async def _filter_live_source_memories(
     if not source_memory_ids:
         return []
     store = get_memories()
-    if store.writes_memory_rows_in_sql:
+    if store.writes_memory_rows_in_sql_for(bank_id):
         rows = await conn.fetch(
             f"SELECT id FROM {fq_table('memory_units')} WHERE id = ANY($1::uuid[]) AND bank_id = $2 FOR SHARE",
             source_memory_ids,
@@ -612,7 +612,7 @@ async def _any_live_source_memory(
     if not source_memory_ids:
         return False
     store = get_memories()
-    if store.writes_memory_rows_in_sql:
+    if store.writes_memory_rows_in_sql_for(bank_id):
         found = await conn.fetchval(
             f"SELECT 1 FROM {fq_table('memory_units')} WHERE id = ANY($1::uuid[]) AND bank_id = $2 LIMIT 1",
             source_memory_ids,
@@ -732,7 +732,7 @@ async def _count_observations_for_scope(
     Observations with no tags are not counted (the limit does not apply to them).
     """
     store = get_memories()
-    if store.writes_memory_rows_in_sql:
+    if store.writes_memory_rows_in_sql_for(bank_id):
         return await conn.fetchval(
             f"SELECT COUNT(*) FROM {fq_table('memory_units')} "
             f"WHERE bank_id = $1 AND fact_type = 'observation' AND tags @> $2::varchar[]",
@@ -2245,7 +2245,7 @@ async def _execute_update_action(
             merged_tags = list(existing_tags | source_tags)
 
             t0 = time.time()
-            if store.writes_memory_rows_in_sql:
+            if store.writes_memory_rows_in_sql_for(bank_id):
                 updated_rows = await conn.execute_rows_affected(
                     f"""
                     UPDATE {fq_table("memory_units")}
@@ -2396,7 +2396,7 @@ async def _execute_delete_action(
 ) -> None:
     """Delete a superseded or contradicted observation."""
     store = get_memories()
-    if store.writes_memory_rows_in_sql:
+    if store.writes_memory_rows_in_sql_for(bank_id):
         await conn.execute(
             f"DELETE FROM {fq_table('memory_units')} WHERE id = $1 AND bank_id = $2 AND fact_type = 'observation'",
             uuid.UUID(observation_id),
@@ -2782,7 +2782,7 @@ async def _create_observation_directly(
             source_memory_ids = live_source_memory_ids
 
             t0 = time.time()
-            if store.writes_memory_rows_in_sql:
+            if store.writes_memory_rows_in_sql_for(bank_id):
                 # Query varies based on text search backend.
                 from ..schema import _is_oracle  # noqa: PLC0415
 

hindsight-api-slim/hindsight_api/engine/db/ops.py

@@ -25,6 +25,68 @@ from .base import DatabaseConnection
 from .result import ResultRow
 
 
+def graph_maintenance_bank_serialization_sql(table: str, alias: str) -> str:
+    """SQL predicate serialising ``graph_maintenance`` claims per bank (#3230).
+
+    Every graph_maintenance run is the same bank-wide sweep — the payload carries
+    only ``bank_id``, and ``run_graph_maintenance_job`` drains the whole queue —
+    so a second concurrent run for one bank adds no work. It is worse than
+    useless: ``claim_graph_maintenance_batch`` locks queue rows ``FOR UPDATE``
+    *without* ``SKIP LOCKED`` (it is written assuming a single runner per bank),
+    so the runs convoy on each other's row locks while each holds a worker slot.
+
+    Same guarantee ``consolidation`` already gets from its ``bank_id != ALL(busy)``
+    exclusion, and the same caveat: a row wedged in 'processing' holds its bank
+    until something releases it (``hindsight-admin recover``, or a restart with a
+    stable ``HINDSIGHT_API_WORKER_ID`` so ``recover_own_tasks`` matches it). That
+    is a general gap in claim recovery, not specific to graph_maintenance.
+
+    Two differences from the consolidation form, both forced by the shape of this
+    problem:
+
+    * It is a **predicate**, not a separate claim phase. Pulling graph_maintenance
+      into its own phase after the generic shared-pool query would drop it below
+      every other operation type: it has no reserved-slot floor
+      (``WORKER_SLOT_TYPE_DEFAULTS`` gives consolidation 2 and graph_maintenance
+      0), and the poller's fairness pass calls ``claim_tasks`` with
+      ``shared_limit=1``, so a single pending retain would starve it indefinitely.
+      As a predicate it keeps competing by ``created_at``.
+    * It also suppresses every same-bank row but the oldest **within one batch**.
+      Excluding busy banks alone does not: with several pending rows and nothing
+      yet processing, one batch claims them all — the convoy, unchanged. Several
+      pending rows per bank are reachable through the recovery paths
+      (``_reclaim_own_processing_tasks`` resets *all* of a worker's processing
+      rows in one statement, from ``recover_own_tasks`` at startup and
+      ``release_own_tasks`` at shutdown, plus ``_schedule_retry`` /
+      ``_defer_operation`` / ``hindsight-admin recover``).
+
+    The candidate row is always 'pending' and the 'pending' branch is
+    strictly-older, so the subquery can never match the candidate itself. The
+    fragment carries no SQL comments on purpose — it is rewritten for Oracle by
+    regex (``db/oracle.py``).
+
+    Args:
+        table: Fully-qualified async_operations table.
+        alias: Alias of the outer candidate row in the calling query.
+    """
+    return f"""
+        ({alias}.operation_type <> 'graph_maintenance' OR NOT EXISTS (
+            SELECT 1 FROM {table} gm_peer
+            WHERE gm_peer.bank_id = {alias}.bank_id
+              AND gm_peer.operation_type = 'graph_maintenance'
+              AND (
+                  gm_peer.status = 'processing'
+                  OR (gm_peer.status = 'pending'
+                      AND gm_peer.task_payload IS NOT NULL
+                      AND (gm_peer.next_retry_at IS NULL OR gm_peer.next_retry_at <= NOW())
+                      AND (gm_peer.created_at < {alias}.created_at
+                           OR (gm_peer.created_at = {alias}.created_at
+                               AND gm_peer.operation_id < {alias}.operation_id)))
+              )
+        ))
+    """
+
+
 @dataclass
 class TagListingParts:
     """Backend-specific SQL fragments for the tag listing query."""
@@ -182,6 +244,7 @@ class DataAccessOps(ABC):
         table: str,
         sorted_links: list[tuple],
         bank_id: str,
+        nil_entity_uuid: str,
         exists_clause: str,
         chunk_size: int = 5000,
     ) -> None:
@@ -600,6 +663,10 @@ class DataAccessOps(ABC):
         Oracle implementation uses two-step claims (query busy banks first, then
         claim excluding them) to avoid ORA-02014.
 
+        Implementations must apply :func:`graph_maintenance_bank_serialization_sql`
+        to every query that can return a ``graph_maintenance`` row, so at most one
+        such row per bank is ever in flight.
+
         Args:
             consolidation_bank_priority: Per-bank priority for consolidation scheduling.
                 Maps bank name patterns to integer priorities (higher = claimed first).

hindsight-api-slim/hindsight_api/engine/db/ops_oracle.py

@@ -10,7 +10,13 @@ import uuid as uuid_mod
 from datetime import UTC, datetime
 
 from .base import DatabaseConnection
-from .ops import DataAccessOps, LinkExpansionRows, TagListingParts, UpdatedWindow
+from .ops import (
+    DataAccessOps,
+    LinkExpansionRows,
+    TagListingParts,
+    UpdatedWindow,
+    graph_maintenance_bank_serialization_sql,
+)
 from .result import DictResultRow as ResultRow
 
 ORACLE_IN_LIST_LIMIT = 1000
@@ -142,6 +148,7 @@ class OracleOps(DataAccessOps):
         table: str,
         sorted_links: list[tuple],
         bank_id: str,
+        nil_entity_uuid: str,
         exists_clause: str,
         chunk_size: int = 5000,
     ) -> None:
@@ -152,16 +159,18 @@ class OracleOps(DataAccessOps):
         to_ids = [lnk[1] for lnk in sorted_links]
         types = [lnk[2] for lnk in sorted_links]
         weights = [lnk[3] for lnk in sorted_links]
+        entity_ids = [lnk[4] for lnk in sorted_links]
 
         await conn.executemany(
             f"""
             INSERT INTO {table}
-                (from_unit_id, to_unit_id, link_type, weight, bank_id)
-            VALUES ($1, $2, $3, $4, $5)
-            ON CONFLICT (from_unit_id, to_unit_id, link_type)
+                (from_unit_id, to_unit_id, link_type, weight, entity_id, bank_id)
+            VALUES ($1, $2, $3, $4, $5, $6)
+            ON CONFLICT (from_unit_id, to_unit_id, link_type,
+                         COALESCE(entity_id, '{nil_entity_uuid}'::uuid))
             DO NOTHING
             """,
-            [(from_ids[i], to_ids[i], types[i], weights[i], bank_id) for i in range(len(sorted_links))],
+            [(from_ids[i], to_ids[i], types[i], weights[i], entity_ids[i], bank_id) for i in range(len(sorted_links))],
         )
 
     async def bulk_insert_entities(
@@ -1329,13 +1338,14 @@ class OracleOps(DataAccessOps):
             else:
                 rows = await conn.fetch(
                     f"""
-                    SELECT operation_id, operation_type, task_payload, retry_count
-                    FROM {table}
-                    WHERE status = 'pending'
-                      AND task_payload IS NOT NULL
-                      AND operation_type = $1
-                      AND (next_retry_at IS NULL OR next_retry_at <= NOW())
-                    ORDER BY created_at
+                    SELECT o.operation_id, o.operation_type, o.task_payload, o.retry_count
+                    FROM {table} o
+                    WHERE o.status = 'pending'
+                      AND o.task_payload IS NOT NULL
+                      AND o.operation_type = $1
+                      AND (o.next_retry_at IS NULL OR o.next_retry_at <= NOW())
+                      AND {graph_maintenance_bank_serialization_sql(table, "o")}
+                    ORDER BY o.created_at
                     LIMIT $2
                     FOR UPDATE SKIP LOCKED
                     """,
@@ -1350,18 +1360,21 @@ class OracleOps(DataAccessOps):
         # --- Phase 2: claim from shared pool ---
         remaining_shared = shared_limit
         if remaining_shared > 0:
-            # 2a. Non-consolidation tasks
+            # 2a. Non-consolidation tasks. graph_maintenance stays in this
+            # created_at-ordered query — see graph_maintenance_bank_serialization_sql
+            # for why it is a predicate rather than a phase of its own.
             if claimed_ids:
                 rows = await conn.fetch(
                     f"""
-                    SELECT operation_id, operation_type, task_payload, retry_count
-                    FROM {table}
-                    WHERE status = 'pending'
-                      AND task_payload IS NOT NULL
-                      AND operation_type != 'consolidation'
-                      AND (next_retry_at IS NULL OR next_retry_at <= NOW())
-                      AND operation_id != ALL($1::uuid[])
-                    ORDER BY created_at
+                    SELECT o.operation_id, o.operation_type, o.task_payload, o.retry_count
+                    FROM {table} o
+                    WHERE o.status = 'pending'
+                      AND o.task_payload IS NOT NULL
+                      AND o.operation_type != 'consolidation'
+                      AND (o.next_retry_at IS NULL OR o.next_retry_at <= NOW())
+                      AND o.operation_id != ALL($1::uuid[])
+                      AND {graph_maintenance_bank_serialization_sql(table, "o")}
+                    ORDER BY o.created_at
                     LIMIT $2
                     FOR UPDATE SKIP LOCKED
                     """,
@@ -1371,13 +1384,14 @@ class OracleOps(DataAccessOps):
             else:
                 rows = await conn.fetch(
                     f"""
-                    SELECT operation_id, operation_type, task_payload, retry_count
-                    FROM {table}
-                    WHERE status = 'pending'
-                      AND task_payload IS NOT NULL
-                      AND operation_type != 'consolidation'
-                      AND (next_retry_at IS NULL OR next_retry_at <= NOW())
-                    ORDER BY created_at
+                    SELECT o.operation_id, o.operation_type, o.task_payload, o.retry_count
+                    FROM {table} o
+                    WHERE o.status = 'pending'
+                      AND o.task_payload IS NOT NULL
+                      AND o.operation_type != 'consolidation'
+                      AND (o.next_retry_at IS NULL OR o.next_retry_at <= NOW())
+                      AND {graph_maintenance_bank_serialization_sql(table, "o")}
+                    ORDER BY o.created_at
                     LIMIT $1
                     FOR UPDATE SKIP LOCKED
                     """,

hindsight-api-slim/hindsight_api/engine/db/ops_postgresql.py

@@ -4,10 +4,17 @@ Uses unnest(), LATERAL, DISTINCT ON, and native array operations for
 efficient batch operations.
 """
 
+import asyncio
 from datetime import datetime
 
 from .base import DatabaseConnection
-from .ops import DataAccessOps, LinkExpansionRows, TagListingParts, UpdatedWindow
+from .ops import (
+    DataAccessOps,
+    LinkExpansionRows,
+    TagListingParts,
+    UpdatedWindow,
+    graph_maintenance_bank_serialization_sql,
+)
 from .result import ResultRow
 
 
@@ -16,24 +23,34 @@ def pg_search_vector_expr(
     *,
     text_col: str = "text",
     context_col: str = "context",
-    signals_col: str = "text_signals",
+    signals_col: str | None = "text_signals",
+    native_inline: bool = True,
 ) -> str | None:
     """SQL expression that builds ``search_vector`` for the configured PG text-search backend.
 
-    Single source of truth shared by the batch insert (over the ``input_data``
-    CTE columns) and the curation revert recompute (over a ``memory_units`` row),
-    so the two can never drift. Returns ``None`` for backends that leave
-    ``search_vector`` unpopulated — pgroonga / pg_textsearch / pg_search index the
-    base text columns directly and keep only a dummy column, so there is nothing
-    to build.
+    Single source of truth shared by ``memory_units`` (the batch insert over the
+    ``input_data`` CTE columns and the curation-revert recompute) and
+    ``mental_models`` (the knowledge-page writes), so the per-backend tokenization
+    can never drift between the two tables. Returns ``None`` for backends that
+    leave ``search_vector`` unpopulated — pgroonga / pg_textsearch / pg_search
+    index the base text columns directly and keep only a dummy column, so there is
+    nothing to build.
+
+    The ``*_col`` arguments are the SQL for each text source (a column name or a
+    bind placeholder); pass ``signals_col=None`` for a two-column table like
+    ``mental_models`` (name + content). Pass ``native_inline=False`` when the
+    table's native ``search_vector`` is a GENERATED column that populates itself
+    (``mental_models``) — writing it inline would fail; only vchord's plain
+    bm25vector column then needs an explicit value.
 
     ``text_search_extension_native_language`` is validated as a PG identifier in
     ``HindsightConfig.validate()``, so embedding it as a SQL literal is safe.
     """
-    combined = f"COALESCE({text_col}, '') || ' ' || COALESCE({context_col}, '') || ' ' || COALESCE({signals_col}, '')"
+    cols = [text_col, context_col] + ([signals_col] if signals_col is not None else [])
+    combined = " || ' ' || ".join(f"COALESCE({c}, '')" for c in cols)
     if config.text_search_extension == "vchord":
         return f"tokenize({combined}, 'llmlingua2')::bm25_catalog.bm25vector"
-    if config.text_search_extension == "native":
+    if config.text_search_extension == "native" and native_inline:
         return f"to_tsvector('{config.text_search_extension_native_language}'::regconfig, {combined})"
     return None
 
@@ -41,6 +58,23 @@ def pg_search_vector_expr(
 class PostgreSQLOps(DataAccessOps):
     """PostgreSQL-specific data access operations using unnest and LATERAL."""
 
+    def __init__(self) -> None:
+        # Per-table serialization of per-bank vector-index DDL within this
+        # process. Concurrent index DDL on one relation deadlocks by design:
+        # DROP INDEX CONCURRENTLY holds ShareUpdateExclusive while it waits out
+        # every transaction whose snapshot could still see the index — including
+        # other sessions' index DDL queued on that same lock — so many banks
+        # deleted at once form a wait cycle Postgres resolves by killing one.
+        # A session advisory lock would serialize this across processes too, but
+        # advisory locks are banned here (poolers hand sessions around; see the
+        # Database Locking standard). In-process the asyncio lock removes the
+        # cycle outright; across processes the callers' retry-with-backoff
+        # absorbs the (now much rarer) collisions.
+        self._index_ddl_locks: dict[str, asyncio.Lock] = {}
+
+    def _index_ddl_lock(self, table: str) -> asyncio.Lock:
+        return self._index_ddl_locks.setdefault(table, asyncio.Lock())
+
     @property
     def uses_observation_sources_table(self) -> bool:
         return False  # PG uses native array ops on source_memory_ids
@@ -182,6 +216,7 @@ class PostgreSQLOps(DataAccessOps):
         table: str,
         sorted_links: list[tuple],
         bank_id: str,
+        nil_entity_uuid: str,
         exists_clause: str,
         chunk_size: int = 5000,
     ) -> None:
@@ -206,6 +241,7 @@ class PostgreSQLOps(DataAccessOps):
         to_ids = [lnk[1] for lnk in sorted_links]
         types = [lnk[2] for lnk in sorted_links]
         weights = [lnk[3] for lnk in sorted_links]
+        entity_ids = [lnk[4] for lnk in sorted_links]
 
         for chunk_start in range(0, len(sorted_links), chunk_size):
             chunk_end = min(chunk_start + chunk_size, len(sorted_links))
@@ -219,23 +255,25 @@ class PostgreSQLOps(DataAccessOps):
                 f"""
                 WITH locked AS (
                     SELECT id FROM {mu_table}
-                    WHERE id = ANY($6::uuid[])
+                    WHERE id = ANY($7::uuid[])
                     ORDER BY id
                     FOR KEY SHARE
                 )
                 INSERT INTO {table}
-                    (from_unit_id, to_unit_id, link_type, weight, bank_id)
-                SELECT f, t, tp, w, $5
-                FROM unnest($1::uuid[], $2::uuid[], $3::text[], $4::float8[])
-                    AS u(f, t, tp, w)
+                    (from_unit_id, to_unit_id, link_type, weight, entity_id, bank_id)
+                SELECT f, t, tp, w, e, $6
+                FROM unnest($1::uuid[], $2::uuid[], $3::text[], $4::float8[], $5::uuid[])
+                    AS u(f, t, tp, w, e)
                 WHERE f IN (SELECT id FROM locked) AND t IN (SELECT id FROM locked)
-                ON CONFLICT (from_unit_id, to_unit_id, link_type)
+                ON CONFLICT (from_unit_id, to_unit_id, link_type,
+                             COALESCE(entity_id, '{nil_entity_uuid}'::uuid))
                 DO NOTHING
                 """,
                 chunk_from,
                 chunk_to,
                 types[chunk_start:chunk_end],
                 weights[chunk_start:chunk_end],
+                entity_ids[chunk_start:chunk_end],
                 bank_id,
                 referenced,
                 timeout=300,
@@ -853,14 +891,15 @@ class PostgreSQLOps(DataAccessOps):
         fact_types: dict[str, str],
     ) -> None:
         escaped = bank_id.replace("'", "''")
-        for ft, suffix in fact_types.items():
-            uid = str(internal_id).replace("-", "")[:16]
-            idx = f"idx_mu_emb_{suffix}_{uid}"
-            await conn.execute(
-                f"CREATE INDEX IF NOT EXISTS {idx} "
-                f"ON {table} {index_clause} "
-                f"WHERE fact_type = '{ft}' AND bank_id = '{escaped}'"
-            )
+        async with self._index_ddl_lock(table):
+            for ft, suffix in fact_types.items():
+                uid = str(internal_id).replace("-", "")[:16]
+                idx = f"idx_mu_emb_{suffix}_{uid}"
+                await conn.execute(
+                    f"CREATE INDEX IF NOT EXISTS {idx} "
+                    f"ON {table} {index_clause} "
+                    f"WHERE fact_type = '{ft}' AND bank_id = '{escaped}'"
+                )
 
     async def drop_bank_vector_indexes(
         self,
@@ -875,10 +914,13 @@ class PostgreSQLOps(DataAccessOps):
         # table; CONCURRENTLY does not conflict with DML. The caller
         # (delete_bank) runs this on an autocommit connection after its delete
         # transaction has committed — CONCURRENTLY cannot run inside a tx.
-        for ft, suffix in fact_types.items():
-            uid = str(internal_id).replace("-", "")[:16]
-            idx = f"idx_mu_emb_{suffix}_{uid}"
-            await conn.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}.{idx}")
+        # The lock key must match create_bank_vector_indexes', whose `table`
+        # is the fq name this reconstructs from `schema`.
+        async with self._index_ddl_lock(f"{schema}.memory_units"):
+            for ft, suffix in fact_types.items():
+                uid = str(internal_id).replace("-", "")[:16]
+                idx = f"idx_mu_emb_{suffix}_{uid}"
+                await conn.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}.{idx}")
 
     def get_entity_resolution_strategy(self) -> str:
         return "trigram"
@@ -1385,13 +1427,14 @@ class PostgreSQLOps(DataAccessOps):
             else:
                 rows = await conn.fetch(
                     f"""
-                    SELECT operation_id, operation_type, task_payload, retry_count
-                    FROM {table}
-                    WHERE status = 'pending'
-                      AND task_payload IS NOT NULL
-                      AND operation_type = $1
-                      AND (next_retry_at IS NULL OR next_retry_at <= NOW())
-                    ORDER BY created_at
+                    SELECT o.operation_id, o.operation_type, o.task_payload, o.retry_count
+                    FROM {table} o
+                    WHERE o.status = 'pending'
+                      AND o.task_payload IS NOT NULL
+                      AND o.operation_type = $1
+                      AND (o.next_retry_at IS NULL OR o.next_retry_at <= NOW())
+                      AND {graph_maintenance_bank_serialization_sql(table, "o")}
+                    ORDER BY o.created_at
                     LIMIT $2
                     FOR UPDATE SKIP LOCKED
                     """,
@@ -1406,18 +1449,21 @@ class PostgreSQLOps(DataAccessOps):
         # --- Phase 2: claim from shared pool ---
         remaining_shared = shared_limit
         if remaining_shared > 0:
-            # 2a. Non-consolidation tasks
+            # 2a. Non-consolidation tasks. graph_maintenance stays in this
+            # created_at-ordered query — see graph_maintenance_bank_serialization_sql
+            # for why it is a predicate rather than a phase of its own.
             if claimed_ids:
                 rows = await conn.fetch(
                     f"""
-                    SELECT operation_id, operation_type, task_payload, retry_count
-                    FROM {table}
-                    WHERE status = 'pending'
-                      AND task_payload IS NOT NULL
-                      AND operation_type != 'consolidation'
-                      AND (next_retry_at IS NULL OR next_retry_at <= NOW())
-                      AND operation_id != ALL($1::uuid[])
-                    ORDER BY created_at
+                    SELECT o.operation_id, o.operation_type, o.task_payload, o.retry_count
+                    FROM {table} o
+                    WHERE o.status = 'pending'
+                      AND o.task_payload IS NOT NULL
+                      AND o.operation_type != 'consolidation'
+                      AND (o.next_retry_at IS NULL OR o.next_retry_at <= NOW())
+                      AND o.operation_id != ALL($1::uuid[])
+                      AND {graph_maintenance_bank_serialization_sql(table, "o")}
+                    ORDER BY o.created_at
                     LIMIT $2
                     FOR UPDATE SKIP LOCKED
                     """,
@@ -1427,13 +1473,14 @@ class PostgreSQLOps(DataAccessOps):
             else:
                 rows = await conn.fetch(
                     f"""
-                    SELECT operation_id, operation_type, task_payload, retry_count
-                    FROM {table}
-                    WHERE status = 'pending'
-                      AND task_payload IS NOT NULL
-                      AND operation_type != 'consolidation'
-                      AND (next_retry_at IS NULL OR next_retry_at <= NOW())
-                    ORDER BY created_at
+                    SELECT o.operation_id, o.operation_type, o.task_payload, o.retry_count
+                    FROM {table} o
+                    WHERE o.status = 'pending'
+                      AND o.task_payload IS NOT NULL
+                      AND o.operation_type != 'consolidation'
+                      AND (o.next_retry_at IS NULL OR o.next_retry_at <= NOW())
+                      AND {graph_maintenance_bank_serialization_sql(table, "o")}
+                    ORDER BY o.created_at
                     LIMIT $1
                     FOR UPDATE SKIP LOCKED
                     """,

hindsight-api-slim/hindsight_api/engine/graph_maintenance.py

@@ -29,6 +29,12 @@ The worker dedupes on bank: a second job for the same bank is dropped
 while one is pending. Once processing starts, a new job becomes the
 *next* pending slot — so work enqueued during processing gets picked up
 by the follow-up run.
+
+That follow-up run is *deferred*, not parallel: ``claim_tasks`` will not claim a
+graph_maintenance row for a bank that already has one in flight (#3230). Two
+concurrent runs would do no extra work anyway — each is this same bank-wide
+sweep — while convoying on each other's row locks and holding a worker slot
+each.
 """
 
 from __future__ import annotations

hindsight-api-slim/hindsight_api/engine/llm_wrapper.py

@@ -29,6 +29,7 @@ from ..config import (
     ENV_REFLECT_LLM_MAX_CONCURRENT,
     ENV_RETAIN_LLM_MAX_CONCURRENT,
 )
+from .cache_affinity import parse_cache_affinity
 from .llm_interface import (
     LLM_TOOL_CHOICE_AUTO,
     LLMToolChoice,
@@ -270,6 +271,7 @@ _PROVIDERS_WITHOUT_API_KEY = frozenset(
         "litellmrouter",
         "bedrock",
         "nous",
+        "xai-oauth",
     }
 )
 
@@ -310,6 +312,8 @@ def create_llm_provider(
     gemini_service_tier: str | None = None,
     timeout: float | None = None,
     ollama_num_ctx: int | None = None,
+    cache_affinity: str | None = None,
+    structured_output_forced_tool: bool = False,
 ) -> Any:  # Returns LLMInterface
     """
     Factory function to create the appropriate LLM provider implementation.
@@ -333,9 +337,20 @@ def create_llm_provider(
             for OpenAI/Anthropic vs ``max_output_tokens`` for Gemini).
         default_headers: Custom headers passed to provider SDK clients (used by operators
             routing through proxies / request-tracing middleware). Wired into the Anthropic
-            provider (SDK ``default_headers``) and the LiteLLM-backed providers — ``litellm``,
-            ``litellmrouter`` and ``bedrock`` — as the LiteLLM ``extra_headers`` completion
-            kwarg; other providers may opt in as needed.
+            provider, the ``OpenAICompatibleLLM`` branch, ``fireworks``, ``nous`` and the
+            Responses API (SDK ``default_headers``), and into the LiteLLM-backed providers —
+            ``litellm``, ``litellmrouter`` and ``bedrock`` — as the LiteLLM ``extra_headers``
+            completion kwarg; other providers may opt in as needed.
+        cache_affinity: Backend prompt-cache pinning mode, forwarded to the
+            ``OpenAICompatibleLLM`` branch, ``fireworks`` and ``nous`` (all three share the
+            OpenAI-compatible wire format): "none" (default), "xai_conv_id",
+            "openai_prompt_cache_key", or "auto". Providers on other branches do their own
+            cache work or none at all. See ``engine/cache_affinity.py``.
+        structured_output_forced_tool: Ask the LiteLLM-backed providers (``litellm``,
+            ``litellmrouter``, ``bedrock``) for structured output via a forced tool call
+            instead of ``response_format``. For backends that reject the response_format
+            route — see ``HINDSIGHT_API_LLM_STRUCTURED_OUTPUT_FORCED_TOOL``. Other
+            providers ignore it.
         vertexai_project_id: Vertex AI project ID (for VertexAI provider).
         vertexai_region: Vertex AI region (for VertexAI provider).
         vertexai_credentials: Vertex AI credentials object (for VertexAI provider).
@@ -381,6 +396,7 @@ def create_llm_provider(
             base_url=base_url,
             model=model,
             reasoning_effort=reasoning_effort,
+            extra_body=extra_body,
         )
 
     elif provider_lower == "claude-code":
@@ -447,6 +463,7 @@ def create_llm_provider(
             extra_body=extra_body,
             default_headers=default_headers,
             timeout=timeout,
+            structured_output_forced_tool=structured_output_forced_tool,
         )
 
     elif provider_lower == "litellmrouter":
@@ -467,6 +484,7 @@ def create_llm_provider(
             extra_body=extra_body,
             default_headers=default_headers,
             timeout=timeout,
+            structured_output_forced_tool=structured_output_forced_tool,
         )
 
     elif provider_lower == "bedrock":
@@ -482,6 +500,7 @@ def create_llm_provider(
             default_headers=default_headers,
             bedrock_service_tier=bedrock_service_tier,
             timeout=timeout,
+            structured_output_forced_tool=structured_output_forced_tool,
         )
 
     elif provider_lower == "llamacpp":
@@ -513,12 +532,16 @@ def create_llm_provider(
             model=model,
             reasoning_effort=reasoning_effort,
             extra_body=extra_body,
+            default_headers=default_headers,
+            cache_affinity=cache_affinity,
         )
 
     elif provider_lower == "nous":
         # Nous Portal is OpenAI-compatible on the wire; NousLLM adds rotating
         # inference:invoke JWT auth read natively from ~/.hermes/auth.json
         # (no static api_key, no hermes_cli dependency — same shape as Codex).
+        # default_headers/cache_affinity ride NousLLM's **kwargs passthrough to
+        # OpenAICompatibleLLM.__init__ unchanged (see NousLLM.__init__).
         from hindsight_api.engine.providers.nous_llm import NousLLM
 
         return NousLLM(
@@ -528,6 +551,25 @@ def create_llm_provider(
             model=model,
             reasoning_effort=reasoning_effort,
             extra_body=extra_body,
+            default_headers=default_headers,
+            cache_affinity=cache_affinity,
+            timeout=timeout,
+        )
+
+    elif provider_lower == "xai-oauth":
+        # SuperGrok subscription lane: api.x.ai spoken plainly, but the
+        # credential is a device-code OAuth grant with proactive/reactive
+        # refresh over a shared on-disk store, and xAI's 403 shapes need their
+        # own classification — neither fits the OpenAI SDK client, hence its
+        # own provider.
+        from hindsight_api.engine.providers.xai_oauth_llm import XaiOAuthLLM
+
+        return XaiOAuthLLM(
+            provider=provider,
+            api_key=api_key,
+            base_url=base_url,
+            model=model,
+            reasoning_effort=reasoning_effort,
             timeout=timeout,
         )
 
@@ -571,6 +613,8 @@ def create_llm_provider(
             groq_service_tier=groq_service_tier,
             openai_service_tier=openai_service_tier,
             extra_body=extra_body,
+            default_headers=default_headers,
+            cache_affinity=cache_affinity,
             ollama_num_ctx=ollama_num_ctx,
             timeout=timeout,
         )
@@ -610,6 +654,8 @@ class LLMProvider:
         initial_backoff: float | None = None,
         max_backoff: float | None = None,
         ollama_num_ctx: int | None = None,
+        cache_affinity: str | None = None,
+        structured_output_forced_tool: bool = False,
     ):
         """
         Initialize LLM provider.
@@ -631,6 +677,11 @@ class LLMProvider:
                 (OpenAI-compatible, Fireworks, Anthropic, Gemini/VertexAI, LiteLLM).
             default_headers: Custom headers passed as ``default_headers`` to provider SDK clients.
                 Used by operators routing through proxies / request-tracing middleware.
+            cache_affinity: Backend prompt-cache pinning mode for the OpenAI-compatible and
+                Fireworks providers ("none", "xai_conv_id", "openai_prompt_cache_key",
+                "auto"). Validated here for every provider so a typo never fails silently;
+                providers on other factory branches ignore it. Used verbatim — callers
+                resolve the per-operation/global fallback.
             litellmrouter_config: Provider-specific config for ``provider="litellmrouter"``.
                 JSON object passed verbatim to ``litellm.Router(**config)`` — see
                 https://docs.litellm.ai/docs/routing. Ignored unless ``provider == "litellmrouter"``.
@@ -651,6 +702,9 @@ class LLMProvider:
                 ``max_retries``. ``None`` keeps each method's own fallback.
             max_backoff: Default maximum retry backoff (seconds), same resolution as
                 ``max_retries``. ``None`` keeps each method's own fallback.
+            structured_output_forced_tool: Structured output via a forced tool call
+                instead of ``response_format``, for the LiteLLM-backed providers - from
+                config (``HINDSIGHT_API_LLM_STRUCTURED_OUTPUT_FORCED_TOOL``).
 
         This constructor uses every argument as passed and does not read global
         ``HindsightConfig``: resolving the server-level default for a ``None`` argument is the
@@ -679,6 +733,9 @@ class LLMProvider:
         self.openai_service_tier = openai_service_tier
         self.bedrock_service_tier = bedrock_service_tier
         self.gemini_service_tier = gemini_service_tier
+        # Structured-output transport for the LiteLLM-backed providers. Used verbatim —
+        # the caller resolves the server-level default, like the fields above.
+        self.structured_output_forced_tool = structured_output_forced_tool
         self.ollama_num_ctx = _validate_ollama_num_ctx(ollama_num_ctx)
         # Gemini safety settings (instance default; can be overridden per-request via context var)
         self.gemini_safety_settings = gemini_safety_settings
@@ -693,6 +750,11 @@ class LLMProvider:
         # Used verbatim — callers resolve the global fallback (see _member_to_llm /
         # the per-op builds in MemoryEngine, and LLMProvider.from_env).
         self.default_headers = default_headers
+        # Backend prompt-cache pinning mode. Validated here rather than only at the
+        # provider so a typo fails for every provider, not just the ones that act on
+        # it — the setting has no visible effect in the response, so a silent
+        # fallback to "none" would be indistinguishable from it working.
+        self.cache_affinity = parse_cache_affinity(cache_affinity).value
 
         # Validate provider
         valid_providers = [
@@ -723,6 +785,7 @@ class LLMProvider:
             "atlas",
             "fireworks",
             "nous",
+            "xai-oauth",
         ]
         if self.provider not in valid_providers:
             raise ValueError(f"Invalid LLM provider: {self.provider}. Must be one of: {', '.join(valid_providers)}")
@@ -827,6 +890,8 @@ class LLMProvider:
             litellmrouter_config=router_config,
             ollama_num_ctx=self.ollama_num_ctx,
             timeout=self.timeout,
+            cache_affinity=self.cache_affinity,
+            structured_output_forced_tool=self.structured_output_forced_tool,
         )
 
         # Backward compatibility: Keep mock provider properties
@@ -1366,15 +1431,18 @@ class LLMProvider:
         # does so without building the full HindsightConfig, keeping from_env() a
         # lightweight env-only loader (see test_llm_provider_from_env_keeps_lightweight_loader).
         from ..config import (
+            DEFAULT_LLM_CACHE_AFFINITY,
             DEFAULT_LLM_GROQ_SERVICE_TIER,
             DEFAULT_LLM_OPENAI_SERVICE_TIER,
             DEFAULT_LLM_PROMPT_CACHE_ENABLED,
             DEFAULT_LLM_PROVIDER,
             DEFAULT_LLM_REASONING_EFFORT,
+            DEFAULT_LLM_STRUCTURED_OUTPUT_FORCED_TOOL,
             DEFAULT_LLM_TIMEOUT,
             ENV_LLM_API_KEY,
             ENV_LLM_BASE_URL,
             ENV_LLM_BEDROCK_SERVICE_TIER,
+            ENV_LLM_CACHE_AFFINITY,
             ENV_LLM_DEFAULT_HEADERS,
             ENV_LLM_EXTRA_BODY,
             ENV_LLM_GEMINI_SAFETY_SETTINGS,
@@ -1387,11 +1455,13 @@ class LLMProvider:
             ENV_LLM_PROMPT_CACHE_ENABLED,
             ENV_LLM_PROVIDER,
             ENV_LLM_REASONING_EFFORT,
+            ENV_LLM_STRUCTURED_OUTPUT_FORCED_TOOL,
             ENV_LLM_TIMEOUT,
             ENV_LLM_VERTEXAI_PROJECT_ID,
             ENV_LLM_VERTEXAI_REGION,
             ENV_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY,
             _get_default_model_for_provider,
+            _parse_boolean_env,
             _parse_llm_router_config,
             _parse_optional_positive_int,
             parse_gemini_service_tier,
@@ -1411,6 +1481,9 @@ class LLMProvider:
         model = os.getenv(ENV_LLM_MODEL) or _get_default_model_for_provider(provider)
         extra_body = json.loads(os.getenv(ENV_LLM_EXTRA_BODY, "null"))
         default_headers = json.loads(os.getenv(ENV_LLM_DEFAULT_HEADERS, "null"))
+        # Same default as HindsightConfig.from_env: this entry point must not
+        # resolve to a different mode than the engine's own config path.
+        cache_affinity = os.getenv(ENV_LLM_CACHE_AFFINITY, DEFAULT_LLM_CACHE_AFFINITY) or None
         prompt_cache_enabled = os.getenv(
             ENV_LLM_PROMPT_CACHE_ENABLED, str(DEFAULT_LLM_PROMPT_CACHE_ENABLED)
         ).lower() in (
@@ -1428,6 +1501,7 @@ class LLMProvider:
             reasoning_effort=os.getenv(ENV_LLM_REASONING_EFFORT, DEFAULT_LLM_REASONING_EFFORT),
             extra_body=extra_body,
             default_headers=default_headers,
+            cache_affinity=cache_affinity,
             groq_service_tier=os.getenv(ENV_LLM_GROQ_SERVICE_TIER, DEFAULT_LLM_GROQ_SERVICE_TIER),
             openai_service_tier=os.getenv(ENV_LLM_OPENAI_SERVICE_TIER, DEFAULT_LLM_OPENAI_SERVICE_TIER),
             bedrock_service_tier=os.getenv(ENV_LLM_BEDROCK_SERVICE_TIER) or None,
@@ -1444,6 +1518,10 @@ class LLMProvider:
             vertexai_region=os.getenv(ENV_LLM_VERTEXAI_REGION) or None,
             vertexai_service_account_key=os.getenv(ENV_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY) or None,
             timeout=float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT))),
+            structured_output_forced_tool=_parse_boolean_env(
+                ENV_LLM_STRUCTURED_OUTPUT_FORCED_TOOL,
+                DEFAULT_LLM_STRUCTURED_OUTPUT_FORCED_TOOL,
+            ),
         )
 
 

hindsight-api-slim/hindsight_api/engine/maintenance.py

@@ -138,7 +138,12 @@ class MaintenanceLoop:
     @staticmethod
     def _cross_store_recovery_enabled() -> bool:
         """True when the memories store keeps memories outside SQL and therefore has
-        cross-store write-group txns a crashed writer could leave undecided."""
+        cross-store write-group txns a crashed writer could leave undecided.
+
+        Deliberately reads the PROCESS-LEVEL class attribute, not the per-bank
+        ``writes_memory_rows_in_sql_for(bank_id)`` — this only decides whether the recovery LOOP
+        needs to run at all. A store that routes some banks outside SQL keeps the class attribute
+        False so the loop runs, then ``recover_pending_txns`` is bank-scoped inside it."""
         try:
             from .memories import get_memories
 
@@ -282,6 +287,9 @@ class MaintenanceLoop:
             try:
                 table = fq_table_explicit("async_operations", schema)
                 async with acquire_with_retry(backend, max_retries=1) as conn:
+                    # Delete export archives owned by rows about to be pruned first,
+                    # so the file-storage blobs don't outlive their operation row.
+                    await engine.purge_expired_export_archives(conn, table, cutoff)
                     async with conn.transaction():
                         deleted = await backend.ops.prune_terminal_operations(
                             conn, table, cutoff, batch_size=cfg.operation_cleanup_batch_size

hindsight-api-slim/hindsight_api/engine/memories/base.py

@@ -394,6 +394,20 @@ class MemoriesExtension(Extension, ABC):
     #: the inline SQL. Cold, never-searched, key-based — see docs/documents-chunks.md.
     owns_document_store: bool = False
 
+    def writes_memory_rows_in_sql_for(self, bank_id: str) -> bool:
+        """Per-bank form of :attr:`writes_memory_rows_in_sql`. Defaults to the class attribute, so a
+        single-store extension needs no override. A store that keeps different banks in different
+        backends (some in SQL, some not) overrides this to answer PER BANK; every *bank-scoped* call
+        site consults this instead of the class attribute, so mixed banks each take the correct path.
+        (The few process-level gates — e.g. "is cross-store txn recovery relevant at all" — keep
+        reading the class attribute.)"""
+        return self.writes_memory_rows_in_sql
+
+    def owns_document_store_for(self, bank_id: str) -> bool:
+        """Per-bank form of :attr:`owns_document_store`. Defaults to the class attribute; a store
+        that keeps some banks in a separate backend overrides it. See :meth:`writes_memory_rows_in_sql_for`."""
+        return self.owns_document_store
+
     # ------------------------------------------------------------------ lifecycle
 
     async def initialize(self) -> None:

hindsight-api-slim/hindsight_api/engine/memories/pg/graph.py

@@ -486,9 +486,9 @@ async def enqueue_relink_victims(
     affected_str_set = {str(uid) for uid in affected_uuids}
 
     # Find units (other than the affected ones) that have an outgoing
-    # temporal/semantic link pointing at an affected unit. Only those two link
-    # types are relinked by graph maintenance; entity edges are not stored in
-    # memory_links (they're derived from unit_entities), so nothing else applies.
+    # temporal/semantic link pointing at an affected unit. Entity links are
+    # intentionally excluded — they're scheduled for removal and would only
+    # add noise to the recompute job.
     victim_rows = await conn.fetch(
         f"""
         SELECT DISTINCT from_unit_id
@@ -531,9 +531,14 @@ async def relink_pass(
 ) -> dict:
     """Drain ``graph_maintenance_queue`` for ``bank_id``, topping up lost links.
 
-    Per-iteration loop: claim → top up → commit. We rely on submit-time
-    dedup to keep at most one job per bank running, so no need for
-    SKIP LOCKED.
+    Per-iteration loop: claim → top up → commit. We rely on at most one job per
+    bank running, so no need for SKIP LOCKED. Submit-time dedup alone does NOT
+    give that — it only inspects 'pending' rows — so the guarantee comes from
+    ``claim_tasks``, which refuses to claim a graph_maintenance row for a bank
+    that already has one in flight (``graph_maintenance_bank_serialization_sql``,
+    #3230). Without it these claims convoy: they lock queue rows ``FOR UPDATE``
+    with no ``SKIP LOCKED``, so a second run blocks on the first while holding a
+    worker slot.
 
     Takes ``backend`` rather than a connection because the loop spans several
     transactions — one per claimed batch, plus a separate connection for the ANN
@@ -662,7 +667,7 @@ async def _relink_batch(
                 if time_diff_h > 24:
                     continue
                 weight = max(0.3, 1.0 - (time_diff_h / 24))
-                new_links.append((row["from_id"], str(row["id"]), "temporal", weight))
+                new_links.append((row["from_id"], str(row["id"]), "temporal", weight, None))
 
     # --- Semantic top-up ---
     # ANN must run on its own connection: it opens a nested transaction with

hindsight-api-slim/hindsight_api/engine/providers/codex_llm.py

@@ -127,6 +127,7 @@ class CodexLLM(LLMInterface):
         base_url: str,
         model: str,
         reasoning_effort: str = "low",
+        extra_body: dict[str, Any] | None = None,
         **kwargs: Any,
     ):
         """Initialize Codex LLM provider."""
@@ -177,6 +178,7 @@ class CodexLLM(LLMInterface):
         # Reasoning summary controls presentation separately from the backend's
         # reasoning effort, which is sent unchanged in each request payload.
         self.reasoning_summary = self._map_reasoning_effort(reasoning_effort)
+        self._extra_body = dict(extra_body or {})
 
         # HTTP client for SSE streaming
         self._client = httpx.AsyncClient(timeout=120.0)
@@ -457,6 +459,7 @@ class CodexLLM(LLMInterface):
             "include": ["reasoning.encrypted_content"],
             "prompt_cache_key": str(uuid.uuid4()),
         }
+        payload.update(self._extra_body)
 
         if use_forced_tool and schema is not None:
             # Single function tool whose parameters ARE the response schema;
@@ -845,6 +848,7 @@ class CodexLLM(LLMInterface):
             "include": ["reasoning.encrypted_content"],
             "prompt_cache_key": str(uuid.uuid4()),
         }
+        payload.update(self._extra_body)
 
         headers = self._build_request_headers()
 

hindsight-api-slim/hindsight_api/engine/providers/gemini_llm.py

@@ -576,23 +576,14 @@ class GeminiLLM(LLMInterface):
                     logger.error(f"Gemini auth error (HTTP {e.code}), not retrying: {str(e)}")
                     raise
 
-                # Diagnostic dump (opt-in) of the exact request behind any 4xx, captured
-                # before the cache-drop retry below rebuilds the config so we see what failed.
-                dump_request_on_4xx(
-                    scope=scope,
-                    provider=self.provider,
-                    model=self.model,
-                    err=e,
-                    request=generation_config,
-                    messages=gemini_contents,
-                )
-
                 # Cached-request safety net: a stale/invalid/expired CachedContent
                 # (or an incompatibility like cache + tool_config) surfaces as a 400.
                 # Retrying the same cached request can't recover, so on the first
                 # such failure drop the cache, invalidate it so later operations
                 # recreate it, and retry THIS call inline with the prefix inlined.
-                # Caching must never break a request.
+                # Caching must never break a request. Handled before the 400
+                # fail-fast below so a recoverable cache-400 isn't mistaken for a
+                # deterministic rejection.
                 if cache_active and e.code == 400:
                     logger.warning(f"Gemini cached call failed (400); retrying uncached. Reason: {str(e)}")
                     if self._cache_manager is not None and cached_prefix is not None:
@@ -601,8 +592,31 @@ class GeminiLLM(LLMInterface):
                     generation_config = _build_generation_config(cache_active)
                     continue
 
-                # Retry on retryable errors (rate limits, server errors, client errors)
-                if e.code in (400, 429, 500, 502, 503, 504) or (e.code and e.code >= 500):
+                # Diagnostic dump of the exact request behind any 4xx. Forced on for a
+                # non-recoverable 400 (see below) so its content-free structural profile
+                # is always in the log on first occurrence; other 4xx dump only under
+                # the opt-in HINDSIGHT_API_LLM_DEBUG_DUMP_4XX flag.
+                dump_request_on_4xx(
+                    scope=scope,
+                    provider=self.provider,
+                    model=self.model,
+                    err=e,
+                    request=generation_config,
+                    messages=gemini_contents,
+                    force=e.code == 400,
+                )
+
+                # HTTP 400 INVALID_ARGUMENT is a deterministic client-side rejection
+                # (malformed schema, oversized prompt section, bad generation param).
+                # Now that the recoverable cache-400 is ruled out, retrying it — and
+                # the batch retry ladder above — just repeats an identical rejected
+                # call, so fail fast instead of burning the retry budget (#3256).
+                if e.code == 400:
+                    logger.error(f"Gemini rejected request (HTTP 400 INVALID_ARGUMENT), not retrying: {str(e)}")
+                    raise
+
+                # Retry on retryable errors (rate limits, server errors)
+                if e.code in (429, 500, 502, 503, 504) or (e.code and e.code >= 500):
                     last_exception = e
                     if attempt < max_retries:
                         backoff = min(initial_backoff * (2**attempt), max_backoff)
@@ -885,21 +899,12 @@ class GeminiLLM(LLMInterface):
                     logger.error(f"Gemini auth error (HTTP {e.code}), not retrying: {str(e)}")
                     raise
 
-                # Diagnostic dump (opt-in) of the exact request behind any 4xx, captured
-                # before the cache-drop retry below rebuilds the config so we see what failed.
-                dump_request_on_4xx(
-                    scope=scope,
-                    provider=self.provider,
-                    model=self.model,
-                    err=e,
-                    request=config,
-                    messages=active_contents,
-                )
-
                 # Cached-request safety net (see ``call``): a stale/invalid cache or
                 # a cache+tool_config conflict surfaces as a 400. Drop the cache,
                 # invalidate it for later operations, and retry THIS call inline
                 # with the prefix + tools re-sent. Caching must never break a call.
+                # Handled before the 400 fail-fast below so a recoverable cache-400
+                # isn't mistaken for a deterministic rejection.
                 if cache_active and e.code == 400:
                     logger.warning(f"Gemini cached tool call failed (400); retrying uncached. Reason: {str(e)}")
                     if self._cache_manager is not None and cached_prefix is not None:
@@ -908,6 +913,28 @@ class GeminiLLM(LLMInterface):
                     config = _build_tools_config(cache_active)
                     continue
 
+                # Diagnostic dump of the exact request behind any 4xx. Forced on for a
+                # non-recoverable 400 so its content-free structural profile is always
+                # in the log on first occurrence; other 4xx dump only under the opt-in
+                # HINDSIGHT_API_LLM_DEBUG_DUMP_4XX flag.
+                dump_request_on_4xx(
+                    scope=scope,
+                    provider=self.provider,
+                    model=self.model,
+                    err=e,
+                    request=config,
+                    messages=active_contents,
+                    force=e.code == 400,
+                )
+
+                # HTTP 400 INVALID_ARGUMENT is a deterministic client-side rejection;
+                # now that the recoverable cache-400 is ruled out, retrying it — and
+                # the batch retry ladder above — just repeats an identical rejected
+                # call, so fail fast instead of burning the retry budget (#3256).
+                if e.code == 400:
+                    logger.error(f"Gemini rejected tool request (HTTP 400 INVALID_ARGUMENT), not retrying: {str(e)}")
+                    raise
+
                 # Retry on retryable errors
                 last_exception = e
                 if attempt < max_retries:

hindsight-api-slim/hindsight_api/engine/providers/litellm_llm.py

@@ -40,6 +40,10 @@ from hindsight_api.worker.stage import set_stage
 
 logger = logging.getLogger(__name__)
 
+# Name of the single tool used when structured output is routed through a forced
+# tool call instead of ``response_format`` (see ``structured_output_forced_tool``).
+_STRUCTURED_TOOL_NAME = "structured_response"
+
 
 def _usage_from_litellm_response(response: Any) -> LLMResponseUsage:
     """Extract prompt/completion/cached token counts from a LiteLLM (OpenAI-shaped) usage block."""
@@ -57,6 +61,22 @@ def _usage_from_litellm_response(response: Any) -> LLMResponseUsage:
     )
 
 
+def _forced_tool_arguments(message: Any) -> str | None:
+    """Return the structured-output tool call's arguments as a JSON string.
+
+    ``None`` when the model answered with plain text instead — some gateways drop
+    ``tool_choice`` — so the caller can fall back to parsing the message content.
+    """
+    for tool_call in message.tool_calls or []:
+        if tool_call.function.name != _STRUCTURED_TOOL_NAME:
+            continue
+        # LiteLLM normalizes to the OpenAI shape (a JSON string), but some
+        # providers hand back an already-decoded object.
+        arguments = tool_call.function.arguments
+        return arguments if isinstance(arguments, str) else json.dumps(arguments)
+    return None
+
+
 class LiteLLMLLM(LLMInterface):
     """
     LLM provider using the LiteLLM SDK for universal model support.
@@ -82,6 +102,7 @@ class LiteLLMLLM(LLMInterface):
         extra_body: dict[str, Any] | None = None,
         bedrock_service_tier: str | None = None,
         default_headers: dict[str, Any] | None = None,
+        structured_output_forced_tool: bool = False,
         **kwargs: Any,
     ):
         super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs)
@@ -103,6 +124,12 @@ class LiteLLMLLM(LLMInterface):
         # copy is handed to each call below to avoid cross-request contamination.
         self._default_headers: dict[str, Any] = dict(default_headers or {})
         self.bedrock_service_tier = bedrock_service_tier
+        # Ask for structured output via a single forced tool call instead of
+        # ``response_format``. Opt-in (HINDSIGHT_API_LLM_STRUCTURED_OUTPUT_FORCED_TOOL)
+        # for backends that reject the response_format route — Bedrock Claude's
+        # Converse layer refuses the translated ``outputConfig`` in some regions but
+        # accepts the same schema as a tool (#3300).
+        self.structured_output_forced_tool = structured_output_forced_tool
 
         try:
             import litellm
@@ -243,16 +270,38 @@ class LiteLLMLLM(LLMInterface):
         call_kwargs = self._build_common_kwargs(messages, max_completion_tokens, temperature)
 
         # Add JSON schema response format if provided
+        use_forced_tool = False
         if response_format is not None and hasattr(response_format, "model_json_schema"):
             schema = strict_json_schema(response_format) if strict_schema else response_format.model_json_schema()
-            call_kwargs["response_format"] = {
-                "type": "json_schema",
-                "json_schema": {
-                    "name": response_format.__name__ if hasattr(response_format, "__name__") else "response",
-                    "schema": schema,
-                    "strict": strict_schema,
-                },
-            }
+            schema_name = response_format.__name__ if hasattr(response_format, "__name__") else "response"
+            if self.structured_output_forced_tool:
+                # The schema travels as the tool's parameters and the model is forced
+                # to call it; the arguments are substituted for the message content
+                # below, so the parse/validate, retry and usage paths are unchanged.
+                use_forced_tool = True
+                call_kwargs["tools"] = [
+                    {
+                        "type": "function",
+                        "function": {
+                            "name": _STRUCTURED_TOOL_NAME,
+                            "description": f"Return the structured response ({schema_name}).",
+                            "parameters": schema,
+                        },
+                    }
+                ]
+                call_kwargs["tool_choice"] = {
+                    "type": "function",
+                    "function": {"name": _STRUCTURED_TOOL_NAME},
+                }
+            else:
+                call_kwargs["response_format"] = {
+                    "type": "json_schema",
+                    "json_schema": {
+                        "name": schema_name,
+                        "schema": schema,
+                        "strict": strict_schema,
+                    },
+                }
 
         last_exception = None
 
@@ -269,10 +318,19 @@ class LiteLLMLLM(LLMInterface):
                 # these tokens (#2387).
                 stash_response_usage(_usage_from_litellm_response(response))
 
-                content = response.choices[0].message.content or ""
+                message = response.choices[0].message
+                content = message.content or ""
                 finish_reason = response.choices[0].finish_reason
                 model_name = self._resolve_completion_model(response)
 
+                if use_forced_tool:
+                    # Forced tool call: its arguments ARE the structured response.
+                    # Absent (a gateway that drops tool_choice) -> keep the text
+                    # content so the existing parse path still has a chance.
+                    forced_arguments = _forced_tool_arguments(message)
+                    if forced_arguments is not None:
+                        content = forced_arguments
+
                 # Check for length-limited output
                 if finish_reason == "length":
                     raise OutputTooLongError("LiteLLM response was truncated due to token limit")

hindsight-api-slim/hindsight_api/engine/providers/llm_debug.py

@@ -135,25 +135,41 @@ def dump_request_on_4xx(
     err: Any,
     request: Any = None,
     messages: Any = None,
+    force: bool = False,
 ) -> None:
     """Log the exact request behind an LLM 4xx when the diagnostic is enabled.
 
-    No-op unless ``HINDSIGHT_API_LLM_DEBUG_DUMP_4XX`` is truthy and ``err`` carries a
-    4xx status. ``request`` is whatever the provider assembled (a Pydantic config, a
-    kwargs dict, ...); ``messages`` overrides where the per-message previews come from
+    No-op unless ``err`` carries a 4xx status AND either the
+    ``HINDSIGHT_API_LLM_DEBUG_DUMP_4XX`` flag is truthy or ``force`` is set.
+    ``request`` is whatever the provider assembled (a Pydantic config, a kwargs
+    dict, ...); ``messages`` overrides where the per-message previews come from
     (defaults to the message list found inside ``request``).
+
+    ``force`` is for deterministic rejections a retry can't fix (e.g. a 400
+    ``INVALID_ARGUMENT``): the structural profile — request config, per-message
+    sizes — is logged on the first (and only) failure so an otherwise-opaque
+    black box is diagnosable in production without flipping a flag and
+    reproducing (#3256). Message *previews* stay gated behind the opt-in flag, so
+    the forced structural dump never spills user content — only per-part sizes.
     """
-    if not _enabled():
+    enabled = _enabled()
+    if not (enabled or force):
         return
     code = status_code_of(err)
     if code is None or not (400 <= code < 500):
         return
+    # Previews carry user content, so they ride only on the explicit opt-in; a
+    # forced dump logs the structural profile (config + per-part sizes) alone.
+    include_previews = enabled
     try:
         cfg_repr = _serialize_config(request)
         summary = []
         for msg in _resolve_messages(request, messages) or []:
             m = _message_preview(msg)
-            summary.append({"role": m.role, "chars": len(m.text), "preview": m.text[:_PREVIEW_CHARS]})
+            entry: dict[str, Any] = {"role": m.role, "chars": len(m.text)}
+            if include_previews:
+                entry["preview"] = m.text[:_PREVIEW_CHARS]
+            summary.append(entry)
         logger.error(
             "[LLM_4XX_DUMP] provider=%s model=%s scope=%s code=%s err=%s config=%s contents=%s",
             provider,

hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py

@@ -37,6 +37,12 @@ from openai import APIConnectionError, APIStatusError, AsyncOpenAI, LengthFinish
 
 from hindsight_api.config import DEFAULT_LLM_TIMEOUT, ENV_LLM_TIMEOUT
 from hindsight_api.engine.bank_attribution import apply_bank_attribution
+from hindsight_api.engine.cache_affinity import (
+    CacheAffinityMode,
+    apply_cache_affinity,
+    parse_cache_affinity,
+    resolve_cache_affinity,
+)
 from hindsight_api.engine.llm_interface import (
     LLM_TOOL_CHOICE_AUTO,
     LLMInterface,
@@ -531,6 +537,8 @@ class OpenAICompatibleLLM(LLMInterface):
         groq_service_tier: str | None = None,
         extra_body: dict[str, Any] | None = None,
         *,
+        default_headers: dict[str, str] | None = None,
+        cache_affinity: str | None = None,
         ollama_num_ctx: int | None = None,
         **kwargs: Any,
     ):
@@ -549,6 +557,11 @@ class OpenAICompatibleLLM(LLMInterface):
             timeout: Request timeout in seconds (uses env var or 120s default).
             groq_service_tier: Groq service tier ("on_demand", "flex", "auto").
             extra_body: Extra body params merged into every API call.
+            default_headers: Custom headers passed to the AsyncOpenAI client (proxies,
+                request-tracing middleware). None sends no extra headers.
+            cache_affinity: Backend prompt-cache pinning mode — "none" (default),
+                "xai_conv_id", "openai_prompt_cache_key", or "auto" (resolved once here
+                from the provider + base-URL host). See ``engine/cache_affinity.py``.
             ollama_num_ctx: Native Ollama context window override. None lets Ollama use
                 the model/server default.
             **kwargs: Additional provider-specific parameters.
@@ -643,8 +656,18 @@ class OpenAICompatibleLLM(LLMInterface):
         # Get timeout config
         self.timeout = timeout or float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT)))
 
+        # Backend prompt-cache pinning. "auto" is resolved ONCE here rather than
+        # per call: base_url is immutable after construction, so the answer can
+        # never change, and resolving per call would re-parse the URL on every
+        # request. Invalid values raise here so a typo fails at startup.
+        self._cache_affinity: CacheAffinityMode = resolve_cache_affinity(
+            parse_cache_affinity(cache_affinity), self.provider, self.base_url
+        )
+
         # Create OpenAI client — extract query params from base_url (e.g. Azure api-version)
         client_kwargs: dict[str, Any] = {"api_key": self.api_key, "max_retries": 0}
+        if default_headers:
+            client_kwargs["default_headers"] = default_headers
         if self.base_url:
             parsed = urlparse(self.base_url)
             if parsed.query:
@@ -663,6 +686,10 @@ class OpenAICompatibleLLM(LLMInterface):
             f"OpenAI-compatible client initialized: provider={self.provider}, model={self.model}, "
             f"base_url={self.base_url or 'default'}"
         )
+        logger.debug(
+            f"Cache affinity resolved: provider={self.provider}, base_url={self.base_url or 'default'}, "
+            f"mode={self._cache_affinity.value}"
+        )
 
     def _drops_tool_choice_required(self) -> bool:
         """Whether this endpoint silently ignores ``tool_choice="required"``.
@@ -898,6 +925,14 @@ class OpenAICompatibleLLM(LLMInterface):
                     call_params["response_format"] = {"type": "json_object"}
 
         apply_bank_attribution(call_params)
+        # Cache pinning, alongside the other identity injection above and, like
+        # call_params itself, built ONCE before the retry loop so every attempt
+        # carries it. Note the hash-point: when no trace context is bound the id
+        # falls back to hashing the first message, and the soft-schema branch
+        # above has already appended the response schema to it. That is
+        # deterministic (the schema text is fixed per response_format), so the id
+        # stays stable across the calls of one run.
+        apply_cache_affinity(call_params, self._cache_affinity)
 
         last_exception = None
 
@@ -1281,6 +1316,7 @@ class OpenAICompatibleLLM(LLMInterface):
             call_params["extra_body"] = extra_body
 
         apply_bank_attribution(call_params)
+        apply_cache_affinity(call_params, self._cache_affinity)
 
         last_exception = None
 
@@ -1463,7 +1499,9 @@ class OpenAICompatibleLLM(LLMInterface):
             "model": self.model,
             "messages": messages,
             "stream": False,
-            "think": False,  # Disable thinking for reasoning models (qwen3.5, etc.)
+            # Disable thinking by default (qwen3.5, etc.). Override via
+            # extra_body, e.g. {"think": "low"} for gpt-oss models (see #3246).
+            "think": False,
         }
 
         # Add schema as format parameter for structured output
@@ -1480,6 +1518,16 @@ class OpenAICompatibleLLM(LLMInterface):
             options["num_predict"] = max_completion_tokens
         if temperature is not None:
             options["temperature"] = temperature
+
+        # Merge configured extra_body into the native payload. Ollama's native
+        # /api/chat body has two tiers, unlike the OpenAI-compatible endpoint
+        # where the SDK flattens everything to top-level: native top-level
+        # fields (think, keep_alive, ...) pass through directly, while an
+        # "options" sub-dict merges into Ollama's generation options
+        # (seed, top_p, num_ctx, ...). User values win over the defaults above.
+        extra_body = dict(self._config_extra_body)
+        options.update(extra_body.pop("options", {}))
+        payload.update(extra_body)
         payload["options"] = options
 
         last_exception = None

hindsight-api-slim/hindsight_api/engine/providers/xai_oauth_auth.py

@@ -0,0 +1,923 @@
+"""
+xAI OAuth credential manager for the ``xai-oauth`` subscription provider.
+
+Serves a consumer SuperGrok subscription — flat-rate, no API key — by holding a
+user-consented OAuth grant obtained from xAI's own OIDC issuer at
+``https://auth.x.ai`` through the RFC 8628 device-code flow, and refreshing it
+with a plain ``grant_type=refresh_token`` POST. For API-key access to the same
+``api.x.ai`` endpoint, use ``provider: openai`` with a base URL; this module is
+the subscription lane's credential half.
+
+Relationship to the other subscription providers
+------------------------------------------------
+``codex_auth.py`` and ``nous_auth.py`` read a credential file that a vendor CLI
+created and keep it fresh. This module follows their store conventions — one
+JSON file, ``0600``, temp-file + ``os.replace`` rewrite, a per-store
+``fcntl.flock`` advisory lock — but owns the login itself, because it must never
+read or write the Grok CLI's ``~/.grok/auth.json``. The login is therefore an
+explicit, interactive entrypoint (``python -m
+hindsight_api.engine.providers.xai_oauth_auth login``) and nothing on the
+request path can reach it.
+
+Store
+-----
+``~/.hindsight/xai_oauth.json`` by default (the directory ``daemon.py`` and
+``llamacpp_llm.py`` already use), overridable with
+``HINDSIGHT_API_XAI_OAUTH_TOKEN_PATH``. It holds the access token, the refresh
+token, ``expires_at``, ``obtained_at``, the granted scope, and the discovered
+``token_endpoint``.
+
+xAI issues a **new refresh token with every refresh** and retires the old one,
+so the store is not a read-only credential drop: whichever process refreshes
+must be able to write back, and every process serving this provider must read
+the same file rather than its own copy of one login's output.
+
+Multi-instance safety
+---------------------
+Refresh state lives in the store, not on the manager: several provider
+instances (one per configured lane) share one credential. Each refresh re-reads
+the store after taking the lock and skips when a sibling's refresh already
+landed, and the anti-spin minimum gap is measured from the store's
+``obtained_at`` rather than an in-memory clock.
+
+Logging
+-------
+Token values, refresh tokens, user codes and Authorization headers are never
+passed to the logger at any level; log lines carry byte counts, expiry
+timestamps, scope names and status codes only.
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import json
+import logging
+import os
+import sys
+import tempfile
+import threading
+import time
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+
+try:
+    import fcntl
+except ImportError:  # pragma: no cover - Windows
+    fcntl = None  # type: ignore[assignment]
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    "DEFAULT_CLIENT_ID",
+    "DEFAULT_MIN_REFRESH_GAP_SECONDS",
+    "DEFAULT_REFRESH_SKEW_SECONDS",
+    "DEFAULT_REFRESH_TIMEOUT_SECONDS",
+    "DEFAULT_SCOPE",
+    "DEVICE_CODE_GRANT_TYPE",
+    "ENV_CLIENT_ID",
+    "ENV_REFRESH_SKEW_SECONDS",
+    "ENV_REFRESH_TIMEOUT_SECONDS",
+    "ENV_SCOPE",
+    "ENV_TOKEN_PATH",
+    "LOGIN_COMMAND",
+    "StoredCredential",
+    "XaiOAuthDiscoveryError",
+    "XaiOAuthError",
+    "XaiOAuthLoginRequiredError",
+    "XaiOAuthManager",
+    "XaiOAuthRefreshError",
+    "default_token_path",
+    "device_code_login",
+    "discover_endpoints",
+    "read_credential",
+    "request_device_code",
+    "poll_device_token",
+    "write_credential",
+]
+
+
+# ---------------------------------------------------------------------------
+# Environment surface
+# ---------------------------------------------------------------------------
+
+#: Relocate the token store. The read and the write both follow it, so this is
+#: a full relocation rather than a read-side override.
+ENV_TOKEN_PATH = "HINDSIGHT_API_XAI_OAUTH_TOKEN_PATH"
+
+#: Override the OAuth client id used for both login and refresh.
+ENV_CLIENT_ID = "HINDSIGHT_API_XAI_OAUTH_CLIENT_ID"
+
+#: Override the scope string requested at login.
+ENV_SCOPE = "HINDSIGHT_API_XAI_OAUTH_SCOPE"
+
+#: Per-HTTP-call timeout for discovery, device-code and refresh requests.
+ENV_REFRESH_TIMEOUT_SECONDS = "HINDSIGHT_API_XAI_OAUTH_REFRESH_TIMEOUT_SECONDS"
+
+#: How long before ``expires_at`` a token counts as due for refresh.
+ENV_REFRESH_SKEW_SECONDS = "HINDSIGHT_API_XAI_OAUTH_REFRESH_SKEW_SECONDS"
+
+
+# ---------------------------------------------------------------------------
+# Vendor constants
+# ---------------------------------------------------------------------------
+
+XAI_OAUTH_ISSUER = "https://auth.x.ai"
+XAI_OAUTH_DISCOVERY_URL = f"{XAI_OAUTH_ISSUER}/.well-known/openid-configuration"
+
+#: Fallback device-authorization endpoint, used only when the discovery
+#: document omits ``device_authorization_endpoint``.
+XAI_OAUTH_DEVICE_CODE_URL = f"{XAI_OAUTH_ISSUER}/oauth2/device/code"
+
+#: Public OAuth client id published in xAI's Apache-2.0 Grok CLI sources
+#: (``crates/codegen/xai-grok-shell/src/auth/config.rs`` in
+#: github.com/xai-org/grok-build). Not a secret: a device-code public client has
+#: no client secret by construction.
+DEFAULT_CLIENT_ID = "b1a00492-073a-47ea-816f-4c329264a828"
+
+#: Scope string the vendor's own device-code login requests.
+DEFAULT_SCOPE = "openid profile email offline_access grok-cli:access api:access"
+
+DEVICE_CODE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:device_code"
+
+#: Refresh this many seconds before ``expires_at``. 60s matches
+#: ``codex_auth.py``'s skew; deployments that touch the provider rarely (a cron
+#: or gateway shape) can widen it with ``ENV_REFRESH_SKEW_SECONDS``.
+DEFAULT_REFRESH_SKEW_SECONDS = 60.0
+
+#: Hard per-request timeout for every call this module makes.
+DEFAULT_REFRESH_TIMEOUT_SECONDS = 20.0
+
+#: Anti-spin floor: a credential obtained less than this many seconds ago is
+#: not refreshed again. Measured from the store's ``obtained_at``.
+DEFAULT_MIN_REFRESH_GAP_SECONDS = 30.0
+
+#: Wait this long for the cross-process store lock before giving up.
+AUTH_LOCK_TIMEOUT_SECONDS = 20.0
+
+#: OAuth token-endpoint statuses that mean the grant itself is dead.
+#:
+#: RFC 6749 section 5.2 answers a spent or revoked ``refresh_token`` with 400
+#: ``invalid_grant``, and 401 covers client authentication. A 403 is not a
+#: token-endpoint error shape at all — it is what an edge or policy layer in
+#: front of the issuer returns — so it stays off this set: quarantining on one
+#: would trade a transient upstream refusal for a mandatory interactive
+#: re-login, which is the same reasoning ``xai_oauth_llm`` applies to a 403
+#: from the API side.
+TERMINAL_REFRESH_STATUSES = frozenset({400, 401})
+
+LOGIN_COMMAND = "python -m hindsight_api.engine.providers.xai_oauth_auth login"
+
+_STORE_LOCKS_GUARD = threading.Lock()
+_STORE_LOCKS: dict[Path, threading.Lock] = {}
+
+
+# ---------------------------------------------------------------------------
+# Errors
+# ---------------------------------------------------------------------------
+
+
+class XaiOAuthError(RuntimeError):
+    """Base class for every credential-side failure in this module."""
+
+
+class XaiOAuthLoginRequiredError(XaiOAuthError):
+    """Raised when only an interactive login can restore service.
+
+    Carries the exact command to run. Nothing on the request path ever starts
+    the device-code flow itself, so this is the terminus of the unattended
+    path rather than a prompt.
+    """
+
+
+class XaiOAuthDiscoveryError(XaiOAuthError):
+    """Raised when OIDC discovery fails or returns an endpoint off the xAI origin."""
+
+
+class XaiOAuthRefreshError(XaiOAuthError):
+    """Raised when a refresh fails in a way a later attempt might survive."""
+
+
+# ---------------------------------------------------------------------------
+# Store
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True, slots=True)
+class StoredCredential:
+    """One credential as it exists on disk.
+
+    ``expires_at`` is ``None`` when the store never recorded one. Such a
+    credential is treated as due for refresh, never as valid.
+    """
+
+    access_token: str
+    refresh_token: str
+    expires_at: float | None
+    obtained_at: float
+    scope: str
+    token_endpoint: str
+
+    def seconds_left(self, now: float | None = None) -> float:
+        """Seconds until expiry, or ``0.0`` when the store recorded no expiry."""
+        if self.expires_at is None:
+            return 0.0
+        return self.expires_at - (time.time() if now is None else now)
+
+
+def default_token_path() -> Path:
+    """Return the token-store path, honoring ``ENV_TOKEN_PATH``.
+
+    Resolved on each call rather than cached at import, so the environment is
+    read at the point of use.
+    """
+    configured = os.environ.get(ENV_TOKEN_PATH, "").strip()
+    if configured:
+        return Path(configured).expanduser()
+    return Path.home() / ".hindsight" / "xai_oauth.json"
+
+
+def _path_scoped_lock(token_path: Path) -> threading.Lock:
+    """Return the process-wide lock for one store path."""
+    key = token_path.expanduser().resolve(strict=False)
+    with _STORE_LOCKS_GUARD:
+        lock = _STORE_LOCKS.get(key)
+        if lock is None:
+            lock = threading.Lock()
+            _STORE_LOCKS[key] = lock
+        return lock
+
+
+@contextlib.contextmanager
+def token_store_lock(token_path: Path, timeout_seconds: float = AUTH_LOCK_TIMEOUT_SECONDS) -> Iterator[None]:
+    """Hold the advisory lock for one token store.
+
+    An in-process lock keyed by the resolved path serialises this process's own
+    managers; a ``fcntl.flock`` on ``<store>.lock`` extends that across
+    processes. Where ``fcntl`` is unavailable (Windows) only the in-process lock
+    applies — the same degradation ``codex_auth.py`` and ``nous_auth.py`` take.
+    """
+    with _path_scoped_lock(token_path):
+        if fcntl is None:  # pragma: no cover - Windows
+            logger.debug("fcntl unavailable; xai-oauth refresh proceeds without a cross-process lock.")
+            yield
+            return
+
+        lock_path = token_path.with_suffix(".lock")
+        lock_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(lock_path, "a+") as lock_file:
+            deadline = time.monotonic() + max(1.0, timeout_seconds)
+            while True:
+                try:
+                    fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+                    break
+                except (BlockingIOError, OSError):
+                    if time.monotonic() >= deadline:
+                        raise TimeoutError("Timed out waiting for the xai-oauth token store lock") from None
+                    time.sleep(0.05)
+            try:
+                yield
+            finally:
+                with contextlib.suppress(OSError):
+                    fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
+
+
+def _coerce_float(value: Any) -> float | None:
+    if isinstance(value, bool) or not isinstance(value, (int, float, str)):
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def read_credential(token_path: Path) -> StoredCredential:
+    """Load the credential from ``token_path``.
+
+    Raises
+    ------
+    XaiOAuthLoginRequiredError:
+        When the file is missing, unreadable, malformed, or carries no
+        ``refresh_token`` — every shape from which only a login recovers.
+    """
+    if not token_path.exists():
+        raise XaiOAuthLoginRequiredError(
+            f"No xai-oauth credential found at {token_path}. "
+            f"Run the xai-oauth login on the host that owns the token store: {LOGIN_COMMAND}"
+        )
+
+    try:
+        with open(token_path) as handle:
+            data = json.load(handle)
+    except (OSError, json.JSONDecodeError) as exc:
+        raise XaiOAuthLoginRequiredError(
+            f"The xai-oauth credential at {token_path} is unreadable ({type(exc).__name__}). "
+            f"Run the xai-oauth login on the host that owns the token store: {LOGIN_COMMAND}"
+        ) from exc
+
+    tokens = data.get("tokens") if isinstance(data, dict) else None
+    tokens = tokens if isinstance(tokens, dict) else {}
+    access_token = tokens.get("access_token")
+    refresh_token = tokens.get("refresh_token")
+
+    if not isinstance(refresh_token, str) or not refresh_token:
+        raise XaiOAuthLoginRequiredError(
+            f"The xai-oauth credential at {token_path} has no refresh_token. "
+            f"Run the xai-oauth login on the host that owns the token store: {LOGIN_COMMAND}"
+        )
+
+    return StoredCredential(
+        access_token=access_token if isinstance(access_token, str) else "",
+        refresh_token=refresh_token,
+        expires_at=_coerce_float(data.get("expires_at")),
+        obtained_at=_coerce_float(data.get("obtained_at")) or 0.0,
+        scope=str(data.get("scope") or ""),
+        token_endpoint=str(data.get("token_endpoint") or ""),
+    )
+
+
+def write_credential(token_path: Path, credential: StoredCredential) -> None:
+    """Persist ``credential`` to ``token_path`` as one atomic replacement.
+
+    The payload is written to a sibling temp file, fsynced, chmod-ed ``0600``
+    where the platform supports it, and moved into place with ``os.replace`` —
+    so a reader never observes a partially written store.
+    """
+    payload = {
+        "auth_mode": "xai-oauth-device-code",
+        "tokens": {
+            "access_token": credential.access_token,
+            "refresh_token": credential.refresh_token,
+        },
+        "expires_at": credential.expires_at,
+        "obtained_at": credential.obtained_at,
+        "scope": credential.scope,
+        "token_endpoint": credential.token_endpoint,
+    }
+
+    parent = token_path.parent
+    parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp_path = tempfile.mkstemp(prefix=".xai_oauth.", suffix=".json.tmp", dir=str(parent))
+    try:
+        with os.fdopen(fd, "w") as handle:
+            json.dump(payload, handle, indent=2)
+            handle.flush()
+            os.fsync(handle.fileno())
+        with contextlib.suppress(OSError):
+            os.chmod(tmp_path, 0o600)
+        os.replace(tmp_path, token_path)
+    except Exception:
+        with contextlib.suppress(OSError):
+            os.unlink(tmp_path)
+        raise
+    with contextlib.suppress(OSError):
+        os.chmod(token_path, 0o600)
+
+
+def quarantine_credential(token_path: Path, *, code: str, reason: str) -> None:
+    """Strip the dead tokens from the store and record why.
+
+    A grant the token endpoint has rejected terminally cannot be revived by
+    retrying, so the tokens are removed and the next process fails fast on the
+    login remediation instead of re-attempting a refresh that cannot succeed.
+    Only the error code and reason are recorded — never a token value. Must be
+    called while holding :func:`token_store_lock`.
+    """
+    try:
+        with open(token_path) as handle:
+            data = json.load(handle)
+        current: dict[str, Any] = data if isinstance(data, dict) else {}
+    except (OSError, json.JSONDecodeError):
+        current = {}
+
+    current["tokens"] = {}
+    current["expires_at"] = None
+    current["last_auth_error"] = {
+        "code": code,
+        "reason": reason,
+        "at": time.time(),
+    }
+
+    parent = token_path.parent
+    parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp_path = tempfile.mkstemp(prefix=".xai_oauth.", suffix=".json.tmp", dir=str(parent))
+    try:
+        with os.fdopen(fd, "w") as handle:
+            json.dump(current, handle, indent=2)
+            handle.flush()
+            os.fsync(handle.fileno())
+        with contextlib.suppress(OSError):
+            os.chmod(tmp_path, 0o600)
+        os.replace(tmp_path, token_path)
+    except Exception:
+        with contextlib.suppress(OSError):
+            os.unlink(tmp_path)
+        raise
+
+
+# ---------------------------------------------------------------------------
+# Discovery
+# ---------------------------------------------------------------------------
+
+
+def validate_oauth_endpoint(url: str, *, field: str) -> str:
+    """Return ``url`` if it is HTTPS on the xAI origin, else raise.
+
+    The discovery result is cached in the store and every later refresh posts
+    the refresh token to the cached ``token_endpoint``, so a substituted
+    endpoint would turn one bad discovery response into a standing credential
+    leak. Pinning scheme and host to ``x.ai`` (or a ``*.x.ai`` subdomain) is
+    what stops the substitution outliving the request that carried it.
+    """
+    parsed = urlparse(url)
+    if parsed.scheme != "https":
+        raise XaiOAuthDiscoveryError(f"xAI OIDC discovery returned a non-HTTPS {field}: {url!r}")
+    host = (parsed.hostname or "").lower()
+    if not host:
+        raise XaiOAuthDiscoveryError(f"xAI OIDC discovery {field} has no hostname: {url!r}")
+    if host != "x.ai" and not host.endswith(".x.ai"):
+        raise XaiOAuthDiscoveryError(
+            f"xAI OIDC discovery {field} host {host!r} is not on the xAI origin (expected x.ai or a *.x.ai subdomain)"
+        )
+    return url
+
+
+def discover_endpoints(client: httpx.Client) -> dict[str, str]:
+    """Fetch the OIDC discovery document and return the endpoints we use.
+
+    Returns ``token_endpoint`` and ``device_authorization_endpoint``. Both are
+    validated against the xAI origin. When the document omits
+    ``device_authorization_endpoint``, :data:`XAI_OAUTH_DEVICE_CODE_URL` is used
+    instead — RFC 8628 registers the metadata key but does not require issuers
+    to publish it.
+    """
+    try:
+        response = client.get(XAI_OAUTH_DISCOVERY_URL, headers={"Accept": "application/json"})
+    except httpx.RequestError as exc:
+        raise XaiOAuthDiscoveryError(f"xAI OIDC discovery request failed: {type(exc).__name__}") from exc
+
+    if response.status_code != 200:
+        raise XaiOAuthDiscoveryError(f"xAI OIDC discovery returned HTTP {response.status_code}")
+
+    try:
+        payload = response.json()
+    except (json.JSONDecodeError, ValueError) as exc:
+        raise XaiOAuthDiscoveryError("xAI OIDC discovery returned a non-JSON body") from exc
+
+    if not isinstance(payload, dict):
+        raise XaiOAuthDiscoveryError("xAI OIDC discovery body was not a JSON object")
+
+    token_endpoint = str(payload.get("token_endpoint") or "").strip()
+    if not token_endpoint:
+        raise XaiOAuthDiscoveryError("xAI OIDC discovery body has no token_endpoint")
+    device_endpoint = str(payload.get("device_authorization_endpoint") or "").strip() or XAI_OAUTH_DEVICE_CODE_URL
+
+    return {
+        "token_endpoint": validate_oauth_endpoint(token_endpoint, field="token_endpoint"),
+        "device_authorization_endpoint": validate_oauth_endpoint(
+            device_endpoint, field="device_authorization_endpoint"
+        ),
+    }
+
+
+# ---------------------------------------------------------------------------
+# Device-code login (interactive only)
+# ---------------------------------------------------------------------------
+
+
+def request_device_code(client: httpx.Client, *, device_endpoint: str, client_id: str, scope: str) -> dict[str, Any]:
+    """Start the device-code flow and return the authorization response.
+
+    Raises :class:`XaiOAuthError` when the endpoint answers non-200 or omits a
+    field RFC 8628 section 3.2 makes required.
+    """
+    response = client.post(
+        device_endpoint,
+        headers={"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json"},
+        data={"client_id": client_id, "scope": scope},
+    )
+    if response.status_code != 200:
+        raise XaiOAuthError(f"xAI device-code request failed with HTTP {response.status_code}")
+
+    try:
+        payload = response.json()
+    except (json.JSONDecodeError, ValueError) as exc:
+        raise XaiOAuthError("xAI device-code response was not JSON") from exc
+    if not isinstance(payload, dict):
+        raise XaiOAuthError("xAI device-code response was not a JSON object")
+
+    missing = [key for key in ("device_code", "user_code", "verification_uri", "expires_in") if key not in payload]
+    if missing:
+        raise XaiOAuthError(f"xAI device-code response is missing fields: {', '.join(missing)}")
+    return payload
+
+
+def poll_device_token(
+    client: httpx.Client,
+    *,
+    token_endpoint: str,
+    device_code: str,
+    client_id: str,
+    expires_in: int,
+    interval: int,
+    sleeper: Callable[[float], None] = time.sleep,
+    monotonic: Callable[[], float] = time.monotonic,
+) -> dict[str, Any]:
+    """Poll the token endpoint until the user approves, or the code dies.
+
+    Follows RFC 8628 section 3.5: waits ``interval`` seconds between polls,
+    widens that interval by 5 seconds on ``slow_down``, keeps polling on
+    ``authorization_pending``, and raises on ``expired_token`` or any other
+    error code. Stops once ``expires_in`` seconds have elapsed.
+
+    ``sleeper`` and ``monotonic`` are injected so the timing rules can be
+    asserted without wall-clock waits.
+    """
+    deadline = monotonic() + max(1, int(expires_in))
+    current_interval = max(1, int(interval))
+
+    while monotonic() < deadline:
+        response = client.post(
+            token_endpoint,
+            headers={"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json"},
+            data={
+                "grant_type": DEVICE_CODE_GRANT_TYPE,
+                "client_id": client_id,
+                "device_code": device_code,
+            },
+        )
+        if response.status_code == 200:
+            payload = response.json()
+            if not isinstance(payload, dict) or not payload.get("access_token"):
+                raise XaiOAuthError("xAI device-code token response carried no access_token")
+            if not payload.get("refresh_token"):
+                raise XaiOAuthError("xAI device-code token response carried no refresh_token")
+            return payload
+
+        try:
+            error_payload = response.json()
+        except (json.JSONDecodeError, ValueError) as exc:
+            raise XaiOAuthError(
+                f"xAI device-code polling returned a non-JSON HTTP {response.status_code} body"
+            ) from exc
+
+        error_code = str(error_payload.get("error") or "") if isinstance(error_payload, dict) else ""
+        if error_code == "authorization_pending":
+            sleeper(current_interval)
+            continue
+        if error_code == "slow_down":
+            current_interval += 5
+            sleeper(current_interval)
+            continue
+        if error_code == "expired_token":
+            raise XaiOAuthError("The xAI device code expired before it was approved. Start the login again.")
+        raise XaiOAuthError(f"xAI device-code polling failed with error {error_code or 'unknown'}")
+
+    raise XaiOAuthError("Timed out waiting for xAI device-code approval.")
+
+
+def device_code_login(
+    *,
+    token_path: Path | None = None,
+    client_id: str | None = None,
+    scope: str | None = None,
+    timeout_seconds: float | None = None,
+    writer: Callable[[str], None] = print,
+) -> Path:
+    """Run the interactive device-code login and write the credential store.
+
+    Prints the verification URL and user code for the operator to approve in a
+    browser, then polls until the grant lands. The user code is written through
+    ``writer`` (stdout by default) and is never passed to the logger.
+
+    Returns the store path that was written. This function is only reachable
+    from the module's ``login`` entrypoint — no request path calls it.
+    """
+    path = token_path or default_token_path()
+    resolved_client_id = client_id or os.environ.get(ENV_CLIENT_ID, "").strip() or DEFAULT_CLIENT_ID
+    resolved_scope = scope or os.environ.get(ENV_SCOPE, "").strip() or DEFAULT_SCOPE
+    timeout = (
+        timeout_seconds
+        if timeout_seconds is not None
+        else _env_float(ENV_REFRESH_TIMEOUT_SECONDS, DEFAULT_REFRESH_TIMEOUT_SECONDS)
+    )
+
+    with httpx.Client(timeout=httpx.Timeout(max(20.0, timeout)), headers={"Accept": "application/json"}) as client:
+        endpoints = discover_endpoints(client)
+        device = request_device_code(
+            client,
+            device_endpoint=endpoints["device_authorization_endpoint"],
+            client_id=resolved_client_id,
+            scope=resolved_scope,
+        )
+        verification = str(device.get("verification_uri_complete") or device["verification_uri"])
+        writer("")
+        writer("To authorize Hindsight against your SuperGrok subscription:")
+        writer(f"  1. Open: {verification}")
+        writer(f"  2. If prompted, enter code: {device['user_code']}")
+        writer("Waiting for approval...")
+
+        payload = poll_device_token(
+            client,
+            token_endpoint=endpoints["token_endpoint"],
+            device_code=str(device["device_code"]),
+            client_id=resolved_client_id,
+            expires_in=int(device["expires_in"]),
+            interval=int(device.get("interval") or 5),
+        )
+
+    now = time.time()
+    expires_in = _coerce_float(payload.get("expires_in"))
+    credential = StoredCredential(
+        access_token=str(payload["access_token"]),
+        refresh_token=str(payload["refresh_token"]),
+        expires_at=(now + expires_in) if expires_in is not None else None,
+        obtained_at=now,
+        scope=str(payload.get("scope") or resolved_scope),
+        token_endpoint=endpoints["token_endpoint"],
+    )
+    with token_store_lock(path):
+        write_credential(path, credential)
+    logger.info(
+        "xai-oauth credential stored at %s (scope=%s, expires_at=%s)",
+        path,
+        credential.scope,
+        credential.expires_at,
+    )
+    writer(f"Stored the xai-oauth credential at {path}")
+    return path
+
+
+def _env_float(name: str, default: float) -> float:
+    raw = os.environ.get(name, "").strip()
+    if not raw:
+        return default
+    try:
+        return float(raw)
+    except ValueError:
+        logger.warning("Ignoring non-numeric %s; using %s", name, default)
+        return default
+
+
+# ---------------------------------------------------------------------------
+# Manager
+# ---------------------------------------------------------------------------
+
+
+class XaiOAuthManager:
+    """Keeps one xAI OAuth credential usable for unattended requests.
+
+    Proactively refreshes a token inside the skew window and reactively
+    refreshes once when the API rejects one. Never starts the device-code flow:
+    a credential that only a login can restore raises
+    :class:`XaiOAuthLoginRequiredError` instead.
+    """
+
+    def __init__(
+        self,
+        token_path: Path | None = None,
+        *,
+        client_id: str | None = None,
+        refresh_skew_seconds: float | None = None,
+        refresh_timeout_seconds: float | None = None,
+        min_refresh_gap_seconds: float = DEFAULT_MIN_REFRESH_GAP_SECONDS,
+        http_client: httpx.Client | None = None,
+    ) -> None:
+        self._token_path = token_path or default_token_path()
+        self._client_id = client_id or os.environ.get(ENV_CLIENT_ID, "").strip() or DEFAULT_CLIENT_ID
+        self._skew_seconds = (
+            refresh_skew_seconds
+            if refresh_skew_seconds is not None
+            else _env_float(ENV_REFRESH_SKEW_SECONDS, DEFAULT_REFRESH_SKEW_SECONDS)
+        )
+        self._timeout_seconds = (
+            refresh_timeout_seconds
+            if refresh_timeout_seconds is not None
+            else _env_float(ENV_REFRESH_TIMEOUT_SECONDS, DEFAULT_REFRESH_TIMEOUT_SECONDS)
+        )
+        self._min_refresh_gap_seconds = min_refresh_gap_seconds
+        self._owns_client = http_client is None
+        self._http_client = http_client or httpx.Client(timeout=httpx.Timeout(self._timeout_seconds))
+
+    @property
+    def token_path(self) -> Path:
+        return self._token_path
+
+    # ------------------------------------------------------------------
+    # Read path
+    # ------------------------------------------------------------------
+
+    def get_access_token(self, min_ttl_seconds: float | None = None) -> str:
+        """Return an access token good for at least ``min_ttl_seconds``.
+
+        ``min_ttl_seconds`` defaults to the configured skew. A credential whose
+        store recorded no expiry counts as having none left, so it is refreshed
+        rather than used on trust.
+        """
+        required_ttl = self._skew_seconds if min_ttl_seconds is None else max(min_ttl_seconds, 0.0)
+        credential = read_credential(self._token_path)
+        if credential.access_token and credential.seconds_left() > required_ttl:
+            return credential.access_token
+        return self.refresh(reason="proactive (token inside the refresh window)", required_ttl=required_ttl)
+
+    def refresh(
+        self,
+        *,
+        reason: str = "",
+        required_ttl: float | None = None,
+        rejected_token: str | None = None,
+    ) -> str:
+        """Refresh the stored credential and return the new access token.
+
+        Takes the store lock, then re-reads the store: when a sibling manager's
+        refresh already produced a token that satisfies ``required_ttl`` (or,
+        for a reactive refresh, any token other than ``rejected_token``), that
+        token is returned and no request is sent. This is what keeps N provider
+        instances sharing one credential down to one refresh.
+
+        Raises
+        ------
+        XaiOAuthLoginRequiredError:
+            When the token endpoint terminally rejects the grant, or the store
+            holds nothing to refresh.
+        XaiOAuthRefreshError:
+            On a transient failure, and when the anti-spin minimum gap has not
+            elapsed since the stored credential was obtained.
+        """
+        ttl = self._skew_seconds if required_ttl is None else required_ttl
+
+        with token_store_lock(self._token_path, timeout_seconds=max(AUTH_LOCK_TIMEOUT_SECONDS, self._timeout_seconds)):
+            credential = read_credential(self._token_path)
+
+            if rejected_token is None:
+                if credential.access_token and credential.seconds_left() > ttl:
+                    logger.debug("xai-oauth refresh skipped: the store already holds a token outside the window")
+                    return credential.access_token
+            elif credential.access_token and credential.access_token != rejected_token:
+                logger.debug("xai-oauth refresh skipped: the store already holds a token newer than the rejected one")
+                return credential.access_token
+
+            gap = time.time() - credential.obtained_at
+            if gap < self._min_refresh_gap_seconds:
+                raise XaiOAuthRefreshError(
+                    f"xai-oauth refused to refresh again {gap:.1f}s after the stored credential was obtained "
+                    f"(minimum gap {self._min_refresh_gap_seconds:.0f}s). The upstream is rejecting a token that "
+                    "was just issued."
+                )
+
+            return self._refresh_locked(credential, reason=reason)
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _token_endpoint(self, credential: StoredCredential) -> str:
+        if credential.token_endpoint:
+            return validate_oauth_endpoint(credential.token_endpoint, field="token_endpoint")
+        return discover_endpoints(self._http_client)["token_endpoint"]
+
+    def _refresh_locked(self, credential: StoredCredential, *, reason: str) -> str:
+        """Exchange the refresh token and persist the result. Lock must be held.
+
+        A terminal status (see :data:`TERMINAL_REFRESH_STATUSES`) is retried
+        exactly once — a single network-level oddity should not cost the
+        operator a re-login — and then, unless the store has meanwhile moved to
+        a different grant, quarantines the store and raises the login
+        remediation.
+        """
+        endpoint = self._token_endpoint(credential)
+        log_reason = f" ({reason})" if reason else ""
+        logger.info("Refreshing the xai-oauth access token%s", log_reason)
+
+        last_status: int | None = None
+        for attempt in (1, 2):
+            try:
+                response = self._http_client.post(
+                    endpoint,
+                    headers={"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json"},
+                    data={
+                        "grant_type": "refresh_token",
+                        "client_id": self._client_id,
+                        "refresh_token": credential.refresh_token,
+                    },
+                    timeout=self._timeout_seconds,
+                )
+            except httpx.RequestError as exc:
+                raise XaiOAuthRefreshError(f"xai-oauth refresh network error: {type(exc).__name__}") from exc
+
+            if response.status_code == 200:
+                return self._persist_refreshed(credential, response)
+
+            last_status = response.status_code
+            if response.status_code not in TERMINAL_REFRESH_STATUSES:
+                raise XaiOAuthRefreshError(f"xai-oauth refresh failed with HTTP {response.status_code}")
+
+            logger.warning(
+                "xai-oauth refresh rejected with HTTP %d (attempt %d/2)",
+                response.status_code,
+                attempt,
+            )
+
+        # xAI rotates the refresh token on every successful refresh, so a
+        # rejection can simply mean some other writer spent this one first and
+        # the store already holds its replacement. Quarantining then would
+        # destroy a live grant and force an interactive login for nothing. The
+        # in-process lock plus the recheck in refresh() rule that out among
+        # this host's managers, but not against a store shared with a host
+        # whose filesystem does not honour flock — so re-read before wiping.
+        superseded = self._superseded_access_token(credential)
+        if superseded is not None:
+            logger.info("xai-oauth refresh rejection ignored: the store already holds a different grant")
+            return superseded
+
+        quarantine_credential(
+            self._token_path,
+            code=f"refresh_rejected_{last_status}",
+            reason="the token endpoint rejected the refresh_token twice",
+        )
+        raise XaiOAuthLoginRequiredError(
+            f"The xAI token endpoint rejected the stored grant (HTTP {last_status}). "
+            f"Run the xai-oauth login on the host that owns the token store: {LOGIN_COMMAND}"
+        )
+
+    def _superseded_access_token(self, rejected: StoredCredential) -> str | None:
+        """Return a usable access token when the store moved on under us.
+
+        ``None`` when the store still holds the grant that was just rejected,
+        or holds nothing readable — both cases the caller must quarantine.
+        """
+        try:
+            current = read_credential(self._token_path)
+        except XaiOAuthError:
+            return None
+        if current.refresh_token == rejected.refresh_token or not current.access_token:
+            return None
+        return current.access_token
+
+    def _persist_refreshed(self, credential: StoredCredential, response: httpx.Response) -> str:
+        try:
+            payload = response.json()
+        except (json.JSONDecodeError, ValueError) as exc:
+            raise XaiOAuthRefreshError("xai-oauth refresh returned a non-JSON body") from exc
+        if not isinstance(payload, dict):
+            raise XaiOAuthRefreshError("xai-oauth refresh body was not a JSON object")
+
+        access_token = str(payload.get("access_token") or "")
+        if not access_token:
+            raise XaiOAuthRefreshError("xai-oauth refresh returned no access_token")
+
+        now = time.time()
+        expires_in = _coerce_float(payload.get("expires_in"))
+        refreshed = replace(
+            credential,
+            access_token=access_token,
+            refresh_token=str(payload.get("refresh_token") or credential.refresh_token),
+            expires_at=(now + expires_in) if expires_in is not None else None,
+            obtained_at=now,
+            scope=str(payload.get("scope") or credential.scope),
+        )
+        write_credential(self._token_path, refreshed)
+        logger.info(
+            "xai-oauth access token refreshed (%d bytes, expires_at=%s, scope=%s)",
+            len(access_token),
+            refreshed.expires_at,
+            refreshed.scope,
+        )
+        return access_token
+
+    def close(self) -> None:
+        """Close the HTTP client when this manager created it."""
+        if self._owns_client:
+            self._http_client.close()
+
+
+# ---------------------------------------------------------------------------
+# Entrypoint
+# ---------------------------------------------------------------------------
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Command-line entrypoint: ``python -m ...xai_oauth_auth login``."""
+    parser = argparse.ArgumentParser(prog="xai_oauth_auth", description="Manage the xai-oauth credential store.")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    login = subparsers.add_parser("login", help="Run the interactive xAI device-code login.")
+    login.add_argument("--token-path", default=None, help="Write the credential here instead of the default store.")
+
+    args = parser.parse_args(argv)
+    if args.command == "login":
+        try:
+            device_code_login(token_path=Path(args.token_path) if args.token_path else None)
+        except XaiOAuthError as exc:
+            print(f"xai-oauth login failed: {exc}", file=sys.stderr)
+            return 1
+        return 0
+    return 1  # pragma: no cover - argparse rejects unknown commands first
+
+
+if __name__ == "__main__":  # pragma: no cover - process entrypoint
+    raise SystemExit(main())

hindsight-api-slim/hindsight_api/engine/reflect/prompts.py

@@ -8,6 +8,7 @@ The reflect agent uses hierarchical retrieval:
 """
 
 import json
+from datetime import datetime, timezone
 from typing import Any
 
 from .tokenization import count_cl100k_tokens
@@ -20,6 +21,20 @@ _DEFAULT_ROLE = "You are a reflection agent that answers questions by reasoning
 _DEFAULT_FINAL_ROLE = "You are a thoughtful assistant that synthesizes answers from retrieved memories."
 
 
+def _current_utc_datetime() -> str:
+    """Return the current UTC date and time for time-relative reflect reasoning.
+
+    Minute precision (not seconds) so requests within the same minute share an
+    identical prompt string — the finest granularity that still keeps prompt
+    caching viable for bursty traffic.
+    """
+    return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+
+
+def _current_datetime_section() -> str:
+    return f"## Current Date and Time\nThe current date and time is {_current_utc_datetime()}."
+
+
 def _extract_directive_rules(directives: list[dict[str, Any]]) -> list[str]:
     """Extract directive rules as a list of strings."""
     rules = []
@@ -392,6 +407,13 @@ def build_system_prompt_for_tools(
         ]
     )
 
+    # Volatile "now" reference goes here — after all the static instructions and
+    # right before the bank-specific/custom data. Everything above is identical
+    # across banks and requests, so it stays a cacheable prefix; only this
+    # timestamp and the custom tail below fall outside the cache.
+    parts.append("")
+    parts.append(_current_datetime_section())
+
     parts.append("")
     parts.append(f"## Memory Bank: {name}")
 
@@ -570,6 +592,9 @@ def build_final_system_prompt(
     parts.append(_FINAL_SYSTEM_PROMPT_BASE.format(role_section=role_section))
     parts.append(_FINAL_LANGUAGE_RULE)
     parts.append(build_directives_reminder(directives) if directives else "")
+    # Volatile "now" reference last, so the static/per-bank instructions above
+    # remain a cacheable prefix and only this timestamp falls outside the cache.
+    parts.append(_current_datetime_section())
 
     return "\n\n".join(p.strip() for p in parts if p.strip()) + output_language_directive(llm_output_language)
 
@@ -586,7 +611,7 @@ You will be given:
 2. CURRENT DOCUMENT (JSON) — the existing structured mental model. Each section
    has a stable ``id``, a ``heading``, a ``level`` (1..6), and an ordered list
    of ``blocks``. Blocks are typed: ``paragraph``, ``bullet_list``,
-   ``ordered_list``, or ``code``.
+   ``ordered_list``, ``code``, or ``table``.
 3. NEW INFORMATION SYNTHESIS (markdown) — a synthesis showing how the new facts
    relate to the document's topic. Use it to understand context and relevance,
    but do NOT copy its formatting or wording wholesale.
@@ -648,6 +673,7 @@ Block shapes
 - ``{"type": "bullet_list", "items": ["...", "..."]}``
 - ``{"type": "ordered_list", "items": ["...", "..."]}``
 - ``{"type": "code", "language": "json", "text": "..."}``
+- ``{"type": "table", "headers": ["col1", "col2"], "rows": [["a", "b"], ["c", "d"]]}``
 
 OUTPUT FORMAT
 Return ONLY a single JSON object on its own, with no prose before or after,

hindsight-api-slim/hindsight_api/engine/reflect/structured_doc.py

@@ -24,11 +24,11 @@ A document is an ordered list of ``Section``s.  Each section has:
 - ``heading``: the markdown heading text (without the ``#`` prefix).
 - ``level`` : 1 (``#``) … 6 (``######``).  Default 2.
 - ``blocks``: ordered list of typed blocks — paragraph, bullet_list,
-              ordered_list, code.
+              ordered_list, code, table.
 
 The schema is intentionally narrow: it covers what real mental-model documents
 actually contain (the kind a coding agent writes for itself or a user writes as
-a "skill" doc).  Tables, images, and raw HTML are out of scope until needed.
+a "skill" doc).  Images and raw HTML are out of scope until needed.
 """
 
 from __future__ import annotations
@@ -66,8 +66,15 @@ class CodeBlock(BaseModel):
     text: str
 
 
+class TableBlock(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    type: Literal["table"] = "table"
+    headers: list[str] = Field(default_factory=list)
+    rows: list[list[str]] = Field(default_factory=list)
+
+
 Block = Annotated[
-    Union[ParagraphBlock, BulletListBlock, OrderedListBlock, CodeBlock],
+    Union[ParagraphBlock, BulletListBlock, OrderedListBlock, CodeBlock, TableBlock],
     Field(discriminator="type"),
 ]
 
@@ -142,9 +149,38 @@ def render_block(block: Block) -> str:
     if isinstance(block, CodeBlock):
         fence_lang = block.language or ""
         return f"```{fence_lang}\n{block.text}\n```"
+    if isinstance(block, TableBlock):
+        # Width is the widest row, not just the header: a row with more cells
+        # than there are headers would otherwise render cells that GFM drops.
+        width = max(len(block.headers), *(len(row) for row in block.rows), 0)
+        if width == 0:
+            return ""
+        lines = [_render_table_row(block.headers, width)]
+        lines.append("| " + " | ".join("---" for _ in range(width)) + " |")
+        lines.extend(_render_table_row(row, width) for row in block.rows)
+        return "\n".join(lines)
     raise TypeError(f"Unknown block type: {type(block)!r}")
 
 
+def _escape_table_cell(cell: str) -> str:
+    """Escape a cell so it survives a markdown table round-trip.
+
+    An unescaped ``|`` would start a new column and a newline would start a new
+    row, so both are neutralised: pipes are backslash-escaped (the GFM
+    convention, undone again by :func:`_parse_table_row`) and newlines collapse
+    to a space, since a table cell is a single line by construction.
+    """
+    escaped = cell.replace("\\", "\\\\").replace("|", "\\|")
+    return " ".join(escaped.splitlines()).strip()
+
+
+def _render_table_row(cells: list[str], width: int) -> str:
+    """Render one table row, padded with empty cells to ``width`` columns."""
+    rendered = [_escape_table_cell(cell) for cell in cells]
+    rendered.extend("" for _ in range(width - len(rendered)))
+    return "| " + " | ".join(rendered) + " |"
+
+
 def render_section(section: Section) -> str:
     """Render a section: heading + blank line + blocks separated by blank lines."""
     parts = ["#" * section.level + " " + section.heading.strip()]
@@ -178,6 +214,8 @@ _BULLET_RX = re.compile(r"^\s*[-*+]\s+(.*)$")
 _ORDERED_RX = re.compile(r"^\s*\d+[.)]\s+(.*)$")
 _FENCE_RX = re.compile(r"^```([A-Za-z0-9_+-]*)\s*$")
 _SEPARATOR_RX = re.compile(r"\s*([-*_])\1{2,}\s*")
+_TABLE_ROW_RX = re.compile(r"^\s*\|.*\|\s*$")
+_TABLE_SEPARATOR_RX = re.compile(r"^\s*\|?[\s:]*-{2,}[\s:|-]*\|?\s*$")
 
 
 def _split_blocks(lines: list[str]) -> list[list[str]]:
@@ -210,6 +248,60 @@ def _split_blocks(lines: list[str]) -> list[list[str]]:
     return chunks
 
 
+def _parse_table_row(line: str) -> list[str]:
+    """Split a ``| a | b |`` line into cells, honouring backslash escapes.
+
+    Splitting on every ``|`` would tear a cell whose text contains an escaped
+    pipe (``\\|``, what :func:`_escape_table_cell` emits) into two columns, so
+    the line is scanned instead: ``\\|`` and ``\\\\`` unescape to ``|`` and
+    ``\\``, and any other backslash sequence is left verbatim so hand-written
+    content such as ``C:\\path`` survives.
+    """
+    cells: list[str] = []
+    buf: list[str] = []
+    escaped = False
+    for ch in line.strip():
+        if escaped:
+            buf.append(ch if ch in "|\\" else "\\" + ch)
+            escaped = False
+        elif ch == "\\":
+            escaped = True
+        elif ch == "|":
+            cells.append("".join(buf))
+            buf = []
+        else:
+            buf.append(ch)
+    if escaped:
+        buf.append("\\")
+    cells.append("".join(buf))
+
+    # The leading and trailing pipes of a fully-delimited row produce an empty
+    # cell on each end that is punctuation, not content.
+    if cells and cells[0] == "":
+        cells = cells[1:]
+    if cells and cells[-1] == "":
+        cells = cells[:-1]
+    return [cell.strip() for cell in cells]
+
+
+def _parse_table_block(chunk: list[str]) -> TableBlock:
+    """Parse table lines into a :class:`TableBlock`.
+
+    ``_parse_block`` only routes here when the chunk contains a separator line,
+    so the separator is what splits headers from data. Rows above it other than
+    the first (malformed input: GFM allows exactly one header row) are kept as
+    data rather than dropped — the parser never silently loses content.
+    """
+    rows_raw = [_parse_table_row(line) for line in chunk]
+    separator_idx = next(i for i, line in enumerate(chunk) if _TABLE_SEPARATOR_RX.match(line))
+    if separator_idx == 0:
+        return TableBlock(headers=[], rows=rows_raw[1:])
+    return TableBlock(
+        headers=rows_raw[0],
+        rows=rows_raw[1:separator_idx] + rows_raw[separator_idx + 1 :],
+    )
+
+
 def _parse_block(chunk: list[str]) -> Block:
     """Parse a single non-empty chunk into a block."""
     if chunk and _FENCE_RX.match(chunk[0]):
@@ -236,6 +328,11 @@ def _parse_block(chunk: list[str]) -> Block:
             items.append(m.group(1).strip())
         return OrderedListBlock(items=items)
 
+    if len(chunk) >= 2 and all(_TABLE_ROW_RX.match(line) for line in chunk):
+        has_separator = any(_TABLE_SEPARATOR_RX.match(line) for line in chunk)
+        if has_separator:
+            return _parse_table_block(chunk)
+
     return ParagraphBlock(text=" ".join(line.strip() for line in chunk).strip())
 
 

hindsight-api-slim/hindsight_api/engine/reflect/tools.py

@@ -23,6 +23,35 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+#: Retrieval plumbing that the reflect agent never reads, dropped from tool
+#: results before they reach the model.
+#:
+#: These are scoring and provenance internals, not evidence: the agent cites by
+#: ``id``, ``based_on`` persists only id/text/type/context, and the expand tool
+#: takes ``memory_ids`` and resolves chunks server-side -- so nothing downstream
+#: needs them, while on real banks they measure several times the size of the
+#: observation text they accompany.
+#:
+#: Identity, text, dates, tags and ``source_fact_ids`` are deliberately kept.
+#: So is ``entities``: it carries canonical entity *names* (not ids), which are
+#: semantically useful retrieval handles -- the canonical name can differ from
+#: the surface text ("Bob" in the text vs canonical "Robert Smith"). Reflect's
+#: recalls don't populate it today (``include_entities`` defaults to False), but
+#: trimming it would bake in dropping the names if that ever flips on.
+_UNREAD_RESULT_FIELDS = ("scores", "metadata", "chunk_id", "document_id")
+
+
+def _drop_unread_fields(d: dict[str, Any]) -> dict[str, Any]:
+    """Strip retrieval plumbing from one serialized tool result.
+
+    Mutates and returns ``d``, which is always a fresh ``model_dump()`` by the
+    time it gets here -- never a caller's dict.
+    """
+    for k in _UNREAD_RESULT_FIELDS:
+        d.pop(k, None)
+    return d
+
+
 def _prune_nulls(d: dict[str, Any]) -> dict[str, Any]:
     """Drop keys whose value is None or an empty collection (``""``, ``[]``, ``{}``).
 
@@ -228,6 +257,11 @@ async def tool_search_observations(
         tags_match=tags_match,
         tag_groups=tag_groups,
         include_source_facts=include_source_facts,
+        # Canonical entity names are semantic signal the surface text may lack
+        # ("Bob" in the text vs canonical "Robert Smith"): they populate each
+        # result's `entities` field, giving the agent resolved names to cite
+        # and to pivot follow-up queries on.
+        include_entities=True,
         created_after=created_after,
         created_before=created_before,
         _connection_budget=1,
@@ -246,8 +280,10 @@ async def tool_search_observations(
     return {
         "query": query,
         "count": len(result.results),
-        "observations": [_prune_nulls(m.model_dump()) for m in result.results],
-        "source_facts": {k: _prune_nulls(v.model_dump()) for k, v in (result.source_facts or {}).items()},
+        "observations": [_drop_unread_fields(_prune_nulls(m.model_dump())) for m in result.results],
+        "source_facts": {
+            k: _drop_unread_fields(_prune_nulls(v.model_dump())) for k, v in (result.source_facts or {}).items()
+        },
         "is_stale": is_stale,
         "freshness": freshness,
     }
@@ -306,6 +342,9 @@ async def tool_recall(
         tag_groups=tag_groups,
         created_after=created_after,
         created_before=created_before,
+        # See tool_search_observations: resolved entity names on each result
+        # are worth the one extra lookup query.
+        include_entities=True,
         _connection_budget=connection_budget,
         _quiet=True,  # Suppress logging for internal operations
         include_chunks=include_chunks,
@@ -314,7 +353,11 @@ async def tool_recall(
 
     return {
         "query": query,
-        "memories": [_prune_nulls(m.model_dump()) for m in result.results],
+        "memories": [_drop_unread_fields(_prune_nulls(m.model_dump())) for m in result.results],
+        # ``chunks`` is deliberately not trimmed: ChunkInfo carries only
+        # chunk_text / chunk_index / truncated, so it holds none of the fields
+        # above and the call would be a no-op. Pinned by
+        # test_chunk_info_carries_no_unread_fields.
         "chunks": {k: _prune_nulls(v.model_dump()) for k, v in (result.chunks or {}).items()},
     }
 
@@ -363,7 +406,7 @@ async def tool_expand(
     from ..memories import get_memories
 
     _store = get_memories()
-    if _store.writes_memory_rows_in_sql:
+    if _store.writes_memory_rows_in_sql_for(bank_id):
         memories = await conn.fetch(
             f"""
             SELECT id, text, chunk_id, document_id, fact_type, context

hindsight-api-slim/hindsight_api/engine/retain/bank_utils.py

@@ -499,7 +499,7 @@ async def list_banks(pool) -> list:
             last_write = max(write_times) if write_times else None
 
             fact_count = row["fact_count"]
-            if not _store.writes_memory_rows_in_sql:
+            if not _store.writes_memory_rows_in_sql_for(row["bank_id"]):
                 fact_count = sum(
                     (await _store.count_memories(conn=conn, fq_table=fq_table, bank_id=row["bank_id"])).values()
                 )

hindsight-api-slim/hindsight_api/engine/retain/chunk_storage.py

@@ -14,6 +14,9 @@ from .types import ChunkMetadata
 
 logger = logging.getLogger(__name__)
 
+# Page size for walking the facts a chunk owns out of a store that keeps memories outside SQL.
+_OUTGOING_PAGE = 500
+
 
 def compute_chunk_hash(chunk_text: str) -> str:
     """Compute SHA256 hash of chunk text for delta comparison."""
@@ -55,7 +58,54 @@ async def load_existing_chunks(conn, bank_id: str, document_id: str) -> list[Exi
     ]
 
 
-async def delete_chunks_by_ids(conn, chunk_ids: list[str], bank_id: str | None = None, txn=None) -> None:
+async def memory_ids_for_chunks(conn, bank_id: str, chunk_ids: list[str]) -> list[str]:
+    """Ids of the facts these chunks own, asked of whichever store holds them.
+
+    The SQL store keeps ``chunk_id`` as a column; a store that keeps memories outside SQL
+    carries it in the metadata bag (the same key its ``delete_where`` predicate matches on),
+    so the two are read differently. Only ``experience``/``world`` units are returned:
+    observations are not chunk-scoped, and feeding one back as a *source* id would be
+    meaningless. Paged to exhaustion — every id is about to be deleted, and a chunk whose
+    facts overflow one page must not keep half of them.
+    """
+    from ..memories import META_CHUNK_ID, get_memories
+
+    store = get_memories()
+    if store.writes_memory_rows_in_sql:
+        rows = await conn.fetch(
+            f"""
+            SELECT id
+            FROM {fq_table("memory_units")}
+            WHERE bank_id = $1
+              AND chunk_id = ANY($2::text[])
+              AND fact_type IN ('experience', 'world')
+            """,
+            bank_id,
+            chunk_ids,
+        )
+        return [str(row["id"]) for row in rows]
+
+    unit_ids: list[str] = []
+    for chunk_id in chunk_ids:
+        page_token = ""
+        while True:
+            page = await store.scan_memories(
+                conn=conn,
+                fq_table=fq_table,
+                bank_id=bank_id,
+                fact_types=["experience", "world"],
+                metadata_equals={META_CHUNK_ID: chunk_id},
+                limit=_OUTGOING_PAGE,
+                page_token=page_token,
+            )
+            unit_ids.extend(m.unit_id for m in page.memories)
+            page_token = page.next_page_token
+            if not page_token:
+                break
+    return unit_ids
+
+
+async def delete_chunks_by_ids(conn, chunk_ids: list[str], bank_id: str | None = None, txn=None, ops=None) -> int:
     """
     Delete specific chunks by their IDs.
 
@@ -66,9 +116,29 @@ async def delete_chunks_by_ids(conn, chunk_ids: list[str], bank_id: str | None =
     the store's tombstones must ride the same txn as the replacement writes so they commit
     (become visible) together — otherwise an aborted re-ingest could drop the old memories
     without landing the new ones.
+
+    ``ops`` is the backend-specific DataAccessOps the observation sweep below needs to choose
+    the PG (native array) vs Oracle (junction table) read path — pass ``pool.ops``.
+
+    Returns the number of observations invalidated by the sweep, so the caller can log it.
     """
     if not chunk_ids:
-        return
+        return 0
+
+    # Delete the observations derived from the facts these chunks own, BEFORE the facts
+    # themselves go. Nothing can reach those observations afterwards: consolidation batches
+    # are built from facts, so an observation whose sources are all deleted is never selected
+    # into a batch again, and it stays valid and recallable — stale knowledge from the previous
+    # version of the document surviving the replace (issue #3294). The full-replace path does
+    # this in ``handle_document_tracking``; the delta path deletes facts through this cascade
+    # instead, which is why the sweep has to live here rather than at one of the call sites.
+    invalidated = 0
+    if bank_id:
+        outgoing_unit_ids = await memory_ids_for_chunks(conn, bank_id, chunk_ids)
+        if outgoing_unit_ids:
+            from .fact_storage import delete_stale_observations_for_memories
+
+            invalidated = await delete_stale_observations_for_memories(conn, bank_id, outgoing_unit_ids, ops=ops)
 
     # The chunks->memory_units FK cascade below does not reach a store that keeps memories
     # outside SQL (its memory_units is empty), so drop the memories carrying each deleted
@@ -76,7 +146,7 @@ async def delete_chunks_by_ids(conn, chunk_ids: list[str], bank_id: str | None =
     from ..memories import META_CHUNK_ID, DeletePredicate, get_memories
 
     _store = get_memories()
-    if bank_id and not _store.writes_memory_rows_in_sql:
+    if bank_id and not _store.writes_memory_rows_in_sql_for(bank_id):
         for _cid in chunk_ids:
             await _store.delete_where(bank_id, DeletePredicate(metadata_equals={META_CHUNK_ID: _cid}), txn=txn)
 
@@ -103,7 +173,8 @@ async def delete_chunks_by_ids(conn, chunk_ids: list[str], bank_id: str | None =
             ORDER BY
                 LEAST(ml.from_unit_id, ml.to_unit_id),
                 GREATEST(ml.from_unit_id, ml.to_unit_id),
-                ml.link_type
+                ml.link_type,
+                COALESCE(ml.entity_id, '00000000-0000-0000-0000-000000000000'::uuid)
             FOR UPDATE OF ml
         )
         DELETE FROM {fq_table("memory_links")} ml
@@ -127,6 +198,7 @@ async def delete_chunks_by_ids(conn, chunk_ids: list[str], bank_id: str | None =
         """,
         chunk_ids,
     )
+    return invalidated
 
 
 async def store_chunks_batch(
@@ -167,7 +239,7 @@ async def store_chunks_batch(
     # same shape as store_document_text=False, and idempotency is unaffected (content_hash stays).
     from ..memories import get_memories
 
-    if get_memories().owns_document_store:
+    if get_memories().owns_document_store_for(bank_id):
         store_text = False
 
     # Prepare chunk data for batch insert

hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py

@@ -270,6 +270,14 @@ async def handle_document_tracking(
                     f"[RETAIN] Document {document_id} re-ingested: invalidated "
                     f"{invalidated} observation(s) derived from {len(existing_unit_ids)} outgoing memory_units"
                 )
+            else:
+                # Logged even at zero: "the sweep matched nothing" and "the sweep never ran"
+                # are the two candidates whenever orphan observations are reported, and
+                # without this line they look identical from the outside (issue #3294).
+                logger.debug(
+                    f"[RETAIN] Document {document_id} re-ingested: no observations derived from "
+                    f"{len(existing_unit_ids)} outgoing memory_units"
+                )
             # Capture link-recompute victims BEFORE the cascade. Same staleness
             # applies on upsert as on explicit delete: surviving units in OTHER
             # documents that linked to these doomed units are about to lose
@@ -372,7 +380,7 @@ async def _upsert_document_row(
     # the bulky body is written to the store up front (orchestrator._store_document_bodies).
     from ..memories import get_memories
 
-    if get_memories().owns_document_store:
+    if get_memories().owns_document_store_for(bank_id):
         original_text = None
     await conn.execute(
         f"""
@@ -414,7 +422,7 @@ async def update_memory_units_metadata_and_tags(
     from ..memories import MemoryPatch, get_memories
 
     store = get_memories()
-    if not store.writes_memory_rows_in_sql:
+    if not store.writes_memory_rows_in_sql_for(bank_id):
         # A store that keeps memories outside SQL: page the document's memories and patch each
         # one's tags through the store — the UPDATE below is a no-op on its empty memory_units.
         page = await store.scan_memories(

hindsight-api-slim/hindsight_api/engine/retain/link_utils.py

@@ -3,6 +3,7 @@ Link creation utilities for temporal, semantic, and entity links.
 """
 
 import logging
+import re
 import time
 from datetime import UTC
 
@@ -21,6 +22,27 @@ from .types import CausalRelation, EntityResolutionResult
 
 logger = logging.getLogger(__name__)
 
+# Sentinel UUID used in the unique index to represent NULL entity_id
+_NIL_ENTITY_UUID = "00000000-0000-0000-0000-000000000000"
+
+# Any run of whitespace, including the \n / \r / \t that extraction sometimes
+# leaves inside a candidate entity name.
+_WHITESPACE_RUN_RE = re.compile(r"\s+")
+
+
+def _normalize_entity_name(name: str) -> str:
+    """Collapse internal whitespace runs to a single space and strip the ends.
+
+    Extraction can hand back names carrying embedded newlines/tabs, which then
+    become ``entities.canonical_name`` values that shear every line-oriented
+    consumer (``psql -A`` output, log lines, exports) — issue #3275. Case is
+    deliberately untouched: the entity registry already matches on
+    ``LOWER(canonical_name)``, so lowercasing here would only lose the display
+    form.
+    """
+    return _WHITESPACE_RUN_RE.sub(" ", name).strip()
+
+
 # Maximum number of temporal links to keep per unit (from_unit_id).
 # Retrieval only reads top 10-20 per unit via LATERAL join, so keeping
 # more is wasted storage and write amplification.
@@ -31,7 +53,7 @@ def _cap_links_per_unit(links: list[tuple], max_per_unit: int = MAX_TEMPORAL_LIN
     """Keep only the top-N links per from_unit_id, ranked by weight descending.
 
     Args:
-        links: List of (from_unit_id, to_unit_id, link_type, weight) tuples.
+        links: List of (from_unit_id, to_unit_id, link_type, weight, entity_id) tuples.
         max_per_unit: Maximum number of links to retain per from_unit_id.
 
     Returns:
@@ -72,7 +94,7 @@ async def _bulk_insert_links(
 
     Args:
         conn: Database connection (must be inside a transaction).
-        links: List of (from_unit_id, to_unit_id, link_type, weight) tuples.
+        links: List of (from_unit_id, to_unit_id, link_type, weight, entity_id) tuples.
         bank_id: Bank identifier stored on memory_links for fast filtering.
         chunk_size: Max rows per INSERT statement to avoid query timeouts on
                     very large tables (100M+ rows).
@@ -100,6 +122,7 @@ async def _bulk_insert_links(
         fq_table("memory_links"),
         sorted_links,
         bank_id,
+        _NIL_ENTITY_UUID,
         exists_clause,
         chunk_size,
     )
@@ -147,6 +170,12 @@ def _prepare_entities_for_resolution(
     """
     Convert LLM entities into the flat format expected by entity resolver.
 
+    Candidate names are whitespace-normalized here (see ``_normalize_entity_name``)
+    and names that are empty afterwards are dropped, so no downstream stage has to
+    cope with an entity whose canonical name is blank or spans several lines.
+    Both happen before the flat list and ``entity_to_unit`` are derived, keeping
+    the resolver's positional invariant (output index-aligned with input) intact.
+
     Returns:
         Tuple of (all_entities_flat, all_entities, entity_to_unit) where:
         - all_entities_flat: flat list of entity dicts ready for resolve_entities_batch
@@ -155,15 +184,45 @@ def _prepare_entities_for_resolution(
     """
     substep_start = time.time()
     all_entities = []
+    dropped_empty = 0
     for entity_list in llm_entities:
         formatted_entities = []
+        # Normalization can make two candidates that reached here as distinct
+        # strings ("Acme\nCorp" from extraction, "Acme Corp" from the caller's
+        # own entity list) identical, and the upstream dedup in
+        # entity_processing runs on the raw text. Without this, the same entity
+        # would be resolved twice for one fact and its mention_count bumped twice.
+        seen_in_fact: set[str] = set()
         for ent in entity_list:
             if hasattr(ent, "text"):
-                formatted_entities.append({"text": ent.text, "type": "CONCEPT"})
+                raw_text, entity_type = ent.text, "CONCEPT"
             elif isinstance(ent, dict):
-                formatted_entities.append({"text": ent.get("text", ""), "type": ent.get("type", "CONCEPT")})
+                raw_text, entity_type = ent.get("text", ""), ent.get("type", "CONCEPT")
+            else:
+                continue
+
+            normalized_text = _normalize_entity_name(raw_text)
+            if not normalized_text:
+                # A blank or whitespace-only candidate would otherwise be created
+                # as an entity with an empty canonical_name — the resolver has no
+                # guard of its own.
+                dropped_empty += 1
+                continue
+
+            if normalized_text.lower() in seen_in_fact:
+                continue
+            seen_in_fact.add(normalized_text.lower())
+
+            formatted_entities.append({"text": normalized_text, "type": entity_type})
         all_entities.append(formatted_entities)
 
+    if dropped_empty:
+        _log(
+            log_buffer,
+            f"  [6.1] Dropped {dropped_empty} empty candidate entity name(s)",
+            level="debug",
+        )
+
     total_entities = sum(len(ents) for ents in all_entities)
     _log(
         log_buffer,
@@ -371,7 +430,7 @@ async def create_temporal_links_batch_per_fact(
         for row in rows:
             time_diff_h = float(row["time_diff_hours"])
             weight = max(0.3, 1.0 - (time_diff_h / time_window_hours))
-            links.append((row["from_id"], str(row["id"]), "temporal", weight))
+            links.append((row["from_id"], str(row["id"]), "temporal", weight, None))
 
         # Also compute temporal links WITHIN the new batch (new units to each other)
         if len(new_units) > 1:
@@ -396,8 +455,8 @@ async def create_temporal_links_batch_per_fact(
                     if time_diff_hours <= time_window_hours:
                         weight = max(0.3, 1.0 - (time_diff_hours / time_window_hours))
                         # Create bidirectional links
-                        links.append((unit_id, other_id, "temporal", weight))
-                        links.append((other_id, unit_id, "temporal", weight))
+                        links.append((unit_id, other_id, "temporal", weight, None))
+                        links.append((other_id, unit_id, "temporal", weight, None))
 
         # Cap temporal links per unit to avoid write amplification;
         # retrieval only reads top 10-20 per unit anyway.
@@ -454,7 +513,7 @@ async def compute_semantic_links_ann(
         log_buffer: Optional logging buffer
 
     Returns:
-        List of (from_id, to_id, "semantic", similarity) tuples
+        List of (from_id, to_id, "semantic", similarity, None) tuples
         where from_id uses placeholder IDs.
     """
     if not unit_ids or not embeddings:
@@ -555,7 +614,7 @@ async def compute_semantic_links_ann(
     for row in rows:
         sim = float(min(1.0, max(0.0, row["similarity"])))
         if sim >= threshold:
-            links.append((row["from_id"], row["to_id"], "semantic", sim))
+            links.append((row["from_id"], row["to_id"], "semantic", sim, None))
 
     _log(
         log_buffer,
@@ -584,7 +643,7 @@ def compute_semantic_links_within_batch(
         threshold: Minimum cosine similarity
 
     Returns:
-        List of (from_id, to_id, "semantic", similarity) tuples
+        List of (from_id, to_id, "semantic", similarity, None) tuples
     """
     if len(unit_ids) < 2:
         return []
@@ -619,7 +678,7 @@ def compute_semantic_links_within_batch(
                 other_idx = other_indices[local_idx]
                 other_id = unit_ids[other_idx]
                 similarity = float(min(1.0, max(0.0, similarities[local_idx])))
-                links.append((unit_id, other_id, "semantic", similarity))
+                links.append((unit_id, other_id, "semantic", similarity, None))
 
     return links
 
@@ -797,7 +856,7 @@ async def _write_causal_links_batch(
                 if from_unit_id == to_unit_id:
                     continue
 
-                links.append((from_unit_id, to_unit_id, relation_type, 1.0))
+                links.append((from_unit_id, to_unit_id, relation_type, 1.0, None))
 
         if links:
             insert_start = time_mod.time()
@@ -908,6 +967,7 @@ async def rematerialize_causal_links(
             descriptor.to_unit_id,
             descriptor.link_type,
             descriptor.weight,
+            None,
         )
         for descriptor in parsed
         if descriptor is not None

hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py

@@ -51,15 +51,20 @@ def utcnow():
     return datetime.now(UTC)
 
 
-def _redact_document_body(body: str, config: Any) -> str:
+def redact_document_body(body: str, config: Any) -> str:
     """Apply Memory Defense redaction to a document body.
 
     Per-item screening only scrubs the chunked content that goes through
-    `screen()`. When a sub-batch carries `document_body_override` (the full
-    original text of an oversized item — see `_split_contents_into_sub_batches`),
-    that override bypasses screening and would persist verbatim into
-    `documents.original_text`. Apply the same redactor here so the document
-    body is scrubbed regardless of which path produced it.
+    `screen()`. A `document_body_override` (the full original text of an
+    oversized item — see `_split_contents_into_sub_batches`) never goes through
+    `screen()` and would otherwise persist verbatim into
+    `documents.original_text`, so the splitting caller runs it through this
+    redactor once, before handing the same body to every slice.
+
+    **Callers of this module must pass an override that is already screened.**
+    The retain path here deliberately does not re-screen it: every slice of an
+    oversized item carries the identical body, so re-screening would rescan the
+    whole document once per sub-batch (issue #3282).
     """
     try:
         policy = parse_policy(getattr(config, "memory_defense", None))
@@ -75,19 +80,17 @@ def _redact_document_body(body: str, config: Any) -> str:
 def _is_strict_append_of_stored_document(
     stored_original_text: str | None,
     document_body_override: str | None,
-    config: Any,
 ) -> bool:
     """Return whether an oversized document body strictly appends stored text.
 
-    ``documents.original_text`` is sanitized and may also be Memory Defense
-    redacted before persistence. Apply those same transformations to the
-    complete incoming body before comparing it with the stored prefix.
+    ``documents.original_text`` is sanitized before persistence (the override
+    arrives Memory Defense redacted — see ``redact_document_body``), so apply
+    the same sanitization before comparing it with the stored prefix.
     """
     if stored_original_text is None or document_body_override is None:
         return False
 
-    redacted_body = _redact_document_body(document_body_override, config)
-    sanitized_body = fact_extraction._sanitize_text(redacted_body) or ""
+    sanitized_body = fact_extraction._sanitize_text(document_body_override) or ""
     return len(sanitized_body) > len(stored_original_text) and sanitized_body.startswith(stored_original_text)
 
 
@@ -467,7 +470,7 @@ def _remap_phase1_results(
 
     # Remap semantic ANN links (from_id uses placeholder)
     remapped_semantic = [
-        (placeholder_to_actual.get(lnk[0], lnk[0]), lnk[1], lnk[2], lnk[3]) for lnk in semantic_ann_links
+        (placeholder_to_actual.get(lnk[0], lnk[0]), lnk[1], lnk[2], lnk[3], lnk[4]) for lnk in semantic_ann_links
     ]
 
     return remapped_entity_to_unit, remapped_unit_to_entity_ids, remapped_semantic
@@ -1275,7 +1278,7 @@ async def _store_document_bodies(
     from ..memories import get_memories
 
     store = get_memories()
-    if not store.owns_document_store:
+    if not store.owns_document_store_for(bank_id):
         return
     # The record's content_hash must equal what the SQL documents row stores, so a read is
     # consistent whichever it comes from: sanitize + sha256 the same combined_content. The
@@ -1367,9 +1370,9 @@ async def _streaming_retain_batch(
     # so documents.original_text stores the complete payload, not just this
     # slice (issue #1838).
     if document_body_override is not None:
-        # The override is the unmodified original body — apply redaction so
-        # secrets in oversized inputs don't bypass screening.
-        combined_content = _redact_document_body(document_body_override, config)
+        # Already Memory Defense screened by the caller that produced it
+        # (see redact_document_body) — do not rescan it per slice.
+        combined_content = document_body_override
     else:
         combined_content = "\n".join([c.get("content", "") for c in contents_dicts])
     # Memory: contents_dicts content strings are now captured in combined_content.
@@ -2337,7 +2340,6 @@ async def _try_delta_retain(
         if _is_strict_append_of_stored_document(
             original_text_at_load,
             document_body_override,
-            config,
         ):
             log_buffer.append(
                 "[delta] First oversized slice has no stored chunk match, but "
@@ -2526,10 +2528,10 @@ async def _try_delta_retain(
                 step_start = time.time()
                 # When this sub-batch is one slice of an oversized item
                 # split across multiple sub-batches, store the full body
-                # (issue #1838) instead of just the slice. Redact the
-                # override since it bypassed per-chunk screening.
+                # (issue #1838) instead of just the slice. The override
+                # arrives already screened (see redact_document_body).
                 if document_body_override is not None:
-                    combined_content = _redact_document_body(document_body_override, config)
+                    combined_content = document_body_override
                 else:
                     combined_content = "\n".join([c.get("content", "") for c in contents_dicts])
                 retain_params, merged_tags = _build_retain_params(contents_dicts, document_tags)
@@ -2571,10 +2573,13 @@ async def _try_delta_retain(
                     for idx in changed_indices + removed_indices
                     if idx in existing_by_index
                 ]
-                await chunk_storage.delete_chunks_by_ids(conn, chunks_to_delete, bank_id, txn=_group_txn)
+                invalidated_obs = await chunk_storage.delete_chunks_by_ids(
+                    conn, chunks_to_delete, bank_id, txn=_group_txn, ops=pool.ops
+                )
                 log_buffer.append(
                     f"  Deleted {len(chunks_to_delete)} chunks "
-                    f"({len(changed_indices)} changed + {len(removed_indices)} removed) "
+                    f"({len(changed_indices)} changed + {len(removed_indices)} removed), "
+                    f"invalidated {invalidated_obs} observation(s) "
                     f"in {time.time() - step_start:.3f}s"
                 )
 
@@ -2718,10 +2723,10 @@ async def _delta_metadata_only(
                 )
                 return None
             # When this sub-batch is a slice of an oversized item, write the
-            # full original body (issue #1838) instead of just the slice.
-            # Redact the override since it bypassed per-chunk screening.
+            # full original body (issue #1838) instead of just the slice. The
+            # override arrives already screened (see redact_document_body).
             if document_body_override is not None:
-                combined_content = _redact_document_body(document_body_override, config)
+                combined_content = document_body_override
             else:
                 combined_content = "\n".join([c.get("content", "") for c in contents_dicts])
             retain_params, merged_tags = _build_retain_params(contents_dicts, document_tags)

hindsight-api-slim/hindsight_api/engine/sql/postgresql.py

@@ -5,9 +5,122 @@ vector distance (pgvector), full-text search (VectorChord BM25 / tsvector),
 and other non-portable patterns.
 """
 
+from dataclasses import dataclass
+
+from ..._text_search import mental_models_text_document
 from .base import SQLDialect
 
 
+@dataclass(frozen=True)
+class KnowledgeBm25Arm:
+    """Backend-specific BM25 clauses for the knowledge-page full-text arm.
+
+    ``score_expr`` is a relevance score where higher = more relevant (used in the
+    SELECT list of the BM25-only fallback). ``order_by`` is the arm's ranking
+    expression (kept separate so distance-based backends can order by the raw,
+    index-friendly ``ASC`` distance). ``match_filter`` is a WHERE predicate that
+    keeps only genuine term matches, already prefixed with ``AND `` — empty when
+    the backend ranks every row and needs no gate.
+    """
+
+    score_expr: str
+    order_by: str
+    match_filter: str
+
+
+def knowledge_bm25_arm(
+    text_search_extension: str,
+    *,
+    table_alias: str,
+    text_param: str,
+) -> KnowledgeBm25Arm:
+    """BM25 clauses for ``search_knowledge_pages`` on a given text-search backend.
+
+    Mirrors :meth:`PostgreSQLDialect.build_bm25_arm` (the memory-recall BM25 arm)
+    but targets the ``mental_models`` BM25 index (``idx_mental_models_text_search``
+    over the page ``name`` + ``content``) that backs knowledge pages. Without this
+    dispatch the native tsvector SQL (``ts_rank_cd`` / ``@@``) is sent to every
+    backend and 500s wherever ``mental_models.search_vector`` is not a tsvector
+    (see issue #3268).
+
+    ``table_alias`` is the alias the ``mental_models`` row carries in the query
+    (``mm``); ``text_param`` is the bind placeholder holding the query text.
+
+    ``pgroonga`` queries the multilingual expression index over ``name +
+    content``; ``ensure_text_search_extension`` reconciles ``mental_models`` to
+    that shape (dummy TEXT ``search_vector``) like every other pgroonga table.
+    """
+    a = table_alias
+    p = text_param
+
+    if text_search_extension == "vchord":
+        # VectorChord BM25 over the bm25vector search_vector column, identical to
+        # build_bm25_arm's vchord form. This only returns rows because the
+        # mental_models write path now tokenizes search_vector for vchord
+        # (pg_search_vector_expr, native_inline=False) the same way memory_units
+        # does — the column is plain, not generated, so it must be filled on write.
+        # <&> is the NEGATIVE score (lower = more relevant); negate it.
+        expr = f"-({a}.search_vector <&> to_bm25query('idx_mental_models_text_search', tokenize({p}, 'llmlingua2')))"
+        return KnowledgeBm25Arm(
+            score_expr=expr,
+            order_by=f"{expr} DESC",
+            # Gate on a positive score: the operator ranks every row, so a bare
+            # LIMIT would pad the arm with zero-score non-matches.
+            match_filter=f"AND {expr} > 0",
+        )
+
+    if text_search_extension == "pg_search":
+        # ParadeDB pg_search: BM25 index over (id, name, content), key_field='id'.
+        # Fan the query across both indexed text fields with paradedb.boolean.
+        score = f"paradedb.score({a}.id)"
+        return KnowledgeBm25Arm(
+            score_expr=score,
+            order_by=f"{score} DESC",
+            match_filter=(
+                f"AND {a}.id @@@ paradedb.boolean(should => ARRAY["
+                f"paradedb.match('name', {p}), paradedb.match('content', {p})])"
+            ),
+        )
+
+    if text_search_extension == "pg_textsearch":
+        # Timescale pg_textsearch: BM25 index over the `content` column. `<@>` is a
+        # distance (lower = closer), so negate it for a higher-is-better score and
+        # order by the raw ASC distance so the index drives the ordering.
+        distance = f"{a}.content <@> to_bm25query({p}, 'idx_mental_models_text_search')"
+        return KnowledgeBm25Arm(
+            score_expr=f"-({distance})",
+            order_by=f"{distance} ASC",
+            match_filter="",
+        )
+
+    if text_search_extension == "pgroonga":
+        # Same operator/score pair as build_bm25_arm's pgroonga form. The filter
+        # repeats idx_mental_models_text_search's indexed expression verbatim (via
+        # the shared helper) so the planner can select that expression index —
+        # pgroonga_score() only returns a real score off a pgroonga index scan and
+        # silently reads 0 for every row otherwise, so the id tiebreak keeps the
+        # arm's ordering deterministic if the planner ever picks another plan.
+        # pgroonga_query_escape neutralises operator characters in user text.
+        score = f"pgroonga_score({a}.tableoid, {a}.ctid)"
+        document = mental_models_text_document(a)
+        return KnowledgeBm25Arm(
+            score_expr=score,
+            order_by=f"{score} DESC, {a}.id",
+            match_filter=f"AND {document} &@~ pgroonga_query_escape({p})",
+        )
+
+    # native: generated tsvector over name + content.
+    # The generating expression hard-codes the 'english' config (see the
+    # learnings/pinned_reflections migration), so query with 'english' regardless
+    # of the configured native language.
+    score = f"ts_rank_cd({a}.search_vector, websearch_to_tsquery('english', {p}))"
+    return KnowledgeBm25Arm(
+        score_expr=score,
+        order_by=f"{score} DESC",
+        match_filter=f"AND {a}.search_vector @@ websearch_to_tsquery('english', {p})",
+    )
+
+
 class PostgreSQLDialect(SQLDialect):
     """SQL dialect for PostgreSQL (asyncpg)."""
 

hindsight-api-slim/hindsight_api/engine/transfer/export.py

@@ -13,12 +13,15 @@ import io
 import json
 import logging
 import zipfile
+from collections.abc import Iterator
 from dataclasses import dataclass, field
 from datetime import UTC, date, datetime
 from decimal import Decimal
 from typing import Any
 from uuid import UUID
 
+import anyio.to_thread
+
 from ..causal_links import CAUSAL_LINK_TYPES
 from ..db_utils import acquire_with_retry
 from ..schema import fq_table
@@ -31,6 +34,7 @@ from .schema import (
     TransferChunk,
     TransferDocument,
     TransferFact,
+    TransferKnowledgePage,
     TransferManifest,
     TransferObservation,
     TransferObservationSource,
@@ -88,12 +92,6 @@ _SKIP_TABLES = frozenset(
         # to fresh ids, so carrying them would only produce dangling associations.
         # Revert anything worth keeping on the source before migrating.
         "invalidated_memory_units",
-        # Knowledge-base folder/page tree (client-managed metadata over the carried
-        # mental models). Not carried yet: its self-referential parent_id FK needs a
-        # parents-first (topological) restore order, which the generic per-row
-        # _restore_rows doesn't provide — a follow-up. The mental models themselves
-        # ARE carried, so the target can recreate the tree.
-        "knowledge_pages",
     }
 )
 # Derived columns dropped from carried rows so the target regenerates them with
@@ -208,11 +206,44 @@ async def export_documents(
         documents = loaded.documents
         observations = await _load_observations(conn, bank_id, loaded.unit_index) if include_observations else []
 
+    fact_total = sum(len(document.facts) for document in documents)
+    manifest = TransferManifest(
+        schema_version=SCHEMA_VERSION,
+        source_bank_id=bank_id,
+        exported_at=datetime.now(UTC),
+        document_count=len(documents),
+        fact_count=fact_total,
+        observation_count=len(observations),
+    )
+
+    # ZIP compression and per-document JSON serialisation are CPU-bound and, on a
+    # large bank, would block the event loop for seconds (issue #3321). Run the
+    # assembly in a worker thread so unrelated requests/tasks keep progressing.
+    archive_bytes = await anyio.to_thread.run_sync(_build_archive_bytes, documents, observations, manifest)
+
+    logger.info(
+        "[transfer] Exported %d document(s), %d fact(s), %d observation(s) from bank %s",
+        len(documents),
+        fact_total,
+        len(observations),
+        bank_id,
+    )
+    return archive_bytes
+
+
+def _build_archive_bytes(
+    documents: list[TransferDocument],
+    observations: list[TransferObservation],
+    manifest: TransferManifest,
+) -> bytes:
+    """Serialise the loaded documents/observations/manifest into a ZIP archive.
+
+    Pure CPU work (DEFLATE + Pydantic JSON dumps) with no I/O, so it runs off the
+    event loop via :func:`anyio.to_thread.run_sync`.
+    """
     archive = io.BytesIO()
-    fact_total = 0
     with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as zf:
         for index, document in enumerate(documents):
-            fact_total += len(document.facts)
             zf.writestr(
                 f"documents/{index:06d}.json",
                 document.model_dump_json(indent=2, exclude_none=False),
@@ -222,23 +253,8 @@ async def export_documents(
             payload = "[\n" + ",\n".join(o.model_dump_json(indent=2) for o in observations) + "\n]\n"
             zf.writestr("observations.json", payload)
 
-        manifest = TransferManifest(
-            schema_version=SCHEMA_VERSION,
-            source_bank_id=bank_id,
-            exported_at=datetime.now(UTC),
-            document_count=len(documents),
-            fact_count=fact_total,
-            observation_count=len(observations),
-        )
         zf.writestr("manifest.json", manifest.model_dump_json(indent=2))
 
-    logger.info(
-        "[transfer] Exported %d document(s), %d fact(s), %d observation(s) from bank %s",
-        len(documents),
-        fact_total,
-        len(observations),
-        bank_id,
-    )
     return archive.getvalue()
 
 
@@ -268,6 +284,47 @@ async def _dump_bank_rows(conn: Any, table: str, bank_id: str) -> list[dict]:
     return [{k: v for k, v in dict(row).items() if k not in _DERIVED_COLUMNS} for row in rows]
 
 
+async def _load_knowledge_pages(conn: Any, bank_id: str) -> list[TransferKnowledgePage]:
+    """Load the knowledge-base tree (folders + pages) as typed rows.
+
+    Ordered parents-before-children (root folders first) via a recursive walk of
+    ``parent_id`` so the archive is deterministic and import can insert in list
+    order; import re-derives a safe order regardless. IDs, ``parent_id``,
+    ``mental_model_id``, ``managed`` and ``sort_order`` are all preserved.
+    """
+    rows = await conn.fetch(
+        f"""
+        WITH RECURSIVE tree AS (
+            SELECT kp.*, 0 AS depth
+            FROM {fq_table("knowledge_pages")} kp
+            WHERE kp.bank_id = $1 AND kp.parent_id IS NULL
+            UNION ALL
+            SELECT kp.*, t.depth + 1
+            FROM {fq_table("knowledge_pages")} kp
+            JOIN tree t ON kp.parent_id = t.id AND kp.bank_id = $1
+        )
+        SELECT id, parent_id, kind, name, mental_model_id, sort_order, managed, created_at, updated_at
+        FROM tree
+        ORDER BY depth, sort_order, id
+        """,
+        bank_id,
+    )
+    return [
+        TransferKnowledgePage(
+            id=row["id"],
+            parent_id=row["parent_id"],
+            kind=row["kind"],
+            name=row["name"],
+            mental_model_id=row["mental_model_id"],
+            sort_order=row["sort_order"],
+            managed=row["managed"],
+            created_at=row["created_at"],
+            updated_at=row["updated_at"],
+        )
+        for row in rows
+    ]
+
+
 async def _dump_history_rows(conn: Any, table: str, bank_id: str) -> list[dict]:
     """Dump a bank-scoped child-history table for carrying across instances.
 
@@ -314,6 +371,7 @@ async def export_bank(
     bank_rows = {table: await _dump_bank_rows(conn, table, bank_id) for table in _BANK_ROW_TABLES}
     for table in CARRIED_HISTORY_TABLES:
         bank_rows[table] = await _dump_history_rows(conn, table, bank_id)
+    knowledge_pages = await _load_knowledge_pages(conn, bank_id)
     history_rows: dict[str, list[dict]] = {}
     if include_history:
         history_rows = {table: await _dump_bank_rows(conn, table, bank_id) for table in HISTORY_TABLES}
@@ -331,6 +389,14 @@ async def export_bank(
 
         for table, rows in bank_rows.items():
             zf.writestr(f"{table}.json", json.dumps(rows, indent=2, default=_row_json_default))
+        # Typed knowledge-page tree (parent-first). Written even when empty so the
+        # importer can distinguish "no pages" from a pre-tree archive.
+        zf.writestr(
+            "knowledge_pages.json",
+            "[\n" + ",\n".join(p.model_dump_json(indent=2) for p in knowledge_pages) + "\n]\n"
+            if knowledge_pages
+            else "[]\n",
+        )
         for table, rows in history_rows.items():
             zf.writestr(f"history/{table}.json", json.dumps(rows, indent=2, default=_row_json_default))
 
@@ -343,6 +409,7 @@ async def export_bank(
             observation_count=len(observations),
             archive_type="bank",
             mental_model_count=len(bank_rows.get("mental_models", [])),
+            knowledge_page_count=len(knowledge_pages),
             directive_count=len(bank_rows.get("directives", [])),
             webhook_count=len(bank_rows.get("webhooks", [])),
             includes_history=include_history,
@@ -352,12 +419,13 @@ async def export_bank(
 
     logger.info(
         "[transfer] Exported bank %s: %d document(s), %d fact(s), %d observation(s), "
-        "%d mental model(s), %d directive(s), %d webhook(s)%s",
+        "%d mental model(s), %d knowledge page(s), %d directive(s), %d webhook(s)%s",
         bank_id,
         len(documents),
         fact_total,
         len(observations),
         len(bank_rows.get("mental_models", [])),
+        len(knowledge_pages),
         len(bank_rows.get("directives", [])),
         len(bank_rows.get("webhooks", [])),
         " (with history)" if include_history else "",
@@ -539,25 +607,41 @@ async def _load_facts(conn: Any, bank_id: str, doc_ids: list[str], include_lifec
     return loaded
 
 
+# A whole-bank export can index hundreds of thousands of memory units. Passing
+# every unit id as a single ``ANY($1)`` parameter (issue #3321) inflates the
+# query, spikes memory, and pins the connection while one enormous scan runs.
+# Fetch the attach queries in bounded batches instead: each unit id belongs to
+# exactly one batch, so per-fact ordering is preserved, and awaiting between
+# batches yields the event loop.
+_ATTACH_BATCH_SIZE = 5000
+
+
+def _iter_id_batches(ids: list[Any], batch_size: int = _ATTACH_BATCH_SIZE) -> Iterator[list[Any]]:
+    """Yield ``ids`` in fixed-size chunks (bounds SQL ``ANY`` parameter size)."""
+    for start in range(0, len(ids), batch_size):
+        yield ids[start : start + batch_size]
+
+
 async def _attach_entities(conn: Any, loaded: _LoadedFacts) -> None:
     """Populate each fact's ``entities`` list with its entities' canonical names."""
     if not loaded.unit_index:
         return
-    rows = await conn.fetch(
-        f"""
-        SELECT ue.unit_id, e.canonical_name
-        FROM {fq_table("unit_entities")} ue
-        JOIN {fq_table("entities")} e ON e.id = ue.entity_id
-        WHERE ue.unit_id = ANY($1)
-        ORDER BY e.canonical_name
-        """,
-        list(loaded.unit_index.keys()),
-    )
-    for row in rows:
-        location = loaded.unit_index.get(row["unit_id"])
-        if location is None:
-            continue
-        loaded.facts_by_doc[location.document_id][location.ordinal].entities.append(row["canonical_name"])
+    for batch in _iter_id_batches(list(loaded.unit_index.keys())):
+        rows = await conn.fetch(
+            f"""
+            SELECT ue.unit_id, e.canonical_name
+            FROM {fq_table("unit_entities")} ue
+            JOIN {fq_table("entities")} e ON e.id = ue.entity_id
+            WHERE ue.unit_id = ANY($1)
+            ORDER BY e.canonical_name
+            """,
+            batch,
+        )
+        for row in rows:
+            location = loaded.unit_index.get(row["unit_id"])
+            if location is None:
+                continue
+            loaded.facts_by_doc[location.document_id][location.ordinal].entities.append(row["canonical_name"])
 
 
 async def _attach_causal_relations(conn: Any, loaded: _LoadedFacts) -> None:
@@ -570,27 +654,32 @@ async def _attach_causal_relations(conn: Any, loaded: _LoadedFacts) -> None:
     """
     if not loaded.unit_index:
         return
-    rows = await conn.fetch(
-        f"""
-        SELECT from_unit_id, to_unit_id, link_type
-        FROM {fq_table("memory_links")}
-        WHERE link_type = ANY($1)
-          AND from_unit_id = ANY($2)
-          AND to_unit_id = ANY($2)
-        """,
-        list(CAUSAL_LINK_TYPES),
-        list(loaded.unit_index.keys()),
-    )
-    for row in rows:
-        source = loaded.unit_index.get(row["from_unit_id"])
-        target = loaded.unit_index.get(row["to_unit_id"])
-        if source is None or target is None:
-            continue
-        if source.document_id != target.document_id:
-            continue
-        loaded.facts_by_doc[source.document_id][source.ordinal].causal_relations.append(
-            TransferCausalRelation(
-                relation_type=row["link_type"],
-                target_fact_index=target.ordinal,
-            )
+    # Batch on ``from_unit_id`` only. The old query also constrained
+    # ``to_unit_id = ANY(<full set>)``, but splitting one list across both bounds
+    # would drop edges whose endpoints fall in different batches. Instead we
+    # filter the target endpoint in Python (``target is None``) exactly as before,
+    # which keeps every in-set edge while bounding the parameter size.
+    for batch in _iter_id_batches(list(loaded.unit_index.keys())):
+        rows = await conn.fetch(
+            f"""
+            SELECT from_unit_id, to_unit_id, link_type
+            FROM {fq_table("memory_links")}
+            WHERE link_type = ANY($1)
+              AND from_unit_id = ANY($2)
+            """,
+            list(CAUSAL_LINK_TYPES),
+            batch,
         )
+        for row in rows:
+            source = loaded.unit_index.get(row["from_unit_id"])
+            target = loaded.unit_index.get(row["to_unit_id"])
+            if source is None or target is None:
+                continue
+            if source.document_id != target.document_id:
+                continue
+            loaded.facts_by_doc[source.document_id][source.ordinal].causal_relations.append(
+                TransferCausalRelation(
+                    relation_type=row["link_type"],
+                    target_fact_index=target.ordinal,
+                )
+            )

hindsight-api-slim/hindsight_api/engine/transfer/importer.py

@@ -20,6 +20,7 @@ from datetime import UTC, date, datetime
 from typing import Any, Literal
 
 from ..causal_links import CANONICAL_CAUSAL_LINK_TYPE, LEGACY_CAUSAL_LINK_TYPES
+from ..db.ops_postgresql import pg_search_vector_expr
 from ..db_utils import acquire_with_retry
 from ..retain import bank_utils, chunk_storage, embedding_processing, fact_storage, link_utils, orchestrator
 from ..retain.types import (
@@ -37,6 +38,7 @@ from .schema import (
     BankRowsJSONEncoding,
     TransferDocument,
     TransferFact,
+    TransferKnowledgePage,
     TransferManifest,
     TransferObservation,
 )
@@ -105,12 +107,36 @@ class ParsedArchive:
     observations: list[TransferObservation] = field(default_factory=list)
 
 
+def _open_archive(archive_bytes: bytes, *, produced_by: str) -> zipfile.ZipFile:
+    """Open a transfer archive, rejecting non-archives as caller errors.
+
+    Both failure modes here are a wrong file, not a server fault, so they must
+    surface as ``ValueError`` (the API maps that to a 400 with the message) —
+    a bare ``zipfile.BadZipFile`` or a missing manifest would otherwise escape
+    as an opaque 500 or an unexplained "manifest.json is missing".
+
+    Args:
+        archive_bytes: The uploaded bytes.
+        produced_by: How the caller obtains a valid archive, named in the error.
+    """
+    try:
+        zf = zipfile.ZipFile(io.BytesIO(archive_bytes), "r")
+    except zipfile.BadZipFile as e:
+        raise ValueError(f"Invalid transfer archive: the uploaded file is not a readable .zip ({e})") from e
+    if "manifest.json" not in set(zf.namelist()):
+        zf.close()
+        raise ValueError(
+            f"Invalid transfer archive: manifest.json is missing. This endpoint only accepts a .zip produced "
+            f"by {produced_by} — it is not a way to upload a zip of ordinary files (PDF, text, Markdown). "
+            f"Use the file upload / retain endpoint for those."
+        )
+    return zf
+
+
 def parse_archive(archive_bytes: bytes) -> ParsedArchive:
     """Parse and validate a transfer ZIP archive produced by ``export_documents``."""
-    with zipfile.ZipFile(io.BytesIO(archive_bytes), "r") as zf:
+    with _open_archive(archive_bytes, produced_by="the document export endpoint") as zf:
         names = set(zf.namelist())
-        if "manifest.json" not in names:
-            raise ValueError("Invalid transfer archive: manifest.json is missing")
         manifest = TransferManifest.model_validate_json(zf.read("manifest.json"))
         if manifest.schema_version != SCHEMA_VERSION:
             raise ValueError(
@@ -246,6 +272,7 @@ class BankImportResult:
     observations_imported: int = 0
     mental_models_imported: int = 0
     mental_model_history_imported: int = 0
+    knowledge_pages_imported: int = 0
     directives_imported: int = 0
     webhooks_imported: int = 0
     history_rows_imported: int = 0
@@ -258,16 +285,16 @@ class ParsedBankArchive:
     manifest: TransferManifest
     # table name -> list of verbatim row dicts (banks, mental_models, directives, webhooks)
     bank_rows: dict[str, list[dict]] = field(default_factory=dict)
+    # Typed knowledge-base tree (folders + pages), restored parent-first.
+    knowledge_pages: list[TransferKnowledgePage] = field(default_factory=list)
     # table name -> rows (audit_log, llm_requests), present only with --include-history
     history_rows: dict[str, list[dict]] = field(default_factory=dict)
 
 
 def parse_bank_archive(archive_bytes: bytes) -> ParsedBankArchive:
     """Parse the bank-level sections of a whole-bank archive (``archive_type='bank'``)."""
-    with zipfile.ZipFile(io.BytesIO(archive_bytes), "r") as zf:
+    with _open_archive(archive_bytes, produced_by="the bank export endpoint") as zf:
         names = set(zf.namelist())
-        if "manifest.json" not in names:
-            raise ValueError("Invalid transfer archive: manifest.json is missing")
         manifest = TransferManifest.model_validate_json(zf.read("manifest.json"))
         if manifest.archive_type != "bank":
             raise ValueError(
@@ -277,12 +304,20 @@ def parse_bank_archive(archive_bytes: bytes) -> ParsedBankArchive:
         for table in ("banks", *_BANK_CHILD_TABLES, *CARRIED_HISTORY_TABLES):
             fname = f"{table}.json"
             bank_rows[table] = json.loads(zf.read(fname)) if fname in names else []
+        # Typed tree — absent on pre-tree archives, which restore with no pages.
+        knowledge_pages: list[TransferKnowledgePage] = []
+        if "knowledge_pages.json" in names:
+            knowledge_pages = [
+                TransferKnowledgePage.model_validate(p) for p in json.loads(zf.read("knowledge_pages.json"))
+            ]
         history_rows: dict[str, list[dict]] = {}
         for table in HISTORY_TABLES:
             fname = f"history/{table}.json"
             if fname in names:
                 history_rows[table] = json.loads(zf.read(fname))
-    return ParsedBankArchive(manifest=manifest, bank_rows=bank_rows, history_rows=history_rows)
+    return ParsedBankArchive(
+        manifest=manifest, bank_rows=bank_rows, knowledge_pages=knowledge_pages, history_rows=history_rows
+    )
 
 
 def _resolve_bank_rows_json_encoding(manifest: TransferManifest) -> BankRowsJSONEncoding:
@@ -349,6 +384,108 @@ async def _restore_rows(
     return inserted
 
 
+async def _regenerate_mental_model_embeddings(embeddings_model: Any, mm_rows: list[dict]) -> dict[str, str]:
+    """Re-embed each restored mental model with the *target* model.
+
+    Export strips the source embedding (target-derived). Embeds the same
+    ``"{name} {content}"`` text ``create_mental_model`` embeds so a restored model
+    ranks identically to a freshly written one. Runs off-connection (no DB conn is
+    held across the embedding call — see the retain path); returns id -> vector
+    literal for the caller to apply in the restore transaction.
+    """
+    if not mm_rows:
+        return {}
+    texts = [f"{(r.get('name') or '')} {(r.get('content') or '')}" for r in mm_rows]
+    vectors = await embedding_processing.generate_embeddings_batch(embeddings_model, texts)
+    return {r["id"]: str(v) for r, v in zip(mm_rows, vectors, strict=True)}
+
+
+async def _apply_mental_model_derived_state(
+    conn: Any,
+    bank_id: str,
+    mm_embeddings: dict[str, str],
+    config: Any,
+) -> None:
+    """Write the regenerated embedding (and vchord lexical state) onto restored models.
+
+    ``search_vector`` is rebuilt only for vchord: native's column is GENERATED and
+    already repopulated when the row was inserted, and pg_search / pg_textsearch /
+    pgroonga index the base ``name`` / ``content`` columns directly. Same
+    per-backend expression the live mental-model writes use (``pg_search_vector_expr``).
+    """
+    if not mm_embeddings:
+        return
+    sv_expr = pg_search_vector_expr(
+        config, text_col="name", context_col="content", signals_col=None, native_inline=False
+    )
+    sv_clause = f", search_vector = {sv_expr}" if sv_expr else ""
+    for mm_id, vector in mm_embeddings.items():
+        await conn.execute(
+            f"UPDATE {fq_table('mental_models')} SET embedding = $3::vector{sv_clause} WHERE bank_id = $1 AND id = $2",
+            bank_id,
+            mm_id,
+            vector,
+        )
+
+
+def _topological_page_order(pages: list[TransferKnowledgePage]) -> list[TransferKnowledgePage]:
+    """Order nodes parents-before-children so the self-referential ``parent_id`` FK
+    always resolves on insert. A node whose parent is absent from the archive (only
+    possible in a corrupt export) or part of a cycle is emitted last so the FK, not
+    a silent drop, surfaces it."""
+    by_id = {p.id: p for p in pages}
+    ordered: list[TransferKnowledgePage] = []
+    placed: set[str] = set()
+    remaining = list(pages)
+    while remaining:
+        ready = [p for p in remaining if p.parent_id is None or p.parent_id not in by_id or p.parent_id in placed]
+        if not ready:
+            # Unresolvable parents (cycle / dangling) — emit the rest as-is.
+            ordered.extend(remaining)
+            break
+        for p in ready:
+            ordered.append(p)
+            placed.add(p.id)
+        ready_ids = {p.id for p in ready}
+        remaining = [p for p in remaining if p.id not in ready_ids]
+    return ordered
+
+
+async def _restore_knowledge_pages(conn: Any, bank_id: str, pages: list[TransferKnowledgePage]) -> int:
+    """Restore the knowledge-base tree into ``bank_id`` parents-first.
+
+    IDs, ``parent_id``, ``mental_model_id``, ``managed``, ``sort_order``, name and
+    timestamps are preserved; ``bank_id`` is applied to the target. Pages are
+    restored after their backing mental models (the caller sequences that), and
+    folders before their children (topological order here). ``ON CONFLICT DO
+    NOTHING`` keeps the import idempotent.
+    """
+    if not pages:
+        return 0
+    inserted = 0
+    for page in _topological_page_order(pages):
+        await conn.execute(
+            f"""
+            INSERT INTO {fq_table("knowledge_pages")}
+                (id, bank_id, parent_id, kind, name, mental_model_id, sort_order, managed, created_at, updated_at)
+            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, COALESCE($9, now()), COALESCE($10, now()))
+            ON CONFLICT DO NOTHING
+            """,
+            page.id,
+            bank_id,
+            page.parent_id,
+            page.kind,
+            page.name,
+            page.mental_model_id,
+            page.sort_order,
+            page.managed,
+            page.created_at,
+            page.updated_at,
+        )
+        inserted += 1
+    return inserted
+
+
 async def import_bank(
     *,
     backend: Any,
@@ -395,6 +532,16 @@ async def import_bank(
                 if "bank_id" in row:
                     row["bank_id"] = bank_id
 
+    # `internal_id` is a globally-unique (banks_internal_id_unique) local identifier
+    # used only for per-bank index naming — it is NOT part of the bank's logical
+    # state and nothing in the archive references it. Drop it so the column DEFAULT
+    # (gen_random_uuid) mints a fresh one on insert. Keeping the source value makes
+    # the banks INSERT collide with the source bank on a same-instance re-import,
+    # where `ON CONFLICT DO NOTHING` then silently skips the parent row and every
+    # child (mental_models, …) trips its bank_id foreign key. See #3270.
+    for row in parsed.bank_rows.get("banks", []):
+        row.pop("internal_id", None)
+
     async with acquire_with_retry(backend) as conn:
         # Refuse to import into an existing bank — this restores a whole bank, it
         # does not merge. Merging would silently mix the archive's config/mental
@@ -451,13 +598,22 @@ async def import_bank(
         facts_imported=doc_result.facts_imported,
         observations_imported=doc_result.observations_imported,
     )
+
+    # Re-embed restored mental models off-connection (the source embedding was
+    # stripped on export), so no DB connection is held across the embedding call.
+    mm_rows = parsed.bank_rows.get("mental_models", [])
+    mm_embeddings = await _regenerate_mental_model_embeddings(embeddings_model, mm_rows)
+
     async with acquire_with_retry(backend) as conn:
         result.mental_models_imported = await _restore_rows(
             conn,
             "mental_models",
-            parsed.bank_rows.get("mental_models", []),
+            mm_rows,
             bank_rows_json_encoding=bank_rows_json_encoding,
         )
+        # Apply the regenerated embedding + backend-specific lexical state onto the
+        # restored rows (native search_vector already repopulated on insert).
+        await _apply_mental_model_derived_state(conn, bank_id, mm_embeddings, config)
         # Restored after mental_models so the (mental_model_id, bank_id) FK resolves.
         result.mental_model_history_imported = await _restore_rows(
             conn,
@@ -465,6 +621,9 @@ async def import_bank(
             parsed.bank_rows.get("mental_model_history", []),
             bank_rows_json_encoding=bank_rows_json_encoding,
         )
+        # Knowledge-base tree after its backing mental models exist (page FK) and
+        # parents-first (self-referential parent_id FK).
+        result.knowledge_pages_imported = await _restore_knowledge_pages(conn, bank_id, parsed.knowledge_pages)
         result.directives_imported = await _restore_rows(
             conn,
             "directives",
@@ -488,13 +647,15 @@ async def import_bank(
 
     logger.info(
         "[transfer] Imported bank %s: %d doc(s), %d fact(s), %d observation(s), "
-        "%d mental model(s), %d mm-history row(s), %d directive(s), %d webhook(s), %d history row(s)",
+        "%d mental model(s), %d mm-history row(s), %d knowledge page(s), %d directive(s), "
+        "%d webhook(s), %d history row(s)",
         bank_id,
         result.documents_imported,
         result.facts_imported,
         result.observations_imported,
         result.mental_models_imported,
         result.mental_model_history_imported,
+        result.knowledge_pages_imported,
         result.directives_imported,
         result.webhooks_imported,
         result.history_rows_imported,

hindsight-api-slim/hindsight_api/engine/transfer/schema.py

@@ -27,6 +27,9 @@ SCHEMA_VERSION = 1
 # history is optional and included only when the caller requests it.
 CARRIED_HISTORY_TABLES = ("mental_model_history",)
 HISTORY_TABLES = ("audit_log", "llm_requests")
+# Logical tree carried as typed rows (not raw dicts) and restored parent-first
+# after its backing mental models exist.
+KNOWLEDGE_TABLES = ("knowledge_pages",)
 
 ObservationScopes = Literal["per_tag", "combined", "all_combinations", "shared"] | list[list[str]]
 BankRowsJSONEncoding = Literal["decoded", "serialized"]
@@ -130,6 +133,29 @@ class TransferDocument(BaseModel):
     facts: list[TransferFact] = Field(default_factory=list)
 
 
+class TransferKnowledgePage(BaseModel):
+    """One node of the knowledge-base tree (a folder or a page).
+
+    Carried verbatim across a whole-bank transfer so the folder/page hierarchy,
+    ``managed`` flags, and ordering survive. IDs are preserved (a page's
+    ``mental_model_id`` and a node's ``parent_id`` must still resolve on the
+    target), and ``bank_id`` is re-applied to the target bank on import. A folder
+    has ``kind='folder'`` and ``mental_model_id=None``; a page has ``kind='page'``
+    and points at its backing mental model. No derived search state lives here —
+    that is on ``mental_models`` and regenerated on the target.
+    """
+
+    id: str
+    parent_id: str | None = None
+    kind: Literal["folder", "page"]
+    name: str
+    mental_model_id: str | None = None
+    sort_order: int = 0
+    managed: bool = False
+    created_at: datetime | None = None
+    updated_at: datetime | None = None
+
+
 class TransferManifest(BaseModel):
     """Top-level archive descriptor (``manifest.json``).
 
@@ -148,6 +174,7 @@ class TransferManifest(BaseModel):
     # (also carries bank config, mental models, directives, webhooks).
     archive_type: Literal["documents", "bank"] = "documents"
     mental_model_count: int = 0
+    knowledge_page_count: int = 0
     directive_count: int = 0
     webhook_count: int = 0
     # True when --include-history carried audit_log / llm_requests.

hindsight-api-slim/hindsight_api/extensions/operation_validator.py

@@ -317,6 +317,7 @@ class ConsolidateResult:
 class BankReadOperation(StrEnum):
     """Bank-scoped read operation names passed to validate_bank_read."""
 
+    EXPORT_KNOWLEDGE_BASE = "export_knowledge_base"
     GET_BANK_CONFIG = "get_bank_config"
     GET_BANK_PROFILE = "get_bank_profile"
     GET_BANK_STATS = "get_bank_stats"
@@ -327,6 +328,8 @@ class BankReadOperation(StrEnum):
     GET_ENTITY_GRAPH = "get_entity_graph"
     GET_ENTITY_STATE = "get_entity_state"
     GET_GRAPH_DATA = "get_graph_data"
+    GET_KNOWLEDGE_BASE_TREE = "get_knowledge_base_tree"
+    GET_KNOWLEDGE_PAGE = "get_knowledge_page"
     GET_MEMORIES_TIMESERIES = "get_memories_timeseries"
     GET_MEMORY_UNIT = "get_memory_unit"
     GET_OBSERVATION_HISTORY = "get_observation_history"
@@ -343,6 +346,7 @@ class BankReadOperation(StrEnum):
     LIST_TAGS = "list_tags"
     LIST_WEBHOOK_DELIVERIES = "list_webhook_deliveries"
     LIST_WEBHOOKS = "list_webhooks"
+    SEARCH_KNOWLEDGE_BASE = "search_knowledge_base"
 
 
 class BankWriteOperation(StrEnum):
@@ -353,15 +357,20 @@ class BankWriteOperation(StrEnum):
     CLEAR_OBSERVATIONS = "clear_observations"
     CLEAR_OBSERVATIONS_FOR_MEMORY = "clear_observations_for_memory"
     CREATE_DIRECTIVE = "create_directive"
+    CREATE_KNOWLEDGE_FOLDER = "create_knowledge_folder"
+    CREATE_KNOWLEDGE_PAGE = "create_knowledge_page"
     CREATE_MENTAL_MODEL = "create_mental_model"
     CREATE_WEBHOOK = "create_webhook"
     DELETE_BANK = "delete_bank"
     DELETE_DIRECTIVE = "delete_directive"
     DELETE_DOCUMENT = "delete_document"
+    DELETE_KNOWLEDGE_NODE = "delete_knowledge_node"
     DELETE_MENTAL_MODEL = "delete_mental_model"
     DELETE_OPERATION = "delete_operation"
     DELETE_WEBHOOK = "delete_webhook"
     MERGE_BANK_MISSION = "merge_bank_mission"
+    MOVE_KNOWLEDGE_NODE = "move_knowledge_node"
+    RENAME_KNOWLEDGE_NODE = "rename_knowledge_node"
     REPROCESS_DOCUMENT = "reprocess_document"
     RESET_BANK_CONFIG = "reset_bank_config"
     RETRY_FAILED_CONSOLIDATION = "retry_failed_consolidation"
@@ -375,6 +384,7 @@ class BankWriteOperation(StrEnum):
     UPDATE_BANK_DISPOSITION = "update_bank_disposition"
     UPDATE_DIRECTIVE = "update_directive"
     UPDATE_DOCUMENT = "update_document"
+    UPDATE_KNOWLEDGE_PAGE = "update_knowledge_page"
     UPDATE_MEMORY_UNIT = "update_memory_unit"
     UPDATE_MENTAL_MODEL = "update_mental_model"
     UPDATE_WEBHOOK = "update_webhook"
@@ -858,7 +868,9 @@ class OperationValidatorExtension(Extension, ABC):
         Validate a bank read operation before execution.
 
         Override to implement custom validation logic for bank reads
-        (get_bank_profile, get_bank_stats).
+        (get_bank_profile, get_bank_stats, and the knowledge-base reads —
+        knowledge_base_tree / get_knowledge_page / search_knowledge_base /
+        export_knowledge_base, which expose mental-model content).
 
         Args:
             ctx: Context containing:
@@ -877,7 +889,10 @@ class OperationValidatorExtension(Extension, ABC):
 
         Override to implement custom validation logic for bank writes
         (delete_bank, update_bank, update_bank_disposition, set_bank_mission,
-        merge_bank_mission, clear_observations, clear_observations_for_memory).
+        merge_bank_mission, clear_observations, clear_observations_for_memory,
+        and the knowledge-base writes — create_knowledge_folder /
+        create_knowledge_page / update_knowledge_page / rename_knowledge_node /
+        move_knowledge_node / delete_knowledge_node).
 
         Args:
             ctx: Context containing:

hindsight-api-slim/hindsight_api/liveness.py

@@ -0,0 +1,112 @@
+"""DB-free liveness payloads, shared by the API server and the worker.
+
+Liveness answers exactly one question: *is this process wedged beyond
+recovery, so that a restart is the only fix?* It must never touch the
+database. A liveness probe that runs ``SELECT 1`` turns database
+degradation into an outage: every pod fails the probe at once, is killed
+mid-flight (claimed async operations are requeued with ``retry_count``
+incremented, walking real work toward the permanent-failure cliff), and
+then reconnects to re-warm its pool against a database that is already
+saturated.
+
+Dependency checks belong in *readiness* (``/health``, ``/health/ready``),
+because failing readiness pulls the pod out of the Service instead of
+killing it — degradation stays degradation.
+
+Serving this handler at all still proves the one condition a restart does
+fix: the event loop is scheduling coroutines. Hindsight runs request
+handlers and task work on a single loop, so a loop blocked by a synchronous
+call cannot answer even a trivial request within the probe timeout (see
+``loop_watchdog.py``).
+"""
+
+import time
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+# Import time is close enough to process start for a probe payload: this
+# module is imported while the app is being constructed, long before the
+# server binds its port.
+_PROCESS_START = time.monotonic()
+
+
+def uptime_seconds() -> float:
+    """Seconds this process has been running, rounded for readability."""
+    return round(time.monotonic() - _PROCESS_START, 1)
+
+
+class LivenessResponse(BaseModel):
+    """Payload for the API's DB-free liveness probe."""
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "status": "alive",
+                "version": "0.4.0",
+                "uptime_seconds": 812.4,
+            }
+        }
+    )
+
+    status: Literal["alive"] = Field(description='Always "alive" — reaching this handler is the check')
+    version: str = Field(description="Hindsight version this process is running")
+    uptime_seconds: float = Field(description="Seconds since the process started")
+
+
+class WorkerLivenessResponse(LivenessResponse):
+    """Payload for the worker's DB-free liveness probe."""
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "status": "alive",
+                "version": "0.4.0",
+                "uptime_seconds": 812.4,
+                "worker_id": "hindsight-worker-0",
+                "is_shutdown": False,
+                "seconds_since_last_poll": 0.4,
+            }
+        }
+    )
+
+    worker_id: str = Field(description="Identifier of this worker process")
+    is_shutdown: bool = Field(description="Whether graceful shutdown has been signalled")
+    seconds_since_last_poll: float | None = Field(
+        default=None,
+        description=(
+            "Age of the last completed poll cycle, or null before the first one. "
+            "Reported for alerting only — the endpoint stays 200 however stale it "
+            "gets, so a saturated database can never trigger a restart."
+        ),
+    )
+
+
+def liveness_response() -> LivenessResponse:
+    """Build the API liveness payload without touching any dependency."""
+    # Imported lazily: ``hindsight_api/__init__`` pulls in MemoryEngine, so a
+    # module-level import here would make this module unusable from anything the
+    # engine itself imports.
+    from hindsight_api import __version__
+
+    return LivenessResponse(status="alive", version=__version__, uptime_seconds=uptime_seconds())
+
+
+def worker_liveness_response(
+    *,
+    worker_id: str,
+    is_shutdown: bool,
+    seconds_since_last_poll: float | None,
+) -> WorkerLivenessResponse:
+    """Build the worker liveness payload without touching any dependency."""
+    # Lazy for the same reason as liveness_response() above.
+    from hindsight_api import __version__
+
+    return WorkerLivenessResponse(
+        status="alive",
+        version=__version__,
+        uptime_seconds=uptime_seconds(),
+        worker_id=worker_id,
+        is_shutdown=is_shutdown,
+        seconds_since_last_poll=seconds_since_last_poll,
+    )

hindsight-api-slim/hindsight_api/migrations.py

@@ -29,6 +29,7 @@ from sqlalchemy import Connection, create_engine, text
 from sqlalchemy.pool import NullPool
 
 from ._pg_search import normalize_pg_search_tokenizer, pg_search_bm25_columns
+from ._text_search import mental_models_text_document
 from ._vector_index import (
     bootstrap_extension,
     configured_vector_extension,
@@ -783,6 +784,34 @@ def ensure_vector_extension(
         logger.info(f"Successfully reconciled vector indexes for {target_ext}")
 
 
+def _reconcile_needs_no_backfill(
+    text_search_extension: str,
+    table_name: str,
+    current_column_type: str | None,
+    current_index_type: str | None,
+) -> bool:
+    """Is this mismatch safe to reconcile even though the table holds rows?
+
+    Only one transition qualifies: ``mental_models`` sitting on the migration-time
+    native tsvector projection while pgroonga is configured. pgroonga indexes
+    ``name + content`` directly, so the replacement ``search_vector`` is a dummy
+    column with nothing to backfill — dropping the derived tsvector loses no data
+    the reconciler would have to recompute.
+
+    This state exists on every pgroonga deployment because the reconciler used to
+    check the pre-rename ``reflections`` table name and therefore never converted
+    mental models (issue #3307). Every other transition (anything writing
+    ``memory_units``, or a target column that stores a per-row tsvector/bm25vector)
+    needs values only the write path can produce, so it stays fail-closed.
+    """
+    return (
+        text_search_extension == "pgroonga"
+        and table_name == "mental_models"
+        and current_column_type == "tsvector"
+        and current_index_type in {None, "gin"}
+    )
+
+
 def ensure_text_search_extension(
     database_url: str,
     text_search_extension: str = "native",
@@ -796,7 +825,9 @@ def ensure_text_search_extension(
     in the database and adjusts them if necessary:
     - If they match configured extension: no action needed
     - If they differ and tables are empty: drop old column/index, recreate with new type
-    - If they differ and tables have data: raise error with migration guidance
+    - If they differ and tables have data: raise error with migration guidance,
+      except for the mental-model native-to-pgroonga transition, which needs no
+      backfill (pgroonga indexes the base columns) and so is safe while populated
 
     Args:
         database_url: SQLAlchemy database URL
@@ -816,10 +847,7 @@ def ensure_text_search_extension(
     engine = create_engine(to_libpq_url(database_url), poolclass=NullPool)
     with engine.connect() as conn:
         # Tables with search_vector columns to check
-        tables_to_check = [
-            "memory_units",
-            "reflections",  # Renamed from pinned_reflections in p1k2l3m4n5o6 migration
-        ]
+        tables_to_check = ["memory_units", "mental_models"]
 
         # Determine target column type and index type
         if text_search_extension == "vchord":
@@ -829,7 +857,7 @@ def ensure_text_search_extension(
             target_column_type = "text"
             target_index_type = "bm25"
         elif text_search_extension == "pgroonga":
-            # pgroonga indexes the base text column directly. We keep a dummy
+            # pgroonga indexes the base text columns directly. We keep a dummy
             # TEXT column named search_vector for symmetry with pg_textsearch
             # and so the column-type mismatch detection above keeps working.
             target_column_type = "text"
@@ -889,13 +917,19 @@ def ensure_text_search_extension(
                 text("""
                     SELECT am.amname, pi.indexdef
                     FROM pg_indexes pi
-                    JOIN pg_class c ON c.relname = pi.indexname
+                    JOIN pg_class c
+                      ON c.relname = pi.indexname
+                     AND c.relnamespace = to_regnamespace(pi.schemaname)
                     JOIN pg_am am ON am.oid = c.relam
                     WHERE pi.schemaname = :schema
                       AND pi.tablename = :table_name
-                      AND pi.indexname LIKE '%text_search%'
+                      AND pi.indexname = :index_name
                 """),
-                {"schema": schema_name, "table_name": table_name},
+                {
+                    "schema": schema_name,
+                    "table_name": table_name,
+                    "index_name": f"idx_{table_name.replace('.', '_')}_text_search",
+                },
             ).fetchone()
 
             current_index_type = current_index_info[0] if current_index_info else None
@@ -926,7 +960,12 @@ def ensure_text_search_extension(
                 # Check if table has data
                 row_count = conn.execute(text(f"SELECT COUNT(*) FROM {schema_name}.{table_name}")).scalar()
 
-                if row_count > 0:
+                if row_count > 0 and not _reconcile_needs_no_backfill(
+                    text_search_extension,
+                    table_name,
+                    current_column_type,
+                    current_index_type,
+                ):
                     tables_with_data.append((table_name, row_count))
             else:
                 logger.debug(f"Text search OK for {table_name}: {current_column_type}/{current_index_type}")
@@ -960,11 +999,17 @@ def ensure_text_search_extension(
                 f"the following tables contain data: {table_list}. "
                 f"To change text search extension, you must either:\n"
                 f"  1. Clear all data: DELETE FROM {schema_name}.memory_units; "
-                f"DELETE FROM {schema_name}.reflections; then restart\n"
+                f"DELETE FROM {schema_name}.mental_models; then restart\n"
                 f"  2. Use the current text search extension (set HINDSIGHT_API_TEXT_SEARCH_EXTENSION='{current_ext}')"
             )
 
-        # Tables are empty, safe to recreate columns/indexes
+        # Tables are empty, except for the backfill-free mental-model
+        # native-to-pgroonga transition admitted above.
+        #
+        # Every statement below is written to be safely re-executable: replicas
+        # boot concurrently during a rolling restart and each runs this
+        # reconciliation, so a plain CREATE/ADD would crash whichever replica
+        # loses the race to the first one's committed DDL.
         logger.info(f"Recreating text search columns/indexes for {text_search_extension}")
 
         for table_name, current_col_type, current_idx_type, _was_pg_search in mismatched_tables:
@@ -987,14 +1032,17 @@ def ensure_text_search_extension(
                 logger.info(f"Creating bm25vector column on {table_name}")
                 # Note: vchord_bm25 extension creates types in bm25_catalog schema
                 conn.execute(
-                    text(f"ALTER TABLE {schema_name}.{table_name} ADD COLUMN search_vector bm25_catalog.bm25vector")
+                    text(
+                        f"ALTER TABLE {schema_name}.{table_name} "
+                        f"ADD COLUMN IF NOT EXISTS search_vector bm25_catalog.bm25vector"
+                    )
                 )
 
                 # Create BM25 index
                 logger.info(f"Creating BM25 index on {table_name}")
                 conn.execute(
                     text(f"""
-                        CREATE INDEX idx_{table_name.replace(".", "_")}_text_search
+                        CREATE INDEX IF NOT EXISTS idx_{table_name.replace(".", "_")}_text_search
                         ON {schema_name}.{table_name}
                         USING bm25 (search_vector bm25_catalog.bm25_ops)
                     """)
@@ -1002,19 +1050,21 @@ def ensure_text_search_extension(
             elif text_search_extension == "pg_textsearch":
                 logger.info(f"Creating TEXT column on {table_name}")
                 # Dummy TEXT column for consistency (indexes operate on base columns)
-                conn.execute(text(f"ALTER TABLE {schema_name}.{table_name} ADD COLUMN search_vector TEXT"))
+                conn.execute(
+                    text(f"ALTER TABLE {schema_name}.{table_name} ADD COLUMN IF NOT EXISTS search_vector TEXT")
+                )
 
                 # Create BM25 index on expression
                 logger.info(f"Creating BM25 index on {table_name}")
                 # Different expression for each table
                 if table_name == "memory_units":
                     index_expr = "(COALESCE(text, '') || ' ' || COALESCE(context, ''))"
-                else:  # reflections
-                    index_expr = "(COALESCE(name, '') || ' ' || content)"
+                else:  # mental_models
+                    index_expr = mental_models_text_document()
 
                 conn.execute(
                     text(f"""
-                        CREATE INDEX idx_{table_name.replace(".", "_")}_text_search
+                        CREATE INDEX IF NOT EXISTS idx_{table_name.replace(".", "_")}_text_search
                         ON {schema_name}.{table_name}
                         USING bm25({index_expr})
                         WITH (text_config='english')
@@ -1031,18 +1081,20 @@ def ensure_text_search_extension(
                         raise
 
                 logger.info(f"Creating dummy TEXT search_vector on {table_name} for pgroonga")
-                # pgroonga indexes the base text column directly, but we keep a
+                # pgroonga indexes the base text columns directly, but we keep a
                 # dummy search_vector column for symmetry with pg_textsearch and
                 # so the column-type mismatch detection above keeps working.
-                conn.execute(text(f"ALTER TABLE {schema_name}.{table_name} ADD COLUMN search_vector TEXT"))
+                conn.execute(
+                    text(f"ALTER TABLE {schema_name}.{table_name} ADD COLUMN IF NOT EXISTS search_vector TEXT")
+                )
 
                 # pgroonga index expression mirrors pg_textsearch
                 if table_name == "memory_units":
                     index_expr = (
                         "(COALESCE(text, '') || ' ' || COALESCE(context, '') || ' ' || COALESCE(text_signals, ''))"
                     )
-                else:  # reflections
-                    index_expr = "(COALESCE(name, '') || ' ' || content)"
+                else:  # mental_models — knowledge_bm25_arm repeats this verbatim
+                    index_expr = mental_models_text_document()
 
                 logger.info(f"Creating pgroonga index on {table_name}")
                 # TokenBigram is the polyglot default — falls back to whitespace
@@ -1051,7 +1103,7 @@ def ensure_text_search_extension(
                 # case folding, etc.) which materially improves Japanese recall.
                 conn.execute(
                     text(f"""
-                        CREATE INDEX idx_{table_name.replace(".", "_")}_text_search
+                        CREATE INDEX IF NOT EXISTS idx_{table_name.replace(".", "_")}_text_search
                         ON {schema_name}.{table_name}
                         USING pgroonga ({index_expr})
                         WITH (tokenizer='TokenBigram', normalizer='NormalizerNFKC150')
@@ -1060,7 +1112,9 @@ def ensure_text_search_extension(
             elif text_search_extension == "pg_search":
                 logger.info(f"Creating TEXT column on {table_name}")
                 # Dummy TEXT column for schema symmetry; pg_search indexes operate on base columns.
-                conn.execute(text(f"ALTER TABLE {schema_name}.{table_name} ADD COLUMN search_vector TEXT"))
+                conn.execute(
+                    text(f"ALTER TABLE {schema_name}.{table_name} ADD COLUMN IF NOT EXISTS search_vector TEXT")
+                )
 
                 # ParadeDB BM25 index over the table's primary key and text columns.
                 # Column list mirrors what the initial / text_signals migrations create.
@@ -1070,7 +1124,7 @@ def ensure_text_search_extension(
                         ("text", "context", "text_signals"),
                         pg_search_tokenizer,
                     )
-                else:  # reflections
+                else:  # mental_models
                     bm25_cols = pg_search_bm25_columns(
                         "id",
                         ("name", "content"),
@@ -1080,7 +1134,7 @@ def ensure_text_search_extension(
                 logger.info(f"Creating ParadeDB BM25 index on {table_name}")
                 conn.execute(
                     text(f"""
-                        CREATE INDEX idx_{table_name.replace(".", "_")}_text_search
+                        CREATE INDEX IF NOT EXISTS idx_{table_name.replace(".", "_")}_text_search
                         ON {schema_name}.{table_name}
                         USING bm25 ({bm25_cols})
                         WITH (key_field='id')
@@ -1088,17 +1142,35 @@ def ensure_text_search_extension(
                 )
             else:  # native
                 logger.info(f"Creating tsvector column on {table_name}")
-                # Plain tsvector column. The application populates search_vector
-                # at INSERT time via to_tsvector($lang, ...) using the configured
-                # HINDSIGHT_API_TEXT_SEARCH_EXTENSION_NATIVE_LANGUAGE — see
-                # ops_postgresql.insert_facts_batch.
-                conn.execute(text(f"ALTER TABLE {schema_name}.{table_name} ADD COLUMN search_vector tsvector"))
+                if table_name == "mental_models":
+                    # No write path populates mental_models.search_vector for
+                    # native (pg_search_vector_expr passes native_inline=False),
+                    # so it must be GENERATED exactly like the learnings /
+                    # pinned_reflections migration creates it — a plain column
+                    # here would stay NULL and silently empty knowledge search.
+                    # The 'english' config is hard-coded there and in
+                    # knowledge_bm25_arm's native branch; keep all three in step.
+                    conn.execute(
+                        text(f"""
+                            ALTER TABLE {schema_name}.{table_name}
+                            ADD COLUMN IF NOT EXISTS search_vector tsvector
+                            GENERATED ALWAYS AS (
+                                to_tsvector('english', {mental_models_text_document()})
+                            ) STORED
+                        """)
+                    )
+                else:
+                    # memory_units writes populate this plain column with the
+                    # configured native language in ops_postgresql.
+                    conn.execute(
+                        text(f"ALTER TABLE {schema_name}.{table_name} ADD COLUMN IF NOT EXISTS search_vector tsvector")
+                    )
 
                 # Create GIN index
                 logger.info(f"Creating GIN index on {table_name}")
                 conn.execute(
                     text(f"""
-                        CREATE INDEX idx_{table_name.replace(".", "_")}_text_search
+                        CREATE INDEX IF NOT EXISTS idx_{table_name.replace(".", "_")}_text_search
                         ON {schema_name}.{table_name}
                         USING gin(search_vector)
                     """)

hindsight-api-slim/hindsight_api/models.py

@@ -191,6 +191,7 @@ class Entity(Base):
 
     # Relationships
     unit_entities = relationship("UnitEntity", back_populates="entity", cascade="all, delete-orphan")
+    memory_links = relationship("MemoryLink", back_populates="entity", cascade="all, delete-orphan")
     cooccurrences_1 = relationship(
         "EntityCooccurrence",
         foreign_keys="EntityCooccurrence.entity_id_1",
@@ -260,12 +261,7 @@ class EntityCooccurrence(Base):
 
 
 class MemoryLink(Base):
-    """Links between memory units (temporal, semantic, causal).
-
-    Entity edges are not stored here: memory-to-entity associations live in
-    ``unit_entities``, and both the /graph endpoint and recall derive entity
-    edges from that table on demand.
-    """
+    """Links between memory units (temporal, semantic, entity)."""
 
     __tablename__ = "memory_links"
 
@@ -276,24 +272,29 @@ class MemoryLink(Base):
         UUID(as_uuid=True), ForeignKey("memory_units.id", ondelete="CASCADE"), primary_key=True
     )
     link_type: Mapped[str] = mapped_column(Text, primary_key=True)
+    entity_id: Mapped[PyUUID | None] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("entities.id", ondelete="CASCADE"), primary_key=True
+    )
     weight: Mapped[float] = mapped_column(Float, nullable=False, server_default="1.0")
     created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now())
 
     # Relationships
     from_unit = relationship("MemoryUnit", foreign_keys=[from_unit_id], back_populates="outgoing_links")
     to_unit = relationship("MemoryUnit", foreign_keys=[to_unit_id], back_populates="incoming_links")
+    entity = relationship("Entity", back_populates="memory_links")
 
     __table_args__ = (
         # Retain writes ``caused_by`` only. Keep the historical causal values
         # valid so existing rows and transfer archives remain queryable.
         CheckConstraint(
-            "link_type IN ('temporal', 'semantic', 'causes', 'caused_by', 'enables', 'prevents')",
+            "link_type IN ('temporal', 'semantic', 'entity', 'causes', 'caused_by', 'enables', 'prevents')",
             name="memory_links_link_type_check",
         ),
         CheckConstraint("weight >= 0.0 AND weight <= 1.0", name="memory_links_weight_check"),
         Index("idx_memory_links_from", "from_unit_id"),
         Index("idx_memory_links_to", "to_unit_id"),
         Index("idx_memory_links_type", "link_type"),
+        Index("idx_memory_links_entity", "entity_id", postgresql_where=sql_text("entity_id IS NOT NULL")),
         Index(
             "idx_memory_links_from_weight",
             "from_unit_id",

hindsight-api-slim/hindsight_api/worker/exceptions.py

@@ -1,6 +1,19 @@
 from datetime import datetime
 
 
+def format_task_error(e: BaseException) -> str:
+    """Render an exception for a task failure log line / stored error_message.
+
+    Always prefixes the exception class. Plenty of exceptions carry an empty
+    ``str()`` — ``TimeoutError()``, ``CancelledError()``, a bare ``raise
+    SomeError()`` — and the bare interpolation those log lines used produced
+    ``Task execution failed: graph_maintenance, error: ``, which says nothing at
+    all about what went wrong (issue #3218).
+    """
+    message = str(e)
+    return f"{type(e).__name__}: {message}" if message else type(e).__name__
+
+
 class RetryTaskAt(Exception):
     """Raise from a task handler to schedule a retry at a specific time."""
 

hindsight-api-slim/hindsight_api/worker/main.py

@@ -58,6 +58,7 @@ def create_worker_app(poller: WorkerPoller, memory):
     from fastapi.responses import JSONResponse, Response
     from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
 
+    from ..liveness import WorkerLivenessResponse, worker_liveness_response
     from ..metrics import create_metrics_collector, get_metrics_collector, initialize_metrics
 
     app = FastAPI(
@@ -81,20 +82,58 @@ def create_worker_app(poller: WorkerPoller, memory):
         metrics_collector.set_db_pool(memory._pool)
         logger.info("DB pool metrics configured")
 
-    @app.get(
-        "/health",
-        summary="Health check endpoint",
-        description="Returns worker health status including database connectivity",
-        tags=["Monitoring"],
-    )
-    async def health_endpoint():
-        """Health check endpoint."""
+    async def _readiness_response() -> JSONResponse:
+        """Shared body of /health and /health/ready: 200 if healthy, 503 if not."""
         health = await memory.health_check()
         health["worker_id"] = poller.worker_id
         health["is_shutdown"] = poller.is_shutdown
         status_code = 200 if health.get("status") == "healthy" else 503
         return JSONResponse(content=health, status_code=status_code)
 
+    @app.get(
+        "/health",
+        summary="Health check endpoint",
+        description="Readiness check: returns worker health status including database "
+        "connectivity. Alias of /health/ready. Use /health/live for liveness probes.",
+        tags=["Monitoring"],
+    )
+    async def health_endpoint():
+        """Health check endpoint."""
+        return await _readiness_response()
+
+    @app.get(
+        "/health/ready",
+        summary="Readiness probe",
+        description="Returns 200 when the worker can reach the database, 503 otherwise. "
+        "Identical to /health, which stays supported as its alias.",
+        tags=["Monitoring"],
+    )
+    async def readiness_endpoint():
+        """Readiness probe that verifies database connectivity."""
+        return await _readiness_response()
+
+    @app.get(
+        "/health/live",
+        response_model=WorkerLivenessResponse,
+        summary="Liveness probe",
+        description="Returns 200 whenever the worker process can serve a request. Performs "
+        "no database access, so a slow database never restarts the worker and never "
+        "requeues its claimed operations. Point livenessProbe here.",
+        tags=["Monitoring"],
+    )
+    async def liveness_endpoint() -> WorkerLivenessResponse:
+        """Liveness probe: in-process only, never touches the database.
+
+        ``seconds_since_last_poll`` exposes poller progress for alerting; it never
+        changes the status code, because a stalled poll cycle under database
+        pressure is precisely the case where restarting makes things worse.
+        """
+        return worker_liveness_response(
+            worker_id=poller.worker_id,
+            is_shutdown=poller.is_shutdown,
+            seconds_since_last_poll=poller.seconds_since_last_poll,
+        )
+
     @app.get(
         "/metrics",
         summary="Prometheus metrics endpoint",

hindsight-api-slim/hindsight_api/worker/poller.py

@@ -13,7 +13,6 @@ import io
 import json
 import logging
 import time
-import traceback
 from collections import Counter
 from collections.abc import Awaitable, Callable, Iterable
 from dataclasses import dataclass
@@ -22,7 +21,7 @@ from typing import TYPE_CHECKING, Any
 from ..config import get_config
 from ..engine.schema import fq_table_explicit as fq_table
 from ..metrics import get_metrics_collector
-from .exceptions import DeferOperation, RetryTaskAt
+from .exceptions import DeferOperation, RetryTaskAt, format_task_error
 from .stage import StageHolder, bind_holder
 
 # Map DB operation_type -> metric `operation` label, collapsing the retain
@@ -255,6 +254,10 @@ class WorkerPoller:
         self._in_flight_lock = asyncio.Lock()
         self._last_progress_log = 0.0
         self._tasks_completed_since_log = 0
+        # Monotonic stamp of the last completed claim cycle. Reported by the
+        # liveness probe so operators can alert on a poller that stopped making
+        # progress; None until the first cycle finishes.
+        self._last_poll_at: float | None = None
         # Track active tasks locally: operation_id -> ActiveTaskInfo
         self._active_tasks: dict[str, ActiveTaskInfo] = {}
         # Track in-flight tasks by operation type
@@ -851,10 +854,12 @@ class WorkerPoller:
             # Retry is not a terminal outcome — do not record a completion.
             await self._schedule_retry(task.operation_id, e.retry_at, str(e), task.schema)
         except Exception as e:
-            logger.error(f"Task {task.operation_id} failed: {e}")
-            traceback.print_exc()
+            # exc_info rather than print_exc(): the stderr copy carries no task id
+            # and is the first thing lost to log rotation (issue #3218).
+            error_message = format_task_error(e)
+            logger.error(f"Task {task.operation_id} failed: {error_message}", exc_info=True)
             try:
-                await self._mark_failed(task.operation_id, str(e), task.schema)
+                await self._mark_failed(task.operation_id, error_message, task.schema)
             except Exception:
                 # Marking a task failed is itself a DB write, and it can fail
                 # (pool exhausted, connection reset, statement timeout). Without
@@ -1251,6 +1256,7 @@ class WorkerPoller:
             try:
                 # Claim a batch of tasks (respecting slot limits)
                 tasks = await self.claim_batch()
+                self._last_poll_at = time.monotonic()
 
                 if tasks:
                     # Log batch info
@@ -1298,8 +1304,7 @@ class WorkerPoller:
                 logger.info(f"Worker {self._worker_id} polling loop cancelled")
                 break
             except Exception as e:
-                logger.error(f"Worker {self._worker_id} error in polling loop: {e}")
-                traceback.print_exc()
+                logger.error(f"Worker {self._worker_id} error in polling loop: {format_task_error(e)}", exc_info=True)
                 # Backoff on error
                 await asyncio.sleep(1)
 
@@ -1741,3 +1746,10 @@ class WorkerPoller:
     def is_shutdown(self) -> bool:
         """Check if shutdown has been signaled."""
         return self._shutdown.is_set()
+
+    @property
+    def seconds_since_last_poll(self) -> float | None:
+        """Age of the last completed claim cycle, or None before the first one."""
+        if self._last_poll_at is None:
+            return None
+        return round(time.monotonic() - self._last_poll_at, 1)

hindsight-api-slim/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-api-slim"
-version = "0.8.6"
+version = "0.9.0"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 license = "MIT"
 readme = "README.md"

hindsight-api-slim/tests/test_async_batch_retain.py

@@ -46,21 +46,134 @@ async def test_duplicate_document_ids_rejected_async(memory, request_context):
 
 
 @pytest.mark.asyncio
-async def test_duplicate_document_ids_rejected_sync(memory, request_context):
-    """Test that sync retain also rejects batches with duplicate document_ids."""
-    bank_id = "test_duplicate_sync"
-    contents = [
-        {"content": "First item", "document_id": "doc1"},
-        {"content": "Second item", "document_id": "doc1"},  # Duplicate!
-    ]
+async def test_shared_document_id_folds_sync(memory, request_context):
+    """Sync retain accepts several items sharing one document_id and folds them
+    into a single document, in request order (the documented RetainRequest
+    example — see issue #3363)."""
+    bank_id = f"test_shared_doc_sync_{uuid.uuid4().hex}"
+    try:
+        contents = [
+            {"content": "Alice works at Google", "context": "work", "document_id": "conversation_123"},
+            {"content": "Bob went hiking yesterday", "document_id": "conversation_123"},
+        ]
 
-    # Should raise ValueError due to duplicate document_ids
-    with pytest.raises(ValueError, match="duplicate document_ids.*doc1"):
-        await memory.retain_batch_async(
+        # Must NOT raise (this used to be a 400).
+        result = await memory.retain_batch_async(
             bank_id=bank_id,
             contents=contents,
             request_context=request_context,
         )
+        # One result slot per input content is preserved.
+        assert len(result) == 2
+
+        # The items folded into exactly one document.
+        docs = await memory.list_documents(bank_id=bank_id, request_context=request_context)
+        assert docs["total"] == 1
+        assert docs["items"][0]["id"] == "conversation_123"
+
+        # Both items' content lands in that one document's body, in order.
+        doc = await memory.get_document("conversation_123", bank_id, request_context=request_context)
+        body = doc["original_text"]
+        assert "Alice works at Google" in body
+        assert "Bob went hiking yesterday" in body
+        assert body.index("Alice works at Google") < body.index("Bob went hiking yesterday")
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_batch_level_document_id_folds_sync(memory, request_context):
+    """The deprecated batch-level document_id (applied to every item without its
+    own) folds those items into one document instead of tripping the guard."""
+    bank_id = f"test_batch_doc_id_sync_{uuid.uuid4().hex}"
+    try:
+        result = await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {"content": "Alice works at Google"},
+                {"content": "Bob loves Python"},
+            ],
+            document_id="meeting-2024-01-15",
+            request_context=request_context,
+        )
+        assert len(result) == 2
+
+        docs = await memory.list_documents(bank_id=bank_id, request_context=request_context)
+        assert docs["total"] == 1
+        assert docs["items"][0]["id"] == "meeting-2024-01-15"
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_shared_and_distinct_document_ids_sync(memory, request_context):
+    """A batch mixing a shared document_id with a distinct one folds only the
+    shared items, leaving the distinct document on its own."""
+    bank_id = f"test_mixed_doc_ids_sync_{uuid.uuid4().hex}"
+    try:
+        result = await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {"content": "Alice works at Google", "document_id": "docA"},
+                {"content": "Bob loves Python", "document_id": "docB"},
+                {"content": "Alice also mentors interns", "document_id": "docA"},
+            ],
+            request_context=request_context,
+        )
+        assert len(result) == 3
+
+        docs = await memory.list_documents(bank_id=bank_id, request_context=request_context)
+        assert docs["total"] == 2
+
+        doc_a = await memory.get_document("docA", bank_id, request_context=request_context)
+        assert "Alice works at Google" in doc_a["original_text"]
+        assert "Alice also mentors interns" in doc_a["original_text"]
+
+        doc_b = await memory.get_document("docB", bank_id, request_context=request_context)
+        assert "Bob loves Python" in doc_b["original_text"]
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_shared_document_id_folds_sync_large_batch(memory, request_context):
+    """A shared-document batch large enough to exceed the auto-split token
+    threshold still folds into one document with NO lost content. Splitting one
+    document across sub-batches would trip the streaming pipeline's content-hash
+    ownership check and drop later sub-batches, so shared groups take a single
+    pass — this guards that decision (issue #3363)."""
+    from hindsight_api.engine.memory_engine import count_tokens
+
+    bank_id = f"test_shared_doc_large_sync_{uuid.uuid4().hex}"
+    try:
+        # Two ~5.5k-token items sharing one document_id → ~11k tokens, over the
+        # 10k default split threshold. Distinct markers pin each item's presence.
+        filler = "The quick brown fox jumps over the lazy dog. " * 500
+        contents = [
+            {"content": f"MARKER_ALPHA at the start. {filler}", "document_id": "big_conversation"},
+            {"content": f"{filler} MARKER_OMEGA at the end.", "document_id": "big_conversation"},
+        ]
+        assert sum(count_tokens(c["content"]) for c in contents) > 10_000
+
+        result = await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=contents,
+            request_context=request_context,
+        )
+        assert len(result) == 2
+
+        docs = await memory.list_documents(bank_id=bank_id, request_context=request_context)
+        assert docs["total"] == 1
+
+        # Both items survive — neither sub-batch was dropped by a takeover abort.
+        doc = await memory.get_document("big_conversation", bank_id, request_context=request_context)
+        assert "MARKER_ALPHA" in doc["original_text"]
+        assert "MARKER_OMEGA" in doc["original_text"]
+
+        chunks = await memory.list_document_chunks(bank_id, "big_conversation", request_context=request_context)
+        assert chunks["total"] > 1
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
 
 
 @pytest.mark.asyncio

hindsight-api-slim/tests/test_bank_config_value_types.py

@@ -0,0 +1,160 @@
+"""Tests for type validation of bank config values (write side + read side).
+
+The bank-config API accepted `dict[str, Any]` updates without ever checking the
+values against the declared ``HindsightConfig`` field types, so a client could
+store a JSON object in a string-typed field such as ``observations_mission``.
+The write succeeded; the bank then failed *every* consolidation with
+``expected string or bytes-like object, got 'dict'`` raised by ``re.sub`` inside
+prompt assembly, deterministically, forever. See issue #3218.
+
+Two halves are covered here:
+  * write side — the value is rejected with a message naming field and type;
+  * read side  — a bank already carrying the bad value resolves to something
+    usable instead of wedging.
+"""
+
+import pytest
+
+from hindsight_api.config_resolver import (
+    _coerce_stored_bank_overrides,
+    _configurable_field_types,
+    _validate_config_value_types,
+)
+from hindsight_api.engine.consolidation.prompts import build_consolidation_input
+from hindsight_api.worker.exceptions import format_task_error
+
+
+# The fields found holding JSON objects in the field report on #3218.
+_STRING_FIELDS = (
+    "observations_mission",
+    "retain_mission",
+    "reflect_mission",
+    "retain_custom_instructions",
+)
+
+_BAD_MISSION = {"rules": ["only keep preferences"], "budget": 3}
+
+
+class TestValidateConfigValueTypes:
+    def test_every_configurable_field_has_a_type_contract(self):
+        from hindsight_api.config import HindsightConfig
+
+        # A field with no derivable contract would silently accept anything.
+        assert _configurable_field_types().keys() == HindsightConfig.get_configurable_fields()
+
+    def test_no_op_passes(self):
+        _validate_config_value_types({})
+        # Non-configurable keys are rejected by the caller, not here.
+        _validate_config_value_types({"unrelated_field": {"any": "shape"}})
+
+    def test_dict_in_string_field_raises(self):
+        for field in _STRING_FIELDS:
+            with pytest.raises(ValueError, match=field) as exc:
+                _validate_config_value_types({field: _BAD_MISSION})
+            # The message must name the expected and the actual type, so the 400
+            # tells the caller what to send instead.
+            assert "must be a string" in str(exc.value)
+            assert "got dict" in str(exc.value)
+
+    def test_valid_values_pass(self):
+        _validate_config_value_types(
+            {
+                "observations_mission": "Keep preferences and skills.",
+                "retain_chunk_size": 4000,
+                "enable_observations": True,
+                "mcp_enabled_tools": ["recall"],
+                "memory_defense": {"mode": "off"},
+                "recall_budget_adaptive_low": 0.05,
+            }
+        )
+
+    def test_none_clears_override(self):
+        for field in _STRING_FIELDS:
+            _validate_config_value_types({field: None})
+
+    def test_int_accepted_for_float_field(self):
+        # JSON draws no int/float distinction; a ratio of 1 must not 400.
+        _validate_config_value_types({"recall_budget_adaptive_high": 1})
+
+    def test_bool_rejected_for_numeric_field(self):
+        # bool is an int subclass and would sneak past a naive isinstance check.
+        with pytest.raises(ValueError, match="retain_chunk_size"):
+            _validate_config_value_types({"retain_chunk_size": True})
+
+    def test_string_rejected_for_numeric_and_bool_fields(self):
+        with pytest.raises(ValueError, match="retain_chunk_size"):
+            _validate_config_value_types({"retain_chunk_size": "4000"})
+        with pytest.raises(ValueError, match="enable_observations"):
+            _validate_config_value_types({"enable_observations": "true"})
+
+    def test_entity_labels_accepts_both_supported_shapes(self):
+        # Annotated `list | None`, but parse_entity_labels() also takes the
+        # {"attributes": [...]} envelope — the contract must not narrow that.
+        _validate_config_value_types({"entity_labels": []})
+        _validate_config_value_types({"entity_labels": {"attributes": []}})
+        with pytest.raises(ValueError, match="entity_labels"):
+            _validate_config_value_types({"entity_labels": "person"})
+
+
+class TestCoerceStoredBankOverrides:
+    def test_clean_overrides_pass_through_unchanged(self):
+        overrides = {"observations_mission": "Keep preferences.", "retain_chunk_size": 4000}
+        assert _coerce_stored_bank_overrides("bank1", overrides) == overrides
+
+    def test_dict_in_string_field_is_json_encoded(self):
+        coerced = _coerce_stored_bank_overrides("bank1", {"observations_mission": _BAD_MISSION})
+        mission = coerced["observations_mission"]
+        assert isinstance(mission, str)
+        # Semantics preserved: the structure still reaches the prompt, as text.
+        assert "only keep preferences" in mission
+
+    def test_non_coercible_override_is_dropped(self):
+        # An int field cannot be salvaged; the bank must fall back to the
+        # server default rather than resolve to a value nothing can use.
+        coerced = _coerce_stored_bank_overrides("bank1", {"retain_chunk_size": {"size": 4000}})
+        assert "retain_chunk_size" not in coerced
+
+    def test_coercion_warns_with_the_bank_and_field(self, caplog):
+        with caplog.at_level("WARNING"):
+            _coerce_stored_bank_overrides("bank1", {"observations_mission": _BAD_MISSION})
+        assert "bank1" in caplog.text
+        assert "observations_mission" in caplog.text
+
+    def test_strategy_overrides_are_coerced_too(self):
+        # apply_strategy() splices these onto the resolved config, so a nested
+        # bad value wedges the bank exactly as a top-level one does.
+        coerced = _coerce_stored_bank_overrides(
+            "bank1",
+            {"retain_strategies": {"chat": {"retain_mission": _BAD_MISSION, "retain_chunk_size": 4000}}},
+        )
+        chat = coerced["retain_strategies"]["chat"]
+        assert isinstance(chat["retain_mission"], str)
+        assert chat["retain_chunk_size"] == 4000
+
+    def test_strategy_null_override_is_preserved(self):
+        # Inside a strategy, null is a deliberate override to None (unlike a
+        # top-level null, which is the "use the server default" tombstone).
+        coerced = _coerce_stored_bank_overrides(
+            "bank1", {"retain_strategies": {"chat": {"retain_structured_chunk_size": None}}}
+        )
+        assert coerced["retain_strategies"]["chat"] == {"retain_structured_chunk_size": None}
+
+    def test_coerced_mission_survives_prompt_assembly(self):
+        """The exact reported failure: a dict mission reaching prompt assembly."""
+        with pytest.raises(TypeError, match="expected string or bytes-like object"):
+            build_consolidation_input("facts", "observations", observations_mission=_BAD_MISSION)
+
+        coerced = _coerce_stored_bank_overrides("bank1", {"observations_mission": _BAD_MISSION})
+        prompt = build_consolidation_input(
+            "facts", "observations", observations_mission=coerced["observations_mission"]
+        )
+        assert "only keep preferences" in prompt
+
+
+class TestFormatTaskError:
+    def test_message_is_prefixed_with_the_exception_class(self):
+        assert format_task_error(ValueError("boom")) == "ValueError: boom"
+
+    def test_empty_message_still_identifies_the_exception(self):
+        # The reported `Task execution failed: graph_maintenance, error: ` case.
+        assert format_task_error(TimeoutError()) == "TimeoutError"

hindsight-api-slim/tests/test_bank_template_full_roundtrip.py

@@ -0,0 +1,222 @@
+"""Every exportable bank field survives a full export -> import round-trip.
+
+The existing coverage leaves a gap in the middle:
+
+* ``test_bank_template_configurable_fields`` sets fields **one at a time** and
+  never exports — it proves import writes a field, not that export reads it back.
+* ``TestExport::test_export_reimport_roundtrip`` does export then import, but
+  with a single config field, and it asserts only the response flags
+  (``config_applied is True``) — never that any value survived.
+
+So a field that import accepts but export drops (or that export reshapes into
+something import rejects) passes both. This module closes that: it sets *every*
+field ``BankTemplateConfig`` declares on one bank, exports it, imports the
+exported manifest into a fresh bank, and asserts the second bank's overrides
+match the first's.
+
+``BankTemplateConfig.model_fields`` is the exportable surface — the export
+endpoint filters bank overrides through exactly that set — so the sample table
+below is checked against it. A newly added template field fails
+``test_sample_values_cover_every_exportable_field`` until it gets a value here,
+which is the point: the round-trip must not silently stop covering it.
+
+Adding a per-bank config field is a multi-step flow, and each step here fails
+until the previous one is done — so a half-wired field cannot land quietly:
+
+1. add it to ``_CONFIGURABLE_FIELDS`` → ``test_every_configurable_field_is_exportable``
+   fails until it is declared on ``BankTemplateConfig``;
+2. declare it there → ``test_sample_values_cover_every_exportable_field`` fails
+   until it has a value in ``_SAMPLE_VALUES``;
+3. give it a value → the round-trip below actually exercises it end to end;
+4. changing ``BankTemplateConfig`` also moves the OpenAPI spec, the generated
+   clients and ``bank-template-schema.json``, so CI's ``verify-generated-files``
+   fails until those are regenerated.
+
+(``test_bank_config_value_types`` adds a fifth: every configurable field must
+have a derivable type contract.) See #3218 for what a half-wired config field
+costs in production.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+import httpx
+import pytest
+import pytest_asyncio
+
+from hindsight_api.api import create_app
+from hindsight_api.api.http import BankTemplateConfig
+from hindsight_api.config import HindsightConfig
+
+
+# One value per BankTemplateConfig field, each chosen to differ visibly from the
+# server default so a value that silently reverts is caught rather than matching
+# by luck. Cross-field constraints enforced by validate_bank_template() and the
+# config validators are respected:
+#   * retain_custom_instructions requires retain_extraction_mode == "custom"
+#   * retain_default_strategy names a key in retain_strategies
+#   * recall_budget_min <= recall_budget_max
+_SAMPLE_VALUES: dict[str, Any] = {
+    "reflect_mission": "Answer as a careful archivist.",
+    "retain_mission": "Keep only decisions and the reasoning behind them.",
+    "retain_extraction_mode": "custom",
+    "retain_custom_instructions": "Extract one fact per decision, dated.",
+    "retain_chunk_size": 2500,
+    "retain_structured_chunk_size": 1800,
+    "enable_observations": False,
+    "observations_mission": "Observations cover preferences and skills only.",
+    "enable_temporal_retrieval": False,
+    "enable_graph_retrieval": False,
+    "enable_reranking": False,
+    "disposition_skepticism": 4,
+    "disposition_literalism": 2,
+    "disposition_empathy": 5,
+    "entity_labels": [
+        {
+            "key": "team",
+            "description": "Owning team",
+            "type": "value",
+            "optional": False,
+            "tag": True,
+            "values": [
+                {"value": "platform", "description": "Platform team"},
+                {"value": "growth", "description": "Growth team"},
+            ],
+        }
+    ],
+    "entities_allow_free_form": False,
+    "retain_default_strategy": "meetings",
+    "retain_strategies": {"meetings": {"retain_chunk_size": 1200, "retain_extraction_mode": "verbose"}},
+    "retain_chunk_batch_size": 7,
+    "mcp_enabled_tools": ["recall", "retain"],
+    "consolidation_llm_batch_size": 11,
+    "consolidation_source_facts_max_tokens": 2048,
+    "consolidation_source_facts_max_tokens_per_observation": 256,
+    "max_observations_per_scope": 13,
+    "observation_scope_limits": [{"scope": ["run_*"], "limit": 2}],
+    "reflect_source_facts_max_tokens": 4096,
+    "llm_gemini_safety_settings": [{"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"}],
+    "recall_budget_function": "adaptive",
+    "recall_budget_fixed_low": 50,
+    "recall_budget_fixed_mid": 250,
+    "recall_budget_fixed_high": 800,
+    "recall_budget_adaptive_low": 0.05,
+    "recall_budget_adaptive_mid": 0.1,
+    "recall_budget_adaptive_high": 0.4,
+    "recall_budget_min": 30,
+    "recall_budget_max": 1500,
+    "audit_log_enabled": True,
+    "store_document_text": False,
+    "enable_auto_consolidation": False,
+    "consolidation_max_memories_per_round": 42,
+    "consolidation_llm_parallelism": 3,
+    "recall_include_chunks": True,
+    "recall_max_tokens": 9000,
+    "recall_chunks_max_tokens": 4500,
+    # Validated against the DefensePolicy schema on write (parse_policy), so this
+    # must be a real policy — and carrying a rule means the round-trip covers the
+    # nested list, not just the top-level flag.
+    "memory_defense": {"enabled": True, "rules": [{"on": "sensitive_data", "action": "redact"}]},
+}
+
+
+@pytest_asyncio.fixture
+async def api_client(memory):
+    """In-process ASGI client — matches the fixture in tests/test_bank_templates.py."""
+    app = create_app(memory, initialize_memory=False)
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
+        yield client
+
+
+@pytest.fixture
+def bank_id():
+    return f"tmpl_roundtrip_{datetime.now().timestamp()}"
+
+
+async def _read_overrides(api_client: httpx.AsyncClient, bank_id: str) -> dict[str, Any]:
+    """Per-bank overrides only — the resolved config would hide a dropped field
+    behind the server default and make the round-trip look successful."""
+    resp = await api_client.get(f"/v1/default/banks/{bank_id}/config")
+    assert resp.status_code == 200, resp.text
+    return resp.json()["overrides"]
+
+
+def test_every_configurable_field_is_exportable():
+    """Every per-bank config field must be part of the template engine.
+
+    A field in ``_CONFIGURABLE_FIELDS`` but not on ``BankTemplateConfig`` is
+    settable per bank yet invisible to export/import: cloning a bank silently
+    drops it, and the clone runs on the server default while looking correctly
+    configured. The two sets must match exactly — an intentional exclusion is a
+    decision to record here, in an explicit set with a reason, not an omission.
+    """
+    configurable = HindsightConfig.get_configurable_fields()
+    exportable = set(BankTemplateConfig.model_fields)
+    assert configurable == exportable, (
+        f"bank config and the template engine have drifted:\n"
+        f"  configurable but not exportable (add to BankTemplateConfig): "
+        f"{sorted(configurable - exportable)}\n"
+        f"  exportable but not configurable (remove, or add to _CONFIGURABLE_FIELDS): "
+        f"{sorted(exportable - configurable)}"
+    )
+
+
+def test_sample_values_cover_every_exportable_field():
+    """A new BankTemplateConfig field must be given a value in _SAMPLE_VALUES.
+
+    Guards the round-trip test below from quietly narrowing: without this, adding
+    a template field leaves it untested and nothing says so.
+    """
+    declared = set(BankTemplateConfig.model_fields)
+    sampled = set(_SAMPLE_VALUES)
+    assert declared == sampled, (
+        f"_SAMPLE_VALUES is out of sync with BankTemplateConfig: "
+        f"missing values for {sorted(declared - sampled)}, "
+        f"stale entries for {sorted(sampled - declared)}"
+    )
+
+
+@pytest.mark.asyncio
+async def test_every_exportable_field_survives_export_then_import(api_client, bank_id):
+    """Set every exportable field, export it, and import into a fresh bank."""
+    source_id = f"{bank_id}_source"
+    clone_id = f"{bank_id}_clone"
+
+    resp = await api_client.post(
+        f"/v1/default/banks/{source_id}/import",
+        json={"version": "1", "bank": dict(_SAMPLE_VALUES)},
+    )
+    assert resp.status_code == 200, resp.text
+    assert resp.json()["config_applied"] is True
+
+    # The source bank must actually carry all of them before the round-trip can
+    # prove anything — a field the import path drops would otherwise show up as
+    # a clean "match" between two equally empty banks.
+    source_overrides = await _read_overrides(api_client, source_id)
+    missing_after_import = sorted(set(_SAMPLE_VALUES) - set(source_overrides))
+    assert not missing_after_import, f"import did not persist: {missing_after_import}"
+
+    export_resp = await api_client.get(f"/v1/default/banks/{source_id}/export")
+    assert export_resp.status_code == 200, export_resp.text
+    exported = export_resp.json()
+
+    exported_bank = exported.get("bank") or {}
+    dropped_by_export = sorted(f for f in _SAMPLE_VALUES if exported_bank.get(f) is None)
+    assert not dropped_by_export, f"export dropped fields that were set: {dropped_by_export}"
+
+    import_resp = await api_client.post(f"/v1/default/banks/{clone_id}/import", json=exported)
+    assert import_resp.status_code == 200, import_resp.text
+    assert import_resp.json()["config_applied"] is True
+
+    # Compare the two banks' overrides rather than the literal input: some fields
+    # are normalized on the way in (entity_labels migrates legacy shapes), and the
+    # property under test is that a clone ends up configured like its source.
+    clone_overrides = await _read_overrides(api_client, clone_id)
+    for field in sorted(_SAMPLE_VALUES):
+        assert clone_overrides.get(field) == source_overrides.get(field), (
+            f"round-trip mismatch for {field}: "
+            f"source has {source_overrides.get(field)!r}, clone has {clone_overrides.get(field)!r}"
+        )

hindsight-api-slim/tests/test_bank_vector_index_ddl_lock.py

@@ -0,0 +1,127 @@
+"""Per-bank vector-index DDL is serialized per table within a process.
+
+Concurrent index DDL on the shared ``memory_units`` table deadlocks by design:
+DROP INDEX CONCURRENTLY holds ShareUpdateExclusive while waiting out every
+other session whose snapshot could still see the index — including other
+sessions' queued index DDL. CI's end-of-run teardown (all xdist workers
+deleting their banks at once) forms exactly that cycle, and the delete path's
+default retry budget (~2.4s) could not outlast the storm (run 31195108586,
+test-api 3/3). Advisory locks are banned in this codebase (poolers), so the
+fix is an in-process asyncio lock on ``PostgreSQLOps`` plus a much larger
+jittered retry budget for the cross-process residue.
+
+Two layers are proven here:
+
+* unit (fake conn): create and drop DDL for one table never interleave, the
+  create and drop sides contend on the same lock, and the lock is released
+  when a statement raises;
+* integration: a many-bank concurrent ``delete_bank`` storm — the CI failure
+  shape — completes without ``DeadlockDetectedError``.
+"""
+
+import asyncio
+import uuid
+
+import pytest
+
+from hindsight_api.engine.db.ops_postgresql import PostgreSQLOps
+from hindsight_api.engine.retain.bank_utils import _BANK_INDEX_FACT_TYPES, _bank_index_name, _vector_index_clause
+
+_SCHEMA = "public"
+_INDEX_CLAUSE = "USING hnsw (embedding vector_cosine_ops)"
+
+
+class _OverlapTrackingConn:
+    """Fake DatabaseConnection asserting no two DDL statements ever overlap.
+
+    Each execute parks on the event loop long enough that unserialized callers
+    would interleave, and records the highest number of in-flight statements.
+    """
+
+    def __init__(self, fail_on: str | None = None):
+        self.calls: list[str] = []
+        self.max_in_flight = 0
+        self._in_flight = 0
+        self._fail_on = fail_on
+
+    async def execute(self, query, *args, **kwargs):
+        self.calls.append(query)
+        self._in_flight += 1
+        self.max_in_flight = max(self.max_in_flight, self._in_flight)
+        try:
+            await asyncio.sleep(0.01)
+            if self._fail_on and self._fail_on in query:
+                raise RuntimeError(f"boom on: {query}")
+        finally:
+            self._in_flight -= 1
+        return "OK"
+
+
+async def test_concurrent_create_and_drop_on_one_table_serialize():
+    ops = PostgreSQLOps()
+    conn = _OverlapTrackingConn()
+    table = f"{_SCHEMA}.memory_units"
+
+    await asyncio.gather(
+        ops.drop_bank_vector_indexes(conn, _SCHEMA, uuid.uuid4().hex, _BANK_INDEX_FACT_TYPES),
+        ops.drop_bank_vector_indexes(conn, _SCHEMA, uuid.uuid4().hex, _BANK_INDEX_FACT_TYPES),
+        # The drop side reconstructs the create side's fq-table key from
+        # `schema`, so a create must queue behind the drops too.
+        ops.create_bank_vector_indexes(conn, table, "bank-1", uuid.uuid4().hex, _INDEX_CLAUSE, _BANK_INDEX_FACT_TYPES),
+    )
+
+    assert len(conn.calls) == 3 * len(_BANK_INDEX_FACT_TYPES)
+    assert conn.max_in_flight == 1, "vector-index DDL statements overlapped despite the per-table lock"
+
+
+async def test_different_tables_do_not_contend():
+    ops = PostgreSQLOps()
+    assert ops._index_ddl_lock("a.memory_units") is ops._index_ddl_lock("a.memory_units")
+    assert ops._index_ddl_lock("a.memory_units") is not ops._index_ddl_lock("b.memory_units")
+
+
+async def test_lock_released_when_ddl_raises():
+    ops = PostgreSQLOps()
+    conn = _OverlapTrackingConn(fail_on="DROP INDEX CONCURRENTLY")
+    with pytest.raises(RuntimeError):
+        await ops.drop_bank_vector_indexes(conn, _SCHEMA, uuid.uuid4().hex, _BANK_INDEX_FACT_TYPES)
+
+    # A subsequent create must not hang on a lock the failed drop never released.
+    await asyncio.wait_for(
+        ops.create_bank_vector_indexes(
+            conn, f"{_SCHEMA}.memory_units", "bank-1", uuid.uuid4().hex, _INDEX_CLAUSE, _BANK_INDEX_FACT_TYPES
+        ),
+        timeout=2.0,
+    )
+
+
+async def test_concurrent_bank_delete_storm_does_not_deadlock(memory, request_context):
+    """The CI failure shape: every worker tears down its banks at once.
+
+    Eight banks (24 partial indexes) dropped concurrently previously wedged
+    into a DROP INDEX CONCURRENTLY wait cycle; serialized behind the per-table
+    lock the storm must complete without DeadlockDetectedError.
+    """
+    if _vector_index_clause() is None:
+        pytest.skip("backend does not use per-bank vector indexes")
+
+    bank_ids = [f"test-ddl-lock-{uuid.uuid4().hex[:8]}" for _ in range(8)]
+    for bank_id in bank_ids:
+        await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
+
+    backend = await memory._get_backend()
+    async with backend.acquire() as conn:
+        internal_ids = {
+            bank_id: str(await conn.fetchval("SELECT internal_id FROM banks WHERE bank_id = $1", bank_id))
+            for bank_id in bank_ids
+        }
+
+    await asyncio.gather(*(memory.delete_bank(bank_id, request_context=request_context) for bank_id in bank_ids))
+
+    async with backend.acquire() as conn:
+        for bank_id in bank_ids:
+            for ft in _BANK_INDEX_FACT_TYPES:
+                idx = _bank_index_name(ft, internal_ids[bank_id])
+                assert not await conn.fetchval(
+                    "SELECT 1 FROM pg_indexes WHERE schemaname = $1 AND indexname = $2", _SCHEMA, idx
+                ), f"index {idx} survived delete_bank"

hindsight-api-slim/tests/test_batch_chunking.py

@@ -1,6 +1,7 @@
 """Test automatic batch chunking based on character count."""
 
 import asyncio
+import json
 import os
 
 import pytest
@@ -11,6 +12,7 @@ from hindsight_api.engine.memory_engine import (
     _split_contents_into_sub_batches,
     count_tokens,
 )
+from hindsight_api.engine.retain.fact_extraction import chunk_text
 
 # ---------------------------------------------------------------------------
 # Regression tests for issue #1571: the splitter must actually chunk an
@@ -31,6 +33,7 @@ def test_split_single_oversized_item_produces_multiple_sub_batches():
     split = _split_contents_into_sub_batches(
         [{"content": big_content, "document_id": "doc-oversize"}],
         tokens_per_batch,
+        chunk_size=3000,
     )
 
     assert len(split.sub_batches) > 1, (
@@ -59,7 +62,7 @@ def test_split_oversized_item_preserves_document_id_and_metadata():
         "tags": ["t1", "t2"],
     }
 
-    split = _split_contents_into_sub_batches([item], tokens_per_batch)
+    split = _split_contents_into_sub_batches([item], tokens_per_batch, chunk_size=3000)
 
     assert len(split.sub_batches) > 1
     for batch in split.sub_batches:
@@ -85,7 +88,7 @@ def test_split_mixed_batch_chunks_only_oversized_items():
         {"content": small_b, "document_id": "doc-c"},
     ]
 
-    split = _split_contents_into_sub_batches(contents, tokens_per_batch)
+    split = _split_contents_into_sub_batches(contents, tokens_per_batch, chunk_size=3000)
 
     # We expect: [small_a packed] then N chunks of big, then [small_b packed].
     # At minimum: > 2 sub-batches (a + multiple big chunks + c).
@@ -104,6 +107,110 @@ def test_split_mixed_batch_chunks_only_oversized_items():
     assert big_origin_count > small_a_origin_count
 
 
+def _conversation_payload(turns: int = 120) -> str:
+    return json.dumps(
+        [{"role": "user" if i % 2 == 0 else "assistant", "content": f"turn {i} " + "blah " * 60} for i in range(turns)]
+    )
+
+
+def _jsonl_payload(lines: int = 200) -> str:
+    return "\n".join(json.dumps({"i": i, "text": "line " + "x " * 80}) for i in range(lines))
+
+
+@pytest.mark.parametrize(
+    "body",
+    [
+        pytest.param(
+            "\n\n".join(f"Section {i}. " + f"marker{i} filler word here. " * 117 for i in range(10)), id="prose"
+        ),
+        pytest.param(_conversation_payload(), id="json-conversation"),
+        pytest.param(_jsonl_payload(), id="jsonl"),
+    ],
+)
+def test_split_slices_are_whole_native_chunks(body):
+    """Every slice holds a whole number of native chunks and re-chunks back to
+    exactly those chunks (issue #3282).
+
+    This is the invariant the rest of the retain path is built on: the chunks
+    stored for a document must depend only on its body, never on how transport
+    split it. Delta retain and the streaming recovery pass both match stored
+    chunks by content hash, so a slice cutting mid-chunk silently re-extracts
+    unchanged history; ``chunk_index`` bookkeeping assumes whole chunks too.
+    """
+    chunk_size = 3000
+    native_chunks = chunk_text(body, chunk_size)
+    assert len(native_chunks) > 3, "test payload must span several native chunks"
+
+    split = _split_contents_into_sub_batches(
+        [{"content": body, "document_id": "doc-align"}],
+        tokens_per_batch=1_500,
+        chunk_size=chunk_size,
+    )
+
+    assert len(split.sub_batches) > 1, "payload should have been sliced"
+    rechunked: list[str] = []
+    for sub_batch, count in zip(split.sub_batches, split.chunk_counts):
+        slice_chunks = chunk_text(sub_batch[0]["content"], chunk_size)
+        assert len(slice_chunks) == count, "chunk_counts must match what the slice re-chunks to"
+        rechunked.extend(slice_chunks)
+
+    assert rechunked == native_chunks, "slices did not reproduce the document's native chunks"
+
+
+def test_split_falls_back_to_one_chunk_per_slice_when_no_faithful_join_exists(monkeypatch):
+    """When a run of chunks cannot be rejoined faithfully, each chunk ships alone.
+
+    Packing several chunks into one slice is only safe while the joined text
+    re-chunks back to exactly those chunks. If no candidate join does (a
+    content shape whose chunker is not reconstructible from its own output),
+    the splitter must degrade to one chunk per sub-batch — trivially aligned by
+    ``chunk_text``'s idempotency — rather than ship a slice whose boundaries
+    disagree with what gets stored (issue #3282).
+    """
+    from hindsight_api.engine.retain import fact_extraction
+
+    # Over the budget so it takes the oversized-item path; the stubbed chunks
+    # are small enough that the packer wants all three in a single slice.
+    body = "unjoinable body. " * 500
+    chunks = ["chunk one", "chunk two", "chunk three"]
+
+    def _fake_chunk_text(text, max_chars, structured_chunk_size=None):
+        # Only the original body chunks into `chunks`; every rejoin attempt
+        # comes back as something else, so verification always fails.
+        return list(chunks) if text == body else ["something else entirely"]
+
+    monkeypatch.setattr(fact_extraction, "chunk_text", _fake_chunk_text)
+
+    split = _split_contents_into_sub_batches(
+        [{"content": body, "document_id": "doc-unjoinable"}],
+        tokens_per_batch=100,
+        chunk_size=3000,
+    )
+
+    assert [b[0]["content"] for b in split.sub_batches] == chunks
+    assert split.chunk_counts == [1, 1, 1]
+
+
+def test_split_slice_never_cuts_a_native_chunk_under_a_tiny_budget():
+    """A budget below one native chunk cannot shrink the slice past that chunk.
+
+    ``retain_chunk_size`` is the real bound on a slice; honouring a smaller
+    token budget would mean cutting mid-chunk, which is exactly what breaks
+    hash-based chunk reuse (issue #3282).
+    """
+    body = "The quick brown fox jumps over the lazy dog. " * 1_000
+    native_chunks = chunk_text(body, 3000)
+
+    split = _split_contents_into_sub_batches(
+        [{"content": body, "document_id": "doc-tiny-budget"}],
+        tokens_per_batch=10,
+        chunk_size=3000,
+    )
+
+    assert [b[0]["content"] for b in split.sub_batches] == native_chunks
+    assert split.chunk_counts == [1] * len(native_chunks)
+
+
 def test_split_small_batch_returns_single_sub_batch():
     """A batch under the budget stays as a single sub-batch."""
     tokens_per_batch = 10_000
@@ -112,7 +219,7 @@ def test_split_small_batch_returns_single_sub_batch():
         {"content": "Bob loves Python", "document_id": "doc-2"},
     ]
 
-    split = _split_contents_into_sub_batches(contents, tokens_per_batch)
+    split = _split_contents_into_sub_batches(contents, tokens_per_batch, chunk_size=3000)
 
     assert len(split.sub_batches) == 1
     assert split.sub_batches[0] == contents

hindsight-api-slim/tests/test_cache_affinity.py

@@ -0,0 +1,630 @@
+"""Tests for backend prompt-cache affinity on the OpenAI-compatible provider family.
+
+Covers the `cache_affinity` knob end to end: mode parsing and `auto` host
+resolution, the affinity id (operation trace identity, first-message hash
+fallback), injection into both the plain and tool-calling call paths with
+user-wins semantics, and the config plumbing that carries the setting from env
+to the wire.
+
+The decisive test here is `test_end_to_end_*`: direct-construction tests cannot
+catch a break between config and the provider, which is exactly how a first
+version of this feature compiled, tested green, and sent nothing on the reflect
+lane.
+
+All deterministic — no network, mocked clients only.
+"""
+
+import re
+from contextlib import contextmanager
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from hindsight_api.engine.cache_affinity import (
+    OPENAI_PROMPT_CACHE_KEY_PARAM,
+    XAI_CONV_ID_HEADER,
+    CacheAffinityMode,
+    apply_cache_affinity,
+    cache_affinity_id,
+    parse_cache_affinity,
+    resolve_cache_affinity,
+)
+from hindsight_api.engine.llm_trace import LLMTraceContext, reset_trace_context, set_trace_context
+from hindsight_api.engine.llm_wrapper import LLMProvider, create_llm_provider
+from hindsight_api.engine.providers.nous_auth import NousAuthManager
+from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
+
+_HEX32 = re.compile(r"\A[0-9a-f]{32}\Z")
+
+
+# ── helpers ───────────────────────────────────────────────────────────────────
+
+
+def _llm(cache_affinity: str | None = None, **kwargs) -> OpenAICompatibleLLM:
+    kwargs.setdefault("base_url", "https://example.test/v1")
+    kwargs.setdefault("provider", "openai")
+    kwargs.setdefault("api_key", "test-key")
+    kwargs.setdefault("model", "gpt-4o-mini")
+    return OpenAICompatibleLLM(cache_affinity=cache_affinity, **kwargs)
+
+
+class _FakeNousAuth:
+    """Minimal `NousAuthManager` stand-in: a fresh token, refresh never needed.
+
+    Mirrors the fake in `test_nous_provider.py` but trimmed to what construction
+    and a no-401 `call()` touch — `access_token`, `base_url`, `_token_is_stale()`.
+    """
+
+    def __init__(self, token: str = "tok-1", base_url: str = "https://inference-api.nousresearch.com/v1"):
+        self.access_token = token
+        self.base_url = base_url
+
+    def _token_is_stale(self) -> bool:
+        return False
+
+
+def _nous_llm(cache_affinity: str | None = None, **kwargs):
+    """Construct a NousLLM through the real `create_llm_provider` factory — the
+    exact path that silently dropped `cache_affinity`/`default_headers` — with
+    Nous's ~/.hermes/auth.json read stubbed out so the test needs no real login."""
+    kwargs.setdefault("base_url", "https://inference-api.nousresearch.com/v1")
+    kwargs.setdefault("api_key", "ignored")
+    kwargs.setdefault("model", "deepseek/deepseek-v4-flash")
+    kwargs.setdefault("reasoning_effort", "low")
+    with patch.object(NousAuthManager, "from_file", return_value=_FakeNousAuth()):
+        return create_llm_provider(provider="nous", cache_affinity=cache_affinity, **kwargs)
+
+
+def _chat_response(content: str = "hello"):
+    choice = SimpleNamespace(
+        finish_reason="stop",
+        message=SimpleNamespace(content=content, tool_calls=None, refusal=None, reasoning_content=None),
+    )
+    return SimpleNamespace(choices=[choice], usage=None, error=None)
+
+
+async def _call(llm: OpenAICompatibleLLM, create: AsyncMock, **kwargs):
+    """Drive `call()` against a mocked client, returning the mock for assertions."""
+    llm._client.chat.completions.create = create
+    with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector"):
+        await llm.call(messages=[{"role": "user", "content": "ping"}], max_retries=0, **kwargs)
+    return create
+
+
+async def _call_with_tools(llm: OpenAICompatibleLLM, create: AsyncMock, **kwargs):
+    """Drive `call_with_tools()` against a mocked client."""
+    llm._client.chat.completions.create = create
+    with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector"):
+        await llm.call_with_tools(
+            messages=[{"role": "user", "content": "ping"}],
+            tools=[{"type": "function", "function": {"name": "noop", "parameters": {}}}],
+            max_retries=0,
+            **kwargs,
+        )
+    return create
+
+
+@contextmanager
+def _bound_trace(trace_id: str):
+    """Bind an operation trace context, as `ConfiguredLLMProvider` does per call."""
+    token = set_trace_context(LLMTraceContext(bank_id="bank-1", operation="reflect", trace_id=trace_id))
+    try:
+        yield
+    finally:
+        reset_trace_context(token)
+
+
+# ── AC1: xai_conv_id header ───────────────────────────────────────────────────
+
+
+async def test_xai_header_is_32_lowercase_hex():
+    llm = _llm("xai_conv_id")
+    create = await _call(llm, AsyncMock(return_value=_chat_response()))
+    header = create.call_args.kwargs["extra_headers"][XAI_CONV_ID_HEADER]
+    assert _HEX32.match(header)
+
+
+async def test_xai_header_is_stable_within_one_trace_context():
+    """Every LLM call of one reflect/retain run must pin to the same backend."""
+    llm = _llm("xai_conv_id")
+    create = AsyncMock(return_value=_chat_response())
+    with _bound_trace("11111111-1111-1111-1111-111111111111"):
+        await _call(llm, create)
+        first = create.call_args.kwargs["extra_headers"][XAI_CONV_ID_HEADER]
+        # Different message content: the id comes from operation identity, not payload.
+        llm._client.chat.completions.create = create
+        with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector"):
+            await llm.call(messages=[{"role": "user", "content": "a different prompt"}], max_retries=0)
+        second = create.call_args.kwargs["extra_headers"][XAI_CONV_ID_HEADER]
+    assert first == second
+
+
+async def test_xai_header_differs_across_trace_contexts():
+    llm = _llm("xai_conv_id")
+    create = AsyncMock(return_value=_chat_response())
+    with _bound_trace("11111111-1111-1111-1111-111111111111"):
+        await _call(llm, create)
+        first = create.call_args.kwargs["extra_headers"][XAI_CONV_ID_HEADER]
+    with _bound_trace("22222222-2222-2222-2222-222222222222"):
+        await _call(llm, create)
+        second = create.call_args.kwargs["extra_headers"][XAI_CONV_ID_HEADER]
+    assert first != second
+
+
+# ── AC2: openai_prompt_cache_key ──────────────────────────────────────────────
+
+
+async def test_openai_prompt_cache_key_is_sent():
+    """`prompt_cache_key` is a first-class named param on chat.completions.create,
+    so it is sent at the top level rather than through extra_body."""
+    llm = _llm("openai_prompt_cache_key")
+    create = await _call(llm, AsyncMock(return_value=_chat_response()))
+    assert _HEX32.match(create.call_args.kwargs[OPENAI_PROMPT_CACHE_KEY_PARAM])
+
+
+async def test_openai_prompt_cache_key_coexists_with_extra_body():
+    """The provider's own extra_body defaults and the operator's configured
+    extra_body must both survive — the affinity hint merges, never replaces."""
+    llm = _llm(
+        "openai_prompt_cache_key",
+        provider="minimax",
+        base_url="https://api.minimax.io/v1",
+        model="MiniMax-M3",
+        extra_body={"top_p": 0.9},
+    )
+    create = await _call(llm, AsyncMock(return_value=_chat_response()))
+    kwargs = create.call_args.kwargs
+    assert _HEX32.match(kwargs[OPENAI_PROMPT_CACHE_KEY_PARAM])
+    assert kwargs["extra_body"]["thinking"] == {"type": "disabled"}  # minimax default
+    assert kwargs["extra_body"]["top_p"] == 0.9  # operator config
+
+
+# ── AC3: default / none is byte-compatible with a pre-affinity request ────────
+
+
+@pytest.mark.parametrize("mode", [None, "none"])
+async def test_no_affinity_key_by_default(mode):
+    llm = _llm(mode)
+    create = await _call(llm, AsyncMock(return_value=_chat_response()))
+    kwargs = create.call_args.kwargs
+    assert "extra_headers" not in kwargs
+    assert OPENAI_PROMPT_CACHE_KEY_PARAM not in kwargs
+    assert XAI_CONV_ID_HEADER not in str(kwargs)
+
+
+def test_invalid_mode_raises():
+    """The setting has no visible effect in the response, so a typo that silently
+    disabled it would be indistinguishable from it working."""
+    with pytest.raises(ValueError, match="Invalid cache_affinity"):
+        _llm("xai-conv-id")
+
+
+# ── AC4: auto resolution table ────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    ("provider", "base_url", "expected"),
+    [
+        ("openai", "https://api.x.ai/v1", CacheAffinityMode.XAI_CONV_ID),
+        ("openai", "https://cli-chat-proxy.grok.com/v1", CacheAffinityMode.XAI_CONV_ID),
+        ("openai", None, CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY),
+        ("openai", "", CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY),
+        ("openai", "https://api.openai.com/v1", CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY),
+        ("openai", "https://my-res.openai.azure.com/openai/deployments/x", CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY),
+        ("fireworks", "https://api.fireworks.ai/inference/v1", CacheAffinityMode.NONE),
+        ("fireworks", "", CacheAffinityMode.NONE),
+        ("openai", "https://llm.internal.example/v1", CacheAffinityMode.NONE),
+    ],
+)
+def test_auto_resolution_table(provider, base_url, expected):
+    assert resolve_cache_affinity(CacheAffinityMode.AUTO, provider, base_url) is expected
+
+
+@pytest.mark.parametrize("base_url", ["https://api.vertex.ai/v1", "https://x.ai.evil.example/v1"])
+def test_auto_does_not_substring_match_xai(base_url):
+    """`"x.ai" in base_url` would false-match both of these; the host is parsed."""
+    assert resolve_cache_affinity(CacheAffinityMode.AUTO, "openai", base_url) is CacheAffinityMode.NONE
+
+
+def test_auto_resolves_once_at_construction():
+    llm = _llm("auto", base_url="https://api.x.ai/v1")
+    assert llm._cache_affinity is CacheAffinityMode.XAI_CONV_ID
+
+
+@pytest.mark.parametrize("mode", ["none", "xai_conv_id", "openai_prompt_cache_key"])
+def test_explicit_modes_are_not_re_resolved(mode):
+    parsed = parse_cache_affinity(mode)
+    assert resolve_cache_affinity(parsed, "openai", "https://api.x.ai/v1") is parsed
+
+
+# ── AC5: no-context fallback (first-message hash) ─────────────────────────────
+
+
+def test_fallback_hashes_the_first_message():
+    messages = [{"role": "system", "content": "you are helpful"}, {"role": "user", "content": "hi"}]
+    assert _HEX32.match(cache_affinity_id(messages))
+
+
+def test_fallback_is_stable_as_the_message_list_grows():
+    """The agent loop appends turns; the pin must not rotate mid-conversation."""
+    base = [{"role": "system", "content": "you are helpful"}]
+    grown = [*base, {"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]
+    assert cache_affinity_id(base) == cache_affinity_id(grown)
+
+
+def test_fallback_differs_for_a_different_first_message():
+    a = cache_affinity_id([{"role": "system", "content": "prompt A"}])
+    b = cache_affinity_id([{"role": "system", "content": "prompt B"}])
+    assert a != b
+
+
+@pytest.mark.parametrize("messages", [None, [], "bogus", ["not a dict"], [42]])
+def test_fallback_returns_none_for_malformed_messages(messages):
+    """A bare string would index to its first character and mint an id from
+    garbage; anything unexpected sends no hint at all."""
+    assert cache_affinity_id(messages) is None
+
+
+async def test_no_header_when_messages_are_malformed():
+    """Fail-open all the way to the wire: no id means no header, not an error."""
+    request = {"messages": "bogus"}
+    apply_cache_affinity(request, CacheAffinityMode.XAI_CONV_ID)
+    assert request == {"messages": "bogus"}
+
+
+async def test_call_uses_the_fallback_hash_without_a_trace_context():
+    llm = _llm("xai_conv_id")
+    create = await _call(llm, AsyncMock(return_value=_chat_response()))
+    sent = create.call_args.kwargs["extra_headers"][XAI_CONV_ID_HEADER]
+    assert sent == cache_affinity_id(create.call_args.kwargs["messages"])
+
+
+# ── AC6: call_with_tools parity ───────────────────────────────────────────────
+
+
+async def test_tools_path_sends_xai_header():
+    llm = _llm("xai_conv_id")
+    create = await _call_with_tools(llm, AsyncMock(return_value=_chat_response("done")))
+    assert _HEX32.match(create.call_args.kwargs["extra_headers"][XAI_CONV_ID_HEADER])
+
+
+async def test_tools_path_sends_prompt_cache_key():
+    llm = _llm("openai_prompt_cache_key")
+    create = await _call_with_tools(llm, AsyncMock(return_value=_chat_response("done")))
+    assert _HEX32.match(create.call_args.kwargs[OPENAI_PROMPT_CACHE_KEY_PARAM])
+
+
+async def test_tools_path_sends_nothing_by_default():
+    llm = _llm()
+    create = await _call_with_tools(llm, AsyncMock(return_value=_chat_response("done")))
+    kwargs = create.call_args.kwargs
+    assert "extra_headers" not in kwargs
+    assert OPENAI_PROMPT_CACHE_KEY_PARAM not in kwargs
+
+
+# ── AC13: fallback hash on the tools path ─────────────────────────────────────
+
+
+async def test_tools_path_fallback_hash_is_stable():
+    """`call_with_tools` builds its message list differently; its fallback id must
+    still derive from the first message and survive a growing conversation."""
+    llm = _llm("xai_conv_id")
+    create = AsyncMock(return_value=_chat_response("done"))
+    llm._client.chat.completions.create = create
+    tools = [{"type": "function", "function": {"name": "noop", "parameters": {}}}]
+    system = {"role": "system", "content": "you are helpful"}
+    with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector"):
+        await llm.call_with_tools(messages=[system], tools=tools, max_retries=0)
+        first = create.call_args.kwargs["extra_headers"][XAI_CONV_ID_HEADER]
+        await llm.call_with_tools(
+            messages=[system, {"role": "user", "content": "and now?"}], tools=tools, max_retries=0
+        )
+        second = create.call_args.kwargs["extra_headers"][XAI_CONV_ID_HEADER]
+    assert _HEX32.match(first)
+    assert first == second
+
+
+# ── AC7: adjacent fix — default_headers reaches the SDK client ────────────────
+
+
+def test_default_headers_reach_the_openai_compatible_client():
+    """`HINDSIGHT_API_LLM_DEFAULT_HEADERS` used to no-op for all 15 providers on
+    this branch: the factory accepted the field but never forwarded it."""
+    llm = create_llm_provider(
+        provider="openai",
+        api_key="test-key",
+        base_url="https://example.test/v1",
+        model="gpt-4o-mini",
+        reasoning_effort="low",
+        default_headers={"x-test": "1"},
+    )
+    assert llm._client.default_headers["x-test"] == "1"
+
+
+def test_default_headers_reach_the_fireworks_client():
+    """Fireworks subclasses OpenAICompatibleLLM and had the same gap."""
+    llm = create_llm_provider(
+        provider="fireworks",
+        api_key="test-key",
+        base_url="https://api.fireworks.ai/inference/v1",
+        model="accounts/fireworks/models/llama-v3p1-8b-instruct",
+        reasoning_effort="low",
+        default_headers={"x-test": "2"},
+    )
+    assert llm._client.default_headers["x-test"] == "2"
+
+
+def test_fireworks_branch_forwards_cache_affinity():
+    llm = create_llm_provider(
+        provider="fireworks",
+        api_key="test-key",
+        base_url="https://api.fireworks.ai/inference/v1",
+        model="accounts/fireworks/models/llama-v3p1-8b-instruct",
+        reasoning_effort="low",
+        cache_affinity="xai_conv_id",
+    )
+    assert llm._cache_affinity is CacheAffinityMode.XAI_CONV_ID
+
+
+def test_default_headers_reach_the_nous_client():
+    """NousLLM subclasses OpenAICompatibleLLM and had the same gap: the `nous`
+    factory branch forwarded neither `default_headers` nor `cache_affinity`,
+    even though `NousLLM.__init__` already passes both through **kwargs."""
+    llm = _nous_llm(default_headers={"x-test": "3"})
+    assert llm._client.default_headers["x-test"] == "3"
+
+
+async def test_nous_branch_forwards_cache_affinity():
+    """AC7-style factory-branch check, taken all the way to the wire (AC1 style)
+    rather than stopping at `_cache_affinity`, since the missing kwarg here is a
+    silent no-op — the same failure shape AC10's end-to-end test exists for."""
+    llm = _nous_llm(cache_affinity="xai_conv_id")
+    create = await _call(llm, AsyncMock(return_value=_chat_response()))
+    header = create.call_args.kwargs["extra_headers"][XAI_CONV_ID_HEADER]
+    assert _HEX32.match(header)
+
+
+# ── AC8: config plumbing (env round-trip) ─────────────────────────────────────
+
+
+@pytest.fixture
+def clean_llm_env(monkeypatch):
+    """Strip all HINDSIGHT_API_*LLM* env so each test sets only what it needs."""
+    import os
+
+    from hindsight_api.config import clear_config_cache
+
+    for key in list(os.environ):
+        if key.startswith("HINDSIGHT_API_") and "LLM" in key:
+            monkeypatch.delenv(key, raising=False)
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "openai")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_API_KEY", "sk-primary")
+    monkeypatch.setenv("HINDSIGHT_API_SKIP_LLM_VERIFICATION", "true")
+    clear_config_cache()
+    yield monkeypatch
+    clear_config_cache()
+
+
+def test_config_reads_global_and_per_operation_affinity(clean_llm_env):
+    from hindsight_api.config import HindsightConfig
+
+    clean_llm_env.setenv("HINDSIGHT_API_LLM_CACHE_AFFINITY", "auto")
+    clean_llm_env.setenv("HINDSIGHT_API_REFLECT_LLM_CACHE_AFFINITY", "xai_conv_id")
+    config = HindsightConfig.from_env()
+
+    assert config.llm_cache_affinity == "auto"
+    assert config.reflect_llm_cache_affinity == "xai_conv_id"
+    assert config.retain_llm_cache_affinity is None
+    assert config.consolidation_llm_cache_affinity is None
+
+
+def test_config_affinity_defaults_to_auto(clean_llm_env):
+    """Unset means "auto", not "off".
+
+    "auto" only emits a hint for hosts documented to accept one (see
+    test_auto_default_sends_nothing_to_an_unrecognized_backend), so defaulting
+    it on costs unknown backends nothing while every xAI/OpenAI deployment gets
+    the cache hit it was otherwise silently losing.
+    """
+    from hindsight_api.config import HindsightConfig
+
+    assert HindsightConfig.from_env().llm_cache_affinity == "auto"
+
+
+def test_llm_provider_from_env_defaults_to_auto(clean_llm_env):
+    """The two env entry points must agree; they resolved differently before."""
+    clean_llm_env.setenv("HINDSIGHT_API_LLM_PROVIDER", "openai")
+    clean_llm_env.setenv("HINDSIGHT_API_LLM_API_KEY", "sk-test")
+
+    assert LLMProvider.from_env().cache_affinity == "auto"
+
+
+@pytest.mark.parametrize(
+    ("provider", "base_url"),
+    [
+        ("openai", "https://my-proxy.internal/v1"),
+        ("openai", "http://localhost:8000/v1"),
+        ("ollama", "http://localhost:11434/v1"),
+        ("groq", "https://api.groq.com/openai/v1"),
+        ("deepseek", "https://api.deepseek.com"),
+        ("openrouter", "https://openrouter.ai/api/v1"),
+        ("lmstudio", "http://localhost:1234/v1"),
+        ("", "https://vllm.internal/v1"),
+    ],
+)
+def test_auto_default_sends_nothing_to_an_unrecognized_backend(provider, base_url):
+    """The safety property that makes "auto" viable as a default.
+
+    An OpenAI-compatible backend that never documented either mechanism must
+    receive a byte-identical request. "auto" is an allowlist, so anything off it
+    resolves to none rather than being probed with an unfamiliar field.
+    """
+    assert resolve_cache_affinity(CacheAffinityMode.AUTO, provider, base_url) is CacheAffinityMode.NONE
+
+
+@pytest.mark.parametrize(
+    ("provider", "base_url", "expected"),
+    [
+        ("openai", "https://api.x.ai/v1", CacheAffinityMode.XAI_CONV_ID),
+        ("openai", "https://grok.com/v1", CacheAffinityMode.XAI_CONV_ID),
+        ("openai", None, CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY),
+        ("openai", "https://api.openai.com/v1", CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY),
+        ("openai", "https://myco.openai.azure.com/", CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY),
+    ],
+)
+def test_auto_default_sends_a_hint_only_to_documented_hosts(provider, base_url, expected):
+    assert resolve_cache_affinity(CacheAffinityMode.AUTO, provider, base_url) is expected
+
+
+def test_indexed_member_carries_affinity(clean_llm_env):
+    from hindsight_api.config import _parse_llm_members
+
+    clean_llm_env.setenv("HINDSIGHT_API_REFLECT_LLM_1_PROVIDER", "openai")
+    clean_llm_env.setenv("HINDSIGHT_API_REFLECT_LLM_1_API_KEY", "sk-member")
+    clean_llm_env.setenv("HINDSIGHT_API_REFLECT_LLM_1_CACHE_AFFINITY", "xai_conv_id")
+
+    members = _parse_llm_members("REFLECT_")
+    assert [m.cache_affinity for m in members] == ["xai_conv_id"]
+    assert _parse_llm_members("RETAIN_") == []
+
+
+def test_llm_provider_from_env_carries_affinity(clean_llm_env):
+    clean_llm_env.setenv("HINDSIGHT_API_LLM_CACHE_AFFINITY", "xai_conv_id")
+    clean_llm_env.setenv("HINDSIGHT_API_LLM_BASE_URL", "https://api.x.ai/v1")
+    llm = LLMProvider.from_env()
+    assert llm._provider_impl._cache_affinity is CacheAffinityMode.XAI_CONV_ID
+
+
+# ── AC9: bank attribution is unaffected ───────────────────────────────────────
+
+
+async def test_bank_attribution_and_affinity_coexist(clean_llm_env):
+    from hindsight_api.config import clear_config_cache
+    from hindsight_api.engine.memory_engine import _current_bank_id
+
+    clean_llm_env.setenv("HINDSIGHT_API_LLM_SEND_BANK_AS_USER", "true")
+    clear_config_cache()
+
+    llm = _llm("xai_conv_id")
+    create = AsyncMock(return_value=_chat_response())
+    token = _current_bank_id.set("user-9")
+    try:
+        await _call(llm, create)
+    finally:
+        _current_bank_id.reset(token)
+
+    assert create.call_args.kwargs["user"] == "user-9"
+    assert _HEX32.match(create.call_args.kwargs["extra_headers"][XAI_CONV_ID_HEADER])
+
+
+# ── AC11: user-configured values win ──────────────────────────────────────────
+
+
+async def test_configured_prompt_cache_key_survives():
+    """The operator's escape hatch: an explicit prompt_cache_key in extra_body
+    reaches the same wire field, so ours is suppressed rather than duplicated."""
+    llm = _llm("openai_prompt_cache_key", extra_body={"prompt_cache_key": "operator-value"})
+    create = await _call(llm, AsyncMock(return_value=_chat_response()))
+    kwargs = create.call_args.kwargs
+    assert kwargs["extra_body"]["prompt_cache_key"] == "operator-value"
+    assert OPENAI_PROMPT_CACHE_KEY_PARAM not in kwargs
+
+
+def test_preset_conv_id_header_is_not_clobbered():
+    """Mirrors `apply_bank_attribution`'s "never override an explicit value" rule."""
+    request = {
+        "messages": [{"role": "user", "content": "ping"}],
+        "extra_headers": {XAI_CONV_ID_HEADER: "caller-pinned"},
+    }
+    apply_cache_affinity(request, CacheAffinityMode.XAI_CONV_ID)
+    assert request["extra_headers"][XAI_CONV_ID_HEADER] == "caller-pinned"
+
+
+def test_other_extra_headers_are_preserved():
+    request = {"messages": [{"role": "user", "content": "ping"}], "extra_headers": {"x-other": "keep"}}
+    apply_cache_affinity(request, CacheAffinityMode.XAI_CONV_ID)
+    assert request["extra_headers"]["x-other"] == "keep"
+    assert _HEX32.match(request["extra_headers"][XAI_CONV_ID_HEADER])
+
+
+# ── AC12: the hint survives a retry ───────────────────────────────────────────
+
+
+async def test_affinity_persists_across_a_retry():
+    """`call_params` is built once before the retry loop; attempt 2 must carry it."""
+    llm = _llm("xai_conv_id")
+    # Empty content raises a retryable ProviderResponseError on attempt 1.
+    create = AsyncMock(side_effect=[_chat_response(content=""), _chat_response()])
+    llm._client.chat.completions.create = create
+    with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector"):
+        await llm.call(
+            messages=[{"role": "user", "content": "ping"}],
+            max_retries=1,
+            initial_backoff=0.0,
+        )
+    assert create.await_count == 2
+    for call in create.await_args_list:
+        assert _HEX32.match(call.kwargs["extra_headers"][XAI_CONV_ID_HEADER])
+
+
+# ── AC10: END-TO-END anti-inert ───────────────────────────────────────────────
+
+
+async def test_end_to_end_reflect_lane_sends_the_affinity_header(clean_llm_env):
+    """The decisive test. Builds the reflect provider through the PRODUCTION path
+    — env -> HindsightConfig -> MemoryEngine's reflect base build -> LLMProvider
+    -> create_llm_provider -> OpenAICompatibleLLM -> with_config() -> the wire —
+    and asserts the header lands on the request.
+
+    A direct-construction test passes even when nothing connects config to the
+    provider, which is exactly how the first version of this feature shipped
+    inert for the reflect lane. This one fails in that state.
+    """
+    from hindsight_api import MemoryEngine
+
+    clean_llm_env.setenv("HINDSIGHT_API_LLM_MODEL", "grok-4.5")
+    clean_llm_env.setenv("HINDSIGHT_API_REFLECT_LLM_BASE_URL", "https://api.x.ai/v1")
+    clean_llm_env.setenv("HINDSIGHT_API_REFLECT_LLM_CACHE_AFFINITY", "xai_conv_id")
+
+    engine = MemoryEngine(skip_llm_verification=True)
+    reflect_llm = engine._reflect_llm_config
+    create = AsyncMock(return_value=_chat_response())
+    reflect_llm._provider_impl._client.chat.completions.create = create
+
+    configured = reflect_llm.with_config(
+        SimpleNamespace(llm_gemini_safety_settings=None), bank_id="bank-e2e", operation="reflect"
+    )
+    with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector"):
+        await configured.call(messages=[{"role": "user", "content": "ping"}], max_retries=0)
+
+    assert _HEX32.match(create.call_args.kwargs["extra_headers"][XAI_CONV_ID_HEADER])
+
+
+async def test_end_to_end_failover_member_sends_the_affinity_header(clean_llm_env):
+    """Same production path for an indexed chain member, which is built by
+    `_member_to_llm` rather than the per-operation base build. Without that leg,
+    a failover member would silently drop the pin the primary carries."""
+    from hindsight_api import MemoryEngine
+    from hindsight_api.engine.multi_llm import MultiLLMProvider
+
+    clean_llm_env.setenv("HINDSIGHT_API_LLM_MODEL", "grok-4.5")
+    clean_llm_env.setenv("HINDSIGHT_API_LLM_CACHE_AFFINITY", "xai_conv_id")
+    clean_llm_env.setenv("HINDSIGHT_API_REFLECT_LLM_1_PROVIDER", "openai")
+    clean_llm_env.setenv("HINDSIGHT_API_REFLECT_LLM_1_API_KEY", "sk-member")
+    clean_llm_env.setenv("HINDSIGHT_API_REFLECT_LLM_1_MODEL", "grok-4.5-fallback")
+    clean_llm_env.setenv("HINDSIGHT_API_REFLECT_LLM_1_BASE_URL", "https://api.x.ai/v1")
+    clean_llm_env.setenv("HINDSIGHT_API_REFLECT_LLM_STRATEGY", '{"mode": "failover"}')
+
+    engine = MemoryEngine(skip_llm_verification=True)
+    chain = engine._reflect_llm_config
+    assert isinstance(chain, MultiLLMProvider)
+    member = chain._members[1]
+    assert member.model == "grok-4.5-fallback"
+
+    create = AsyncMock(return_value=_chat_response())
+    member._provider_impl._client.chat.completions.create = create
+    with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector"):
+        await member.call(messages=[{"role": "user", "content": "ping"}], max_retries=0)
+
+    assert _HEX32.match(create.call_args.kwargs["extra_headers"][XAI_CONV_ID_HEADER])

hindsight-api-slim/tests/test_codex_extra_body.py

@@ -0,0 +1,75 @@
+"""Regression tests for Codex extra request-body parameters."""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from hindsight_api.engine.llm_wrapper import create_llm_provider
+from hindsight_api.engine.providers.codex_llm import CodexLLM
+
+
+def build_llm(extra_body: dict | None = None) -> CodexLLM:
+    with (
+        patch.object(CodexLLM, "_load_codex_auth", return_value=("token", "account")),
+        patch.object(CodexLLM, "_load_codex_refresh_token", return_value=None),
+    ):
+        return CodexLLM(
+            provider="openai-codex",
+            api_key="ignored",
+            base_url="https://chatgpt.com/backend-api",
+            model="gpt-5.6-luna",
+            extra_body=extra_body,
+        )
+
+
+def test_factory_forwards_extra_body() -> None:
+    with (
+        patch.object(CodexLLM, "_load_codex_auth", return_value=("token", "account")),
+        patch.object(CodexLLM, "_load_codex_refresh_token", return_value=None),
+    ):
+        llm = create_llm_provider(
+            provider="openai-codex",
+            api_key="ignored",
+            base_url="https://chatgpt.com/backend-api",
+            model="gpt-5.6-luna",
+            reasoning_effort="low",
+            extra_body={"service_tier": "priority"},
+        )
+
+    assert llm._extra_body == {"service_tier": "priority"}
+
+
+@pytest.mark.asyncio
+async def test_call_merges_extra_body_into_request() -> None:
+    llm = build_llm({"service_tier": "priority"})
+    response = MagicMock()
+    response.raise_for_status.return_value = None
+
+    with (
+        patch.object(llm._client, "post", new_callable=AsyncMock) as mock_post,
+        patch.object(llm, "_parse_sse_stream", new_callable=AsyncMock, return_value="ok"),
+    ):
+        mock_post.return_value = response
+        await llm.call(messages=[{"role": "user", "content": "hello"}], max_retries=0)
+
+    assert mock_post.call_args.kwargs["json"]["service_tier"] == "priority"
+
+
+@pytest.mark.asyncio
+async def test_call_with_tools_merges_extra_body_into_request() -> None:
+    llm = build_llm({"service_tier": "priority"})
+    response = MagicMock(status_code=200)
+    response.raise_for_status.return_value = None
+
+    with (
+        patch.object(llm._client, "post", new_callable=AsyncMock) as mock_post,
+        patch.object(llm, "_parse_sse_tool_stream", new_callable=AsyncMock, return_value=(None, [])),
+    ):
+        mock_post.return_value = response
+        await llm.call_with_tools(
+            messages=[{"role": "user", "content": "hello"}],
+            tools=[],
+            max_retries=0,
+        )
+
+    assert mock_post.call_args.kwargs["json"]["service_tier"] == "priority"

hindsight-api-slim/tests/test_delta_oversized_replacement.py

@@ -0,0 +1,262 @@
+"""
+Regression tests for https://github.com/vectorize-io/hindsight/issues/3282
+
+When a replacement body for an existing ``document_id`` exceeds
+``retain_batch_tokens``, ``retain_batch_async`` slices it into sub-batches
+*before* the orchestrator gets a chance to classify the whole replacement
+against the stored chunks. Delta retain only runs on the first sub-batch (and
+only sees that slice), so a one-section edit plus an appended tail re-extracts
+the entire unchanged history instead of just the changed/new native chunks.
+
+The control test pins the behaviour with a transport budget large enough to
+keep the replacement in one piece (delta works there today); the repro test
+lowers only ``retain_batch_tokens`` and asserts the same document edit still
+skips the unchanged chunks.
+"""
+
+from datetime import datetime, timezone
+
+import pytest
+
+from hindsight_api.config import clear_config_cache
+from hindsight_api.engine.memory_engine import count_tokens
+from hindsight_api.engine.retain import fact_extraction, orchestrator
+
+# Sections are separated by blank lines and each is just under
+# retain_chunk_size (3000 chars), so the chunker emits exactly one native chunk
+# per section. That keeps the chunk boundaries stable across versions: the edit
+# below is length-preserving, so only the edited section's chunk changes.
+_SECTION_REPEATS = 117  # ~2.5 KB per section
+_BASE_SECTIONS = 10
+_APPENDED_SECTIONS = 3
+
+# Transport budget for the replacement retain. The splitter slices an oversized
+# item at ``3 * retain_batch_tokens`` chars, so this is deliberately below the
+# native chunk size: the slices cut across native chunk boundaries.
+_OVERSIZED_BATCH_TOKENS = 300
+
+
+def _ts() -> float:
+    return datetime.now(timezone.utc).timestamp()
+
+
+def _section(idx: int, *, edited: bool = False) -> str:
+    marker = f"MARKER{idx:02d}"
+    payload = "bbbbb" if edited else "aaaaa"
+    return f"Section {idx:02d} {marker}. Payload {payload}. " + f"{marker} filler word here. " * _SECTION_REPEATS
+
+
+def _body(*, edited_idx: int | None = None, appended: int = 0) -> str:
+    sections = [_section(i, edited=(i == edited_idx)) for i in range(_BASE_SECTIONS)]
+    sections += [_section(100 + j) for j in range(appended)]
+    return "\n\n".join(sections)
+
+
+def _unchanged_markers(edited_idx: int) -> list[str]:
+    return [f"MARKER{i:02d}" for i in range(_BASE_SECTIONS) if i != edited_idx]
+
+
+class _ExtractionSpy:
+    """Records the content fed to LLM fact extraction on each call."""
+
+    def __init__(self) -> None:
+        self.texts: list[str] = []
+
+    def install(self, monkeypatch) -> None:
+        original = fact_extraction.extract_facts_from_contents
+
+        async def _spy(contents, *args, **kwargs):
+            self.texts.extend(c.content for c in contents)
+            return await original(contents, *args, **kwargs)
+
+        monkeypatch.setattr(fact_extraction, "extract_facts_from_contents", _spy)
+
+    def markers_seen(self, markers: list[str]) -> list[str]:
+        blob = "\n".join(self.texts)
+        return [m for m in markers if m in blob]
+
+    @property
+    def extracted_tokens(self) -> int:
+        return sum(count_tokens(t) for t in self.texts)
+
+
+@pytest.fixture(autouse=True)
+def _fast_retain_env(monkeypatch):
+    # Keep the tests focused on the retain path.
+    monkeypatch.setenv("HINDSIGHT_API_ENABLE_AUTO_CONSOLIDATION", "false")
+    monkeypatch.setenv("HINDSIGHT_API_ENABLE_OBSERVATIONS", "false")
+    clear_config_cache()
+    yield
+    clear_config_cache()
+
+
+async def _retain_v1_then_v2(
+    memory,
+    request_context,
+    bank_id: str,
+    document_id: str,
+    *,
+    replacement_batch_tokens: int | None,
+    monkeypatch,
+    spy: _ExtractionSpy,
+    edited_idx: int,
+) -> str:
+    """Retain the base body, then the edited+appended replacement.
+
+    ``replacement_batch_tokens`` is applied only to the second retain, matching
+    the issue's reproduction steps. Returns the replacement body.
+    """
+    v1 = _body()
+    await memory.retain_async(
+        bank_id=bank_id,
+        content=v1,
+        context="notes",
+        document_id=document_id,
+        request_context=request_context,
+    )
+
+    v2 = _body(edited_idx=edited_idx, appended=_APPENDED_SECTIONS)
+
+    if replacement_batch_tokens is not None:
+        monkeypatch.setenv("HINDSIGHT_API_RETAIN_BATCH_TOKENS", str(replacement_batch_tokens))
+        clear_config_cache()
+
+    # Only spy on the replacement — the first retain legitimately extracts everything.
+    spy.install(monkeypatch)
+
+    await memory.retain_async(
+        bank_id=bank_id,
+        content=v2,
+        context="notes",
+        document_id=document_id,
+        request_context=request_context,
+    )
+    return v2
+
+
+@pytest.mark.asyncio
+async def test_replacement_within_batch_budget_skips_unchanged_chunks(memory, request_context, monkeypatch):
+    """Control: with the whole replacement inside the transport budget, delta
+    retain re-extracts only the edited section and the appended tail."""
+    bank_id = f"test_3282_control_{_ts()}"
+    document_id = "doc-3282-control"
+    edited_idx = 1
+    spy = _ExtractionSpy()
+
+    try:
+        await _retain_v1_then_v2(
+            memory,
+            request_context,
+            bank_id,
+            document_id,
+            replacement_batch_tokens=100_000,  # whole replacement fits — no splitting
+            monkeypatch=monkeypatch,
+            spy=spy,
+            edited_idx=edited_idx,
+        )
+
+        re_extracted = spy.markers_seen(_unchanged_markers(edited_idx))
+        assert re_extracted == [], f"unchanged sections were re-extracted: {re_extracted}"
+        assert spy.markers_seen([f"MARKER{edited_idx:02d}", "MARKER100"]), (
+            "the edited section and the appended tail should have been extracted"
+        )
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_oversized_replacement_still_skips_unchanged_chunks(memory, request_context, monkeypatch):
+    """Repro for #3282: the same edit must not re-extract unchanged history just
+    because the complete replacement exceeds ``retain_batch_tokens``.
+
+    The budget is set below both the full replacement and the aggregate token
+    count of the changed/new chunks, so the transport split cannot be satisfied
+    by the delta chunk list either — delta classification must still run against
+    the complete body first.
+    """
+    bank_id = f"test_3282_oversized_{_ts()}"
+    document_id = "doc-3282-oversized"
+    edited_idx = 1
+    spy = _ExtractionSpy()
+    batch_tokens = _OVERSIZED_BATCH_TOKENS
+
+    try:
+        v2 = await _retain_v1_then_v2(
+            memory,
+            request_context,
+            bank_id,
+            document_id,
+            replacement_batch_tokens=batch_tokens,
+            monkeypatch=monkeypatch,
+            spy=spy,
+            edited_idx=edited_idx,
+        )
+
+        # Sanity-check the premise of the repro: the replacement really is over
+        # the transport budget, and so is the changed/new chunk aggregate.
+        assert count_tokens(v2) > batch_tokens
+        changed_chunk_tokens = count_tokens(_section(edited_idx, edited=True)) + sum(
+            count_tokens(_section(100 + j)) for j in range(_APPENDED_SECTIONS)
+        )
+        assert changed_chunk_tokens > batch_tokens
+
+        re_extracted = spy.markers_seen(_unchanged_markers(edited_idx))
+        assert re_extracted == [], (
+            f"unchanged sections {re_extracted} were re-extracted: the oversized "
+            f"replacement bypassed delta retain (issue #3282). Extraction saw "
+            f"{spy.extracted_tokens:,} tokens; only the changed/new chunks "
+            f"(~{changed_chunk_tokens:,} tokens) should have reached it."
+        )
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_oversized_replacement_screens_document_body_once(memory, request_context, monkeypatch):
+    """Companion to #3282: the split fallback must not re-run Memory Defense over
+    the whole document for every sub-batch.
+
+    Each sub-batch carries ``document_body_override`` (the COMPLETE body, so
+    ``documents.original_text`` isn't clobbered with a slice), and the retain
+    path redaction-scans that override before persisting it. With N sub-batches
+    the full body is scanned N times even though only one of them wins the
+    document row.
+    """
+    bank_id = f"test_3282_defense_{_ts()}"
+    document_id = "doc-3282-defense"
+    edited_idx = 1
+    spy = _ExtractionSpy()
+
+    full_body_scans: list[int] = []
+    original_redaction = orchestrator.apply_redaction
+
+    def _counting_redaction(text: str, *args, **kwargs):
+        full_body_scans.append(len(text))
+        return original_redaction(text, *args, **kwargs)
+
+    try:
+        await memory.update_bank_config(
+            bank_id,
+            {"memory_defense": {"enabled": True, "rules": [{"on": "sensitive_data", "action": "redact"}]}},
+            request_context=request_context,
+        )
+        monkeypatch.setattr(orchestrator, "apply_redaction", _counting_redaction)
+
+        v2 = await _retain_v1_then_v2(
+            memory,
+            request_context,
+            bank_id,
+            document_id,
+            replacement_batch_tokens=_OVERSIZED_BATCH_TOKENS,
+            monkeypatch=monkeypatch,
+            spy=spy,
+            edited_idx=edited_idx,
+        )
+
+        replacement_scans = [n for n in full_body_scans if n == len(v2)]
+        assert len(replacement_scans) <= 1, (
+            f"the complete {len(v2):,}-char body was Memory Defense scanned "
+            f"{len(replacement_scans)} times — once per fallback sub-batch (issue #3282)"
+        )
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_delta_retain_orphan_observations.py

@@ -0,0 +1,266 @@
+"""End-to-end regression for issue #3294: delta retain must not orphan observations.
+
+The reporter's sequence, driven through the public engine API rather than by calling
+the storage helpers directly:
+
+    retain(document) -> consolidate -> retain(same document_id, edited) -> consolidate
+
+Before the fix, the second retain took the delta path, which deletes the changed
+chunks and lets the FK cascade drop their facts — with no observation sweep in
+between. The observations derived from those facts stayed behind, still valid and
+still recallable, pointing at ``source_memory_ids`` that no longer resolved. Nothing
+could reach them afterwards: consolidation batches are built from facts, so an
+observation whose sources are all gone is never selected into a batch again.
+
+These tests assert the invariant the lifecycle documents ("removing a document: all
+observations derived from the document's memories are deleted") on the paths delta
+retain actually takes: an edit, a removal, and a no-op re-ingest. A rewrite of *every*
+chunk is deliberately not covered here — with no unchanged chunk left, delta declines
+and the full-replace path (already covered in ``test_observation_invalidation.py``)
+handles it.
+"""
+
+import uuid
+
+import pytest
+
+from hindsight_api import RequestContext
+from hindsight_api.config import _get_raw_config
+from hindsight_api.engine.memories import FactRecord, get_memories
+from hindsight_api.engine.memory_engine import MemoryEngine, fq_table
+
+# Delta retain works per chunk, so the document has to be big enough to produce several
+# (chunk size is 3000 chars) and at least one of them must come out unchanged — with no
+# unchanged chunk delta declines and falls back to a full replace. Keeping the FIRST
+# block byte-identical across a re-ingest is what guarantees that: chunking is greedy
+# from the start of the text, so an edit after chunk 0's boundary cannot move it.
+_BLOCK_A = " ".join(
+    f"Alice shipped the Alpha{i} milestone at Google in the search infrastructure group." for i in range(40)
+)
+_BLOCK_B = " ".join(f"Bob reviewed the Beta{i} rollout at Microsoft in the Azure networking group." for i in range(40))
+_BLOCK_B_EDITED = " ".join(
+    f"Bob reviewed the Beta{i} rollout at Amazon in the AWS networking group." for i in range(40)
+)
+
+_DOCUMENT_V1 = f"{_BLOCK_A} {_BLOCK_B}"
+_DOCUMENT_V2_PARTIAL_EDIT = f"{_BLOCK_A} {_BLOCK_B_EDITED}"
+
+
+@pytest.fixture(autouse=True)
+def enable_observations():
+    config = _get_raw_config()
+    original = config.enable_observations
+    config.enable_observations = True
+    yield
+    config.enable_observations = original
+
+
+async def _scan(memory: MemoryEngine, bank_id: str, fact_types: list[str]) -> list[FactRecord]:
+    """Every stored memory of these types, read through whichever store holds them."""
+    store = get_memories()
+    pool = await memory._get_pool()
+    async with pool.acquire() as conn:
+        page = await store.scan_memories(
+            conn=conn,
+            fq_table=fq_table,
+            bank_id=bank_id,
+            fact_types=fact_types,
+            limit=1_000_000,
+        )
+    return list(page.memories)
+
+
+async def _facts(memory: MemoryEngine, bank_id: str) -> list[FactRecord]:
+    return await _scan(memory, bank_id, ["experience", "world"])
+
+
+async def _observations(memory: MemoryEngine, bank_id: str) -> list[FactRecord]:
+    return await _scan(memory, bank_id, ["observation"])
+
+
+def _broken_source_refs(observations: list[FactRecord], live_fact_ids: set[str]) -> list[tuple[str, list[str]]]:
+    """The reporter's diagnostic: observations whose sources no longer resolve.
+
+    Returns ``(observation_id, unresolvable_source_ids)`` per affected row — what the
+    bug report counted as "broken references" on their bank.
+    """
+    broken = []
+    for obs in observations:
+        missing = [sid for sid in obs.source_memory_ids if sid not in live_fact_ids]
+        if missing:
+            broken.append((obs.unit_id, missing))
+    return broken
+
+
+async def _assert_no_orphans(memory: MemoryEngine, bank_id: str, when: str) -> None:
+    facts = await _facts(memory, bank_id)
+    observations = await _observations(memory, bank_id)
+    broken = _broken_source_refs(observations, {f.unit_id for f in facts})
+    assert broken == [], (
+        f"{when}: {len(broken)} of {len(observations)} observation(s) reference deleted source "
+        f"memories (issue #3294 — delta retain cascaded the facts away without sweeping the "
+        f"observations derived from them): {broken[:5]}"
+    )
+
+
+async def _retain_document(
+    memory: MemoryEngine, bank_id: str, document_id: str, content: str, request_context: RequestContext
+) -> None:
+    await memory.retain_async(
+        bank_id=bank_id,
+        content=content,
+        context="team roster",
+        document_id=document_id,
+        request_context=request_context,
+    )
+
+
+def _facts_by_chunk(facts: list[FactRecord]) -> dict[str, set[str]]:
+    by_chunk: dict[str, set[str]] = {}
+    for fact in facts:
+        if fact.chunk_id:
+            by_chunk.setdefault(fact.chunk_id, set()).add(fact.unit_id)
+    return by_chunk
+
+
+@pytest.mark.asyncio
+async def test_delta_retain_partial_edit_leaves_no_orphan_observations(
+    memory: MemoryEngine, request_context: RequestContext
+):
+    """Editing the tail of a consolidated document orphans nothing.
+
+    Also pins the precision of the sweep: the untouched first chunk keeps its facts
+    AND the observations derived only from them, so a small edit does not
+    re-consolidate the whole document — the case delta retain exists for.
+    """
+    bank_id = f"test_delta_orphan_partial_{uuid.uuid4().hex[:8]}"
+    document_id = "roster-doc"
+
+    try:
+        await _retain_document(memory, bank_id, document_id, _DOCUMENT_V1, request_context)
+        await memory.run_consolidation(bank_id=bank_id, request_context=request_context)
+
+        facts_v1 = await _facts(memory, bank_id)
+        observations_v1 = await _observations(memory, bank_id)
+        by_chunk_v1 = _facts_by_chunk(facts_v1)
+        assert len(by_chunk_v1) >= 2, f"Setup: the document should span several chunks, got {list(by_chunk_v1)}"
+        assert observations_v1, "Setup: consolidation should have produced observations to orphan"
+        await _assert_no_orphans(memory, bank_id, "after the first retain")
+
+        first_chunk = sorted(by_chunk_v1)[0]
+        kept_fact_ids = by_chunk_v1[first_chunk]
+        edited_fact_ids = {fid for chunk, ids in by_chunk_v1.items() if chunk != first_chunk for fid in ids}
+        assert kept_fact_ids and edited_fact_ids
+
+        obs_over_edited = {o.unit_id for o in observations_v1 if edited_fact_ids.intersection(o.source_memory_ids)}
+        obs_only_over_kept = {
+            o.unit_id
+            for o in observations_v1
+            if o.source_memory_ids and set(o.source_memory_ids).issubset(kept_fact_ids)
+        }
+        assert obs_over_edited, "Setup: the chunks being edited should have observations derived from them"
+        assert obs_only_over_kept, "Setup: the unchanged chunk should have observations of its own"
+
+        # Re-ingest with only the tail changed — this is the delta path.
+        await _retain_document(memory, bank_id, document_id, _DOCUMENT_V2_PARTIAL_EDIT, request_context)
+
+        surviving_fact_ids = {f.unit_id for f in await _facts(memory, bank_id)}
+        # Delta really applied: the unchanged chunk's facts were preserved rather than
+        # re-extracted under new ids (a full replace would have changed all of them).
+        assert kept_fact_ids.issubset(surviving_fact_ids), (
+            "Unchanged chunk's facts should survive the delta re-ingest — if they did not, this "
+            "test fell back to the full-replace path and no longer covers the bug"
+        )
+        assert not edited_fact_ids.intersection(surviving_fact_ids), "The edited chunks' facts should be gone"
+
+        await _assert_no_orphans(memory, bank_id, "after the delta re-ingest")
+
+        observation_ids_v2 = {o.unit_id for o in await _observations(memory, bank_id)}
+        assert not observation_ids_v2.intersection(obs_over_edited), (
+            "Observations derived from the edited chunks' facts should have been invalidated"
+        )
+        assert obs_only_over_kept.issubset(observation_ids_v2), (
+            "Observations derived only from the unchanged chunk must survive a partial edit"
+        )
+
+        # And the follow-up consolidation the reporter ran — still no orphans.
+        await memory.run_consolidation(bank_id=bank_id, request_context=request_context)
+        await _assert_no_orphans(memory, bank_id, "after re-consolidating the edited document")
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_delta_retain_removed_chunks_leave_no_orphan_observations(
+    memory: MemoryEngine, request_context: RequestContext
+):
+    """Shortening a document orphans nothing either.
+
+    Delta deletes removed chunks through the same call as changed ones, so this
+    covers the ``removed_indices`` half of that list — a document that shrinks loses
+    facts without any replacement being extracted for them.
+    """
+    bank_id = f"test_delta_orphan_shrink_{uuid.uuid4().hex[:8]}"
+    document_id = "roster-doc"
+
+    try:
+        await _retain_document(memory, bank_id, document_id, _DOCUMENT_V1, request_context)
+        await memory.run_consolidation(bank_id=bank_id, request_context=request_context)
+
+        facts_v1 = await _facts(memory, bank_id)
+        observations_v1 = await _observations(memory, bank_id)
+        by_chunk_v1 = _facts_by_chunk(facts_v1)
+        assert len(by_chunk_v1) >= 2, f"Setup: the document should span several chunks, got {list(by_chunk_v1)}"
+        assert observations_v1, "Setup: consolidation should have produced observations to orphan"
+
+        first_chunk = sorted(by_chunk_v1)[0]
+        kept_fact_ids = by_chunk_v1[first_chunk]
+        dropped_fact_ids = {fid for chunk, ids in by_chunk_v1.items() if chunk != first_chunk for fid in ids}
+        obs_over_dropped = {o.unit_id for o in observations_v1 if dropped_fact_ids.intersection(o.source_memory_ids)}
+        assert obs_over_dropped, "Setup: the chunks being dropped should have observations derived from them"
+
+        # Re-ingest only the first block: every later chunk is removed outright.
+        await _retain_document(memory, bank_id, document_id, _BLOCK_A, request_context)
+
+        surviving_fact_ids = {f.unit_id for f in await _facts(memory, bank_id)}
+        assert kept_fact_ids.issubset(surviving_fact_ids), (
+            "The retained chunk's facts should survive — if they did not, this test fell back "
+            "to the full-replace path and no longer covers the bug"
+        )
+        assert not dropped_fact_ids.intersection(surviving_fact_ids), "The removed chunks' facts should be gone"
+
+        await _assert_no_orphans(memory, bank_id, "after shrinking the document")
+        assert not {o.unit_id for o in await _observations(memory, bank_id)}.intersection(obs_over_dropped), (
+            "Observations derived from the removed chunks' facts should have been invalidated"
+        )
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_delta_retain_unchanged_content_keeps_observations(memory: MemoryEngine, request_context: RequestContext):
+    """Re-submitting identical content deletes no chunk, so it sweeps no observation
+    and requeues nothing for consolidation."""
+    bank_id = f"test_delta_orphan_noop_{uuid.uuid4().hex[:8]}"
+    document_id = "roster-doc"
+
+    try:
+        await _retain_document(memory, bank_id, document_id, _DOCUMENT_V1, request_context)
+        await memory.run_consolidation(bank_id=bank_id, request_context=request_context)
+        observation_ids_v1 = {o.unit_id for o in await _observations(memory, bank_id)}
+        assert observation_ids_v1
+
+        await _retain_document(memory, bank_id, document_id, _DOCUMENT_V1, request_context)
+
+        assert {o.unit_id for o in await _observations(memory, bank_id)} == observation_ids_v1, (
+            "A no-op delta re-ingest must not touch existing observations"
+        )
+        assert all(f.consolidated_at is not None for f in await _facts(memory, bank_id)), (
+            "A no-op delta re-ingest must not requeue facts for consolidation"
+        )
+        await _assert_no_orphans(memory, bank_id, "after a no-op re-ingest")
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_document_transfer.py

@@ -100,6 +100,20 @@ async def _import(memory, bank_id, archive, request_context, on_conflict="skip")
     return status["result_metadata"]
 
 
+async def _export_async(memory, bank_id, request_context, **kwargs):
+    """Submit an async export and return (result_metadata, archive_bytes).
+
+    Export is async; the SyncTaskBackend fixture runs it inline, so the operation
+    is completed (with the archive stashed in file storage) when submit returns.
+    """
+    submission = await memory.submit_export_documents_async(bank_id, request_context, **kwargs)
+    status = await memory.get_operation_status(bank_id, submission["operation_id"], request_context=request_context)
+    assert status["status"] == "completed", status
+    meta = status["result_metadata"]
+    archive = await memory._file_storage.retrieve(meta["storage_key"])
+    return meta, archive
+
+
 @pytest.mark.asyncio
 async def test_import_filters_degenerate_fact_without_shifting_archive_ordinals(memory, request_context):
     """A rejected archive fact must not shift chunks, causal links, or observation sources."""
@@ -191,12 +205,13 @@ def test_export_bank_covers_schema():
     history, or explicitly skipped — so a future migration can't silently drop one."""
     from hindsight_api.admin.cli import BACKUP_TABLES
     from hindsight_api.engine.transfer.export import _BANK_ROW_TABLES, _REPLAYED_TABLES, _SKIP_TABLES
-    from hindsight_api.engine.transfer.schema import CARRIED_HISTORY_TABLES, HISTORY_TABLES
+    from hindsight_api.engine.transfer.schema import CARRIED_HISTORY_TABLES, HISTORY_TABLES, KNOWLEDGE_TABLES
 
     buckets = [
         set(_REPLAYED_TABLES),
         set(_BANK_ROW_TABLES),
         set(CARRIED_HISTORY_TABLES),
+        set(KNOWLEDGE_TABLES),
         set(HISTORY_TABLES),
         set(_SKIP_TABLES),
     ]
@@ -209,6 +224,38 @@ def test_export_bank_covers_schema():
     assert sum(len(b) for b in buckets) == len(classified), "a table is classified in more than one bucket"
 
 
+def test_topological_page_order_is_parent_first():
+    """Nodes always sort so a parent precedes its children (self-FK safe)."""
+    from hindsight_api.engine.transfer.importer import _topological_page_order
+    from hindsight_api.engine.transfer.schema import TransferKnowledgePage
+
+    def _page(pid, parent):
+        kind = "page" if pid.startswith("p") else "folder"
+        return TransferKnowledgePage(id=pid, parent_id=parent, kind=kind, name=pid)
+
+    # Deliberately shuffled: child before parent, grandchild before both.
+    pages = [_page("pC", "fB"), _page("fB", "fA"), _page("fA", None), _page("pRoot", None)]
+    ordered = [p.id for p in _topological_page_order(pages)]
+    assert ordered.index("fA") < ordered.index("fB") < ordered.index("pC")
+    assert ordered.index("fA") < ordered.index("pC")
+    assert set(ordered) == {"pC", "fB", "fA", "pRoot"}
+
+
+def test_topological_page_order_tolerates_cycles_and_dangling_parents():
+    """A cycle or missing parent (only possible in a corrupt export) is emitted
+    rather than dropped, so the DB FK — not a silent loss — surfaces it."""
+    from hindsight_api.engine.transfer.importer import _topological_page_order
+    from hindsight_api.engine.transfer.schema import TransferKnowledgePage
+
+    cycle = [
+        TransferKnowledgePage(id="a", parent_id="b", kind="folder", name="a"),
+        TransferKnowledgePage(id="b", parent_id="a", kind="folder", name="b"),
+    ]
+    assert {p.id for p in _topological_page_order(cycle)} == {"a", "b"}
+    dangling = [TransferKnowledgePage(id="x", parent_id="missing", kind="page", name="x")]
+    assert [p.id for p in _topological_page_order(dangling)] == ["x"]
+
+
 def test_export_jsonb_coercion_preserves_decoded_scalar_string():
     """Admin connections decode JSONB before the transfer exporter sees it."""
     from hindsight_api.engine.transfer.export import _as_jsonb
@@ -630,6 +677,54 @@ async def test_bank_export_import_exact_roundtrip(memory, request_context):
         await memory.delete_bank(bank, request_context=request_context)
 
 
+@pytest.mark.asyncio
+async def test_bank_import_into_new_id_on_same_instance(memory, request_context):
+    """Export a bank and import it RIGHT BACK into a new id on the same instance
+    (source bank left in place). The banks row carries a globally-unique
+    ``internal_id``; if that were kept, the copy's banks INSERT would collide with
+    the still-present source and ``ON CONFLICT DO NOTHING`` would skip the parent
+    row, so the mental_models insert would trip fk_mental_models_bank_id. Import
+    must mint a fresh internal_id so the copy lands cleanly. Regression for #3270."""
+    source = _unique_bank("bank_src")
+    target = _unique_bank("bank_copy")
+    try:
+        await _retain(memory, source, "Alice works at Google.", request_context, "doc-1")
+        await memory.create_mental_model(
+            source,
+            name="Work model",
+            source_query="where do people work",
+            content="User tracks where people work.",
+            mental_model_id="mm-1",
+            request_context=request_context,
+        )
+
+        backend = await memory._get_backend()
+        async with acquire_with_retry(backend) as conn:
+            source_internal_id = await conn.fetchval(
+                f"SELECT internal_id FROM {fq_table('banks')} WHERE bank_id = $1", source
+            )
+
+        from hindsight_api.engine.transfer import export_bank
+
+        async with acquire_with_retry(backend) as conn:
+            archive = await export_bank(conn, source)
+        # Source bank is left in place — this is the same-instance "make a copy" flow.
+        result = await memory.import_bank_async(archive, request_context, target_bank_id=target)
+        assert result.bank_id == target
+        assert result.mental_models_imported == 1
+
+        async with acquire_with_retry(backend) as conn:
+            target_internal_id = await conn.fetchval(
+                f"SELECT internal_id FROM {fq_table('banks')} WHERE bank_id = $1", target
+            )
+        # The copy exists (parent row landed) and got a fresh, non-colliding id.
+        assert target_internal_id is not None
+        assert target_internal_id != source_internal_id
+    finally:
+        await memory.delete_bank(source, request_context=request_context)
+        await memory.delete_bank(target, request_context=request_context)
+
+
 @pytest.mark.asyncio
 async def test_bank_roundtrip_carries_mental_model_history(memory, request_context):
     """Mental-model refresh history survives export/import. Mental models keep a
@@ -667,6 +762,77 @@ async def test_bank_roundtrip_carries_mental_model_history(memory, request_conte
         await memory.delete_bank(bank, request_context=request_context)
 
 
+@pytest.mark.asyncio
+async def test_bank_roundtrip_carries_knowledge_pages(memory, request_context):
+    """A whole-bank archive restores the Knowledge Pages tree — nested folders +
+    pages, parent_id / mental_model_id / managed / sort_order preserved — and
+    regenerates each backing mental model's embedding + lexical state on the
+    target, so pages stay searchable after import (#3308, #3323)."""
+    bank = _unique_bank("bank_kb")
+    try:
+        await memory.get_bank_profile(bank, request_context=request_context)
+        root = await memory.create_knowledge_folder(bank, "Runbooks", managed=True, request_context=request_context)
+        sub = await memory.create_knowledge_folder(
+            bank, "Billing", parent_id=root["id"], request_context=request_context
+        )
+        page = await memory.create_knowledge_page(
+            bank,
+            name="Net-30 policy",
+            source_query="what is our billing policy",
+            content="Invoices are due Net-30. Late payments accrue interest.",
+            parent_id=sub["id"],
+            request_context=request_context,
+        )
+        # A root-level page (NULL parent) exercises the non-nested path too.
+        await memory.create_knowledge_page(
+            bank,
+            name="Overview",
+            source_query="overview",
+            content="Company overview and mission statement.",
+            request_context=request_context,
+        )
+
+        def _tree(nodes):
+            return sorted(
+                (n["id"], n["kind"], n["parent_id"], n["mental_model_id"], n["managed"], n["name"]) for n in nodes
+            )
+
+        before = _tree(await memory.list_knowledge_nodes(bank, request_context=request_context))
+        before_search = await memory.search_knowledge_pages(
+            bank, "net-30 billing", limit=5, request_context=request_context
+        )
+        assert any(r["id"] == page["id"] for r in before_search), "page should be searchable before export"
+
+        from hindsight_api.engine.transfer import export_bank
+
+        backend = await memory._get_backend()
+        async with acquire_with_retry(backend) as conn:
+            archive = await export_bank(conn, bank)
+        # Delete then restore into the same id — exact round-trip, no PK collisions.
+        await memory.delete_bank(bank, request_context=request_context)
+        result = await memory.import_bank_async(archive, request_context)
+        assert result.knowledge_pages_imported == 4  # 2 folders + 2 pages
+
+        # Tree restored exactly: ids, parents, backing mental models, managed flag.
+        after = _tree(await memory.list_knowledge_nodes(bank, request_context=request_context))
+        assert after == before
+
+        # Backing mental models re-embedded on the target (no NULL vectors), so both
+        # the vector and lexical arms of knowledge search work again.
+        async with acquire_with_retry(backend) as conn:
+            null_embeddings = await conn.fetchval(
+                f"SELECT count(*) FROM {fq_table('mental_models')} WHERE bank_id = $1 AND embedding IS NULL",
+                bank,
+            )
+        assert null_embeddings == 0, "restored mental models must be re-embedded"
+        after_search = await memory.search_knowledge_pages(
+            bank, "net-30 billing", limit=5, request_context=request_context
+        )
+        assert any(r["id"] == page["id"] for r in after_search), "page must be searchable after import"
+    finally:
+        await memory.delete_bank(bank, request_context=request_context)
+
+
 @pytest.mark.asyncio
 async def test_import_bank_rejects_documents_archive(memory, request_context):
     """A documents-only archive must be rejected by the bank importer."""
@@ -900,8 +1066,8 @@ async def test_transfer_preserves_legacy_causal_links(memory, request_context):
         async with acquire_with_retry(backend) as conn:
             await conn.executemany(
                 f"INSERT INTO {fq_table('memory_links')} "
-                "(from_unit_id, to_unit_id, link_type, bank_id, weight) "
-                "VALUES ($1, $2, $3, $4, 1.0)",
+                "(from_unit_id, to_unit_id, link_type, entity_id, bank_id, weight) "
+                "VALUES ($1, $2, $3, NULL, $4, 1.0)",
                 [(from_unit_id, to_unit_id, link_type, src) for link_type in legacy_types],
             )
 
@@ -1171,27 +1337,49 @@ async def test_import_on_conflict_modes(memory, request_context):
 
 @pytest.mark.asyncio
 async def test_http_export_import_endpoints(api_client, memory, request_context):
-    """Round trip through the HTTP export (GET) and import (POST multipart) endpoints."""
+    """Round trip through the async HTTP export (POST + poll + download) and import endpoints."""
     src = _unique_bank("transfer_http_src")
     dst = _unique_bank("transfer_http_dst")
     try:
         await _retain(memory, src, "Dana lives in Berlin.", request_context, document_id="doc-http")
 
-        export = await api_client.get(f"/v1/default/banks/{src}/document-transfer")
-        assert export.status_code == 200
-        assert export.headers["content-type"] == "application/zip"
-        archive = export.content
-        assert len(archive) > 0
+        # The old synchronous GET export is removed — it returns 410 pointing at
+        # the async endpoint (it could take down the shared API on large banks).
+        removed = await api_client.get(f"/v1/default/banks/{src}/document-transfer")
+        assert removed.status_code == 410
+        assert "document-transfer/export" in removed.json()["detail"]
 
-        # include_observations + a document_id subset is a 400.
-        bad = await api_client.get(
-            f"/v1/default/banks/{src}/document-transfer",
+        # Async export: POST returns 202 + operation_id, runs inline under the
+        # SyncTaskBackend test fixture, so it's completed by the time we poll.
+        submit = await api_client.post(f"/v1/default/banks/{src}/document-transfer/export")
+        assert submit.status_code == 202
+        export_op = submit.json()["operation_id"]
+
+        export_status = await api_client.get(f"/v1/default/banks/{src}/operations/{export_op}")
+        assert export_status.status_code == 200
+        export_meta = export_status.json()["result_metadata"]
+        assert export_meta["byte_size"] > 0
+        download_url = export_meta["download_url"]
+        assert download_url.startswith("/v1/default/files/download/banks/")
+
+        # Download the finished archive through the download route.
+        download = await api_client.get(download_url)
+        assert download.status_code == 200
+        assert download.headers["content-type"] == "application/zip"
+        archive = download.content
+        assert len(archive) > 0
+        # It is a real transfer archive for this bank.
+        parsed = parse_archive(archive)
+        assert parsed.manifest.source_bank_id == src
+
+        # include_observations + a document_id subset is a 400 (validated up front).
+        bad = await api_client.post(
+            f"/v1/default/banks/{src}/document-transfer/export",
             params={"document_id": "meeting-notes", "include_observations": "true"},
         )
         assert bad.status_code == 400
 
-        # Import is async: returns 202 + operation_id (runs inline under the
-        # SyncTaskBackend test fixture, so it's completed by the time we poll).
+        # Import is async: returns 202 + operation_id.
         imported = await api_client.post(
             f"/v1/default/banks/{dst}/document-transfer",
             files={"file": ("transfer.zip", archive, "application/zip")},
@@ -1208,7 +1396,7 @@ async def test_http_export_import_endpoints(api_client, memory, request_context)
         assert op["result_metadata"]["facts_imported"] >= 1
 
         # Exporting a bank that does not exist is a 404.
-        missing = await api_client.get("/v1/default/banks/does-not-exist-bank/document-transfer")
+        missing = await api_client.post("/v1/default/banks/does-not-exist-bank/document-transfer/export")
         assert missing.status_code == 404
     finally:
         await memory.delete_bank(src, request_context=request_context)
@@ -1225,10 +1413,15 @@ async def test_endpoints_disabled_by_config(api_client, monkeypatch):
     monkeypatch.setenv("HINDSIGHT_API_ENABLE_DOCUMENT_IMPORT_API", "false")
     clear_config_cache()
     try:
-        export = await api_client.get("/v1/default/banks/any-bank/document-transfer")
+        export = await api_client.post("/v1/default/banks/any-bank/document-transfer/export")
         assert export.status_code == 404
         assert "disabled" in export.json()["detail"].lower()
 
+        # The download route serves export archives, so it is gated on the same flag.
+        download = await api_client.get("/v1/default/files/download/banks/any-bank/exports/x/transfer.zip")
+        assert download.status_code == 404
+        assert "disabled" in download.json()["detail"].lower()
+
         imported = await api_client.post(
             "/v1/default/banks/any-bank/document-transfer",
             files={"file": ("x.zip", b"not-a-zip", "application/zip")},
@@ -1257,6 +1450,51 @@ async def test_import_rejects_unsupported_schema_version(memory, request_context
         await memory.import_documents_async("any-bank", buffer.getvalue(), request_context)
 
 
+def test_parse_archive_rejects_a_file_that_is_not_a_zip():
+    """Garbage bytes are a caller error (400), not a zipfile.BadZipFile crash (500)."""
+    with pytest.raises(ValueError, match="not a readable .zip"):
+        parse_archive(b"%PDF-1.7 this is not a zip at all")
+
+
+def test_parse_archive_rejects_a_plain_zip_of_files():
+    """A zip of ordinary documents is refused with a message that names the fix.
+
+    Regression for #3327: users read "Import from zip" as a bulk upload of their
+    own PDFs/text files, so the rejection has to say where that actually lives
+    instead of only naming the missing manifest.
+    """
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, "w") as zf:
+        zf.writestr("notes.txt", "Dana lives in Berlin.")
+        zf.writestr("report.pdf", "%PDF-1.7")
+
+    with pytest.raises(ValueError, match="manifest.json is missing") as excinfo:
+        parse_archive(buffer.getvalue())
+    assert "retain" in str(excinfo.value)
+
+
+@pytest.mark.asyncio
+async def test_http_import_rejects_non_transfer_zip_with_400(api_client):
+    """The wrong zip fails fast with a 400 whose detail explains what to upload."""
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, "w") as zf:
+        zf.writestr("notes.txt", "Dana lives in Berlin.")
+
+    response = await api_client.post(
+        "/v1/default/banks/any-bank/document-transfer",
+        files={"file": ("my-documents.zip", buffer.getvalue(), "application/zip")},
+    )
+    assert response.status_code == 400
+    assert "manifest.json is missing" in response.json()["detail"]
+
+    not_a_zip = await api_client.post(
+        "/v1/default/banks/any-bank/document-transfer",
+        files={"file": ("notes.pdf", b"%PDF-1.7", "application/pdf")},
+    )
+    assert not_a_zip.status_code == 400
+    assert "not a readable .zip" in not_a_zip.json()["detail"]
+
+
 @pytest.mark.asyncio
 async def test_import_rejects_invalid_on_conflict(memory, request_context):
     """An unknown on_conflict mode is rejected with a ValueError."""
@@ -1353,3 +1591,179 @@ async def test_bank_import_classifies_label_entities(memory, request_context):
         assert kinds.get(regular_entity) == "regular", kinds
     finally:
         await memory.delete_bank(bank, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_async_export_roundtrip(memory, request_context):
+    """The async export operation stashes a real archive that re-imports cleanly.
+
+    Mirrors the synchronous round trip, but through submit_export_documents_async:
+    the worker (inline under SyncTaskBackend) builds the ZIP, stores it, and
+    records the storage key / download URL / size in the operation's
+    result_metadata.
+    """
+    src = _unique_bank("async_export_src")
+    dst = _unique_bank("async_export_dst")
+    try:
+        await _retain(memory, src, "Alice works at Google. Bob works at Microsoft.", request_context, "doc-1")
+
+        meta, archive = await _export_async(memory, src, request_context)
+        assert meta["storage_key"].startswith(f"banks/{src}/exports/")
+        assert meta["download_url"] == f"/v1/default/files/download/{meta['storage_key']}"
+        assert meta["byte_size"] == len(archive)
+        assert meta["filename"] == f"{src}-documents.zip"
+
+        parsed = parse_archive(archive)
+        assert parsed.manifest.source_bank_id == src
+        exported_texts = {fact.text for doc in parsed.documents for fact in doc.facts}
+        assert exported_texts
+
+        result = await _import(memory, dst, archive, request_context)
+        assert result["facts_imported"] == parsed.manifest.fact_count
+
+        units = await memory.list_memory_units(dst, request_context=request_context)
+        imported = {u["text"] for u in units["items"] if u["fact_type"] != "observation"}
+        assert imported == exported_texts
+    finally:
+        await memory.delete_bank(src, request_context=request_context)
+        await memory.delete_bank(dst, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_async_export_include_observations_subset_rejected(memory, request_context):
+    """include_observations with a document subset fails fast (before enqueue)."""
+    with pytest.raises(ValueError, match="whole bank"):
+        await memory.submit_export_documents_async(
+            "any-bank", request_context, document_ids=["doc-1"], include_observations=True
+        )
+
+
+@pytest.mark.asyncio
+async def test_export_attach_batching_preserves_entities_and_causal_links(memory, request_context, monkeypatch):
+    """Batched attach queries carry every fact's entities and cross-batch causal edges.
+
+    With _ATTACH_BATCH_SIZE forced to 1 each unit lands in its own batch, so a
+    causal edge whose endpoints fall in different batches is exactly the case the
+    old ``to_unit_id = ANY(<full set>)`` filter covered — the Python-side target
+    check must still attach it.
+    """
+    from hindsight_api.engine.transfer import export as export_mod
+
+    bank = _unique_bank("attach_batch")
+    try:
+        await _retain(memory, bank, "Alice works at Google. Bob works at Microsoft.", request_context, "doc-1")
+
+        # Insert a synthetic caused_by edge between two facts of the same document,
+        # ordered the same way export assigns fact ordinals (created_at, id).
+        backend = await memory._get_backend()
+        async with acquire_with_retry(backend) as conn:
+            rows = await conn.fetch(
+                f"SELECT id FROM {fq_table('memory_units')} WHERE bank_id = $1 AND document_id = 'doc-1' "
+                "AND fact_type IN ('world', 'experience') ORDER BY created_at, id",
+                bank,
+            )
+            assert len(rows) >= 2, "need at least two facts to link"
+            source_id, target_id = rows[0]["id"], rows[1]["id"]
+            await conn.execute(
+                f"INSERT INTO {fq_table('memory_links')} (bank_id, from_unit_id, to_unit_id, link_type) "
+                "VALUES ($1, $2, $3, 'caused_by') ON CONFLICT DO NOTHING",
+                bank,
+                source_id,
+                target_id,
+            )
+
+        monkeypatch.setattr(export_mod, "_ATTACH_BATCH_SIZE", 1)
+        _, archive = await _export_async(memory, bank, request_context)
+        parsed = parse_archive(archive)
+
+        doc = next(d for d in parsed.documents if d.id == "doc-1")
+        # Every fact kept its entities despite one-unit-per-batch fetching.
+        all_entities = {name for fact in doc.facts for name in fact.entities}
+        assert any("alice" in n.lower() for n in all_entities), all_entities
+        assert any("bob" in n.lower() for n in all_entities), all_entities
+        # The cross-batch causal edge survived: fact 0 points at fact 1.
+        relations = doc.facts[0].causal_relations
+        assert any(r.relation_type == "caused_by" and r.target_fact_index == 1 for r in relations), relations
+    finally:
+        await memory.delete_bank(bank, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_delete_operation_removes_export_archive(memory, request_context):
+    """Deleting an export operation also deletes its stored archive (no orphan blob)."""
+    bank = _unique_bank("export_delete")
+    try:
+        await _retain(memory, bank, "Alice works at Google.", request_context, "doc-1")
+        submission = await memory.submit_export_documents_async(bank, request_context)
+        op_id = submission["operation_id"]
+        status = await memory.get_operation_status(bank, op_id, request_context=request_context)
+        storage_key = status["result_metadata"]["storage_key"]
+
+        # The archive exists while the operation does.
+        assert await memory._file_storage.retrieve(storage_key)
+
+        # Deleting the operation deletes the archive with it.
+        await memory.delete_operation(bank, op_id, request_context=request_context)
+        with pytest.raises(FileNotFoundError):
+            await memory._file_storage.retrieve(storage_key)
+    finally:
+        await memory.delete_bank(bank, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_purge_expired_export_archives(memory, request_context):
+    """Retention's archive purge deletes the blobs of export ops past the cutoff."""
+    from datetime import timedelta
+
+    bank = _unique_bank("export_purge")
+    try:
+        await _retain(memory, bank, "Bob works at Microsoft.", request_context, "doc-1")
+        submission = await memory.submit_export_documents_async(bank, request_context)
+        op_id = submission["operation_id"]
+        status = await memory.get_operation_status(bank, op_id, request_context=request_context)
+        storage_key = status["result_metadata"]["storage_key"]
+        assert await memory._file_storage.retrieve(storage_key)
+
+        # The purge is schema-wide (it doesn't take a bank), and this DB is shared
+        # across xdist workers — so backdate THIS op and use a past cutoff to target
+        # it specifically. A future cutoff would purge other concurrent tests' fresh
+        # export archives too (they'd be < cutoff), making both this count and those
+        # tests flaky.
+        backend = await memory._get_backend()
+        old = datetime.now(timezone.utc) - timedelta(days=100)
+        cutoff = datetime.now(timezone.utc) - timedelta(days=1)
+        async with acquire_with_retry(backend) as conn:
+            await conn.execute(
+                f"UPDATE {fq_table('async_operations')} SET updated_at = $1 WHERE operation_id = $2",
+                old,
+                uuid.UUID(op_id),
+            )
+            purged = await memory.purge_expired_export_archives(conn, fq_table("async_operations"), cutoff)
+        assert purged >= 1
+        with pytest.raises(FileNotFoundError):
+            await memory._file_storage.retrieve(storage_key)
+    finally:
+        await memory.delete_bank(bank, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_download_route_rejects_unauthorized_keys(api_client, memory, request_context):
+    """The download route only serves bank-scoped keys for banks the caller can see."""
+    bank = _unique_bank("download_guard")
+    try:
+        await memory.get_bank_profile(bank_id=bank, request_context=request_context)
+
+        # Non-"banks/"-prefixed key: not a downloadable resource.
+        r = await api_client.get("/v1/default/files/download/etc/passwd")
+        assert r.status_code == 404
+        # Path-traversal attempt is rejected structurally.
+        r = await api_client.get("/v1/default/files/download/banks/../secrets/x.zip")
+        assert r.status_code == 404
+        # Well-formed key for a bank that does not exist (IDOR guard via bank read).
+        r = await api_client.get("/v1/default/files/download/banks/no-such-bank/exports/x/transfer.zip")
+        assert r.status_code == 404
+        # Well-formed key for a visible bank but no such stored file: 404, not 500.
+        r = await api_client.get(f"/v1/default/files/download/banks/{bank}/exports/missing/transfer.zip")
+        assert r.status_code == 404
+    finally:
+        await memory.delete_bank(bank, request_context=request_context)

hindsight-api-slim/tests/test_entity_name_hygiene.py

@@ -0,0 +1,175 @@
+"""Candidate entity-name hygiene at resolution intake (issue #3275).
+
+``_prepare_entities_for_resolution`` is the single choke point both entity
+resolution entry paths funnel through (retain via
+``entity_processing.resolve_entities``, and the memory-edit path in
+``MemoryEngine``), so it is where a name is made safe to store:
+
+1. whitespace runs — including the ``\\n`` extraction sometimes leaves behind —
+   collapse to a single space, and the ends are stripped;
+2. a name that is empty afterwards is dropped rather than stored as an entity
+   with a blank ``canonical_name``;
+3. candidates that normalization made identical are deduplicated per fact.
+
+All of it runs before the flat list / ``entity_to_unit`` mapping is derived, so
+the resolver's positional invariant is untouched.
+"""
+
+import pytest
+
+from hindsight_api.engine.retain.link_utils import (
+    _normalize_entity_name,
+    _prepare_entities_for_resolution,
+)
+
+
+class _FakeEntity:
+    """Object-style candidate: exposes ``.text``, like the extraction models."""
+
+    def __init__(self, text: str):
+        self.text = text
+
+
+def _texts(all_entities_flat: list[dict]) -> list[str]:
+    return [e["text"] for e in all_entities_flat]
+
+
+def _prepare(entities: list, unit_ids: list[str] | None = None):
+    """Run one fact's candidate list through intake."""
+    return _prepare_entities_for_resolution(
+        unit_ids=unit_ids or ["u1"],
+        sentences=["fact text"],
+        fact_dates=[None],
+        llm_entities=[entities],
+    )
+
+
+# --- _normalize_entity_name ---
+
+
+@pytest.mark.parametrize(
+    "raw,expected",
+    [
+        ("Acme\nCorp", "Acme Corp"),
+        ("a\r\n b\tc", "a b c"),
+        ("  leading and trailing  ", "leading and trailing"),
+        ("multiple   spaces    inside", "multiple spaces inside"),
+        ("Normal Name", "Normal Name"),
+        ("", ""),
+        ("   \n\t  ", ""),
+    ],
+)
+def test_normalize_entity_name(raw, expected):
+    assert _normalize_entity_name(raw) == expected
+
+
+def test_normalize_entity_name_preserves_case():
+    # The registry matches on LOWER(canonical_name); lowercasing here would only
+    # destroy the display form.
+    assert _normalize_entity_name("MiXeD\nCaSe") == "MiXeD CaSe"
+
+
+def test_normalize_entity_name_leaves_ordinary_punctuation_alone():
+    assert _normalize_entity_name("Dr. Foo-Bar (ACME), Inc.") == "Dr. Foo-Bar (ACME), Inc."
+
+
+# --- intake: normalization reaches the text handed to the resolver ---
+
+
+def test_intake_normalizes_dict_style_candidates():
+    all_entities_flat, _all, _map = _prepare([{"text": "Acme\nCorp", "type": "ORG"}])
+    assert _texts(all_entities_flat) == ["Acme Corp"]
+    assert all_entities_flat[0]["type"] == "ORG"
+
+
+def test_intake_normalizes_object_style_candidates():
+    all_entities_flat, _all, _map = _prepare([_FakeEntity("a\r\n b\tc")])
+    assert _texts(all_entities_flat) == ["a b c"]
+
+
+def test_intake_normalizes_nearby_entities_too():
+    # nearby_entities is the co-occurrence signal the resolver scores against;
+    # it must carry the normalized names, not the raw ones.
+    all_entities_flat, all_entities, _map = _prepare(
+        [{"text": "Acme\nCorp", "type": "CONCEPT"}, {"text": "Alice", "type": "CONCEPT"}]
+    )
+    assert [e["text"] for e in all_entities[0]] == ["Acme Corp", "Alice"]
+    assert [e["text"] for e in all_entities_flat[1]["nearby_entities"]] == ["Acme Corp", "Alice"]
+
+
+# --- intake: empty names are dropped, not stored blank ---
+
+
+@pytest.mark.parametrize("raw", ["", "   ", "\n", " \t\r\n "])
+def test_intake_drops_empty_and_whitespace_only_candidates(raw):
+    all_entities_flat, all_entities, entity_to_unit = _prepare([{"text": raw, "type": "CONCEPT"}])
+    assert all_entities_flat == []
+    assert all_entities == [[]]
+    assert entity_to_unit == []
+
+
+def test_intake_drops_candidate_dict_without_text_key():
+    all_entities_flat, _all, _map = _prepare([{"type": "CONCEPT"}, {"text": "Alice", "type": "CONCEPT"}])
+    assert _texts(all_entities_flat) == ["Alice"]
+
+
+def test_intake_keeps_real_entities_alongside_dropped_empties():
+    all_entities_flat, _all, entity_to_unit = _prepare(
+        [{"text": "  ", "type": "CONCEPT"}, {"text": " Alice ", "type": "CONCEPT"}]
+    )
+    assert _texts(all_entities_flat) == ["Alice"]
+    # entity_to_unit stays index-aligned with the flat list the resolver receives.
+    assert entity_to_unit == [("u1", 0, None)]
+
+
+# --- intake: dedup of candidates that normalization made identical ---
+
+
+def test_intake_dedupes_candidates_normalization_made_identical():
+    # The upstream dedup in entity_processing runs on raw text, so these two
+    # arrive here distinct and collide only after normalization.
+    all_entities_flat, all_entities, entity_to_unit = _prepare(
+        [{"text": "Acme\nCorp", "type": "CONCEPT"}, {"text": "Acme Corp", "type": "CONCEPT"}]
+    )
+    assert _texts(all_entities_flat) == ["Acme Corp"]
+    assert [e["text"] for e in all_entities[0]] == ["Acme Corp"]
+    assert len(entity_to_unit) == 1
+
+
+def test_intake_dedupe_is_case_insensitive_and_keeps_first_spelling():
+    all_entities_flat, _all, _map = _prepare(
+        [{"text": "Acme Corp", "type": "CONCEPT"}, {"text": "acme\ncorp", "type": "CONCEPT"}]
+    )
+    assert _texts(all_entities_flat) == ["Acme Corp"]
+
+
+def test_intake_dedupe_is_scoped_per_fact():
+    # The same entity mentioned by two different facts must still be resolved
+    # for each of them — the dedup is within a fact, not across the batch.
+    all_entities_flat, _all, entity_to_unit = _prepare_entities_for_resolution(
+        unit_ids=["u1", "u2"],
+        sentences=["first", "second"],
+        fact_dates=[None, None],
+        llm_entities=[
+            [{"text": "Acme\nCorp", "type": "CONCEPT"}],
+            [{"text": "Acme Corp", "type": "CONCEPT"}],
+        ],
+    )
+    assert _texts(all_entities_flat) == ["Acme Corp", "Acme Corp"]
+    assert [unit_id for unit_id, _idx, _date in entity_to_unit] == ["u1", "u2"]
+
+
+def test_intake_attaches_fact_dates_after_dropping():
+    from datetime import UTC, datetime
+
+    when = datetime(2026, 8, 8, tzinfo=UTC)
+    all_entities_flat, _all, _map = _prepare_entities_for_resolution(
+        unit_ids=["u1"],
+        sentences=["fact text"],
+        fact_dates=[when],
+        llm_entities=[[{"text": " ", "type": "CONCEPT"}, {"text": "Alice", "type": "CONCEPT"}]],
+    )
+    # The event_date attach loop walks entity_to_unit positionally; a dropped
+    # candidate must not shift it.
+    assert _texts(all_entities_flat) == ["Alice"]
+    assert all_entities_flat[0]["event_date"] == when

hindsight-api-slim/tests/test_gemini_400_fail_fast.py

@@ -0,0 +1,156 @@
+"""Gemini deterministic-400 handling (#3256).
+
+An HTTP 400 ``INVALID_ARGUMENT`` is a deterministic client-side rejection: the
+schema/prompt/generation-config the bank compiled is malformed, so every retry
+repeats an identical rejected call. These tests pin the two fixes:
+
+1. Retry classification — a 400 fails fast; it must NOT consume the LLM retry
+   budget (nor, above it, the batch retry ladder). A retryable 503 still burns
+   the full budget, so the distinction is real.
+2. Diagnosability — a 400 always emits the content-free structural profile
+   (``[LLM_4XX_DUMP]``) even with the opt-in flag off, so the otherwise-opaque
+   failure is diagnosable on first occurrence. A recoverable cache-400 must not
+   be mistaken for a deterministic rejection.
+"""
+
+import logging
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+pytest.importorskip("google.genai")
+
+from google.genai import errors as genai_errors  # noqa: E402
+
+
+def _make_gemini_provider():
+    """Return a GeminiLLM instance with a mocked genai.Client."""
+    with patch("google.genai.Client") as mock_client_cls:
+        mock_client_cls.return_value = MagicMock()
+        from hindsight_api.engine.providers.gemini_llm import GeminiLLM
+
+        provider = GeminiLLM(
+            provider="gemini",
+            api_key="fake-api-key",
+            base_url="",
+            model="gemini-2.5-flash",
+        )
+        provider._client = MagicMock()
+        return provider
+
+
+def _api_error(code: int, status: str = "INVALID_ARGUMENT") -> genai_errors.APIError:
+    return genai_errors.APIError(code, {"error": {"message": f"{status}: rejected", "status": status}})
+
+
+@pytest.mark.asyncio
+async def test_call_400_fails_fast_without_consuming_retries():
+    """A 400 raises after a single attempt — no retry budget is burned."""
+    provider = _make_gemini_provider()
+    generate = AsyncMock(side_effect=_api_error(400))
+    provider._client.aio.models.generate_content = generate
+
+    with pytest.raises(genai_errors.APIError) as excinfo:
+        await provider.call(
+            messages=[{"role": "user", "content": "hi"}],
+            scope="consolidation",
+            max_retries=4,
+            initial_backoff=0.0,
+        )
+
+    assert excinfo.value.code == 400
+    assert generate.call_count == 1  # NOT 5 (1 + 4 retries)
+
+
+@pytest.mark.asyncio
+async def test_call_503_still_consumes_full_retry_budget():
+    """A retryable error still burns the budget — the fail-fast is 400-specific."""
+    provider = _make_gemini_provider()
+    generate = AsyncMock(side_effect=_api_error(503, status="UNAVAILABLE"))
+    provider._client.aio.models.generate_content = generate
+
+    with pytest.raises(genai_errors.APIError):
+        await provider.call(
+            messages=[{"role": "user", "content": "hi"}],
+            scope="consolidation",
+            max_retries=3,
+            initial_backoff=0.0,
+        )
+
+    assert generate.call_count == 4  # 1 + 3 retries
+
+
+@pytest.mark.asyncio
+async def test_call_with_tools_400_fails_fast():
+    """The tool path fails fast on 400 too."""
+    provider = _make_gemini_provider()
+    generate = AsyncMock(side_effect=_api_error(400))
+    provider._client.aio.models.generate_content = generate
+
+    with pytest.raises(genai_errors.APIError) as excinfo:
+        await provider.call_with_tools(
+            messages=[{"role": "user", "content": "hi"}],
+            tools=[
+                {
+                    "type": "function",
+                    "function": {"name": "noop", "description": "n", "parameters": {"type": "object"}},
+                }
+            ],
+            scope="consolidation",
+            max_retries=4,
+            initial_backoff=0.0,
+        )
+
+    assert excinfo.value.code == 400
+    assert generate.call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_call_400_always_dumps_structural_profile(monkeypatch, caplog):
+    """The structural profile is logged on a 400 even with the opt-in flag off,
+    and carries no user content (only per-part sizes)."""
+    from hindsight_api.config import ENV_LLM_DEBUG_DUMP_4XX, clear_config_cache
+
+    monkeypatch.delenv(ENV_LLM_DEBUG_DUMP_4XX, raising=False)
+    clear_config_cache()
+
+    provider = _make_gemini_provider()
+    provider._client.aio.models.generate_content = AsyncMock(side_effect=_api_error(400))
+
+    with caplog.at_level(logging.ERROR):
+        with pytest.raises(genai_errors.APIError):
+            await provider.call(
+                messages=[{"role": "user", "content": "sensitive memory text"}],
+                scope="consolidation",
+                max_retries=4,
+                initial_backoff=0.0,
+            )
+
+    assert "[LLM_4XX_DUMP]" in caplog.text
+    assert "code=400" in caplog.text
+    assert "sensitive memory text" not in caplog.text  # forced dump omits previews
+    clear_config_cache()
+
+
+@pytest.mark.asyncio
+async def test_call_503_does_not_force_dump(monkeypatch, caplog):
+    """A retryable non-4xx never triggers the forced structural dump."""
+    from hindsight_api.config import ENV_LLM_DEBUG_DUMP_4XX, clear_config_cache
+
+    monkeypatch.delenv(ENV_LLM_DEBUG_DUMP_4XX, raising=False)
+    clear_config_cache()
+
+    provider = _make_gemini_provider()
+    provider._client.aio.models.generate_content = AsyncMock(side_effect=_api_error(503, status="UNAVAILABLE"))
+
+    with caplog.at_level(logging.ERROR):
+        with pytest.raises(genai_errors.APIError):
+            await provider.call(
+                messages=[{"role": "user", "content": "hi"}],
+                scope="consolidation",
+                max_retries=1,
+                initial_backoff=0.0,
+            )
+
+    assert "[LLM_4XX_DUMP]" not in caplog.text
+    clear_config_cache()

hindsight-api-slim/tests/test_get_entity_uuid_validation.py

@@ -0,0 +1,65 @@
+"""Regression tests for UUID validation on get_entity and get_observation_history.
+
+Mirrors the pattern in test_delete_memory_units_validation.py: a stub engine
+that bypasses __init__ so the pure-Python validation branches are exercised
+without a DB connection.
+
+get_memory_unit and update_memory_unit already validate their memory_id
+against uuid.UUID (PR #3062, commit in #906). get_entity and
+get_observation_history were missed - a malformed id raised a bare
+ValueError from the stdlib that the HTTP handler mapped to 500 instead
+of 400.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from hindsight_api import RequestContext
+from hindsight_api.engine.memory_engine import MemoryEngine
+
+
+def _stub_engine() -> MemoryEngine:
+    engine = object.__new__(MemoryEngine)
+    engine._authenticate_tenant = AsyncMock()
+    engine._operation_validator = None
+    # 让 _get_backend 返回一个 mock conn，但它永远不会被到达
+    # 因为 uuid.UUID(bad) 会先 raise
+    engine._get_backend = AsyncMock(return_value=MagicMock())
+    return engine
+
+
+BAD_UUID = "not-a-uuid"
+
+
+@pytest.mark.asyncio
+async def test_get_entity_rejects_malformed_uuid():
+    """get_entity 应对畸形 UUID raise ValueError（而非让 uuid.UUID 裸抛）。"""
+    engine = _stub_engine()
+
+    with pytest.raises(ValueError, match="Invalid entity_id"):
+        await engine.get_entity(
+            bank_id="test-bank",
+            entity_id=BAD_UUID,
+            request_context=RequestContext(api_key="anything"),
+        )
+
+    # 不应该到达 DB 层
+    engine._get_backend.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_get_observation_history_rejects_malformed_uuid():
+    """get_observation_history 应对畸形 UUID raise ValueError。"""
+    engine = _stub_engine()
+
+    with pytest.raises(ValueError, match="Invalid memory_id"):
+        await engine.get_observation_history(
+            bank_id="test-bank",
+            memory_id=BAD_UUID,
+            request_context=RequestContext(api_key="anything"),
+        )
+
+    engine._get_backend.assert_not_awaited()

hindsight-api-slim/tests/test_graph_maintenance.py

@@ -191,10 +191,24 @@ class TestEnqueueRelinkVictims:
             assert count == 0
             assert await _queue_unit_ids(conn, bank_id) == []
 
-    # Entity links used to be skipped here by the ``link_type IN ('temporal',
-    # 'semantic')`` filter; they can no longer be stored in memory_links at all
-    # (the CHECK constraint rejects link_type = 'entity'), so there is nothing
-    # left to construct a "skips entity links" scenario from.
+    @pytest.mark.asyncio
+    async def test_skips_entity_links(self, memory: MemoryEngine, request_context: RequestContext):
+        """Entity links are being removed from the product — we don't enqueue for them."""
+        bank_id = f"test-gm-ent-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(memory, bank_id, request_context)
+
+        pool = await memory._get_pool()
+        async with pool.acquire() as conn:
+            doomed = await _insert_unit(conn, bank_id, "doomed")
+            survivor = await _insert_unit(conn, bank_id, "survivor")
+            # Only an entity link — should NOT trigger enqueue.
+            await _insert_link(conn, bank_id, survivor, doomed, "entity")
+
+            backend = await memory._get_backend()
+            async with conn.transaction():
+                count = await enqueue_relink_victims(conn, bank_id, [str(doomed)])
+
+            assert count == 0
 
     @pytest.mark.asyncio
     async def test_dedupes_via_on_conflict(self, memory: MemoryEngine, request_context: RequestContext):

hindsight-api-slim/tests/test_graph_maintenance_claim_serialization.py

@@ -0,0 +1,308 @@
+"""Per-bank serialisation of graph_maintenance at claim time (#3230).
+
+Every graph_maintenance run is the same bank-wide sweep — the payload carries
+only ``bank_id`` and ``run_graph_maintenance_job`` drains the whole queue — so a
+second concurrent run for one bank adds no work while convoying on the first
+run's queue-row locks (``claim_graph_maintenance_batch`` locks ``FOR UPDATE``
+with no ``SKIP LOCKED``) and holding a worker slot.
+
+``claim_tasks`` therefore refuses to claim a graph_maintenance row for a bank
+that already has one in flight, and takes at most one per bank per batch. It
+does that with a predicate on the ordinary shared-pool query rather than a
+claim phase of its own, so graph_maintenance keeps competing by ``created_at``
+instead of dropping below every other operation type — it has no reserved-slot
+floor, and the poller's fairness pass claims with ``shared_limit=1``.
+
+These call ``ops.claim_tasks`` directly rather than ``WorkerPoller.claim_batch``
+so the slot limits under test are exact and not a function of ambient in-flight
+work in the shared test database.
+"""
+
+import json
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import pytest
+import pytest_asyncio
+
+# Use loadgroup to ensure these tests run in the same worker
+# since they share database state
+pytestmark = pytest.mark.xdist_group("worker_tests")
+
+_TABLE = "async_operations"
+
+
+@pytest_asyncio.fixture
+async def backend(pg0_db_url):
+    """Create a DatabaseBackend for claim tests."""
+    from hindsight_api.engine.db import create_database_backend
+    from hindsight_api.pg0 import resolve_database_url
+
+    resolved_url = await resolve_database_url(pg0_db_url)
+
+    b = create_database_backend("postgresql")
+    await b.initialize(resolved_url, min_size=2, max_size=10, command_timeout=30)
+    yield b
+    await b.shutdown()
+
+
+@pytest_asyncio.fixture
+async def pool(backend):
+    """Expose the raw asyncpg pool from the backend for direct DB access in tests."""
+    yield backend.get_pool()
+
+
+@pytest_asyncio.fixture
+async def clean_operations(pool):
+    """Isolate these tests from other pending work in the shared schema.
+
+    Same rationale as test_worker.py's fixture: the claim queries scan the whole
+    schema, so stale pending rows from other tests would be claimed here and
+    consume the (deliberately small) slot limits under test.
+    """
+    await pool.execute("DELETE FROM async_operations WHERE status = 'pending'")
+    yield
+    await pool.execute("DELETE FROM async_operations WHERE bank_id LIKE 'test-gmclaim-%'")
+
+
+async def _make_bank(pool) -> str:
+    """Create a bank with a unique id so concurrent runs can't collide."""
+    bank_id = f"test-gmclaim-{uuid.uuid4().hex[:8]}"
+    await pool.execute(
+        "INSERT INTO banks (bank_id, name) VALUES ($1, $2) ON CONFLICT DO NOTHING",
+        bank_id,
+        bank_id,
+    )
+    return bank_id
+
+
+async def _insert_op(
+    pool,
+    bank_id: str,
+    op_type: str,
+    status: str = "pending",
+    *,
+    created_at: datetime | None = None,
+    claimed_at: datetime | None = None,
+    next_retry_at: datetime | None = None,
+    worker_id: str | None = None,
+) -> uuid.UUID:
+    """Insert one operation row with a claimable payload."""
+    op_id = uuid.uuid4()
+    payload = json.dumps({"type": op_type, "bank_id": bank_id, "operation_id": str(op_id)})
+    await pool.execute(
+        f"""
+        INSERT INTO {_TABLE}
+            (operation_id, bank_id, operation_type, status, task_payload, worker_id, next_retry_at,
+             created_at, claimed_at, updated_at)
+        VALUES ($1, $2, $3, $4, $5::jsonb, $6, $7,
+                COALESCE($8, now()), $9, now())
+        """,
+        op_id,
+        bank_id,
+        op_type,
+        status,
+        payload,
+        worker_id,
+        next_retry_at,
+        created_at,
+        claimed_at,
+    )
+    return op_id
+
+
+async def _claim(backend, *, shared: int = 10, reserved: dict[str, int] | None = None) -> set[str]:
+    """Run one claim cycle and return the claimed operation ids as strings."""
+    async with backend.acquire() as conn:
+        async with conn.transaction():
+            rows = await backend.ops.claim_tasks(
+                conn,
+                _TABLE,
+                "test-gmclaim-worker",
+                reserved or {},
+                shared,
+            )
+    return {str(row["operation_id"]) for row in rows}
+
+
+async def _status_of(pool, op_id: uuid.UUID) -> str:
+    return await pool.fetchval(f"SELECT status FROM {_TABLE} WHERE operation_id = $1", op_id)
+
+
+@pytest.mark.asyncio
+async def test_not_claimed_while_bank_has_run_in_flight(pool, backend, clean_operations):
+    """A pending graph_maintenance is left alone while its bank has one processing."""
+    bank = await _make_bank(pool)
+    await _insert_op(pool, bank, "graph_maintenance", "processing", claimed_at=datetime.now(UTC), worker_id="other")
+    pending = await _insert_op(pool, bank, "graph_maintenance")
+
+    claimed = await _claim(backend)
+
+    assert str(pending) not in claimed
+    assert await _status_of(pool, pending) == "pending"
+
+
+@pytest.mark.asyncio
+async def test_single_batch_claims_at_most_one_per_bank(pool, backend, clean_operations):
+    """One batch takes a single graph_maintenance per bank — the oldest.
+
+    Multiple pending rows for one bank are reachable despite submit-time dedup:
+    ``recover_own_tasks`` resets every processing row for a worker back to
+    pending in one statement, and _schedule_retry / _defer_operation / the admin
+    recover command each restore rows independently.
+    """
+    bank = await _make_bank(pool)
+    base = datetime.now(UTC) - timedelta(minutes=10)
+    op_ids = [
+        await _insert_op(pool, bank, "graph_maintenance", created_at=base + timedelta(seconds=i)) for i in range(5)
+    ]
+
+    claimed = await _claim(backend)
+
+    ours = [op for op in op_ids if str(op) in claimed]
+    assert len(ours) == 1, f"expected exactly one same-bank claim, got {len(ours)}"
+    assert ours[0] == op_ids[0], "the oldest pending row should be the one claimed"
+
+
+@pytest.mark.asyncio
+async def test_idle_bank_still_claimed_while_another_is_busy(pool, backend, clean_operations):
+    """The guard is per bank, not global: a different bank is unaffected."""
+    busy_bank = await _make_bank(pool)
+    idle_bank = await _make_bank(pool)
+    await _insert_op(pool, busy_bank, "graph_maintenance", "processing", claimed_at=datetime.now(UTC))
+    blocked = await _insert_op(pool, busy_bank, "graph_maintenance")
+    claimable = await _insert_op(pool, idle_bank, "graph_maintenance")
+
+    claimed = await _claim(backend)
+
+    assert str(claimable) in claimed
+    assert str(blocked) not in claimed
+
+
+@pytest.mark.asyncio
+async def test_other_operation_types_unaffected(pool, backend, clean_operations):
+    """A busy bank's non-graph_maintenance work is still claimed."""
+    bank = await _make_bank(pool)
+    await _insert_op(pool, bank, "graph_maintenance", "processing", claimed_at=datetime.now(UTC))
+    retain = await _insert_op(pool, bank, "retain")
+
+    claimed = await _claim(backend)
+
+    assert str(retain) in claimed
+
+
+@pytest.mark.asyncio
+async def test_not_starved_by_newer_pending_work(pool, backend, clean_operations):
+    """graph_maintenance still wins the shared slot when it is the oldest row.
+
+    The poller's fairness pass claims with ``shared_limit=1`` and
+    graph_maintenance has no reserved-slot floor, so claiming it in a phase
+    *after* the generic shared-pool query would let any single pending retain
+    starve it indefinitely. As a predicate on that same query it keeps its place
+    in the created_at ordering.
+    """
+    bank = await _make_bank(pool)
+    older = await _insert_op(pool, bank, "graph_maintenance", created_at=datetime.now(UTC) - timedelta(minutes=5))
+    await _insert_op(pool, bank, "retain")
+
+    claimed = await _claim(backend, shared=1)
+
+    assert claimed == {str(older)}
+
+
+@pytest.mark.asyncio
+async def test_retry_blocked_older_row_does_not_block_a_claimable_one(pool, backend, clean_operations):
+    """An older row still in retry backoff must not hold up its bank.
+
+    It cannot be claimed itself, so counting it as "goes first" would stall the
+    bank's graph maintenance for the whole backoff window.
+    """
+    bank = await _make_bank(pool)
+    await _insert_op(
+        pool,
+        bank,
+        "graph_maintenance",
+        created_at=datetime.now(UTC) - timedelta(minutes=5),
+        next_retry_at=datetime.now(UTC) + timedelta(hours=1),
+    )
+    claimable = await _insert_op(pool, bank, "graph_maintenance")
+
+    claimed = await _claim(backend)
+
+    assert str(claimable) in claimed
+
+
+def test_guard_survives_the_oracle_sql_rewrite():
+    """The shared predicate must still be valid Oracle after db/oracle.py rewrites it.
+
+    Oracle tests need an ORACLE_TEST_DSN this suite does not have, so the dialect
+    parity of this guard is otherwise unverified. The rewriter is pure text
+    substitution, so assert on its output directly: PG-only spellings must be
+    gone, and the ROWNUM row limit must land on the *outer* WHERE rather than the
+    subquery's (the rewriter replaces only the first WHERE it sees).
+    """
+    from hindsight_api.engine.db.ops import graph_maintenance_bank_serialization_sql
+    from hindsight_api.engine.db.oracle import _rewrite_pg_to_oracle
+
+    table = "async_operations"
+    rewritten = _rewrite_pg_to_oracle(
+        f"""
+        SELECT o.operation_id FROM {table} o
+        WHERE o.status = 'pending'
+          AND (o.next_retry_at IS NULL OR o.next_retry_at <= NOW())
+          AND o.operation_id != ALL($1::uuid[])
+          AND {graph_maintenance_bank_serialization_sql(table, "o")}
+        ORDER BY o.created_at
+        LIMIT $2
+        FOR UPDATE SKIP LOCKED
+        """
+    ).query
+
+    assert "NOW()" not in rewritten and "SYSTIMESTAMP" in rewritten
+    assert "!= ALL" not in rewritten and "::uuid[]" not in rewritten
+    assert "LIMIT" not in rewritten
+    assert "FOR UPDATE SKIP LOCKED" in rewritten
+    assert "WHERE ROWNUM <= :2 AND o.status = 'pending'" in rewritten
+    # The correlated subquery keeps its own unmodified WHERE.
+    assert "WHERE gm_peer.bank_id = o.bank_id" in rewritten
+
+
+@pytest.mark.asyncio
+async def test_reserved_pool_is_serialised_too(pool, backend, clean_operations):
+    """The guard also applies when graph_maintenance has reserved slots.
+
+    WORKER_SLOT_TYPE_DEFAULTS gives it 0 by default, but an operator can raise
+    HINDSIGHT_API_WORKER_GRAPH_MAINTENANCE_RESERVED_SLOTS, which routes claims
+    through the reserved-pool query instead of the shared one.
+    """
+    bank = await _make_bank(pool)
+    base = datetime.now(UTC) - timedelta(minutes=10)
+    op_ids = [
+        await _insert_op(pool, bank, "graph_maintenance", created_at=base + timedelta(seconds=i)) for i in range(3)
+    ]
+
+    claimed = await _claim(backend, shared=0, reserved={"graph_maintenance": 5})
+
+    ours = [op for op in op_ids if str(op) in claimed]
+    assert len(ours) == 1, f"expected exactly one same-bank claim, got {len(ours)}"
+
+
+@pytest.mark.asyncio
+async def test_reserved_and_shared_phases_do_not_double_claim(pool, backend, clean_operations):
+    """A reserved-pool claim blocks the shared pool from taking a second one.
+
+    The two phases run in one transaction, so the row claimed in phase 1 is still
+    'pending' when the shared query runs and is excluded from it by operation_id.
+    It has to keep blocking through the guard's older-pending branch instead,
+    otherwise one cycle hands the same bank two concurrent runs.
+    """
+    bank = await _make_bank(pool)
+    base = datetime.now(UTC) - timedelta(minutes=10)
+    op_ids = [
+        await _insert_op(pool, bank, "graph_maintenance", created_at=base + timedelta(seconds=i)) for i in range(3)
+    ]
+
+    claimed = await _claim(backend, shared=5, reserved={"graph_maintenance": 1})
+
+    ours = [op for op in op_ids if str(op) in claimed]
+    assert len(ours) == 1, f"expected exactly one same-bank claim across both phases, got {len(ours)}"

hindsight-api-slim/tests/test_health_probes.py

@@ -0,0 +1,179 @@
+"""Liveness must never depend on the database (#3329).
+
+A liveness probe wired to a DB check restarts every pod at once when the
+database is merely slow: in-flight requests die, claimed async operations are
+requeued with ``retry_count`` incremented, and each restarted process reconnects
+to re-warm its pool against a database that is already saturated. These tests
+pin the split — ``/health/live`` answers without touching the database, while
+``/health`` and ``/health/ready`` keep failing closed so readiness gates traffic.
+"""
+
+import asyncio
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from fastapi.testclient import TestClient
+
+_DB_DOWN = {"status": "unhealthy", "database": "error", "error": "connection refused"}
+_DB_UP = {"status": "healthy", "database": "connected", "db_acquire_ms": 0.4}
+
+
+def _api_client(health: dict) -> tuple[TestClient, MagicMock]:
+    """API app whose engine reports ``health``, with no database behind it."""
+    from hindsight_api.api.http import create_app
+
+    memory = MagicMock()
+    # Copy: the worker handler enriches the payload it gets back in place.
+    memory.health_check = AsyncMock(return_value=dict(health))
+    return TestClient(create_app(memory, initialize_memory=False)), memory
+
+
+def _worker_client(health: dict, poller: MagicMock | None = None) -> tuple[TestClient, MagicMock]:
+    """Worker metrics app whose engine reports ``health``."""
+    from hindsight_api.worker.main import create_worker_app
+
+    memory = MagicMock()
+    # Copy: the worker handler enriches the payload it gets back in place.
+    memory.health_check = AsyncMock(return_value=dict(health))
+    if poller is None:
+        poller = MagicMock()
+        poller.worker_id = "w-test"
+        poller.is_shutdown = False
+        poller.seconds_since_last_poll = 0.4
+    return TestClient(create_worker_app(poller, memory)), memory
+
+
+# ---------------------------------------------------------------------------
+# Liveness: DB-free
+# ---------------------------------------------------------------------------
+
+
+def test_api_liveness_stays_200_when_database_is_down():
+    client, memory = _api_client(_DB_DOWN)
+
+    response = client.get("/health/live")
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["status"] == "alive"
+    assert body["version"]
+    assert body["uptime_seconds"] >= 0
+    # The point of the endpoint: it never asks the engine about the database.
+    assert memory.health_check.await_count == 0
+
+
+def test_worker_liveness_stays_200_when_database_is_down():
+    client, memory = _worker_client(_DB_DOWN)
+
+    response = client.get("/health/live")
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["status"] == "alive"
+    assert body["worker_id"] == "w-test"
+    assert body["is_shutdown"] is False
+    assert body["seconds_since_last_poll"] == 0.4
+    assert memory.health_check.await_count == 0
+
+
+def test_worker_liveness_stays_200_when_poller_has_stalled():
+    """A stale poll cycle is reported, never converted into a restart."""
+    poller = MagicMock()
+    poller.worker_id = "w-stalled"
+    poller.is_shutdown = False
+    poller.seconds_since_last_poll = 900.0
+    client, _ = _worker_client(_DB_UP, poller=poller)
+
+    response = client.get("/health/live")
+
+    assert response.status_code == 200
+    assert response.json()["seconds_since_last_poll"] == 900.0
+
+
+def test_worker_liveness_reports_null_poll_age_before_first_cycle():
+    poller = MagicMock()
+    poller.worker_id = "w-fresh"
+    poller.is_shutdown = False
+    poller.seconds_since_last_poll = None
+    client, _ = _worker_client(_DB_UP, poller=poller)
+
+    assert client.get("/health/live").json()["seconds_since_last_poll"] is None
+
+
+# ---------------------------------------------------------------------------
+# Readiness: still gated on the database
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("path", ["/health", "/health/ready"])
+def test_api_readiness_fails_when_database_is_down(path):
+    client, _ = _api_client(_DB_DOWN)
+
+    response = client.get(path)
+
+    assert response.status_code == 503
+    assert response.json()["status"] == "unhealthy"
+
+
+@pytest.mark.parametrize("path", ["/health", "/health/ready"])
+def test_worker_readiness_fails_when_database_is_down(path):
+    client, _ = _worker_client(_DB_DOWN)
+
+    response = client.get(path)
+
+    assert response.status_code == 503
+    assert response.json()["status"] == "unhealthy"
+
+
+def test_api_health_and_ready_return_the_same_payload():
+    """/health is documented as the alias of /health/ready — keep it true."""
+    client, _ = _api_client(_DB_UP)
+
+    health = client.get("/health")
+    ready = client.get("/health/ready")
+
+    assert health.status_code == ready.status_code == 200
+    assert health.json() == ready.json()
+
+
+def test_worker_health_and_ready_return_the_same_payload():
+    client, _ = _worker_client(_DB_UP)
+
+    health = client.get("/health")
+    ready = client.get("/health/ready")
+
+    assert health.status_code == ready.status_code == 200
+    assert health.json() == ready.json() == {**_DB_UP, "worker_id": "w-test", "is_shutdown": False}
+
+
+# ---------------------------------------------------------------------------
+# Poller progress stamp
+# ---------------------------------------------------------------------------
+
+
+def test_poll_age_is_none_until_the_first_cycle_completes():
+    from hindsight_api.worker import WorkerPoller
+
+    poller = WorkerPoller(backend=MagicMock(), worker_id="w-test", executor=AsyncMock())
+
+    assert poller.seconds_since_last_poll is None
+
+
+@pytest.mark.asyncio
+async def test_poll_age_is_stamped_by_the_claim_loop():
+    """One pass of the loop must refresh the stamp, whether or not it claimed work."""
+    from hindsight_api.worker import WorkerPoller
+
+    poller = WorkerPoller(backend=MagicMock(), worker_id="w-test", executor=AsyncMock(), poll_interval_ms=1)
+    poller.recover_own_tasks = AsyncMock()
+    poller.claim_batch = AsyncMock(return_value=[])
+    poller._log_progress_if_due = AsyncMock()
+
+    # Let the loop run a couple of cycles, then signal shutdown.
+    run = asyncio.create_task(poller.run())
+    await asyncio.sleep(0.05)
+    poller._shutdown.set()
+    await run
+
+    assert poller.seconds_since_last_poll is not None
+    assert poller.seconds_since_last_poll >= 0

hindsight-api-slim/tests/test_http_api_integration.py

@@ -1421,6 +1421,28 @@ async def test_patch_config_rejection_does_not_create_bank(api_client):
     await _assert_bank_missing(api_client, test_bank_id)
 
 
+@pytest.mark.asyncio
+async def test_patch_config_rejects_wrong_value_type(api_client):
+    """A JSON object in a string-typed field must 400, not be stored (#3218).
+
+    Storing it succeeded before, and the bank then failed every consolidation
+    with ``expected string or bytes-like object, got 'dict'`` from inside prompt
+    assembly — deterministically, so the bank never recovered.
+    """
+    test_bank_id = f"patch_bad_type_{datetime.now().timestamp()}"
+
+    response = await api_client.patch(
+        f"/v1/default/banks/{test_bank_id}/config",
+        json={"updates": {"observations_mission": {"rules": ["only preferences"], "budget": 3}}},
+    )
+    assert response.status_code == 400, response.text
+    detail = response.json()["detail"]
+    assert "observations_mission" in detail
+    assert "must be a string" in detail
+
+    await _assert_bank_missing(api_client, test_bank_id)
+
+
 @pytest.mark.asyncio
 async def test_patch_config_does_not_apply_client_field_permissions_to_projected_defaults(
     api_client,

hindsight-api-slim/tests/test_knowledge_base.py

@@ -12,12 +12,82 @@ from datetime import datetime, timedelta, timezone
 import pytest_asyncio
 
 from hindsight_api.engine.memory_engine import MemoryEngine, _may_need_refresh
+from hindsight_api.extensions import (
+    BankReadContext,
+    BankReadOperation,
+    BankWriteContext,
+    BankWriteOperation,
+    OperationValidatorExtension,
+    ValidationResult,
+)
 
 
 def _enc(bank_id: str) -> str:
     return urllib.parse.quote(bank_id, safe="")
 
 
+class _RecordingValidator(OperationValidatorExtension):
+    """A validator that records every bank read/write and rejects one operation.
+
+    A concrete subclass (not a MagicMock) so every inherited hook — including the
+    async post-hooks the background mental-model refresh worker fires after a page
+    create — is a real coroutine. Only the named operation is rejected; every other
+    hook (including DELETE_BANK, used by the ``kb_bank`` fixture teardown) accepts.
+    """
+
+    def __init__(
+        self,
+        *,
+        reject_read: BankReadOperation | None = None,
+        reject_write: BankWriteOperation | None = None,
+        reason: str = "operation is forbidden",
+    ) -> None:
+        super().__init__({})
+        self._reject_read = reject_read
+        self._reject_write = reject_write
+        self._reason = reason
+        self.read_ops: list[BankReadOperation] = []
+        self.write_ops: list[BankWriteOperation] = []
+
+    async def validate_retain(self, ctx) -> ValidationResult:
+        return ValidationResult.accept()
+
+    async def validate_recall(self, ctx) -> ValidationResult:
+        return ValidationResult.accept()
+
+    async def validate_reflect(self, ctx) -> ValidationResult:
+        return ValidationResult.accept()
+
+    async def validate_bank_read(self, ctx: BankReadContext) -> ValidationResult:
+        self.read_ops.append(ctx.operation)
+        if ctx.operation is self._reject_read:
+            return ValidationResult.reject(self._reason)
+        return ValidationResult.accept()
+
+    async def validate_bank_write(self, ctx: BankWriteContext) -> ValidationResult:
+        self.write_ops.append(ctx.operation)
+        if ctx.operation is self._reject_write:
+            return ValidationResult.reject(self._reason)
+        return ValidationResult.accept()
+
+
+def _kb_validator(
+    *,
+    reject_read: BankReadOperation | None = None,
+    reject_write: BankWriteOperation | None = None,
+    reason: str = "operation is forbidden",
+) -> _RecordingValidator:
+    return _RecordingValidator(reject_read=reject_read, reject_write=reject_write, reason=reason)
+
+
+def _read_ops(validator: _RecordingValidator) -> list[BankReadOperation]:
+    return list(validator.read_ops)
+
+
+def _write_ops(validator: _RecordingValidator) -> list[BankWriteOperation]:
+    return list(validator.write_ops)
+
+
 class _Seed:
     """Holds the ids created by the seed fixture for assertions."""
 
@@ -376,3 +446,248 @@ class TestMoveRenameDelete:
         # the backing mental model is gone too
         mm = await memory.get_mental_model(bank_id, ids.orders_mm, request_context=request_context)
         assert mm is None
+
+
+class TestAuthorizationReadDenied:
+    """A validator that denies a knowledge-base read blocks it with 403 and leaks
+    nothing — knowledge pages render mental-model content, so this is the sharp
+    edge of #3312 (read-your-neighbour's-synthesized-memories)."""
+
+    async def test_tree_denied(self, api_client, kb_bank, memory, monkeypatch):
+        bank_id, ids = kb_bank
+        validator = _kb_validator(reject_read=BankReadOperation.GET_KNOWLEDGE_BASE_TREE)
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+        resp = await api_client.get(f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/tree")
+        assert resp.status_code == 403, resp.text
+        assert "Orders" not in resp.text
+        assert _read_ops(validator) == [BankReadOperation.GET_KNOWLEDGE_BASE_TREE]
+
+    async def test_get_page_denied(self, api_client, kb_bank, memory, monkeypatch):
+        bank_id, ids = kb_bank
+        validator = _kb_validator(reject_read=BankReadOperation.GET_KNOWLEDGE_PAGE)
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+        resp = await api_client.get(f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/pages/{ids.orders}")
+        assert resp.status_code == 403, resp.text
+        assert "One row per order." not in resp.text
+        assert _read_ops(validator) == [BankReadOperation.GET_KNOWLEDGE_PAGE]
+
+    async def test_search_denied(self, api_client, kb_bank, memory, monkeypatch):
+        bank_id, ids = kb_bank
+        validator = _kb_validator(reject_read=BankReadOperation.SEARCH_KNOWLEDGE_BASE)
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+        resp = await api_client.get(
+            f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/search",
+            params={"q": "orders"},
+        )
+        assert resp.status_code == 403, resp.text
+        assert "Orders" not in resp.text
+        assert _read_ops(validator) == [BankReadOperation.SEARCH_KNOWLEDGE_BASE]
+
+    async def test_export_denied_leaks_nothing_and_gates_once(self, api_client, kb_bank, memory, monkeypatch):
+        bank_id, ids = kb_bank
+        validator = _kb_validator(reject_read=BankReadOperation.EXPORT_KNOWLEDGE_BASE)
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+        resp = await api_client.get(f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/export")
+        assert resp.status_code == 403, resp.text
+        assert "One row per order." not in resp.text
+        # A single export read gate — the per-page reads never run on a denied path.
+        assert _read_ops(validator) == [BankReadOperation.EXPORT_KNOWLEDGE_BASE]
+
+
+class TestAuthorizationWriteDenied:
+    """A validator that denies a knowledge-base write blocks it with 403 and leaves
+    the tree unchanged."""
+
+    async def _tree_names(self, api_client, bank_id) -> set[str]:
+        tree = (await api_client.get(f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/tree")).json()
+
+        def walk(nodes):
+            for n in nodes:
+                yield n["name"]
+                yield from walk(n.get("children", []))
+
+        return set(walk(tree["roots"]))
+
+    async def test_create_folder_denied(self, api_client, kb_bank, memory, monkeypatch):
+        bank_id, ids = kb_bank
+        before = await self._tree_names(api_client, bank_id)
+        validator = _kb_validator(reject_write=BankWriteOperation.CREATE_KNOWLEDGE_FOLDER)
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+        resp = await api_client.post(
+            f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/folders",
+            json={"name": "Guides", "parent_id": None},
+        )
+        assert resp.status_code == 403, resp.text
+        assert _write_ops(validator) == [BankWriteOperation.CREATE_KNOWLEDGE_FOLDER]
+        monkeypatch.setattr(memory, "_operation_validator", None)
+        assert await self._tree_names(api_client, bank_id) == before
+
+    async def test_create_page_denied(self, api_client, kb_bank, memory, monkeypatch):
+        bank_id, ids = kb_bank
+        before = await self._tree_names(api_client, bank_id)
+        validator = _kb_validator(reject_write=BankWriteOperation.CREATE_KNOWLEDGE_PAGE)
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+        resp = await api_client.post(
+            f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/pages",
+            json={"name": "New page", "source_query": "what is new?"},
+        )
+        assert resp.status_code == 403, resp.text
+        # Rejected before the backing mental model is created — a single write hook.
+        assert _write_ops(validator) == [BankWriteOperation.CREATE_KNOWLEDGE_PAGE]
+        monkeypatch.setattr(memory, "_operation_validator", None)
+        assert await self._tree_names(api_client, bank_id) == before
+
+    async def test_rename_denied(self, api_client, kb_bank, memory, monkeypatch):
+        bank_id, ids = kb_bank
+        validator = _kb_validator(reject_write=BankWriteOperation.RENAME_KNOWLEDGE_NODE)
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+        resp = await api_client.patch(
+            f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/nodes/{ids.policies}",
+            json={"name": "Compliance"},
+        )
+        assert resp.status_code == 403, resp.text
+        assert _write_ops(validator) == [BankWriteOperation.RENAME_KNOWLEDGE_NODE]
+        monkeypatch.setattr(memory, "_operation_validator", None)
+        assert "Policies" in await self._tree_names(api_client, bank_id)
+
+    async def test_move_denied(self, api_client, kb_bank, memory, monkeypatch):
+        bank_id, ids = kb_bank
+        validator = _kb_validator(reject_write=BankWriteOperation.MOVE_KNOWLEDGE_NODE)
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+        resp = await api_client.patch(
+            f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/nodes/{ids.loose}",
+            json={"parent_id": ids.policies},
+        )
+        assert resp.status_code == 403, resp.text
+        assert _write_ops(validator) == [BankWriteOperation.MOVE_KNOWLEDGE_NODE]
+
+    async def test_update_page_denied(self, api_client, kb_bank, memory, monkeypatch):
+        bank_id, ids = kb_bank
+        validator = _kb_validator(reject_write=BankWriteOperation.UPDATE_KNOWLEDGE_PAGE)
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+        resp = await api_client.patch(
+            f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/nodes/{ids.orders}",
+            json={"source_query": "changed"},
+        )
+        assert resp.status_code == 403, resp.text
+        # Rejected before touching the backing mental model — a single write hook.
+        assert _write_ops(validator) == [BankWriteOperation.UPDATE_KNOWLEDGE_PAGE]
+
+    async def test_delete_denied(self, api_client, kb_bank, memory, monkeypatch):
+        bank_id, ids = kb_bank
+        validator = _kb_validator(reject_write=BankWriteOperation.DELETE_KNOWLEDGE_NODE)
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+        resp = await api_client.delete(f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/nodes/{ids.runbooks}")
+        assert resp.status_code == 403, resp.text
+        assert _write_ops(validator) == [BankWriteOperation.DELETE_KNOWLEDGE_NODE]
+        monkeypatch.setattr(memory, "_operation_validator", None)
+        assert "Runbooks" in await self._tree_names(api_client, bank_id)
+
+    async def test_denied_create_leaves_no_bank_behind(self, api_client, memory, request_context, monkeypatch):
+        """An unauthorized create must not lazily provision the target bank."""
+        bank_id = f"kb-denied-create-{uuid.uuid4().hex[:8]}"
+        validator = _kb_validator(reject_write=BankWriteOperation.CREATE_KNOWLEDGE_FOLDER)
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+        resp = await api_client.post(
+            f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/folders",
+            json={"name": "Guides", "parent_id": None},
+        )
+        assert resp.status_code == 403, resp.text
+        monkeypatch.setattr(memory, "_operation_validator", None)
+        assert await memory.get_bank_profile(bank_id, request_context=request_context, create_if_missing=False) is None
+
+
+class TestAuthorizationSuccessHookCounts:
+    """Successful knowledge-base routes invoke exactly one validator hook — the
+    knowledge-base operation — and never the nested mental-model hooks."""
+
+    async def test_reads_gate_once(self, api_client, kb_bank, memory, monkeypatch):
+        bank_id, ids = kb_bank
+        validator = _kb_validator()
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+
+        validator.read_ops.clear()
+        await api_client.get(f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/tree")
+        assert _read_ops(validator) == [BankReadOperation.GET_KNOWLEDGE_BASE_TREE]
+
+        validator.read_ops.clear()
+        await api_client.get(f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/pages/{ids.orders}")
+        assert _read_ops(validator) == [BankReadOperation.GET_KNOWLEDGE_PAGE]
+
+        validator.read_ops.clear()
+        await api_client.get(f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/search", params={"q": "orders"})
+        assert _read_ops(validator) == [BankReadOperation.SEARCH_KNOWLEDGE_BASE]
+
+    async def test_export_gates_once_and_suppresses_nested_reads(self, api_client, kb_bank, memory, monkeypatch):
+        bank_id, ids = kb_bank
+        validator = _kb_validator()
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+        validator.read_ops.clear()
+        resp = await api_client.get(f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/export")
+        assert resp.status_code == 200, resp.text
+        # Exactly one gate for the whole bundle; the per-page reads run under it.
+        assert _read_ops(validator) == [BankReadOperation.EXPORT_KNOWLEDGE_BASE]
+
+    async def test_create_page_gates_once_without_nested_mental_model_write(
+        self, kb_bank, memory, request_context, monkeypatch
+    ):
+        # Tested at the engine level: the HTTP route additionally schedules an
+        # async refresh whose background worker later writes the generated content
+        # (a separate, legitimately metered UPDATE_MENTAL_MODEL). Here we assert the
+        # synchronous create in isolation — the backing CREATE_MENTAL_MODEL is
+        # suppressed, so the KB write is the only bank_write hook.
+        bank_id, ids = kb_bank
+        validator = _kb_validator()
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+        validator.write_ops.clear()
+        node = await memory.create_knowledge_page(
+            bank_id,
+            "Fresh page",
+            "what is fresh?",
+            "content",
+            request_context=request_context,
+        )
+        assert node is not None
+        assert _write_ops(validator) == [BankWriteOperation.CREATE_KNOWLEDGE_PAGE]
+
+    async def test_writes_gate_once(self, api_client, kb_bank, memory, monkeypatch):
+        bank_id, ids = kb_bank
+        validator = _kb_validator()
+        monkeypatch.setattr(memory, "_operation_validator", validator)
+
+        validator.write_ops.clear()
+        await api_client.post(f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/folders", json={"name": "Guides"})
+        assert _write_ops(validator) == [BankWriteOperation.CREATE_KNOWLEDGE_FOLDER]
+
+        validator.write_ops.clear()
+        await api_client.patch(
+            f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/nodes/{ids.policies}",
+            json={"name": "Compliance"},
+        )
+        assert _write_ops(validator) == [BankWriteOperation.RENAME_KNOWLEDGE_NODE]
+
+        validator.write_ops.clear()
+        await api_client.patch(
+            f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/nodes/{ids.orders}",
+            json={"tags": ["type:runbook"]},
+        )
+        assert _write_ops(validator) == [BankWriteOperation.UPDATE_KNOWLEDGE_PAGE]
+
+        validator.write_ops.clear()
+        await api_client.delete(f"/v1/default/banks/{_enc(bank_id)}/knowledge-base/nodes/{ids.billing}")
+        assert _write_ops(validator) == [BankWriteOperation.DELETE_KNOWLEDGE_NODE]
+
+
+class TestAuthorizationDisabled:
+    """Without an operation validator (OSS default), knowledge-base routes are
+    unauthenticated-by-tenant and work exactly as before."""
+
+    async def test_engine_calls_do_not_raise(self, memory, request_context):
+        assert memory._operation_validator is None
+        bank_id = f"kb-noauth-{uuid.uuid4().hex[:8]}"
+        folder = await memory.create_knowledge_folder(bank_id, "Docs", request_context=request_context)
+        nodes = await memory.list_knowledge_nodes(bank_id=bank_id, request_context=request_context)
+        assert folder["id"] in {n["id"] for n in nodes}
+        export = await memory.export_knowledge_base(bank_id=bank_id, request_context=request_context)
+        assert any(n["id"] == folder["id"] for n in export.nodes)
+        await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_knowledge_bm25_dispatch.py

@@ -0,0 +1,120 @@
+"""Backend dispatch for the knowledge-page BM25 arm (issue #3268).
+
+``search_knowledge_pages`` used to hard-code the native tsvector SQL
+(``ts_rank_cd`` / ``@@``) for every text-search backend, so it 500'd on
+``pg_search`` / ``pg_textsearch`` / ``vchord`` where ``mental_models.search_vector``
+is not a tsvector. These tests pin the per-backend SQL the read dispatcher emits,
+and the write-side ``search_vector`` tokenization the ``mental_models`` insert/
+update reuse from the memory-recall path. They need no live extension because they
+assert on the generated SQL, the way ``test_multilingual_bm25`` does.
+"""
+
+from types import SimpleNamespace
+
+from hindsight_api._text_search import mental_models_text_document
+from hindsight_api.engine.db.ops_postgresql import pg_search_vector_expr
+from hindsight_api.engine.sql.postgresql import KnowledgeBm25Arm, knowledge_bm25_arm
+
+
+def _cfg(ext: str) -> SimpleNamespace:
+    return SimpleNamespace(text_search_extension=ext, text_search_extension_native_language="english")
+
+
+def _arm(ext: str) -> KnowledgeBm25Arm:
+    return knowledge_bm25_arm(ext, table_alias="mm", text_param="$3")
+
+
+def test_native_uses_tsvector_operators():
+    arm = _arm("native")
+    # The mental_models tsvector is generated with the 'english' config, so the
+    # query must use 'english' regardless of the configured native language.
+    assert "ts_rank_cd(mm.search_vector, websearch_to_tsquery('english', $3))" in arm.score_expr
+    assert arm.match_filter == "AND mm.search_vector @@ websearch_to_tsquery('english', $3)"
+
+
+def test_pgroonga_uses_multilingual_expression_index():
+    arm = _arm("pgroonga")
+    assert arm.score_expr == "pgroonga_score(mm.tableoid, mm.ctid)"
+    assert arm.match_filter == "AND (COALESCE(mm.name, '') || ' ' || mm.content) &@~ pgroonga_query_escape($3)"
+    # pgroonga_score() reads 0 off any plan that did not use the pgroonga index,
+    # so the ordering carries a tiebreak instead of collapsing to input order.
+    assert arm.order_by == "pgroonga_score(mm.tableoid, mm.ctid) DESC, mm.id"
+
+
+def test_pgroonga_filter_repeats_the_indexed_expression_verbatim():
+    """The expression index is only selectable when the query repeats its
+    expression exactly — so both sides must come from the shared helper."""
+    assert mental_models_text_document("mm") in _arm("pgroonga").match_filter
+    assert mental_models_text_document() == "(COALESCE(name, '') || ' ' || content)"
+
+
+def test_pg_search_uses_paradedb_over_base_columns():
+    arm = _arm("pg_search")
+    assert arm.score_expr == "paradedb.score(mm.id)"
+    assert "mm.id @@@ paradedb.boolean(should => ARRAY[" in arm.match_filter
+    assert "paradedb.match('name', $3)" in arm.match_filter
+    assert "paradedb.match('content', $3)" in arm.match_filter
+    # Must not fall back to the native tsvector function.
+    assert "ts_rank_cd" not in arm.score_expr
+    assert "ts_rank_cd" not in arm.order_by
+
+
+def test_pg_textsearch_ranks_content_by_bm25_distance():
+    arm = _arm("pg_textsearch")
+    # `<@>` is a distance (lower = closer): order ASC, negate for the score.
+    assert arm.order_by == "mm.content <@> to_bm25query($3, 'idx_mental_models_text_search') ASC"
+    assert arm.score_expr == "-(mm.content <@> to_bm25query($3, 'idx_mental_models_text_search'))"
+    # It ranks every row, so there is no boolean match gate.
+    assert arm.match_filter == ""
+    assert "ts_rank_cd" not in arm.order_by
+
+
+def test_vchord_ranks_over_bm25vector_search_vector():
+    arm = _arm("vchord")
+    # Negated <&> distance over the bm25vector column and the mental_models index,
+    # gated on a positive score — the same operator build_bm25_arm uses.
+    assert (
+        arm.score_expr
+        == "-(mm.search_vector <&> to_bm25query('idx_mental_models_text_search', tokenize($3, 'llmlingua2')))"
+    )
+    assert arm.order_by.endswith(" DESC")
+    assert arm.match_filter.endswith(" > 0")
+    assert "ts_rank_cd" not in arm.score_expr
+
+
+def test_text_param_and_alias_are_threaded_through():
+    arm = knowledge_bm25_arm("pg_search", table_alias="kbm", text_param="$7")
+    assert "paradedb.score(kbm.id)" == arm.score_expr
+    assert "kbm.id @@@" in arm.match_filter
+    assert "paradedb.match('name', $7)" in arm.match_filter
+
+
+# --- write side: search_vector tokenization for mental_models ---------------
+# mental_models is a two-column (name + content) table whose native search_vector
+# is a GENERATED column, so only vchord needs an inline write (native_inline=False).
+
+
+def test_mm_write_tokenizes_only_for_vchord():
+    for ext in ("native", "pgroonga", "pg_search", "pg_textsearch"):
+        assert (
+            pg_search_vector_expr(_cfg(ext), text_col="$3", context_col="$5", signals_col=None, native_inline=False)
+            is None
+        ), f"{ext} must leave mental_models.search_vector unwritten"
+
+    vchord = pg_search_vector_expr(
+        _cfg("vchord"), text_col="$3", context_col="$5", signals_col=None, native_inline=False
+    )
+    assert vchord == "tokenize(COALESCE($3, '') || ' ' || COALESCE($5, ''), 'llmlingua2')::bm25_catalog.bm25vector"
+
+
+def test_memory_units_default_expr_is_unchanged():
+    # The recall/insert path keeps its three-column, native-inline behaviour.
+    assert pg_search_vector_expr(_cfg("native")) == (
+        "to_tsvector('english'::regconfig, "
+        "COALESCE(text, '') || ' ' || COALESCE(context, '') || ' ' || COALESCE(text_signals, ''))"
+    )
+    assert pg_search_vector_expr(_cfg("vchord")) == (
+        "tokenize(COALESCE(text, '') || ' ' || COALESCE(context, '') || ' ' || COALESCE(text_signals, ''), "
+        "'llmlingua2')::bm25_catalog.bm25vector"
+    )
+    assert pg_search_vector_expr(_cfg("pg_search")) is None

hindsight-api-slim/tests/test_large_document_replacement.py

@@ -168,21 +168,28 @@ async def test_append_after_zero_fact_header_slice_skips_unchanged_history(
     bank_id = f"test_large_append_zero_fact_header_{_ts()}"
     document_id = "chat-session-zero-fact-header"
     header = "# Stable Chat Session\nSession id: session-1\nCreated at: 2026-07-09T00:00:00Z"
+    # Long enough to span several native chunks: slices are cut on chunk
+    # boundaries, so the history has to exceed a few retain_chunk_size windows
+    # for this test to see more than a couple of sub-batches.
     history = "\n".join(
         f"[role: user] turn {turn}: "
         f"{'UNCHANGED_HEAD' if turn == 0 else 'UNCHANGED_MIDDLE' if turn == 30 else 'historical'} "
         "alpha bravo charlie delta echo foxtrot golf hotel india juliet"
-        for turn in range(60)
+        for turn in range(160)
     )
     initial_body = f"{header}\n\n{history}"
     tail_marker = "NEW_APPEND_ONLY_TAIL"
     appended_body = f"{initial_body}\n[role: user] turn 60: {tail_marker} alpha bravo charlie delta"
 
+    # Slices are cut on native chunk boundaries, so a tiny token budget yields
+    # one sub-batch per chunk of the body — the header lands alone in the first
+    # one, followed by the history slices this test replays.
     split = _split_contents_into_sub_batches(
         [{"content": initial_body, "document_id": document_id}],
         100,
+        chunk_size=3000,
     )
-    assert len(split.sub_batches) > 3
+    assert len(split.sub_batches) > 2
     assert "[role:" not in split.sub_batches[0][0]["content"]
 
     extracted_contents: list[str] = []

hindsight-api-slim/tests/test_link_utils.py

@@ -63,15 +63,15 @@ class TestCapLinksPerUnit:
 
     def test_under_cap_unchanged(self):
         links = [
-            ("unit_a", "unit_x", "temporal", 0.9),
-            ("unit_a", "unit_y", "temporal", 0.8),
+            ("unit_a", "unit_x", "temporal", 0.9, None),
+            ("unit_a", "unit_y", "temporal", 0.8, None),
         ]
         result = _cap_links_per_unit(links, max_per_unit=5)
         assert len(result) == 2
 
     def test_caps_to_max_per_unit(self):
         # Create 30 links from the same unit with descending weights
-        links = [("unit_a", f"unit_{i}", "temporal", 1.0 - i * 0.01) for i in range(30)]
+        links = [("unit_a", f"unit_{i}", "temporal", 1.0 - i * 0.01, None) for i in range(30)]
         result = _cap_links_per_unit(links, max_per_unit=10)
         assert len(result) == 10
         # Should keep the highest-weight links
@@ -80,8 +80,8 @@ class TestCapLinksPerUnit:
         assert weights[0] == 1.0  # Highest weight kept
 
     def test_caps_independently_per_unit(self):
-        links_a = [("unit_a", f"target_{i}", "temporal", 0.9 - i * 0.01) for i in range(10)]
-        links_b = [("unit_b", f"target_{i}", "temporal", 0.8 - i * 0.01) for i in range(10)]
+        links_a = [("unit_a", f"target_{i}", "temporal", 0.9 - i * 0.01, None) for i in range(10)]
+        links_b = [("unit_b", f"target_{i}", "temporal", 0.8 - i * 0.01, None) for i in range(10)]
         result = _cap_links_per_unit(links_a + links_b, max_per_unit=5)
         # 5 from unit_a + 5 from unit_b
         assert len(result) == 10
@@ -91,14 +91,14 @@ class TestCapLinksPerUnit:
         assert len(from_b) == 5
 
     def test_default_max_is_temporal_constant(self):
-        links = [("unit_a", f"target_{i}", "temporal", 1.0 - i * 0.01) for i in range(50)]
+        links = [("unit_a", f"target_{i}", "temporal", 1.0 - i * 0.01, None) for i in range(50)]
         result = _cap_links_per_unit(links)
         assert len(result) == MAX_TEMPORAL_LINKS_PER_UNIT
 
     def test_preserves_tuple_structure(self):
-        links = [("from_id", "to_id", "temporal", 0.95)]
+        links = [("from_id", "to_id", "temporal", 0.95, "entity_id")]
         result = _cap_links_per_unit(links, max_per_unit=5)
-        assert result[0] == ("from_id", "to_id", "temporal", 0.95)
+        assert result[0] == ("from_id", "to_id", "temporal", 0.95, "entity_id")
 
 
 class TestComputeSemanticLinksWithinBatch:
@@ -148,6 +148,7 @@ class TestComputeSemanticLinksWithinBatch:
         for lnk in links:
             assert lnk[2] == "semantic"
             assert lnk[3] >= 0.99  # near-1.0 similarity
+            assert lnk[4] is None  # no entity_id
 
     def test_orthogonal_embeddings_no_links(self):
         """Orthogonal embeddings should have similarity=0 (below 0.7 threshold)."""
@@ -218,12 +219,13 @@ class TestComputeSemanticLinksWithinBatch:
             threshold=DEFAULT_SEMANTIC_LINK_MIN_SIMILARITY,
         )
         for lnk in links:
-            assert len(lnk) == 4
-            from_id, to_id, link_type, weight = lnk
+            assert len(lnk) == 5
+            from_id, to_id, link_type, weight, entity_id = lnk
             assert isinstance(from_id, str)
             assert isinstance(to_id, str)
             assert link_type == "semantic"
             assert 0.0 <= weight <= 1.0
+            assert entity_id is None
 
 
 class TestComputeSemanticLinksAnnPgBouncerSafety:

hindsight-api-slim/tests/test_list_banks_non_sql_store.py

@@ -0,0 +1,67 @@
+"""Regression test: list_banks must consult the per-bank capability with the
+*current row's* bank id, and source fact_count from the store when that bank
+keeps its memories outside SQL.
+
+Bug (introduced with per-bank store capabilities, #3350): list_banks called
+``_store.writes_memory_rows_in_sql_for(bank_id)`` with a bare ``bank_id`` name
+that is not in scope inside the per-row loop (the row's id is ``row["bank_id"]``).
+Because the argument is evaluated before the call, this raised
+``NameError: name 'bank_id' is not defined`` for *every* org on the very first
+bank — i.e. GET /banks 500'd outright — regardless of the store's capability.
+
+This test swaps in a store that reports ``writes_memory_rows_in_sql_for -> False``
+(the non-SQL branch the feature added), and asserts list_banks (a) does not raise,
+(b) calls the capability + count_memories with the correct per-bank id, and
+(c) surfaces the store's live count as fact_count.
+
+Runs via: uv run pytest tests/test_list_banks_non_sql_store.py -v
+"""
+
+from __future__ import annotations
+
+import pytest
+
+import hindsight_api.engine.memories as memories_mod
+from hindsight_api.models import RequestContext
+
+
+class _NonSqlStore:
+    """A store that keeps memory rows outside SQL: list_banks must count via the store."""
+
+    def __init__(self):
+        self.capability_calls: list[str] = []
+        self.count_calls: list[str] = []
+
+    def writes_memory_rows_in_sql_for(self, bank_id: str) -> bool:
+        self.capability_calls.append(bank_id)
+        return False
+
+    async def count_memories(self, *, conn, fq_table, bank_id: str) -> dict:
+        self.count_calls.append(bank_id)
+        return {"world": 7}
+
+
+@pytest.mark.asyncio
+async def test_list_banks_counts_via_store_for_non_sql_bank(memory, monkeypatch):
+    bank_id = "list_banks_non_sql_bank"
+    request_context = RequestContext(api_key=None, api_key_id=None, tenant_id=None, internal=False)
+
+    store = _NonSqlStore()
+    monkeypatch.setattr(memories_mod, "get_memories", lambda: store)
+
+    try:
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        # Must not raise NameError; must reach the store's non-SQL count path.
+        banks = await memory.list_banks(request_context=request_context)
+
+        entry = next((b for b in banks if b["bank_id"] == bank_id), None)
+        assert entry is not None, f"bank {bank_id!r} not present in list_banks output"
+
+        # The capability + count were consulted with the row's real bank id.
+        assert bank_id in store.capability_calls
+        assert bank_id in store.count_calls
+        # fact_count came from the store (sum of the per-type counts), not the empty SQL join.
+        assert entry["fact_count"] == 7
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_litellm_forced_tool_structured_output.py

@@ -0,0 +1,198 @@
+"""Structured output via a forced tool call on the LiteLLM-backed providers.
+
+Covers the flag (HINDSIGHT_API_LLM_STRUCTURED_OUTPUT_FORCED_TOOL) end to end:
+config parsing, the config -> LLMProvider -> LiteLLMLLM wiring, the request shape
+it produces, and the response path that substitutes the tool call's arguments for
+the message content. Motivation: Bedrock Claude rejects the ``response_format``
+route outright (``output_config.format: Extra inputs are not permitted``, #3300)
+while accepting the identical schema as a tool.
+"""
+
+from typing import Any
+from unittest.mock import AsyncMock
+
+import pytest
+from pydantic import BaseModel
+
+from hindsight_api.config import ENV_LLM_STRUCTURED_OUTPUT_FORCED_TOOL, HindsightConfig
+from hindsight_api.engine.llm_wrapper import LLMConfig
+from hindsight_api.engine.providers.litellm_llm import LiteLLMLLM
+
+_MESSAGES = [{"role": "user", "content": "hi"}]
+
+
+class _Facts(BaseModel):
+    facts: list[str]
+
+
+def _make_litellm(*, forced_tool: bool) -> LiteLLMLLM:
+    return LiteLLMLLM(
+        provider="bedrock",
+        api_key="",
+        base_url="",
+        model="bedrock/au.anthropic.claude-haiku-4-5-20251001-v1:0",
+        structured_output_forced_tool=forced_tool,
+    )
+
+
+class _Function:
+    def __init__(self, name: str, arguments: Any):
+        self.name = name
+        self.arguments = arguments
+
+
+class _ToolCall:
+    def __init__(self, name: str, arguments: Any):
+        self.id = "call_1"
+        self.function = _Function(name, arguments)
+
+
+class _Message:
+    def __init__(self, content: str | None, tool_calls: list[_ToolCall] | None = None):
+        self.content = content
+        self.tool_calls = tool_calls
+
+
+class _Choice:
+    def __init__(self, message: _Message, finish_reason: str):
+        self.message = message
+        self.finish_reason = finish_reason
+
+
+class _Response:
+    def __init__(self, message: _Message, finish_reason: str = "tool_calls"):
+        self.choices = [_Choice(message, finish_reason)]
+        self.usage = None
+
+
+async def _call_capturing_request(llm: LiteLLMLLM, response: _Response) -> dict[str, Any]:
+    """Run ``call`` against a stubbed completion and return the request kwargs."""
+    completion = AsyncMock(return_value=response)
+    llm._acompletion = completion  # type: ignore[method-assign]
+    result = await llm.call(messages=_MESSAGES, response_format=_Facts, max_retries=0)
+    return {"kwargs": completion.await_args.kwargs, "result": result}
+
+
+# ── config ───────────────────────────────────────────────────────────────────
+
+
+def test_forced_tool_defaults_off(monkeypatch):
+    monkeypatch.delenv(ENV_LLM_STRUCTURED_OUTPUT_FORCED_TOOL, raising=False)
+    assert HindsightConfig.from_env().llm_structured_output_forced_tool is False
+
+
+def test_forced_tool_can_be_enabled(monkeypatch):
+    monkeypatch.setenv(ENV_LLM_STRUCTURED_OUTPUT_FORCED_TOOL, "true")
+    assert HindsightConfig.from_env().llm_structured_output_forced_tool is True
+
+
+@pytest.mark.parametrize("value", ["", "yes", "tru", "enabled"])
+def test_forced_tool_rejects_ambiguous_values(monkeypatch, value):
+    monkeypatch.setenv(ENV_LLM_STRUCTURED_OUTPUT_FORCED_TOOL, value)
+
+    with pytest.raises(ValueError, match=ENV_LLM_STRUCTURED_OUTPUT_FORCED_TOOL):
+        HindsightConfig.from_env()
+
+
+def test_llm_config_threads_flag_to_provider_impl():
+    """LLMConfig -> create_llm_provider -> LiteLLMLLM carries the flag.
+
+    Without this bridge the env var is inert: the provider silently keeps the
+    default ``response_format`` transport.
+    """
+    llm = LLMConfig(
+        provider="bedrock",
+        api_key="",
+        base_url="",
+        model="au.anthropic.claude-haiku-4-5-20251001-v1:0",
+        structured_output_forced_tool=True,
+    )
+    assert llm._provider_impl.structured_output_forced_tool is True
+
+
+# ── request shape ────────────────────────────────────────────────────────────
+
+
+@pytest.mark.asyncio
+async def test_forced_tool_replaces_response_format_with_a_forced_tool():
+    llm = _make_litellm(forced_tool=True)
+    response = _Response(_Message(None, [_ToolCall("structured_response", '{"facts": ["the sky is blue"]}')]))
+
+    captured = await _call_capturing_request(llm, response)
+
+    kwargs = captured["kwargs"]
+    assert "response_format" not in kwargs
+    assert kwargs["tool_choice"] == {"type": "function", "function": {"name": "structured_response"}}
+    assert len(kwargs["tools"]) == 1
+    function = kwargs["tools"][0]["function"]
+    assert function["name"] == "structured_response"
+    assert function["parameters"] == _Facts.model_json_schema()
+    assert captured["result"] == _Facts(facts=["the sky is blue"])
+
+
+@pytest.mark.asyncio
+async def test_flag_off_keeps_response_format():
+    llm = _make_litellm(forced_tool=False)
+    response = _Response(_Message('{"facts": ["the sky is blue"]}'), finish_reason="stop")
+
+    captured = await _call_capturing_request(llm, response)
+
+    kwargs = captured["kwargs"]
+    assert "tools" not in kwargs
+    assert "tool_choice" not in kwargs
+    assert kwargs["response_format"]["json_schema"]["schema"] == _Facts.model_json_schema()
+    assert captured["result"] == _Facts(facts=["the sky is blue"])
+
+
+@pytest.mark.asyncio
+async def test_plain_calls_are_untouched_by_the_flag():
+    """No ``response_format`` -> no tool is forced, so free-text calls still work."""
+    llm = _make_litellm(forced_tool=True)
+    completion = AsyncMock(return_value=_Response(_Message("hello"), finish_reason="stop"))
+    llm._acompletion = completion  # type: ignore[method-assign]
+
+    result = await llm.call(messages=_MESSAGES, max_retries=0)
+
+    assert result == "hello"
+    assert "tools" not in completion.await_args.kwargs
+
+
+# ── response path ────────────────────────────────────────────────────────────
+
+
+@pytest.mark.asyncio
+async def test_forced_tool_accepts_already_decoded_arguments():
+    """Some providers hand back decoded arguments instead of a JSON string."""
+    llm = _make_litellm(forced_tool=True)
+    response = _Response(_Message(None, [_ToolCall("structured_response", {"facts": ["grass is green"]})]))
+
+    captured = await _call_capturing_request(llm, response)
+
+    assert captured["result"] == _Facts(facts=["grass is green"])
+
+
+@pytest.mark.asyncio
+async def test_falls_back_to_text_when_the_tool_call_is_missing():
+    """A gateway that drops ``tool_choice`` must not hard-fail the call."""
+    llm = _make_litellm(forced_tool=True)
+    response = _Response(_Message('{"facts": ["parsed from text"]}'), finish_reason="stop")
+
+    captured = await _call_capturing_request(llm, response)
+
+    assert captured["result"] == _Facts(facts=["parsed from text"])
+
+
+@pytest.mark.asyncio
+async def test_skip_validation_returns_the_raw_tool_arguments():
+    llm = _make_litellm(forced_tool=True)
+    response = _Response(_Message(None, [_ToolCall("structured_response", '{"facts": ["raw"]}')]))
+    llm._acompletion = AsyncMock(return_value=response)  # type: ignore[method-assign]
+
+    result = await llm.call(
+        messages=_MESSAGES,
+        response_format=_Facts,
+        skip_validation=True,
+        max_retries=0,
+    )
+
+    assert result == {"facts": ["raw"]}

hindsight-api-slim/tests/test_llm_4xx_dump.py

@@ -116,6 +116,39 @@ def test_dump_pydantic_config_with_separate_contents(monkeypatch, caplog):
     assert "gemini hi" in caplog.text  # content preview
 
 
+# ── force: always-on structural profile for deterministic rejections ───────────
+
+
+def test_force_dumps_structural_profile_when_flag_off(monkeypatch, caplog):
+    """A forced dump (deterministic 400) logs even with the opt-in flag off, but
+    carries only the structural profile — config + per-part sizes, no previews."""
+    _disable(monkeypatch)
+    request = {"messages": [{"role": "user", "content": "secret memory content"}]}
+    with caplog.at_level(logging.ERROR):
+        _dump(err=SimpleNamespace(status_code=400), request=request, force=True)
+    assert "[LLM_4XX_DUMP]" in caplog.text
+    assert '"chars": 21' in caplog.text  # per-part size is logged
+    assert "secret memory content" not in caplog.text  # but user content is not
+    assert "preview" not in caplog.text
+
+
+def test_force_is_still_noop_on_non_4xx(monkeypatch, caplog):
+    """force bypasses the opt-in flag, not the 4xx gate."""
+    _disable(monkeypatch)
+    with caplog.at_level(logging.ERROR):
+        _dump(err=SimpleNamespace(status_code=500), request={"messages": []}, force=True)
+    assert "[LLM_4XX_DUMP]" not in caplog.text
+
+
+def test_force_includes_previews_when_flag_also_on(monkeypatch, caplog):
+    """With the opt-in flag on, a forced dump still gets the full preview."""
+    _enable(monkeypatch)
+    request = {"messages": [{"role": "user", "content": "visible content"}]}
+    with caplog.at_level(logging.ERROR):
+        _dump(err=SimpleNamespace(status_code=400), request=request, force=True)
+    assert '"preview": "visible content"' in caplog.text
+
+
 # ── safety: truncation + never-raise ───────────────────────────────────────────
 
 

hindsight-api-slim/tests/test_llm_router_provider.py

@@ -195,6 +195,7 @@ def _make_router_provider(config: dict[str, Any], mock_router: Any) -> LiteLLMRo
         provider.reasoning_effort = "low"
         provider.timeout = 300.0
         provider._default_headers = {}
+        provider.structured_output_forced_tool = False
         provider.config = config
         provider._litellm = fake_litellm
         provider._router = mock_router

hindsight-api-slim/tests/test_maintenance_loop.py

@@ -151,6 +151,9 @@ class TestOperationCleanupJob:
         engine = MagicMock()
         engine._backend = backend
         engine._tenant_extension.list_tenants = AsyncMock(return_value=[SimpleNamespace(schema=s) for s in tenants])
+        # The sweep purges expired export archives before pruning each schema's rows;
+        # stub it as a no-op so these tests stay focused on discovery + prune scoping.
+        engine.purge_expired_export_archives = AsyncMock(return_value=0)
         return engine, backend, conn
 
     @staticmethod

hindsight-api-slim/tests/test_memories_extension.py

@@ -601,6 +601,59 @@ async def test_maintenance_passes_are_optional(restore_default_store):
     await store.record_unit_entities(conn=None, ops=None, fq_table=None, unit_ids=["u"], entity_ids=["e"])
 
 
+# ---------------------------------------------------------------------------
+# Per-bank store capabilities. A store may route different banks to different
+# backends, so every BANK-SCOPED call site asks per bank —
+# writes_memory_rows_in_sql_for(bank_id) / owns_document_store_for(bank_id) —
+# rather than reading the process-global class attribute. The class attribute
+# stays the single-store default the _for methods fall back to.
+# ---------------------------------------------------------------------------
+
+
+def test_per_bank_capability_defaults_to_the_class_attribute():
+    """A single-store extension needs no override: the _for methods return the class attr, so
+    every existing store keeps its exact behaviour for every bank."""
+    pg = PostgresMemories({})
+    assert (pg.writes_memory_rows_in_sql, pg.owns_document_store) == (True, False)
+    assert pg.writes_memory_rows_in_sql_for("any-bank") is True
+    assert pg.owns_document_store_for("any-bank") is False
+
+    mem = InMemoryMemories({})  # owns its rows AND its document store
+    assert mem.writes_memory_rows_in_sql_for("any-bank") is False
+    assert mem.owns_document_store_for("any-bank") is True
+
+
+def test_a_store_answers_capabilities_per_bank():
+    """The point of the _for methods: a store that keeps some banks in SQL and others in a
+    separate store answers PER BANK, so mixed banks in one process each take the right path."""
+
+    class PerBankStore(InMemoryMemories):
+        name = "per-bank"
+        # The loop-level class attr stays False so cross-store txn recovery still runs; the
+        # per-bank answer is what every bank-scoped site consults.
+        writes_memory_rows_in_sql = False
+
+        def __init__(self, config=None):
+            super().__init__(config)
+            self.sql_banks = {"legacy-bank"}
+
+        def writes_memory_rows_in_sql_for(self, bank_id):
+            return bank_id in self.sql_banks
+
+        def owns_document_store_for(self, bank_id):
+            return bank_id not in self.sql_banks
+
+    store = PerBankStore({})
+    # A SQL-backed bank looks like Postgres (host does inline SQL, keeps documents in SQL)...
+    assert store.writes_memory_rows_in_sql_for("legacy-bank") is True
+    assert store.owns_document_store_for("legacy-bank") is False
+    # ...a store-backed bank owns its rows and its document store.
+    assert store.writes_memory_rows_in_sql_for("new-bank") is False
+    assert store.owns_document_store_for("new-bank") is True
+    # The process-level gate (cross-store recovery loop) still fires off the class attr.
+    assert store.writes_memory_rows_in_sql is False
+
+
 # ---------------------------------------------------------------------------
 # Interface conformance: the stub must stay a COMPLETE, signature-compatible
 # implementation of every MemoriesExtension method. This is the guard that keeps

hindsight-api-slim/tests/test_migration_drop_memory_links_entity.py

@@ -1,153 +0,0 @@
-"""Regression for the ``memory_links`` entity-schema drop (``c1e7a9d3f5b2``).
-
-Entity edges stopped being materialized in ``memory_links`` once retain moved
-memory-to-entity associations to ``unit_entities`` and the read paths (the /graph
-endpoint and recall) began deriving entity edges from that table. Migration
-``e9b2c7d1f3a4`` deleted the stored entity rows; this migration removes the
-now-dead entity *schema* — the ``entity_id`` column and FK, the entity index,
-``link_type = 'entity'`` from the CHECK, and the ``entity_id`` term in the
-function-based unique index (which collapses to ``(from_unit_id, to_unit_id,
-link_type)``).
-
-Uses a dedicated pg0 instance (mirrors test_migration_drop_access_count) so it
-controls exactly which migrations have run and never stamps the shared test
-instance.
-"""
-
-import asyncio
-from pathlib import Path
-
-import pytest
-from alembic import command
-from alembic.config import Config
-from sqlalchemy import create_engine, text
-
-# One module-scoped pg0 instance shared across tests; pin the module to a single
-# xdist worker so concurrent workers don't race to provision the same instance
-# or re-migrate a DB another worker is reading.
-pytestmark = pytest.mark.xdist_group("migration-drop-memory-links-entity-pg0")
-
-_SCRIPT_LOCATION = str(Path(__file__).parent.parent / "hindsight_api" / "alembic")
-
-_DROP_REVISION = "c1e7a9d3f5b2"
-# Revision immediately before the drop.
-_PRE_DROP_REVISION = "e4a7c1b9d2f6"
-
-
-def _alembic_cfg(db_url: str) -> Config:
-    cfg = Config()
-    cfg.set_main_option("script_location", _SCRIPT_LOCATION)
-    cfg.set_main_option("sqlalchemy.url", db_url)
-    cfg.set_main_option("prepend_sys_path", ".")
-    cfg.set_main_option("path_separator", "os")
-    return cfg
-
-
-def _columns(conn, table: str) -> set[str]:
-    return {
-        r[0]
-        for r in conn.execute(
-            text("SELECT column_name FROM information_schema.columns WHERE table_name = :t"),
-            {"t": table},
-        )
-    }
-
-
-def _index_exists(conn, name: str) -> bool:
-    return bool(conn.execute(text("SELECT 1 FROM pg_indexes WHERE indexname = :n"), {"n": name}).scalar())
-
-
-def _index_def(conn, name: str) -> str:
-    return conn.execute(text("SELECT indexdef FROM pg_indexes WHERE indexname = :n"), {"n": name}).scalar() or ""
-
-
-def _link_type_check_def(conn) -> str:
-    return (
-        conn.execute(
-            text("SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conname = 'memory_links_link_type_check'")
-        ).scalar()
-        or ""
-    )
-
-
-@pytest.fixture(scope="module")
-def head_db_url():
-    """pg0 instance migrated to head (includes the entity-schema drop)."""
-    from hindsight_api.pg0 import EmbeddedPostgres
-
-    # port=None lets pg0 auto-assign a free port; a hardcoded port is not xdist-safe.
-    pg0 = EmbeddedPostgres(name="hindsight-drop-ml-entity-test", port=None)
-    loop = asyncio.new_event_loop()
-    try:
-        url = loop.run_until_complete(pg0.ensure_running())
-    finally:
-        loop.close()
-
-    command.upgrade(_alembic_cfg(url), "heads")
-    return url
-
-
-def test_entity_schema_is_gone(head_db_url):
-    engine = create_engine(head_db_url)
-    try:
-        with engine.connect() as conn:
-            assert "entity_id" not in _columns(conn, "memory_links"), (
-                "memory_links.entity_id still exists at head — entity edges are derived from unit_entities"
-            )
-            assert not _index_exists(conn, "idx_memory_links_entity"), "the entity index should be gone with the column"
-    finally:
-        engine.dispose()
-
-
-def test_unique_index_is_three_columns(head_db_url):
-    engine = create_engine(head_db_url)
-    try:
-        with engine.connect() as conn:
-            definition = _index_def(conn, "idx_memory_links_unique").lower()
-            assert definition, "idx_memory_links_unique is missing"
-            assert "unique" in definition
-            for col in ("from_unit_id", "to_unit_id", "link_type"):
-                assert col in definition, f"{col} missing from idx_memory_links_unique"
-            # The old expression key coalesced a nullable entity_id; both must be gone.
-            assert "entity_id" not in definition
-            assert "coalesce" not in definition
-    finally:
-        engine.dispose()
-
-
-def test_check_rejects_entity_keeps_causal(head_db_url):
-    engine = create_engine(head_db_url)
-    try:
-        with engine.connect() as conn:
-            definition = _link_type_check_def(conn).lower()
-            assert definition, "memory_links_link_type_check is missing"
-            assert "'entity'" not in definition, "CHECK should no longer permit link_type = 'entity'"
-            for keep in ("'temporal'", "'semantic'", "'caused_by'", "'causes'", "'enables'", "'prevents'"):
-                assert keep in definition, f"CHECK dropped a still-supported link_type {keep}"
-    finally:
-        engine.dispose()
-
-
-def test_downgrade_restores_entity_schema(head_db_url):
-    """Downgrade restores the former schema shape (column, index, CHECK), and
-    re-upgrading removes it again — so the migration is not a one-way door."""
-    cfg = _alembic_cfg(head_db_url)
-    engine = create_engine(head_db_url)
-    try:
-        command.downgrade(cfg, _PRE_DROP_REVISION)
-        with engine.connect() as conn:
-            assert "entity_id" in _columns(conn, "memory_links"), "downgrade did not restore memory_links.entity_id"
-            assert _index_exists(conn, "idx_memory_links_entity")
-            assert "'entity'" in _link_type_check_def(conn).lower()
-            assert "entity_id" in _index_def(conn, "idx_memory_links_unique").lower()
-
-        command.upgrade(cfg, _DROP_REVISION)
-        with engine.connect() as conn:
-            assert "entity_id" not in _columns(conn, "memory_links")
-            assert not _index_exists(conn, "idx_memory_links_entity")
-            assert "'entity'" not in _link_type_check_def(conn).lower()
-            assert "entity_id" not in _index_def(conn, "idx_memory_links_unique").lower()
-    finally:
-        # Leave the module fixture's instance at head for any later user.
-        command.upgrade(cfg, "heads")
-        engine.dispose()

hindsight-api-slim/tests/test_observation_invalidation.py

@@ -33,6 +33,7 @@ async def _insert_memory(
     text: str,
     fact_type: str = "experience",
     document_id: str | None = None,
+    chunk_id: str | None = None,
 ) -> uuid.UUID:
     """Seed one memory through the store, bypassing the LLM retain pipeline.
 
@@ -47,7 +48,7 @@ async def _insert_memory(
         tags=[],
         context=None,
         document_id=document_id,
-        chunk_id=None,
+        chunk_id=chunk_id,
         metadata=None,
         observation_scopes=None,
         entities=[],
@@ -435,6 +436,187 @@ class TestDocumentUpsertObservationCleanup:
         await memory.delete_bank(bank_id, request_context=request_context)
 
 
+# ---------------------------------------------------------------------------
+# Tests: delta retain chunk delete (regression for orphan observations, #3294)
+# ---------------------------------------------------------------------------
+
+
+async def _seed_chunked_document(memory: MemoryEngine, conn, bank_id: str, chunk_texts: list[str]) -> tuple[str, list]:
+    """One document with ``len(chunk_texts)`` chunks, each owning one fact.
+
+    Returns the document id and, per chunk, the (chunk_id, fact_id) it owns.
+    """
+    doc_id = str(uuid.uuid4())
+    await conn.execute(
+        """
+        INSERT INTO documents (id, bank_id, original_text, content_hash, created_at, updated_at)
+        VALUES ($1, $2, $3, 'hash-old', NOW(), NOW())
+        """,
+        doc_id,
+        bank_id,
+        "\n\n".join(chunk_texts),
+    )
+    seeded = []
+    for idx, text in enumerate(chunk_texts):
+        chunk_id = f"{bank_id}_{doc_id}_{idx}"
+        await conn.execute(
+            """
+            INSERT INTO chunks (chunk_id, document_id, bank_id, chunk_index, chunk_text, content_hash)
+            VALUES ($1, $2, $3, $4, $5, $6)
+            """,
+            chunk_id,
+            doc_id,
+            bank_id,
+            idx,
+            text,
+            f"chunk-hash-{idx}",
+        )
+        fact_id = await _insert_memory(memory, conn, bank_id, text, "experience", document_id=doc_id, chunk_id=chunk_id)
+        seeded.append((chunk_id, fact_id))
+    return doc_id, seeded
+
+
+class TestDeltaChunkDeleteObservationCleanup:
+    """Regression: delta retain drops facts by deleting their chunks, and that
+    cascade must invalidate the observations derived from them.
+
+    ``handle_document_tracking`` (full replace) sweeps observations before its
+    delete, but the delta path never calls it — it upserts the document row and
+    deletes the changed/removed chunks directly, cascading to memory_units. Every
+    delta re-ingest therefore used to leave the observations of the changed chunks
+    behind, valid and recallable, pointing at ids that no longer exist. Nothing
+    could reach them afterwards: consolidation batches are built from facts, so an
+    observation whose sources are all gone is never selected into a batch again.
+    """
+
+    @pytest.mark.asyncio
+    async def test_chunk_delete_removes_observations_from_outgoing_memories(
+        self, memory: MemoryEngine, request_context: RequestContext
+    ):
+        """The observation of a deleted chunk's fact goes with it; its surviving
+        co-source is requeued for re-consolidation."""
+        from hindsight_api.engine.retain.chunk_storage import delete_chunks_by_ids
+
+        bank_id = f"test-delta-obs-cleanup-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(memory, bank_id, request_context)
+        pool = await memory._get_pool()
+
+        async with pool.acquire() as conn:
+            _doc_id, seeded = await _seed_chunked_document(
+                memory, conn, bank_id, ["Alice works at Google.", "Bob works at Microsoft."]
+            )
+            (outgoing_chunk, outgoing_fact), (_kept_chunk, kept_fact) = seeded
+            standalone_fact = await _insert_memory(memory, conn, bank_id, "Carol works at Netflix.")
+            obs_id = await _insert_observation(
+                memory,
+                conn,
+                bank_id,
+                "The team is spread across Google, Microsoft and Netflix.",
+                [outgoing_fact, kept_fact, standalone_fact],
+            )
+
+        async with pool.acquire() as conn:
+            async with conn.transaction():
+                invalidated = await delete_chunks_by_ids(conn, [outgoing_chunk], bank_id, ops=memory._backend.ops)
+
+        assert invalidated == 1, "delete_chunks_by_ids should report the observation it invalidated"
+
+        async with pool.acquire() as conn:
+            assert str(obs_id) not in await _get_observation_ids(conn, bank_id), (
+                "Observation derived from the deleted chunk's fact should have been invalidated "
+                "(regression #3294: the delta path cascaded the fact away and left the orphan)"
+            )
+            assert await _count_surviving(conn, bank_id, [outgoing_fact]) == 0, "The chunk's fact is gone"
+            # Both surviving co-sources lost an observation, so both are due for re-consolidation.
+            for survivor in (kept_fact, standalone_fact):
+                assert await _get_consolidated_at(conn, survivor, bank_id) is None, (
+                    "Surviving co-source should be reset for re-consolidation"
+                )
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    @pytest.mark.asyncio
+    async def test_chunk_delete_keeps_observations_of_unchanged_chunks(
+        self, memory: MemoryEngine, request_context: RequestContext
+    ):
+        """Precision: an observation sourced only from chunks that stay is untouched.
+
+        A sweep keyed on the document rather than on the deleted chunks would take
+        this one too — and needlessly requeue the whole document for consolidation
+        on every small edit, which is exactly the case delta retain exists for.
+        """
+        from hindsight_api.engine.retain.chunk_storage import delete_chunks_by_ids
+
+        bank_id = f"test-delta-obs-keep-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(memory, bank_id, request_context)
+        pool = await memory._get_pool()
+
+        async with pool.acquire() as conn:
+            _doc_id, seeded = await _seed_chunked_document(
+                memory, conn, bank_id, ["Alice works at Google.", "Bob works at Microsoft."]
+            )
+            (outgoing_chunk, _outgoing_fact), (_kept_chunk, kept_fact) = seeded
+            kept_obs = await _insert_observation(memory, conn, bank_id, "Bob is at Microsoft.", [kept_fact])
+
+        async with pool.acquire() as conn:
+            async with conn.transaction():
+                invalidated = await delete_chunks_by_ids(conn, [outgoing_chunk], bank_id, ops=memory._backend.ops)
+
+        assert invalidated == 0, "No observation of the surviving chunk should have been touched"
+
+        async with pool.acquire() as conn:
+            assert str(kept_obs) in await _get_observation_ids(conn, bank_id), (
+                "Observation of an unchanged chunk must survive the delta delete"
+            )
+            assert await _get_consolidated_at(conn, kept_fact, bank_id) is not None, (
+                "An untouched fact must not be requeued for consolidation"
+            )
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    @pytest.mark.asyncio
+    async def test_chunk_delete_sweeps_every_deleted_chunk(self, memory: MemoryEngine, request_context: RequestContext):
+        """Multiple chunks in one call: each one's observations are swept.
+
+        The reporter's bank lost the observations of a whole document at once
+        (25 fully orphaned from a single replace), so the sweep must cover the
+        entire ``chunks_to_delete`` list, not just the first entry.
+        """
+        from hindsight_api.engine.retain.chunk_storage import delete_chunks_by_ids
+
+        bank_id = f"test-delta-obs-multi-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(memory, bank_id, request_context)
+        pool = await memory._get_pool()
+
+        async with pool.acquire() as conn:
+            _doc_id, seeded = await _seed_chunked_document(
+                memory,
+                conn,
+                bank_id,
+                ["Alice works at Google.", "Bob works at Microsoft.", "Dan works at Apple."],
+            )
+            observations = [
+                await _insert_observation(memory, conn, bank_id, f"Observation of chunk {i}.", [fact])
+                for i, (_chunk, fact) in enumerate(seeded)
+            ]
+
+        async with pool.acquire() as conn:
+            async with conn.transaction():
+                invalidated = await delete_chunks_by_ids(
+                    conn, [chunk for chunk, _fact in seeded], bank_id, ops=memory._backend.ops
+                )
+
+        assert invalidated == 3
+
+        async with pool.acquire() as conn:
+            remaining = await _get_observation_ids(conn, bank_id)
+            assert [o for o in observations if str(o) in remaining] == [], (
+                "Every deleted chunk's observations should be swept, not just the first chunk's"
+            )
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
 # ---------------------------------------------------------------------------
 # Tests: delete_bank with fact_type filter
 # ---------------------------------------------------------------------------

hindsight-api-slim/tests/test_ollama_native_think.py

@@ -0,0 +1,111 @@
+"""
+Regression tests for Ollama native API extra_body handling.
+
+The native /api/chat payload has two tiers: native top-level fields (``think``,
+``keep_alive``, ...) and a nested ``options`` object (``seed``, ``top_p``,
+``num_ctx``, ...). Configured ``extra_body`` must reach both, so operators can
+enable thinking for gpt-oss models (``{"think": "low"}``) or tune generation
+options without a code change (see #3246).
+"""
+
+import json
+from unittest.mock import AsyncMock, patch
+
+import httpx
+import pytest
+from pydantic import BaseModel
+
+from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
+
+
+class _SampleOutput(BaseModel):
+    summary: str
+
+
+def _make_ollama_llm(model: str, extra_body: dict | None = None) -> OpenAICompatibleLLM:
+    return OpenAICompatibleLLM(
+        provider="ollama",
+        api_key="",
+        base_url="http://localhost:11434/v1",
+        model=model,
+        extra_body=extra_body,
+    )
+
+
+def _mock_ollama_response(content: dict) -> httpx.Response:
+    body = {
+        "model": "test-model",
+        "message": {
+            "role": "assistant",
+            "content": json.dumps(content),
+        },
+        "done": True,
+    }
+    request = httpx.Request("POST", "http://localhost:11434/api/chat")
+    return httpx.Response(200, json=body, request=request)
+
+
+async def _capture_payload(llm: OpenAICompatibleLLM) -> dict:
+    mock_client = AsyncMock()
+    mock_client.post.return_value = _mock_ollama_response({"summary": "test"})
+    mock_client.__aenter__.return_value = mock_client
+
+    with patch(
+        "hindsight_api.engine.providers.openai_compatible_llm.httpx.AsyncClient",
+        return_value=mock_client,
+    ):
+        await llm._call_ollama_native(
+            messages=[{"role": "user", "content": "hello"}],
+            response_format=_SampleOutput,
+            max_completion_tokens=512,
+            temperature=0.1,
+            max_retries=0,
+            initial_backoff=1.0,
+            max_backoff=10.0,
+            skip_validation=True,
+        )
+
+    request = mock_client.post.call_args
+    assert request is not None
+    return request.kwargs["json"]
+
+
+@pytest.mark.asyncio
+async def test_ollama_native_think_defaults_false():
+    """Thinking is disabled by default and structured-output format is included."""
+    payload = await _capture_payload(_make_ollama_llm("qwen3.5:2b"))
+
+    assert payload["think"] is False
+    assert "format" in payload
+    assert payload["options"]["num_predict"] == 512
+    assert payload["options"]["temperature"] == 0.1
+
+
+@pytest.mark.asyncio
+async def test_ollama_native_think_override_via_extra_body():
+    """extra_body top-level field overrides the think default (gpt-oss path)."""
+    payload = await _capture_payload(_make_ollama_llm("gpt-oss:20b", extra_body={"think": "low"}))
+
+    assert payload["think"] == "low"
+    # Computed options are preserved alongside the top-level override.
+    assert payload["options"]["num_predict"] == 512
+    assert payload["options"]["temperature"] == 0.1
+
+
+@pytest.mark.asyncio
+async def test_ollama_native_options_merge_via_extra_body():
+    """An extra_body "options" sub-dict merges into native generation options."""
+    payload = await _capture_payload(
+        _make_ollama_llm(
+            "qwen3.5:2b",
+            extra_body={"options": {"seed": 42, "temperature": 0.9}},
+        )
+    )
+
+    # New option added, and a user value wins over the computed default.
+    assert payload["options"]["seed"] == 42
+    assert payload["options"]["temperature"] == 0.9
+    assert payload["options"]["num_predict"] == 512
+    # "options" is not leaked as a top-level payload field.
+    assert "options" in payload
+    assert payload["think"] is False

hindsight-api-slim/tests/test_reflect_prompt_builder.py

@@ -19,10 +19,19 @@ without hiding what each one produces — the ``_assemble`` helper is a
 mechanical join, not a re-implementation of the builder.
 """
 
+import pytest
+
+from hindsight_api.engine.reflect import prompts
 from hindsight_api.engine.reflect.prompts import build_final_system_prompt, build_system_prompt_for_tools
 
 BANK = {"name": "TestBank", "mission": ""}
 
+
+@pytest.fixture(autouse=True)
+def _freeze_current_datetime(monkeypatch):
+    monkeypatch.setattr(prompts, "_current_utc_datetime", lambda: "2026-08-09 14:32 UTC")
+
+
 _HEADER = (
     "CRITICAL: You MUST ONLY use information from retrieved tool results. "
     "NEVER make up names, people, events, or entities."
@@ -30,6 +39,8 @@ _HEADER = (
 
 _DEFAULT_ROLE = "You are a reflection agent that answers questions by reasoning over retrieved memories."
 
+_CURRENT_DATETIME = "## Current Date and Time\nThe current date and time is 2026-08-09 14:32 UTC."
+
 _LANGUAGE_AND_RULES = """\
 ## LANGUAGE RULE (default - directives take precedence)
 - By default, detect the language of the user's question and respond in that SAME language.
@@ -306,6 +317,8 @@ def _assemble(
     parts.append("")
     parts.append(_OUTPUT_FORMAT)
     parts.append("")
+    parts.append(_CURRENT_DATETIME)
+    parts.append("")
     parts.append(_BANK_HEADER + trailer)
     return "\n".join(parts)
 
@@ -560,6 +573,7 @@ _FRENCH_DIRECTIVE = {
 
 def test_final_prompt_always_includes_language_rule():
     prompt = build_final_system_prompt()
+    assert "The current date and time is 2026-08-09 14:32 UTC." in prompt
     assert "## LANGUAGE" in prompt
     assert "SAME language as the user's question" in prompt
 

hindsight-api-slim/tests/test_reflect_recall_entities.py

@@ -0,0 +1,82 @@
+"""Reflect sub-recalls must request entity resolution.
+
+Canonical entity names are semantic signal the surface text may lack ("Bob"
+in the text vs canonical "Robert Smith"). `recall_async` only populates each
+result's `entities` field when `include_entities=True`, and it defaults to
+False — so both reflect retrieval tools must pass it explicitly, and the
+names must survive the serialization into the tool result the agent reads.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from hindsight_api.engine.reflect.tools import tool_recall, tool_search_observations
+from hindsight_api.engine.response_models import MemoryFact, RecallResult
+
+
+@dataclass
+class _FakeRequestContext:
+    """Dataclass stand-in matching the fields used by ``dataclasses.replace``."""
+
+    api_key: str | None = None
+    api_key_id: str | None = None
+    tenant_id: str | None = None
+    internal: bool = False
+    mcp_authenticated: bool = False
+    user_initiated: bool = False
+    allowed_bank_ids: list[str] | None = None
+
+
+def _fact_with_entities() -> MemoryFact:
+    return MemoryFact(
+        id="123e4567-e89b-12d3-a456-426614174000",
+        text="Bob moved the deploy to 09:00 UTC.",
+        fact_type="world",
+        entities=["Robert Smith"],
+    )
+
+
+def _mock_engine(results: list[MemoryFact] | None = None):
+    engine = MagicMock()
+    engine.recall_async = AsyncMock(return_value=RecallResult(results=results or [], source_facts={}))
+    return engine
+
+
+class TestReflectRecallRequestsEntities:
+    @pytest.mark.asyncio
+    async def test_recall_passes_include_entities(self):
+        engine = _mock_engine()
+
+        await tool_recall(engine, "bank-1", "query", _FakeRequestContext())
+
+        assert engine.recall_async.call_args.kwargs["include_entities"] is True
+
+    @pytest.mark.asyncio
+    async def test_search_observations_passes_include_entities(self):
+        engine = _mock_engine()
+
+        await tool_search_observations(engine, "bank-1", "query", _FakeRequestContext())
+
+        assert engine.recall_async.call_args.kwargs["include_entities"] is True
+
+    @pytest.mark.asyncio
+    async def test_entity_names_reach_the_agent(self):
+        """End-to-end through the tool's serialization (null-pruning, field
+        trimming): the canonical names land in the payload the agent reads."""
+        engine = _mock_engine(results=[_fact_with_entities()])
+
+        result = await tool_recall(engine, "bank-1", "query", _FakeRequestContext())
+
+        assert result["memories"][0]["entities"] == ["Robert Smith"]
+
+    @pytest.mark.asyncio
+    async def test_observation_entity_names_reach_the_agent(self):
+        engine = _mock_engine(results=[_fact_with_entities()])
+
+        result = await tool_search_observations(engine, "bank-1", "query", _FakeRequestContext())
+
+        assert result["observations"][0]["entities"] == ["Robert Smith"]

hindsight-api-slim/tests/test_reflect_tool_result_fields.py

@@ -0,0 +1,84 @@
+"""Tests for what reflect tool results carry into the agent's context.
+
+Reflect tool results are dropped into the model's context verbatim, so every
+field in them is spent context. `_drop_unread_fields` removes the retrieval
+plumbing the agent never reads. The risk of removing fields is that something
+downstream quietly needed one, so these tests pin BOTH directions: the heavy
+fields go, and the fields the loop actually depends on stay.
+"""
+
+from __future__ import annotations
+
+import unittest
+
+from hindsight_api.engine.reflect.tools import _UNREAD_RESULT_FIELDS, _drop_unread_fields
+from hindsight_api.engine.response_models import ChunkInfo
+
+
+def _dumped_observation() -> dict:
+    """A serialized result carrying every field the trim targets."""
+    return {
+        "id": "obs-1",
+        "text": "The deploy runs at 09:00 UTC.",
+        "occurred_start": "2026-08-01T00:00:00Z",
+        "tags": ["ops"],
+        "source_fact_ids": ["f1", "f2"],
+        "scores": {"semantic": 0.71, "reranker": 0.93, "final": 1.04},
+        "metadata": {"ingest_batch": "b-17"},
+        "entities": ["Robert Smith"],
+        "chunk_id": "chunk-9",
+        "document_id": "doc-3",
+    }
+
+
+class DropUnreadFieldsTests(unittest.TestCase):
+    def test_every_unread_field_is_removed(self):
+        trimmed = _drop_unread_fields(_dumped_observation())
+        for field in _UNREAD_RESULT_FIELDS:
+            self.assertNotIn(field, trimmed, f"{field} should not reach the agent")
+
+    def test_the_fields_the_loop_depends_on_survive(self):
+        """The direction that actually breaks things.
+
+        Citations key on ``id``; ``based_on`` persists id/text/type/context; the
+        expand tool takes memory_ids. Dropping any of these would silently
+        degrade answers rather than raise.
+
+        ``entities`` survives too: it carries canonical entity *names* (not
+        ids), which are semantic signal the surface text may lack ("Bob" in the
+        text vs canonical "Robert Smith"). Reflect's recalls don't populate it
+        yet, but the trim must not eat the names once ``include_entities`` is
+        turned on.
+        """
+        trimmed = _drop_unread_fields(_dumped_observation())
+        for field in ("id", "text", "occurred_start", "tags", "source_fact_ids", "entities"):
+            self.assertIn(field, trimmed, f"{field} is load-bearing and must survive the trim")
+
+    def test_missing_fields_are_not_an_error(self):
+        """Results legitimately omit these — `_prune_nulls` runs first."""
+        self.assertEqual(_drop_unread_fields({"id": "obs-1"}), {"id": "obs-1"})
+
+    def test_trim_is_idempotent(self):
+        once = _drop_unread_fields(_dumped_observation())
+        self.assertEqual(_drop_unread_fields(dict(once)), once)
+
+
+class ChunkEnvelopeTests(unittest.TestCase):
+    def test_chunk_info_carries_no_unread_fields(self):
+        """Pins why ``chunks`` is left untrimmed in tool_recall.
+
+        ChunkInfo holds only chunk_text / chunk_index / truncated, so trimming
+        it would be a no-op and its absence is not an inconsistency. If a future
+        field lands here that IS plumbing, this test fails and the decision gets
+        revisited instead of silently going stale.
+        """
+        overlap = set(ChunkInfo.model_fields) & set(_UNREAD_RESULT_FIELDS)
+        self.assertEqual(
+            overlap,
+            set(),
+            f"ChunkInfo gained trimmable field(s) {overlap}; tool_recall's chunks should now be trimmed too",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

hindsight-api-slim/tests/test_structured_doc.py

@@ -35,6 +35,7 @@ from hindsight_api.engine.reflect.structured_doc import (
     ParagraphBlock,
     Section,
     StructuredDocument,
+    TableBlock,
     make_unique_id,
     parse_markdown,
     render_block,
@@ -126,6 +127,33 @@ class TestRenderer:
         block = CodeBlock(text="raw text")
         assert render_block(block) == "```\nraw text\n```"
 
+    def test_table_one_row_per_line(self):
+        block = TableBlock(headers=["Layer", "Role"], rows=[["API", "HTTP"], ["Engine", "Memory"]])
+        assert render_block(block) == ("| Layer | Role |\n| --- | --- |\n| API | HTTP |\n| Engine | Memory |")
+
+    def test_table_escapes_pipes_in_cells(self):
+        block = TableBlock(headers=["col"], rows=[["a | b"]])
+        assert render_block(block) == "| col |\n| --- |\n| a \\| b |"
+
+    def test_table_cell_newline_collapses_to_one_line(self):
+        block = TableBlock(headers=["col"], rows=[["first\nsecond"]])
+        assert render_block(block) == "| col |\n| --- |\n| first second |"
+
+    def test_table_short_row_is_padded(self):
+        block = TableBlock(headers=["a", "b", "c"], rows=[["1"]])
+        assert render_block(block).splitlines()[-1] == "| 1 |  |  |"
+
+    def test_table_row_wider_than_headers_keeps_every_cell(self):
+        block = TableBlock(headers=["a"], rows=[["1", "2"]])
+        assert render_block(block) == "| a |  |\n| --- | --- |\n| 1 | 2 |"
+
+    def test_table_without_headers_still_renders_rows(self):
+        block = TableBlock(headers=[], rows=[["1", "2"]])
+        assert render_block(block) == "|  |  |\n| --- | --- |\n| 1 | 2 |"
+
+    def test_empty_table_renders_empty(self):
+        assert render_block(TableBlock()) == ""
+
     def test_section_heading_level(self):
         section = Section(id="purpose", heading="Purpose", level=3, blocks=[ParagraphBlock(text="hi")])
         assert render_section(section).startswith("### Purpose\n\nhi")
@@ -218,6 +246,57 @@ class TestParser:
         doc = parse_markdown(markdown)
         assert [s.id for s in doc.sections] == ["notes", "notes-2"]
 
+    def test_table(self):
+        md = "## Layers\n\n| Layer | Role |\n| --- | --- |\n| API | HTTP |\n| Engine | Memory |\n"
+        block = parse_markdown(md).sections[0].blocks[0]
+        assert isinstance(block, TableBlock)
+        assert block.headers == ["Layer", "Role"]
+        assert block.rows == [["API", "HTTP"], ["Engine", "Memory"]]
+
+    def test_table_alignment_colons_are_a_separator(self):
+        md = "## T\n\n| a | b |\n|:---|---:|\n| 1 | 2 |\n"
+        block = parse_markdown(md).sections[0].blocks[0]
+        assert isinstance(block, TableBlock)
+        assert block.headers == ["a", "b"]
+        assert block.rows == [["1", "2"]]
+
+    def test_table_escaped_pipe_stays_in_one_cell(self):
+        md = "## T\n\n| col | expr |\n| --- | --- |\n| a | x \\| y |\n"
+        block = parse_markdown(md).sections[0].blocks[0]
+        assert isinstance(block, TableBlock)
+        assert block.rows == [["a", "x | y"]]
+
+    def test_table_keeps_other_backslash_sequences_verbatim(self):
+        md = "## T\n\n| col |\n| --- |\n| C:\\path |\n"
+        block = parse_markdown(md).sections[0].blocks[0]
+        assert isinstance(block, TableBlock)
+        assert block.rows == [["C:\\path"]]
+
+    def test_prose_containing_a_pipe_is_not_a_table(self):
+        md = "## T\n\nUse a | b to pipe.\nSecond line.\n"
+        block = parse_markdown(md).sections[0].blocks[0]
+        assert isinstance(block, ParagraphBlock)
+
+    def test_pipe_rows_without_separator_are_not_a_table(self):
+        md = "## T\n\n| a | b |\n| 1 | 2 |\n"
+        block = parse_markdown(md).sections[0].blocks[0]
+        assert isinstance(block, ParagraphBlock)
+
+    def test_table_round_trip_via_render(self):
+        doc = StructuredDocument(
+            sections=[
+                Section(
+                    id="layers",
+                    heading="Layers",
+                    blocks=[TableBlock(headers=["Layer", "Note"], rows=[["API", "a | b"], ["Engine", ""]])],
+                )
+            ]
+        )
+        markdown = render_document(doc)
+        roundtripped = parse_markdown(markdown)
+        assert roundtripped.sections[0].blocks[0] == doc.sections[0].blocks[0]
+        assert render_document(roundtripped) == markdown
+
     def test_round_trip_via_render(self):
         original = _team_overview_doc()
         markdown = render_document(original)

hindsight-api-slim/tests/test_text_search_reconciliation.py

@@ -0,0 +1,246 @@
+"""Regression coverage for runtime text-search reconciliation.
+
+``ensure_text_search_extension`` issues DDL against a live database at startup,
+so these tests drive it through a fake connection that records every statement.
+The assertions are about *which* statements are emitted (and that they are all
+re-executable), not about a real schema — the shapes themselves are covered by
+the migration suites.
+"""
+
+import re
+from contextlib import contextmanager
+from dataclasses import dataclass
+
+import pytest
+
+from hindsight_api import migrations
+from hindsight_api._text_search import mental_models_text_document
+
+_COUNT_TABLE = re.compile(r"SELECT COUNT\(\*\) FROM \S+\.(\w+)")
+# Statements that change the schema; each one must be safely re-executable
+# because replicas boot concurrently and all run this reconciliation.
+_DDL_PREFIXES = ("ALTER", "CREATE", "DROP")
+
+
+class _Result:
+    def __init__(self, *, scalar=None, row=None):
+        self._scalar = scalar
+        self._row = row
+
+    def scalar(self):
+        return self._scalar
+
+    def fetchone(self):
+        return self._row
+
+
+@dataclass(frozen=True)
+class _TableState:
+    column: str | None
+    index: str | None
+    rows: int
+    indexdef: str = ""
+
+
+class _Connection:
+    def __init__(self, tables: dict[str, _TableState]):
+        self.tables = tables
+        self.statements: list[str] = []
+        self.table_checks: list[str] = []
+        self.commits = 0
+
+    def execute(self, statement, params=None):
+        sql = str(statement)
+        self.statements.append(sql)
+        if "information_schema.tables" in sql:
+            table_name = params["table_name"]
+            self.table_checks.append(table_name)
+            return _Result(scalar=table_name in self.tables)
+        if "information_schema.columns" in sql:
+            column_type = self.tables[params["table_name"]].column
+            return _Result(row=("USER-DEFINED", column_type) if column_type else None)
+        if "FROM pg_indexes" in sql:
+            table = self.tables[params["table_name"]]
+            return _Result(row=(table.index, table.indexdef) if table.index else None)
+        count_target = _COUNT_TABLE.search(" ".join(sql.split()))
+        if count_target:
+            return _Result(scalar=self.tables[count_target.group(1)].rows)
+        return _Result()
+
+    def commit(self):
+        self.commits += 1
+
+    @property
+    def ddl(self) -> list[str]:
+        normalized = (" ".join(s.split()) for s in self.statements)
+        return [s for s in normalized if s.startswith(_DDL_PREFIXES)]
+
+
+class _Engine:
+    def __init__(self, conn):
+        self.conn = conn
+
+    @contextmanager
+    def connect(self):
+        yield self.conn
+
+
+def _connect(monkeypatch, tables: dict[str, _TableState]) -> _Connection:
+    conn = _Connection(tables)
+    monkeypatch.setattr(migrations, "create_engine", lambda *args, **kwargs: _Engine(conn))
+    return conn
+
+
+def _run(monkeypatch, tables: dict[str, _TableState], extension="pgroonga") -> _Connection:
+    conn = _connect(monkeypatch, tables)
+    migrations.ensure_text_search_extension("postgresql://unused", text_search_extension=extension)
+    return conn
+
+
+def _assert_no_schema_changes(conn: _Connection) -> None:
+    assert conn.ddl == []
+    assert conn.commits == 0
+
+
+def test_populated_legacy_mental_models_converts_to_pgroonga(monkeypatch):
+    """The one populated transition that is allowed: the derived tsvector that
+    reconciliation used to skip (it checked the pre-rename `reflections` name)
+    is replaced by the pgroonga expression index. Nothing needs a backfill."""
+    conn = _run(
+        monkeypatch,
+        {
+            "memory_units": _TableState(column="text", index="pgroonga", rows=20),
+            "mental_models": _TableState(column="tsvector", index="gin", rows=5),
+        },
+    )
+
+    assert conn.table_checks == ["memory_units", "mental_models"]
+    assert conn.ddl == [
+        "DROP INDEX IF EXISTS public.idx_mental_models_text_search",
+        "ALTER TABLE public.mental_models DROP COLUMN IF EXISTS search_vector",
+        "CREATE EXTENSION IF NOT EXISTS pgroonga CASCADE",
+        "ALTER TABLE public.mental_models ADD COLUMN IF NOT EXISTS search_vector TEXT",
+        "CREATE INDEX IF NOT EXISTS idx_mental_models_text_search ON public.mental_models "
+        f"USING pgroonga ({mental_models_text_document()}) "
+        "WITH (tokenizer='TokenBigram', normalizer='NormalizerNFKC150')",
+    ]
+    # memory_units already matched, so it is left alone entirely.
+    assert not any("memory_units" in statement for statement in conn.ddl)
+    assert conn.commits == 1
+
+
+def test_pgroonga_state_is_idempotent(monkeypatch):
+    conn = _run(
+        monkeypatch,
+        {
+            "memory_units": _TableState(column="text", index="pgroonga", rows=20),
+            "mental_models": _TableState(column="text", index="pgroonga", rows=5),
+        },
+    )
+
+    assert conn.table_checks == ["memory_units", "mental_models"]
+    _assert_no_schema_changes(conn)
+
+
+def test_populated_memory_units_backend_switch_remains_fail_closed(monkeypatch):
+    conn = _connect(
+        monkeypatch,
+        {
+            "memory_units": _TableState(column="tsvector", index="gin", rows=20),
+            "mental_models": _TableState(column="tsvector", index="gin", rows=5),
+        },
+    )
+
+    with pytest.raises(RuntimeError, match=r"memory_units\(20 rows\)"):
+        migrations.ensure_text_search_extension("postgresql://unused", text_search_extension="pgroonga")
+
+    _assert_no_schema_changes(conn)
+
+
+def test_populated_unknown_mental_model_index_remains_fail_closed(monkeypatch):
+    """Only the native tsvector/GIN shape is derived-only; anything else may hold
+    values the reconciler cannot recompute, so it must still refuse."""
+    conn = _connect(
+        monkeypatch,
+        {
+            "memory_units": _TableState(column="text", index="pgroonga", rows=20),
+            "mental_models": _TableState(column="tsvector", index="bm25", rows=5),
+        },
+    )
+
+    with pytest.raises(RuntimeError, match=r"mental_models\(5 rows\)"):
+        migrations.ensure_text_search_extension("postgresql://unused", text_search_extension="pgroonga")
+
+    _assert_no_schema_changes(conn)
+
+
+def test_populated_mental_models_backfill_backend_remains_fail_closed(monkeypatch):
+    """vchord's bm25vector must be tokenized per row on write, so converting a
+    populated table would leave every existing row unsearchable."""
+    conn = _connect(
+        monkeypatch,
+        {
+            "memory_units": _TableState(column="bm25vector", index="bm25", rows=20),
+            "mental_models": _TableState(column="tsvector", index="gin", rows=5),
+        },
+    )
+
+    with pytest.raises(RuntimeError, match=r"mental_models\(5 rows\)"):
+        migrations.ensure_text_search_extension("postgresql://unused", text_search_extension="vchord")
+
+    _assert_no_schema_changes(conn)
+
+
+def test_empty_mental_models_native_reconcile_creates_generated_projection(monkeypatch):
+    """No write path fills mental_models.search_vector for native, so the column
+    has to generate itself (see pg_search_vector_expr's native_inline=False)."""
+    conn = _run(
+        monkeypatch,
+        {
+            "memory_units": _TableState(column="tsvector", index="gin", rows=0),
+            "mental_models": _TableState(column="text", index="pgroonga", rows=0),
+        },
+        extension="native",
+    )
+
+    assert (
+        "ALTER TABLE public.mental_models ADD COLUMN IF NOT EXISTS search_vector tsvector "
+        f"GENERATED ALWAYS AS ( to_tsvector('english', {mental_models_text_document()}) ) STORED" in conn.ddl
+    )
+    assert conn.commits == 1
+
+
+def test_empty_memory_units_native_reconcile_creates_plain_column(monkeypatch):
+    conn = _run(
+        monkeypatch,
+        {
+            "memory_units": _TableState(column="text", index="pgroonga", rows=0),
+            "mental_models": _TableState(column="tsvector", index="gin", rows=0),
+        },
+        extension="native",
+    )
+
+    assert "ALTER TABLE public.memory_units ADD COLUMN IF NOT EXISTS search_vector tsvector" in conn.ddl
+
+
+@pytest.mark.parametrize("extension", ["native", "vchord", "pg_textsearch", "pgroonga", "pg_search"])
+def test_reconcile_ddl_is_re_executable(monkeypatch, extension):
+    """Replicas boot concurrently during a rolling restart and each runs this
+    reconciliation, so the loser of the race must not crash on DDL the winner
+    already committed."""
+    conn = _run(
+        monkeypatch,
+        {
+            "memory_units": _TableState(column=None, index=None, rows=0),
+            "mental_models": _TableState(column=None, index=None, rows=0),
+        },
+        extension=extension,
+    )
+
+    assert conn.ddl, "expected reconciliation to emit DDL"
+    for statement in conn.ddl:
+        assert re.match(
+            r"(CREATE (INDEX|EXTENSION) IF NOT EXISTS|DROP INDEX IF EXISTS"
+            r"|ALTER TABLE \S+ (ADD|DROP) COLUMN IF (NOT )?EXISTS)",
+            statement,
+        ), f"non re-executable DDL: {statement}"

hindsight-api/pyproject.toml

@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hindsight-api"
-version = "0.8.6"
+version = "0.9.0"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 license = "MIT"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "hindsight-api-slim[all]==0.8.6",
+    "hindsight-api-slim[all]==0.9.0",
 ]
 
 [tool.uv.sources]

hindsight-cli/.openapi-coverage.toml

@@ -64,7 +64,15 @@ llm_request_stats = "UI-only endpoint for the control plane LLM Requests stats c
 # Document transfer (export/import) is an admin/ops operation used from the API
 # and the control plane, not the end-user Rust CLI.
 export_documents = "Admin/ops operation, used via the API and control plane, not the end-user CLI"
+export_documents_sync_removed = "Removed sync export endpoint kept as a 410 stub; nothing should call it"
 import_documents = "Admin/ops operation, used via the API and control plane, not the end-user CLI"
+download_file = "Serves async export archives; fetched via the API/control plane, not the end-user CLI"
+
+# Kubernetes probe endpoints. `hindsight health` already calls /health, which is
+# the readiness check; /health/ready is its alias and /health/live is a DB-free
+# liveness signal meant for kubelet, not for a human at a terminal.
+get_liveness = "Liveness probe for orchestrators; `hindsight health` covers the human-facing check"
+get_readiness = "Alias of /health, which `hindsight health` already calls"
 
 # ---------------------------------------------------------------------------
 # Per-operation parameter skips

hindsight-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hindsight-cli"
-version = "0.8.6"
+version = "0.9.0"
 edition = "2021"
 authors = ["Hindsight Team"]
 description = "A beautiful CLI for Hindsight - semantic memory system"

hindsight-clients/go/api/openapi.yaml

@@ -7,13 +7,16 @@ info:
     name: Apache 2.0
     url: https://www.apache.org/licenses/LICENSE-2.0.html
   title: Hindsight HTTP API
-  version: 0.8.6
+  version: 0.9.0
 servers:
 - url: /
 paths:
   /health:
     get:
-      description: Checks the health of the API and database connection
+      description: "Readiness check: verifies the API can reach the database. Alias\
+        \ of /health/ready. Use /health/live for liveness probes — this one fails\
+        \ whenever the database is unreachable, which must gate traffic, not restart\
+        \ the process."
       operationId: health_endpoint_health_get
       responses:
         "200":
@@ -24,6 +27,36 @@ paths:
       summary: Health check endpoint
       tags:
       - Monitoring
+  /health/ready:
+    get:
+      description: "Returns 200 when the API can serve traffic (database reachable),\
+        \ 503 otherwise. Identical to /health, which stays supported as its alias."
+      operationId: get_readiness
+      responses:
+        "200":
+          content:
+            application/json:
+              schema: {}
+          description: Successful Response
+      summary: Readiness probe
+      tags:
+      - Monitoring
+  /health/live:
+    get:
+      description: "Returns 200 whenever the process can serve a request. Performs\
+        \ no database access, so a slow or unreachable database never restarts the\
+        \ pod. Point livenessProbe here and readinessProbe at /health."
+      operationId: get_liveness
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/LivenessResponse'
+          description: Successful Response
+      summary: Liveness probe
+      tags:
+      - Monitoring
   /version:
     get:
       description: Returns API version information and enabled feature flags. Use
@@ -3233,13 +3266,14 @@ paths:
       - Bank Templates
   /v1/default/banks/{bank_id}/document-transfer:
     get:
-      description: "Export documents (extracted facts, entity names, causal links,\
-        \ chunks) from a bank as a transfer ZIP archive. Embeddings and database ids\
-        \ are not included — importing re-embeds with the target bank's model and\
-        \ re-resolves entities. Consolidated observations are excluded unless include_observations=true.\
-        \ Pass document_id query params to export specific documents, or omit to export\
-        \ the whole bank."
-      operationId: export_documents
+      deprecated: true
+      description: "**Removed.** The synchronous whole-bank export loaded the entire\
+        \ bank into memory and held a database connection for the full request, which\
+        \ could exhaust memory and take down the shared API on large banks. Use the\
+        \ asynchronous POST /v1/default/banks/{bank_id}/document-transfer/export instead:\
+        \ it returns an operation_id, runs the export in the background, and exposes\
+        \ a download URL on completion."
+      operationId: export_documents_sync_removed
       parameters:
       - explode: false
         in: path
@@ -3249,28 +3283,6 @@ paths:
           title: Bank Id
           type: string
         style: simple
-      - description: Document id(s) to export; omit for all
-        explode: true
-        in: query
-        name: document_id
-        required: false
-        schema:
-          items:
-            type: string
-          nullable: true
-          type: array
-        style: form
-      - description: Also export consolidated observations (restored on import)
-        explode: true
-        in: query
-        name: include_observations
-        required: false
-        schema:
-          default: false
-          description: Also export consolidated observations (restored on import)
-          title: Include Observations
-          type: boolean
-        style: form
       - explode: false
         in: header
         name: authorization
@@ -3284,15 +3296,14 @@ paths:
           content:
             application/json:
               schema: {}
-            application/zip: {}
-          description: Transfer archive
+          description: Successful Response
         "422":
           content:
             application/json:
               schema:
                 $ref: '#/components/schemas/HTTPValidationError'
           description: Validation Error
-      summary: Export documents
+      summary: Export documents (removed — use POST .../document-transfer/export)
       tags:
       - Document Transfer
     post:
@@ -3353,6 +3364,117 @@ paths:
       summary: Import documents (async)
       tags:
       - Document Transfer
+  /v1/default/banks/{bank_id}/document-transfer/export:
+    post:
+      description: "Submit an async export of a bank's documents (extracted facts,\
+        \ entity names, causal links, chunks) as a transfer ZIP archive. Embeddings\
+        \ and database ids are not included — importing re-embeds with the target\
+        \ bank's model and re-resolves entities. Runs as a background operation to\
+        \ avoid pinning the API on large banks. Returns an operation_id; poll GET\
+        \ /v1/default/banks/{bank_id}/operations/{operation_id}. On completion the\
+        \ operation's result_metadata carries download_url (fetch the ZIP from GET\
+        \ /v1/default/files/download/{key}), storage_key, byte_size, and filename.\
+        \ Pass document_id query params to export specific documents, or omit to export\
+        \ the whole bank; include_observations=true also carries consolidated observations\
+        \ (whole-bank export only)."
+      operationId: export_documents
+      parameters:
+      - explode: false
+        in: path
+        name: bank_id
+        required: true
+        schema:
+          title: Bank Id
+          type: string
+        style: simple
+      - description: Document id(s) to export; omit for all
+        explode: true
+        in: query
+        name: document_id
+        required: false
+        schema:
+          items:
+            type: string
+          nullable: true
+          type: array
+        style: form
+      - description: Also export consolidated observations (restored on import; whole-bank
+          only)
+        explode: true
+        in: query
+        name: include_observations
+        required: false
+        schema:
+          default: false
+          description: Also export consolidated observations (restored on import;
+            whole-bank only)
+          title: Include Observations
+          type: boolean
+        style: form
+      - explode: false
+        in: header
+        name: authorization
+        required: false
+        schema:
+          nullable: true
+          type: string
+        style: simple
+      responses:
+        "202":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/DocumentExportSubmitResponse'
+          description: Successful Response
+        "422":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      summary: Export documents (async)
+      tags:
+      - Document Transfer
+  /v1/default/files/download/{key}:
+    get:
+      description: Stream a file previously written to file storage — currently the
+        transfer ZIP produced by an async document export. The key comes from the
+        export operation's result_metadata (storage_key / download_url). Access is
+        authorized against the bank the key belongs to.
+      operationId: download_file
+      parameters:
+      - explode: false
+        in: path
+        name: key
+        required: true
+        schema:
+          title: Key
+          type: string
+        style: simple
+      - explode: false
+        in: header
+        name: authorization
+        required: false
+        schema:
+          nullable: true
+          type: string
+        style: simple
+      responses:
+        "200":
+          content:
+            application/json:
+              schema: {}
+            application/zip: {}
+          description: Stored file
+        "422":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      summary: Download a stored file (async export archive)
+      tags:
+      - Document Transfer
   /v1/bank-template-schema:
     get:
       description: Returns the JSON Schema for the bank template manifest format.
@@ -5145,6 +5267,27 @@ components:
         store_document_text:
           nullable: true
           type: boolean
+        enable_auto_consolidation:
+          nullable: true
+          type: boolean
+        consolidation_max_memories_per_round:
+          nullable: true
+          type: integer
+        consolidation_llm_parallelism:
+          nullable: true
+          type: integer
+        recall_include_chunks:
+          nullable: true
+          type: boolean
+        recall_max_tokens:
+          nullable: true
+          type: integer
+        recall_chunks_max_tokens:
+          nullable: true
+          type: integer
+        memory_defense:
+          additionalProperties: {}
+          nullable: true
       title: BankTemplateConfig
     BankTemplateDirective:
       description: |-
@@ -6007,6 +6150,28 @@ components:
       - literalism
       - skepticism
       title: DispositionTraits
+    DocumentExportSubmitResponse:
+      description: |-
+        Response for the async document-export endpoint (202).
+
+        The export runs in the background; poll the operations endpoint for status.
+        On completion the operation's ``result_metadata`` carries ``download_url``
+        (fetch the ZIP from GET /v1/default/files/download/{key}), ``storage_key``,
+        ``byte_size``, and ``filename``.
+      example:
+        operation_id: operation_id
+        status: pending
+      properties:
+        operation_id:
+          title: Operation Id
+          type: string
+        status:
+          default: pending
+          title: Status
+          type: string
+      required:
+      - operation_id
+      title: DocumentExportSubmitResponse
     DocumentImportSubmitResponse:
       description: |-
         Response for the async document-import endpoint (202).
@@ -7484,6 +7649,32 @@ components:
       - offset
       - total
       title: ListTagsResponse
+    LivenessResponse:
+      description: Payload for the API's DB-free liveness probe.
+      example:
+        status: alive
+        uptime_seconds: 812.4
+        version: 0.4.0
+      properties:
+        status:
+          description: Always "alive" — reaching this handler is the check
+          enum:
+          - alive
+          title: Status
+          type: string
+        version:
+          description: Hindsight version this process is running
+          title: Version
+          type: string
+        uptime_seconds:
+          description: Seconds since the process started
+          title: Uptime Seconds
+          type: number
+      required:
+      - status
+      - uptime_seconds
+      - version
+      title: LivenessResponse
     LlmOperationHealth:
       description: |-
         LLM connectivity status for a single operation. Status only — no provider/model/

hindsight-clients/go/api_audit.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.8.6
+API version: 0.9.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_bank_templates.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.8.6
+API version: 0.9.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_banks.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.8.6
+API version: 0.9.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_directives.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.8.6
+API version: 0.9.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.