docs: regenerate skill mirror for FAIL_ON_EXTRACTION_ERRORS config row

The Gemini batch request builder only set the native response_schema
(responseJsonSchema) when json_schema.strict was truthy, but
HINDSIGHT_API_LLM_STRICT_SCHEMA defaults to False. The interactive Gemini
path always grammar-enforces via response_schema regardless of strict
(strict is an OpenAI concept, meaningless to Gemini). At default config
batch requests therefore got only responseMimeType + a textual schema hint
and intermittently emitted malformed JSON, dropping every fact in the chunk.

Set responseJsonSchema whenever a schema is present so batch mirrors
interactive. Update the batch translation unit test accordingly.

Fixes #2699

.claude-plugin/marketplace.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
   "name": "hindsight",
-  "version": "0.7.2",
+  "version": "0.7.5",
   "description": "Official Hindsight integrations for Claude Code",
   "owner": {
     "name": "vectorize-io"

.env.example

@@ -59,6 +59,15 @@ HINDSIGHT_API_LLM_BASE_URL=https://api.openai.com/v1
 # HINDSIGHT_API_LLM_BASE_URL=http://localhost:1234/v1
 # HINDSIGHT_API_LLM_MODEL=qwen2.5-32b-instruct
 
+# Example: Ollama local configuration (native provider)
+# HINDSIGHT_API_LLM_PROVIDER=ollama
+# HINDSIGHT_API_LLM_BASE_URL=http://localhost:11434/v1
+# HINDSIGHT_API_LLM_MODEL=gemma3:12b
+# Native Ollama context-window override (num_ctx). Leave unset to let Ollama use
+# the model Modelfile / server default; set a positive integer only to force a
+# specific context size (e.g. 16384 to keep the previous request behavior).
+# HINDSIGHT_API_LLM_OLLAMA_NUM_CTX=16384
+
 # Multi-LLM strategies: configure extra LLMs by index alongside the primary above,
 # then pick a routing strategy. Unset = single primary LLM (default). Members are
 # numbered from 1; indices must be contiguous. Each operation can override with a
@@ -80,6 +89,10 @@ HINDSIGHT_API_LOG_LEVEL=info
 # Unset uses HINDSIGHT_API_RETAIN_CHUNK_SIZE as the structured-chunk limit.
 # HINDSIGHT_API_RETAIN_STRUCTURED_CHUNK_SIZE=
 
+# When true, a retain operation that hit any fact-extraction errors is marked
+# 'failed' (not 'completed'), surfacing silently-dropped facts. Default false.
+# HINDSIGHT_API_FAIL_ON_EXTRACTION_ERRORS=false
+
 # Dry-run extraction preview endpoint (POST /memories/dry-run-extract). Enabled by default; it makes
 # a real LLM call but stores nothing. Set to false to remove the endpoint (returns 404).
 # HINDSIGHT_API_ENABLE_DRY_RUN_EXTRACT=true
@@ -96,6 +109,8 @@ HINDSIGHT_API_LOG_LEVEL=info
 # HINDSIGHT_API_MIGRATION_DATABASE_URL=  # Direct PostgreSQL URL for migrations (bypasses PgBouncer). Falls back to DATABASE_URL.
 # HINDSIGHT_API_DATABASE_SCHEMA=public  # PostgreSQL schema name (default: public)
 # HINDSIGHT_API_MIGRATION_CONCURRENCY=1  # Tenant schemas to migrate concurrently (PG only, each in its own process; per-schema work stays sequential). Each worker has ~1-2s startup cost + uses ~3 DB connections, so it only pays off with many schemas (tens+) or slow migrations; keep concurrency*3 <= spare max_connections. Default: 1 (sequential).
+# HINDSIGHT_API_OPERATION_RETENTION_DAYS=30  # Keep terminal operation rows, payloads, and metadata for this many days; 0 disables automatic pruning.
+# HINDSIGHT_API_OPERATION_CLEANUP_BATCH_SIZE=1000  # Maximum expired terminal rows deleted per tenant schema in each cleanup cycle; must be positive.
 
 # Vector Extension (Optional - uses pgvector by default)
 # Options: "pgvector" (default), "vchord", "pgvectorscale" (DiskANN)
@@ -115,6 +130,11 @@ HINDSIGHT_API_LOG_LEVEL=info
 # chinese_lindera/lindera(chinese), japanese_lindera/lindera(japanese),
 # korean_lindera/lindera(korean), ngram(min,max), edge_ngram(min,max)
 # HINDSIGHT_API_TEXT_SEARCH_EXTENSION_PG_SEARCH_TOKENIZER=
+# Optional cap on the number of terms in the native PostgreSQL BM25 tsquery.
+# Long queries OR-join every normalized token, which can match too much of a
+# large bank. 0 (default) keeps the historical uncapped behavior; a positive
+# value bounds only the native backend (other BM25 backends get the raw query).
+# HINDSIGHT_API_BM25_MAX_QUERY_TERMS=0
 
 # File Parser (Optional - uses markitdown by default)
 # HINDSIGHT_API_FILE_PARSER=markitdown
@@ -133,6 +153,8 @@ HINDSIGHT_API_LOG_LEVEL=info
 # HINDSIGHT_API_EMBEDDINGS_PROVIDER=local
 # For local provider:
 # HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL=BAAI/bge-small-en-v1.5
+# Force CPU if local embeddings hit MPS/XPC instability on macOS:
+# HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU=false
 # For ONNX provider (local CPU embeddings without an Ollama/TEI sidecar):
 # HINDSIGHT_API_EMBEDDINGS_PROVIDER=onnx
 # HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_ID=intfloat/multilingual-e5-small
@@ -174,6 +196,8 @@ HINDSIGHT_API_LOG_LEVEL=info
 # HINDSIGHT_API_RERANKER_PROVIDER=local
 # For local provider:
 # HINDSIGHT_API_RERANKER_LOCAL_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2
+# Force CPU if the local reranker hits MPS/XPC instability on macOS:
+# HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU=false
 # For TEI provider:
 # HINDSIGHT_API_RERANKER_TEI_URL=http://localhost:8081
 

.github/workflows/test.yml

@@ -520,22 +520,17 @@ jobs:
       with:
         ref: ${{ github.event.pull_request.head.sha || '' }}
 
-    - name: Set up Python
-      uses: actions/setup-python@v6
+    - name: Set up Node.js
+      uses: actions/setup-node@v6
       with:
-        python-version: '3.11'
-
-    - name: Install package and pytest
-      working-directory: ./hindsight-integrations/zed
-      # Installs the package (incl. the zstandard runtime dep) so the threads.db
-      # reader tests can decompress Zed's zstd blobs.
-      run: pip install -e . pytest
+        node-version: '22'
 
     - name: Run tests
       working-directory: ./hindsight-integrations/zed
-      # PR CI runs only the deterministic bucket; the real-LLM E2E bucket
-      # (requires_real_llm) needs a live Hindsight server and runs separately.
-      run: python -m pytest tests/ -v -m "not requires_real_llm"
+      # Config-only integration with no dependencies — it uses Node's built-in
+      # test runner. The runtime MCP bridge is `npx mcp-remote` (Node), so this
+      # integration requires only Node.js (no Python).
+      run: npm test
 
   test-omo-integration:
     needs: [detect-changes]

hindsight-api-slim/hindsight_api/alembic/versions/a8c1e4f7b0d3_add_operation_retention_indexes.py

@@ -0,0 +1,82 @@
+"""Add indexes for terminal cleanup and newest-first operation listing.
+
+Revision ID: a8c1e4f7b0d3
+Revises: f2a4b6c8d0e2
+Create Date: 2026-07-14
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "a8c1e4f7b0d3"
+down_revision: str | Sequence[str] | None = "f2a4b6c8d0e2"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _pg_schema_prefix() -> str:
+    """Schema-qualifier for PostgreSQL multi-tenant migration runs."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def _pg_upgrade() -> None:
+    schema = _pg_schema_prefix()
+    # These can be large tables in long-running installations. Concurrent DDL
+    # keeps operation submission, polling, and status reads available.
+    with op.get_context().autocommit_block():
+        op.execute(
+            "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_async_operations_terminal_cleanup "
+            f"ON {schema}async_operations (updated_at, operation_id) "
+            "WHERE status IN ('completed', 'failed', 'cancelled')"
+        )
+        op.execute(
+            "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_async_operations_bank_created_desc "
+            f"ON {schema}async_operations (bank_id, created_at DESC)"
+        )
+
+
+def _pg_downgrade() -> None:
+    schema = _pg_schema_prefix()
+    with op.get_context().autocommit_block():
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_async_operations_bank_created_desc")
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_async_operations_terminal_cleanup")
+
+
+def _oracle_create_index(sql: str) -> None:
+    """Create an index idempotently for rerun-safe Oracle migrations."""
+    block = (
+        "BEGIN "
+        "EXECUTE IMMEDIATE :stmt; "
+        "EXCEPTION WHEN OTHERS THEN "
+        "IF SQLCODE = -955 THEN NULL; ELSE RAISE; END IF; "
+        "END;"
+    )
+    op.get_bind().exec_driver_sql(block, {"stmt": sql})
+
+
+def _oracle_upgrade() -> None:
+    # Oracle migrations run with CURRENT_SCHEMA set to each tenant, so table
+    # and index names intentionally remain unqualified here.
+    _oracle_create_index(
+        "CREATE INDEX idx_async_operations_terminal_cleanup ON async_operations (updated_at, operation_id, status)"
+    )
+    _oracle_create_index(
+        "CREATE INDEX idx_async_operations_bank_created_desc ON async_operations (bank_id, created_at DESC)"
+    )
+
+
+def _oracle_downgrade() -> None:
+    op.execute("DROP INDEX idx_async_operations_bank_created_desc")
+    op.execute("DROP INDEX idx_async_operations_terminal_cleanup")
+
+
+def upgrade() -> None:
+    run_for_dialect(pg=_pg_upgrade, oracle=_oracle_upgrade)
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade, oracle=_oracle_downgrade)

hindsight-api-slim/hindsight_api/alembic/versions/e7c3a9f1b2d5_drop_archive_search_vector_column.py

@@ -0,0 +1,96 @@
+"""Drop the search_vector column from the curation archive (invalidated_memory_units).
+
+The archive is cold storage, never a recall surface, and carries no text-search
+index. Like ``embedding`` (dropped in d4f6a8c2e1b3), ``search_vector`` is a
+recall-surface column whose type follows the configured text-search backend, so
+it has no business living on the archive. Earlier curation code copied the live
+row's ``search_vector`` into ``invalidated_memory_units`` on invalidate; the
+engine now leaves it out on invalidate and recomputes it on revert, so the
+column is dead weight.
+
+Dropping it removes a latent failure mode (#2503): under a non-native backend
+(pgroonga / pg_textsearch / pg_search / vchord) ``ensure_text_search_extension``
+reconciles ``memory_units.search_vector`` to ``text`` / ``bm25vector`` but never
+touched the archive, which the ``LIKE memory_units`` clone (c9a1b2d3e4f5) created
+as ``tsvector``. The type mismatch then broke the curation INSERT … SELECT
+round-trip:
+
+    column "search_vector" is of type tsvector but expression is of type text
+
+With no column at all, there is nothing to mismatch. Unlike ``embedding`` (whose
+creation sites already omit it), the ``LIKE`` clone still adds ``search_vector``,
+so this migration does real work on both fresh and existing PostgreSQL databases.
+
+DROP COLUMN is a metadata-only operation on both PostgreSQL and Oracle 23ai (no
+table rewrite), so it is cheap even across many tenant schemas. The downgrade
+re-adds an empty ``tsvector`` column (its original creation type).
+
+Revision ID: e7c3a9f1b2d5
+Revises: b57a7c9e0d13
+Create Date: 2026-07-02
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "e7c3a9f1b2d5"
+down_revision: str | Sequence[str] | None = "b57a7c9e0d13"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _pg_schema_prefix() -> str:
+    """Schema-qualifier for raw SQL on PG (multi-tenant search_path)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def _pg_upgrade() -> None:
+    schema = _pg_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}invalidated_memory_units DROP COLUMN IF EXISTS search_vector")
+
+
+def _pg_downgrade() -> None:
+    schema = _pg_schema_prefix()
+    # Re-add as the original tsvector creation type; comes back empty regardless.
+    op.execute(f"ALTER TABLE {schema}invalidated_memory_units ADD COLUMN IF NOT EXISTS search_vector tsvector")
+
+
+def _oracle_upgrade() -> None:
+    # Oracle has no `DROP COLUMN IF EXISTS`; swallow ORA-00904 (column does not
+    # exist) so the migration is idempotent and safe on a schema whose baseline
+    # may already omit the column.
+    op.execute(
+        """
+        BEGIN
+            EXECUTE IMMEDIATE 'ALTER TABLE invalidated_memory_units DROP COLUMN search_vector';
+        EXCEPTION WHEN OTHERS THEN
+            IF SQLCODE != -904 THEN RAISE; END IF;
+        END;
+        """
+    )
+
+
+def _oracle_downgrade() -> None:
+    # Swallow ORA-01430 (column already exists) for idempotency. Oracle stores
+    # search_vector as CLOB (see the Oracle baseline), so re-add it as CLOB.
+    op.execute(
+        """
+        BEGIN
+            EXECUTE IMMEDIATE 'ALTER TABLE invalidated_memory_units ADD (search_vector CLOB)';
+        EXCEPTION WHEN OTHERS THEN
+            IF SQLCODE != -1430 THEN RAISE; END IF;
+        END;
+        """
+    )
+
+
+def upgrade() -> None:
+    run_for_dialect(pg=_pg_upgrade, oracle=_oracle_upgrade)
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade, oracle=_oracle_downgrade)

hindsight-api-slim/hindsight_api/alembic/versions/f2a4b6c8d0e2_maintenance_routines_all_schema_runs.py

@@ -0,0 +1,158 @@
+"""Install the shared public.* maintenance routines on every PG run.
+
+``public.banks_needing_consolidation()`` and
+``public.schemas_with_expired_rows(...)`` physically live in ``public`` and are
+consumed by the background maintenance loop through a hardcoded ``public.``
+qualifier (``engine/maintenance.py``). Every prior install/repair
+(``e5f6a7b8c9d0`` → ``b2d4f6a8c1e3`` → ``c7e9f1a3b5d2``) gated creation on
+``_should_install_public_routines(target_schema)`` — i.e. the base run
+(no ``target_schema``) or an explicit ``target_schema='public'`` run.
+
+That leaves a gap for a **single-tenant deployment migrated into a dedicated,
+non-``public`` schema** (``HINDSIGHT_API_DATABASE_SCHEMA=<non-public>``). The
+runtime migrates only that one schema (``run_migrations_for_schemas`` fans out
+per configured schema), so ``target_schema`` is never falsy or ``public`` and
+the routines are never created. The maintenance loop then logs, forever::
+
+    function public.banks_needing_consolidation() does not exist
+    function public.schemas_with_expired_rows(...) does not exist
+
+and the revision is stamped applied, so redeploying the same version does not
+help (issue #2638; #2056 only fixed the ``public``/base-run case).
+
+Fix: install the routines on **every** PG run regardless of ``target_schema``.
+They are schema-agnostic (they enumerate ``pg_class`` and dispatch per schema),
+so a non-``public`` run creating them in ``public`` is correct and idempotent
+via ``CREATE OR REPLACE``. The reason the earlier migrations gated to a single
+run was to avoid ``tuple concurrently updated`` when parallel per-schema
+migration processes race on the same ``CREATE OR REPLACE``; we keep that safety
+with a transaction-scoped advisory lock so exactly one concurrent run performs
+the replace at a time. Function bodies are identical to ``c7e9f1a3b5d2`` (the
+vanishing-schema-resilient versions).
+
+Revision ID: f2a4b6c8d0e2
+Revises: e7c3a9f1b2d5
+Create Date: 2026-07-13
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "f2a4b6c8d0e2"
+down_revision: str | Sequence[str] | None = "e7c3a9f1b2d5"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+# Stable 64-bit key for the transaction-scoped advisory lock that serializes
+# the CREATE OR REPLACE against concurrent per-schema migration processes.
+# Arbitrary constant; only needs to be identical across processes.
+_ROUTINE_INSTALL_LOCK_KEY = 472638_00000001
+
+
+def _pg_upgrade() -> None:
+    # Serialize concurrent per-schema migration processes so only one performs
+    # the CREATE OR REPLACE at a time (avoids `tuple concurrently updated`).
+    # Transaction-scoped: released automatically at commit.
+    op.execute(f"SELECT pg_advisory_xact_lock({_ROUTINE_INSTALL_LOCK_KEY})")
+
+    op.execute(
+        """
+        CREATE OR REPLACE FUNCTION public.banks_needing_consolidation()
+        RETURNS TABLE(schema_name text, bank_id text)
+        LANGUAGE plpgsql STABLE
+        AS $fn$
+        DECLARE
+            sch text;
+        BEGIN
+            FOR sch IN
+                SELECT n.nspname
+                FROM pg_class c
+                JOIN pg_namespace n ON n.oid = c.relnamespace
+                WHERE c.relname = 'memory_units' AND c.relkind = 'r'
+            LOOP
+                BEGIN
+                    RETURN QUERY EXECUTE format($q$
+                        SELECT %1$L::text, m.bank_id
+                        FROM %1$I.memory_units m
+                        JOIN %1$I.banks b ON b.bank_id = m.bank_id
+                        WHERE m.consolidated_at IS NULL
+                          AND m.consolidation_failed_at IS NULL
+                          AND m.fact_type IN ('experience', 'world')
+                          AND COALESCE(b.config -> 'enable_auto_consolidation', 'true'::jsonb) <> 'false'::jsonb
+                          AND NOT EXISTS (
+                              SELECT 1 FROM %1$I.async_operations o
+                              WHERE o.bank_id = m.bank_id
+                                AND o.operation_type = 'consolidation'
+                                AND o.status IN ('pending', 'processing')
+                          )
+                        GROUP BY m.bank_id
+                    $q$, sch);
+                EXCEPTION
+                    -- Schema or its tables vanished between the pg_class
+                    -- snapshot and this query (tenant dropped or migrating).
+                    WHEN undefined_table OR invalid_schema_name OR undefined_column THEN
+                        CONTINUE;
+                END;
+            END LOOP;
+        END;
+        $fn$;
+        """
+    )
+
+    op.execute(
+        """
+        CREATE OR REPLACE FUNCTION public.schemas_with_expired_rows(
+            p_table text, p_ts_col text, p_days int
+        )
+        RETURNS SETOF text
+        LANGUAGE plpgsql STABLE
+        AS $fn$
+        DECLARE
+            sch text;
+            has_expired boolean;
+        BEGIN
+            IF p_days IS NULL OR p_days <= 0 THEN
+                RETURN;
+            END IF;
+            FOR sch IN
+                SELECT n.nspname
+                FROM pg_class c
+                JOIN pg_namespace n ON n.oid = c.relnamespace
+                WHERE c.relname = p_table AND c.relkind = 'r'
+            LOOP
+                BEGIN
+                    EXECUTE format(
+                        'SELECT EXISTS (SELECT 1 FROM %I.%I WHERE %I < NOW() - make_interval(days => $1))',
+                        sch, p_table, p_ts_col
+                    ) INTO has_expired USING p_days;
+                EXCEPTION
+                    -- Schema or its table vanished mid-scan; skip it.
+                    WHEN undefined_table OR invalid_schema_name OR undefined_column THEN
+                        CONTINUE;
+                END;
+                IF has_expired THEN
+                    RETURN NEXT sch;
+                END IF;
+            END LOOP;
+        END;
+        $fn$;
+        """
+    )
+
+
+def _pg_downgrade() -> None:
+    # No-op: e5f6a7b8c9d0 owns these functions' lifecycle and drops them on its
+    # own downgrade. This migration only (re)installs them on more runs, so there
+    # is nothing to undo without racing that migration's DROP.
+    pass
+
+
+def upgrade() -> None:
+    run_for_dialect(pg=_pg_upgrade)
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade)

hindsight-api-slim/hindsight_api/api/http.py

@@ -52,6 +52,7 @@ from fastapi.routing import APIRoute
 from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
 from hindsight_api import MemoryEngine
+from hindsight_api.config import RETAIN_EXTRACTION_MODES
 
 
 def _annotation_is_nullable(annotation: Any) -> bool:
@@ -1245,7 +1246,7 @@ class CreateBankRequest(BaseModel):
     )
     retain_extraction_mode: str | None = Field(
         default=None,
-        description="Fact extraction mode: 'concise' (default), 'verbose', or 'custom'.",
+        description="Fact extraction mode: 'concise' (default), 'verbose', 'custom', 'verbatim', or 'chunks'.",
     )
     retain_custom_instructions: str | None = Field(
         default=None,
@@ -1433,6 +1434,7 @@ class ListMemoryUnitsResponse(BaseModel):
                         "date": "2024-01-15T10:30:00Z",
                         "type": "world",
                         "entities": "Alice (PERSON), Google (ORGANIZATION)",
+                        "metadata": {"source": "slack", "channel": "engineering"},
                     }
                 ],
                 "total": 150,
@@ -1666,8 +1668,8 @@ class UpdateMemoryRequest(BaseModel):
 
     @model_validator(mode="after")
     def _require_an_edit(self) -> "UpdateMemoryRequest":
-        if all(
-            v is None
+        has_value_edit = any(
+            v is not None
             for v in (
                 self.text,
                 self.context,
@@ -1677,7 +1679,9 @@ class UpdateMemoryRequest(BaseModel):
                 self.entities,
                 self.state,
             )
-        ):
+        )
+        has_date_clear = bool({"occurred_start", "occurred_end"} & self.model_fields_set)
+        if not has_value_edit and not has_date_clear:
             raise ValueError("Provide at least one field to update.")
         if self.state is not None and self.state not in ("valid", "invalidated"):
             raise ValueError("state must be 'valid' or 'invalidated'.")
@@ -2203,7 +2207,8 @@ class BankTemplateConfig(BaseModel):
     reflect_mission: str | None = Field(default=None, description="Mission/context for Reflect operations")
     retain_mission: str | None = Field(default=None, description="Steers what gets extracted during retain")
     retain_extraction_mode: str | None = Field(
-        default=None, description="Fact extraction mode: 'concise' (default), 'verbose', or 'custom'"
+        default=None,
+        description="Fact extraction mode: 'concise' (default), 'verbose', 'custom', 'verbatim', or 'chunks'",
     )
     retain_custom_instructions: str | None = Field(
         default=None, description="Custom extraction prompt (when mode='custom')"
@@ -2429,10 +2434,10 @@ def validate_bank_template(manifest: "BankTemplateManifest") -> list[str]:
     if manifest.bank:
         bank = manifest.bank
         if bank.retain_extraction_mode is not None:
-            valid_modes = ("concise", "verbose", "custom", "chunks")
-            if bank.retain_extraction_mode not in valid_modes:
+            if bank.retain_extraction_mode not in RETAIN_EXTRACTION_MODES:
                 errors.append(
-                    f"bank.retain_extraction_mode: must be one of {valid_modes}, got '{bank.retain_extraction_mode}'"
+                    "bank.retain_extraction_mode: "
+                    f"must be one of {RETAIN_EXTRACTION_MODES}, got '{bank.retain_extraction_mode}'"
                 )
         if bank.retain_custom_instructions and bank.retain_extraction_mode != "custom":
             errors.append("bank.retain_custom_instructions: requires retain_extraction_mode='custom'")
@@ -3139,6 +3144,8 @@ def create_app(
                 max_slots=config.worker_max_slots,
                 slot_reservations=config.worker_slot_reservations,
                 consolidation_bank_priority=config.worker_consolidation_bank_priority or None,
+                operation_retention_days=config.operation_retention_days,
+                operation_cleanup_batch_size=config.operation_cleanup_batch_size,
             )
             poller_task = asyncio.create_task(poller.run())
             logging.info(f"Worker poller started (worker_id={worker_id})")
@@ -3750,13 +3757,23 @@ def _register_routes(app: FastAPI):
     ):
         """Curate a single memory unit (edit text / invalidate / revert)."""
         try:
+            occurred_start = (
+                ""
+                if "occurred_start" in request.model_fields_set and request.occurred_start is None
+                else request.occurred_start
+            )
+            occurred_end = (
+                ""
+                if "occurred_end" in request.model_fields_set and request.occurred_end is None
+                else request.occurred_end
+            )
             data = await app.state.memory.update_memory_unit(
                 bank_id=bank_id,
                 memory_id=memory_id,
                 text=request.text,
                 context=request.context,
-                occurred_start=request.occurred_start,
-                occurred_end=request.occurred_end,
+                occurred_start=occurred_start,
+                occurred_end=occurred_end,
                 new_fact_type=request.fact_type,
                 entities=request.entities,
                 state=request.state,
@@ -5407,8 +5424,9 @@ def _register_routes(app: FastAPI):
         "/v1/default/banks/{bank_id}/operations/{operation_id}",
         response_model=OperationStatusResponse,
         summary="Get operation status",
-        description="Get the status of a specific async operation. Returns 'pending', 'completed', or 'failed'. "
-        "Completed operations are removed from storage, so 'completed' means the operation finished successfully.",
+        description="Get the status of a specific async operation. Returns 'pending', 'processing', 'completed', "
+        "'failed', or 'cancelled'. Completed operations remain queryable with their payload for the configured "
+        "retention window and are pruned afterward.",
         operation_id="get_operation_status",
         tags=["Operations"],
     )

hindsight-api-slim/hindsight_api/config.py

@@ -147,6 +147,7 @@ ENV_LLM_EXTRA_BODY = "HINDSIGHT_API_LLM_EXTRA_BODY"
 ENV_LLM_DEFAULT_HEADERS = "HINDSIGHT_API_LLM_DEFAULT_HEADERS"
 ENV_LLM_STRICT_SCHEMA = "HINDSIGHT_API_LLM_STRICT_SCHEMA"
 ENV_LLM_SEND_BANK_AS_USER = "HINDSIGHT_API_LLM_SEND_BANK_AS_USER"
+ENV_LLM_OLLAMA_NUM_CTX = "HINDSIGHT_API_LLM_OLLAMA_NUM_CTX"
 
 # Per-operation sampling temperature. Each internal LLM call uses a temperature
 # tuned for its task (deterministic extraction vs. creative reflection). These
@@ -598,6 +599,8 @@ ENV_WORKER_MAX_RETRIES = "HINDSIGHT_API_WORKER_MAX_RETRIES"
 ENV_WORKER_TASK_RETRY_BACKOFF_SECONDS = "HINDSIGHT_API_WORKER_TASK_RETRY_BACKOFF_SECONDS"
 ENV_WORKER_HTTP_PORT = "HINDSIGHT_API_WORKER_HTTP_PORT"
 ENV_WORKER_MAX_SLOTS = "HINDSIGHT_API_WORKER_MAX_SLOTS"
+ENV_OPERATION_RETENTION_DAYS = "HINDSIGHT_API_OPERATION_RETENTION_DAYS"
+ENV_OPERATION_CLEANUP_BATCH_SIZE = "HINDSIGHT_API_OPERATION_CLEANUP_BATCH_SIZE"
 
 # Per-operation-type slot reservations. Each entry maps an operation_type
 # (as stored in async_operations.operation_type) to its env var and default.
@@ -638,6 +641,7 @@ ENV_RECALL_BUDGET_MAX = "HINDSIGHT_API_RECALL_BUDGET_MAX"
 
 # Recall candidate gating (per-source cap + BM25 score floor)
 ENV_BM25_MIN_SCORE = "HINDSIGHT_API_BM25_MIN_SCORE"
+ENV_BM25_MAX_QUERY_TERMS = "HINDSIGHT_API_BM25_MAX_QUERY_TERMS"
 ENV_RECALL_MAX_CANDIDATES_PER_SOURCE = "HINDSIGHT_API_RECALL_MAX_CANDIDATES_PER_SOURCE"
 # Per-strategy recall boost. Prioritises specific retrieval arms (semantic,
 # bm25, graph, temporal) on recall via a human priority level — e.g.
@@ -661,6 +665,9 @@ ENV_AUDIT_LOG_ENABLED = "HINDSIGHT_API_AUDIT_LOG_ENABLED"
 ENV_AUDIT_LOG_ACTIONS = "HINDSIGHT_API_AUDIT_LOG_ACTIONS"
 ENV_AUDIT_LOG_RETENTION_DAYS = "HINDSIGHT_API_AUDIT_LOG_RETENTION_DAYS"
 
+# Retain reliability settings
+ENV_FAIL_ON_EXTRACTION_ERRORS = "HINDSIGHT_API_FAIL_ON_EXTRACTION_ERRORS"
+
 # LLM request tracing settings
 ENV_LLM_TRACE_ENABLED = "HINDSIGHT_API_LLM_TRACE_ENABLED"
 ENV_LLM_TRACE_SCOPES = "HINDSIGHT_API_LLM_TRACE_SCOPES"
@@ -789,6 +796,9 @@ DEFAULT_SEMANTIC_MIN_SIMILARITY = 0.3
 # zero-score (non-matching) rows on backends — notably VectorChord — whose
 # operator ranks every document rather than pre-filtering to term matches.
 DEFAULT_BM25_MIN_SCORE = 0.0
+# Native tsvector BM25 can optionally cap the OR tsquery built from normalized
+# query tokens. 0 preserves the historical uncapped behavior.
+DEFAULT_BM25_MAX_QUERY_TERMS = 0
 # Per-source candidate cap applied to each retrieval arm (semantic, BM25, graph,
 # temporal) before RRF, so a single over-expanding backend cannot fill the
 # reranker's global candidate budget on its own. 0 disables the cap.
@@ -1050,6 +1060,10 @@ DEFAULT_WORKER_MAX_RETRIES = 3  # Max retries before marking task failed
 DEFAULT_WORKER_TASK_RETRY_BACKOFF_SECONDS = 60  # Seconds between retries on transient task failure
 DEFAULT_WORKER_HTTP_PORT = 8889  # HTTP port for worker metrics/health
 DEFAULT_WORKER_MAX_SLOTS = 10  # Total concurrent tasks per worker
+# Terminal rows keep their payload and metadata for one coherent debug/retry TTL.
+# Zero retention days disables automatic pruning entirely.
+DEFAULT_OPERATION_RETENTION_DAYS = 30
+DEFAULT_OPERATION_CLEANUP_BATCH_SIZE = 1000
 DEFAULT_RETAIN_MAX_CONCURRENT = 4  # Max concurrent retain DB phases (HNSW reads + writes). Limits I/O contention.
 
 # Reflect agent settings
@@ -1094,6 +1108,11 @@ DEFAULT_AUDIT_LOG_ENABLED = False  # Disabled by default
 DEFAULT_AUDIT_LOG_ACTIONS = ""  # Empty = audit all eligible actions
 DEFAULT_AUDIT_LOG_RETENTION_DAYS = -1  # -1 = keep forever
 
+# Retain reliability defaults
+DEFAULT_FAIL_ON_EXTRACTION_ERRORS = (
+    False  # Preserve existing behavior: retain completes even if some chunks fail extraction
+)
+
 # LLM request tracing defaults
 DEFAULT_LLM_TRACE_ENABLED = True  # Enabled by default
 DEFAULT_LLM_TRACE_SCOPES = ""  # Empty = trace all call scopes
@@ -1209,6 +1228,19 @@ def _parse_positive_int(name: str, raw: str | None, default: int) -> int:
     return parsed
 
 
+def _parse_non_negative_int(name: str, raw: str | None, default: int) -> int:
+    """Parse an env var that must be an integer >= 0."""
+    if raw is None or raw == "":
+        return default
+    try:
+        parsed = int(raw)
+    except ValueError as e:
+        raise ValueError(f"{name} must be an integer, got {raw!r}") from e
+    if parsed < 0:
+        raise ValueError(f"{name} must be >= 0, got {parsed}")
+    return parsed
+
+
 def _parse_optional_positive_int(name: str, raw: str | None) -> int | None:
     """Parse an optional env var that must be a positive integer when set."""
     if raw is None or raw == "":
@@ -1577,6 +1609,9 @@ class HindsightConfig:
     # LiteLLM, Helicone) key attribution on the OpenAI `user` field. Opt-in; never
     # overrides a `user` the caller already set.
     llm_send_bank_as_user: bool
+    # Optional native Ollama context window override. Unset lets Ollama use the
+    # model/server default instead of forcing a Hindsight-wide value.
+    llm_ollama_num_ctx: int | None = field(default=None, kw_only=True)
 
     # Per-operation sampling temperature. None means the temperature parameter is
     # omitted from the call (for models that reject explicit temperatures). See
@@ -1908,6 +1943,8 @@ class HindsightConfig:
     worker_max_slots: int
     worker_slot_reservations: dict[str, int]
     worker_consolidation_bank_priority: dict[str, int]
+    operation_retention_days: int
+    operation_cleanup_batch_size: int
     retain_max_concurrent: int
 
     # Reflect agent settings
@@ -1929,6 +1966,11 @@ class HindsightConfig:
     audit_log_actions: list[str]  # Allowlist of action types (empty = all)
     audit_log_retention_days: int  # -1 = keep forever, >0 = delete after N days
 
+    # Retain reliability configuration (static - server-level only)
+    # When True, a retain operation that accumulated any fact-extraction errors is
+    # marked 'failed' instead of 'completed', surfacing silent fact loss to clients.
+    fail_on_extraction_errors: bool
+
     # LLM request tracing configuration (static - server-level only)
     llm_trace_enabled: bool  # Master switch for per-bank LLM request tracing
     llm_trace_scopes: list[str]  # Allowlist of call scopes to trace (empty = all)
@@ -1979,6 +2021,7 @@ class HindsightConfig:
     reflect_llm_strategy: LLMStrategyConfig | None = None
     consolidation_llm_members: list[LLMMemberConfig] = field(default_factory=list)
     consolidation_llm_strategy: LLMStrategyConfig | None = None
+    bm25_max_query_terms: int = DEFAULT_BM25_MAX_QUERY_TERMS
 
     # Class-level sets for configuration categorization
 
@@ -2179,6 +2222,9 @@ class HindsightConfig:
                 f"Invalid semantic_min_similarity: {self.semantic_min_similarity}. Must be between 0.0 and 1.0"
             )
 
+        if self.bm25_max_query_terms < 0:
+            raise ValueError(f"Invalid bm25_max_query_terms: {self.bm25_max_query_terms}. Must be >= 0")
+
         # Validate bedrock_service_tier
         valid_bedrock_tiers = (None, "flex", "priority", "reserved")
         if self.llm_bedrock_service_tier not in valid_bedrock_tiers:
@@ -2268,6 +2314,13 @@ class HindsightConfig:
                 f"Reduce reservations or increase HINDSIGHT_API_WORKER_MAX_SLOTS."
             )
 
+        if self.operation_retention_days < 0:
+            raise ValueError(f"{ENV_OPERATION_RETENTION_DAYS} must be >= 0, got {self.operation_retention_days}")
+        if self.operation_cleanup_batch_size < 1:
+            raise ValueError(
+                f"{ENV_OPERATION_CLEANUP_BATCH_SIZE} must be >= 1, got {self.operation_cleanup_batch_size}"
+            )
+
     @classmethod
     def from_env(cls) -> "HindsightConfig":
         """Create configuration from environment variables."""
@@ -2319,6 +2372,10 @@ class HindsightConfig:
             llm_strict_schema=os.getenv(ENV_LLM_STRICT_SCHEMA, str(DEFAULT_LLM_STRICT_SCHEMA)).lower() in ("true", "1"),
             llm_send_bank_as_user=os.getenv(ENV_LLM_SEND_BANK_AS_USER, str(DEFAULT_LLM_SEND_BANK_AS_USER)).lower()
             in ("true", "1"),
+            llm_ollama_num_ctx=_parse_optional_positive_int(
+                ENV_LLM_OLLAMA_NUM_CTX,
+                os.getenv(ENV_LLM_OLLAMA_NUM_CTX),
+            ),
             llm_temperature_verification=_resolve_operation_temperature(
                 ENV_LLM_TEMPERATURE_VERIFICATION, DEFAULT_LLM_TEMPERATURE_VERIFICATION
             ),
@@ -2608,6 +2665,11 @@ class HindsightConfig:
             reranker_max_candidates=int(os.getenv(ENV_RERANKER_MAX_CANDIDATES, str(DEFAULT_RERANKER_MAX_CANDIDATES))),
             semantic_min_similarity=float(os.getenv(ENV_SEMANTIC_MIN_SIMILARITY, str(DEFAULT_SEMANTIC_MIN_SIMILARITY))),
             bm25_min_score=float(os.getenv(ENV_BM25_MIN_SCORE, str(DEFAULT_BM25_MIN_SCORE))),
+            bm25_max_query_terms=_parse_non_negative_int(
+                ENV_BM25_MAX_QUERY_TERMS,
+                os.getenv(ENV_BM25_MAX_QUERY_TERMS),
+                DEFAULT_BM25_MAX_QUERY_TERMS,
+            ),
             recall_max_candidates_per_source=int(
                 os.getenv(ENV_RECALL_MAX_CANDIDATES_PER_SOURCE, str(DEFAULT_RECALL_MAX_CANDIDATES_PER_SOURCE))
             ),
@@ -2924,6 +2986,16 @@ class HindsightConfig:
             worker_consolidation_bank_priority=_parse_bank_priority(
                 os.getenv(ENV_WORKER_CONSOLIDATION_BANK_PRIORITY, "")
             ),
+            operation_retention_days=_parse_non_negative_int(
+                ENV_OPERATION_RETENTION_DAYS,
+                os.getenv(ENV_OPERATION_RETENTION_DAYS),
+                DEFAULT_OPERATION_RETENTION_DAYS,
+            ),
+            operation_cleanup_batch_size=_parse_positive_int(
+                ENV_OPERATION_CLEANUP_BATCH_SIZE,
+                os.getenv(ENV_OPERATION_CLEANUP_BATCH_SIZE),
+                DEFAULT_OPERATION_CLEANUP_BATCH_SIZE,
+            ),
             retain_max_concurrent=int(os.getenv(ENV_RETAIN_MAX_CONCURRENT, str(DEFAULT_RETAIN_MAX_CONCURRENT))),
             # Reflect agent settings
             reflect_max_iterations=int(os.getenv(ENV_REFLECT_MAX_ITERATIONS, str(DEFAULT_REFLECT_MAX_ITERATIONS))),
@@ -2989,6 +3061,11 @@ class HindsightConfig:
             audit_log_retention_days=int(
                 os.getenv(ENV_AUDIT_LOG_RETENTION_DAYS, str(DEFAULT_AUDIT_LOG_RETENTION_DAYS))
             ),
+            # Retain reliability configuration (static, server-level only)
+            fail_on_extraction_errors=os.getenv(
+                ENV_FAIL_ON_EXTRACTION_ERRORS, str(DEFAULT_FAIL_ON_EXTRACTION_ERRORS)
+            ).lower()
+            == "true",
             # LLM request tracing configuration (static, server-level only)
             llm_trace_enabled=os.getenv(ENV_LLM_TRACE_ENABLED, str(DEFAULT_LLM_TRACE_ENABLED)).lower() == "true",
             llm_trace_scopes=[

hindsight-api-slim/hindsight_api/engine/chinese_temporal_periods.py

@@ -109,6 +109,11 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
             end.replace(hour=23, minute=59, second=59, microsecond=999999),
         )
 
+    def safe_constraint(start: datetime | None, end: datetime | None) -> DateRange | NoTemporalConstraintSentinel:
+        if start is None or end is None:
+            return NO_TEMPORAL_CONSTRAINT
+        return constraint(start, end)
+
     def subtract_months(months: int) -> datetime:
         month_index = reference_date.month - months - 1
         year = reference_date.year + month_index // 12
@@ -126,11 +131,21 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
         day = min(base_date.day, calendar.monthrange(year, month)[1])
         return base_date.replace(year=year, month=month, day=day)
 
-    def add_years(base_date: datetime, years: int) -> datetime:
+    def add_years(base_date: datetime, years: int) -> datetime | None:
         year = base_date.year + years
+        if year < datetime.min.year or year > datetime.max.year:
+            return None
         day = min(base_date.day, calendar.monthrange(year, base_date.month)[1])
         return base_date.replace(year=year, day=day)
 
+    def add_days(base_date: datetime | None, days: int) -> datetime | None:
+        if base_date is None:
+            return None
+        try:
+            return base_date + timedelta(days=days)
+        except OverflowError:
+            return None
+
     def has_chinese_temporal_context(match: re.Match[str]) -> bool:
         if match.end() >= len(query):
             return True
@@ -438,6 +453,11 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
             return NO_TEMPORAL_CONSTRAINT
         return constraint(start, reference_date)
 
+    def safe_since_constraint(start: datetime | None) -> DateRange | NoTemporalConstraintSentinel:
+        if start is None:
+            return NO_TEMPORAL_CONSTRAINT
+        return since_constraint(start)
+
     def since_from_period(
         period: DateRange | None,
     ) -> DateRange | NoTemporalConstraintSentinel | None:
@@ -450,7 +470,7 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
             return None
         return since_constraint(day)
 
-    def relative_offset_datetime(amount: int, unit: str, direction: int) -> datetime:
+    def relative_offset_datetime(amount: int, unit: str, direction: int) -> datetime | None:
         if unit in ("天", "日"):
             return reference_date + timedelta(days=direction * amount)
         if unit in ("周", "星期", "礼拜"):
@@ -459,15 +479,15 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
             return add_months(reference_date, direction * amount)
         return add_years(reference_date, direction * amount)
 
-    def point_constraint_at_offset(amount: int, unit: str, direction: int) -> DateRange:
+    def point_constraint_at_offset(amount: int, unit: str, direction: int) -> DateRange | NoTemporalConstraintSentinel:
         d = relative_offset_datetime(amount, unit, direction)
-        return constraint(d, d)
+        return safe_constraint(d, d)
 
-    def window_to_reference(amount: int, unit: str) -> DateRange:
-        return constraint(relative_offset_datetime(amount, unit, -1), reference_date)
+    def window_to_reference(amount: int, unit: str) -> DateRange | NoTemporalConstraintSentinel:
+        return safe_constraint(relative_offset_datetime(amount, unit, -1), reference_date)
 
-    def window_from_reference(amount: int, unit: str) -> DateRange:
-        return constraint(reference_date, relative_offset_datetime(amount, unit, 1))
+    def window_from_reference(amount: int, unit: str) -> DateRange | NoTemporalConstraintSentinel:
+        return safe_constraint(reference_date, relative_offset_datetime(amount, unit, 1))
 
     # Chinese rule guide
     #
@@ -781,8 +801,8 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
     if relative_year_fixed_day_since_match:
         year = relative_year_number(relative_year_fixed_day_since_match.group(1))
         base = add_years(reference_date, year - reference_date.year)
-        d = base + timedelta(days=fixed_day_offset(relative_year_fixed_day_since_match.group(2)))
-        return since_constraint(d)
+        d = add_days(base, fixed_day_offset(relative_year_fixed_day_since_match.group(2)))
+        return safe_since_constraint(d)
 
     fixed_day_since_match = chinese_search(
         rf"(大大后天|大后天|后天|明天|明日|今天|今日|本日|当日|当天|昨天|昨日|大大前天|大前天|前天)"
@@ -799,7 +819,7 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
         amount = parse_chinese_number(exact_relative_since_match.group(1))
         unit = exact_relative_since_match.group(2)
         if amount is not None:
-            return since_constraint(relative_offset_datetime(amount, unit, -1))
+            return safe_since_constraint(relative_offset_datetime(amount, unit, -1))
 
     weekend_since_match = chinese_search(
         rf"(?<![上下大小每个各隔])"
@@ -899,8 +919,8 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
     if relative_year_daypart_since_match:
         year = relative_year_number(relative_year_daypart_since_match.group(1))
         base = add_years(reference_date, year - reference_date.year)
-        d = base + timedelta(days=daypart_day_offset(relative_year_daypart_since_match.group(2)))
-        return since_constraint(d)
+        d = add_days(base, daypart_day_offset(relative_year_daypart_since_match.group(2)))
+        return safe_since_constraint(d)
 
     daypart_since_match = chinese_search(
         rf"(昨晚|昨夜|前晚|前夜|今晚|今早|今晨|明早|明晚|明夜){chinese_since_suffix_pattern}"
@@ -915,17 +935,17 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
     if relative_year_daypart_match:
         year = relative_year_number(relative_year_daypart_match.group(1))
         base = add_years(reference_date, year - reference_date.year)
-        d = base + timedelta(days=daypart_day_offset(relative_year_daypart_match.group(2)))
-        return constraint(d, d)
+        d = add_days(base, daypart_day_offset(relative_year_daypart_match.group(2)))
+        return safe_constraint(d, d)
 
     # Day-part abbreviations still resolve only to date granularity.
     if chinese_search(r"昨晚|昨夜"):
-        d = reference_date + timedelta(days=daypart_day_offset("昨晚"))
-        return constraint(d, d)
+        d = add_days(reference_date, daypart_day_offset("昨晚"))
+        return safe_constraint(d, d)
 
     if chinese_search(r"前晚|前夜"):
-        d = reference_date + timedelta(days=daypart_day_offset("前晚"))
-        return constraint(d, d)
+        d = add_days(reference_date, daypart_day_offset("前晚"))
+        return safe_constraint(d, d)
 
     if chinese_search(r"今晚|今早|今晨"):
         return constraint(reference_date, reference_date)
@@ -941,8 +961,8 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
     if relative_year_fixed_day_match:
         year = relative_year_number(relative_year_fixed_day_match.group(1))
         base = add_years(reference_date, year - reference_date.year)
-        d = base + timedelta(days=fixed_day_offset(relative_year_fixed_day_match.group(2)))
-        return constraint(d, d)
+        d = add_days(base, fixed_day_offset(relative_year_fixed_day_match.group(2)))
+        return safe_constraint(d, d)
 
     if chinese_search(r"昨天|昨日"):
         d = reference_date - timedelta(days=1)
@@ -1085,7 +1105,7 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
             end_amount = parse_chinese_number(amount_text[-1])
         unit = adjacent_fuzzy_future_match.group(2)
         if start_amount is not None and end_amount is not None:
-            return constraint(
+            return safe_constraint(
                 relative_offset_datetime(start_amount, unit, 1),
                 relative_offset_datetime(end_amount, unit, 1),
             )
@@ -1093,7 +1113,7 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
     few_future_match = chinese_search(rf"[几数]个?(天|日|周|星期|礼拜|月|年){chinese_relative_future_suffix_pattern}")
     if few_future_match:
         unit = few_future_match.group(1)
-        return constraint(relative_offset_datetime(2, unit, 1), relative_offset_datetime(5, unit, 1))
+        return safe_constraint(relative_offset_datetime(2, unit, 1), relative_offset_datetime(5, unit, 1))
 
     exact_future_match = chinese_search(
         rf"(?<![{_CHINESE_NUMERAL_PREFIX_CHARS}])([0-9]+|[{_CHINESE_NUMERAL_CHARS}]+)个?(天|日|周|星期|礼拜|月|年){chinese_relative_future_suffix_pattern}"
@@ -1113,7 +1133,7 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
         second_amount = parse_chinese_number(adjacent_fuzzy_past_match.group(2))
         unit = adjacent_fuzzy_past_match.group(3)
         if first_amount is not None and second_amount is not None and second_amount == first_amount + 1:
-            return constraint(
+            return safe_constraint(
                 relative_offset_datetime(second_amount, unit, -1),
                 relative_offset_datetime(first_amount, unit, -1),
             )
@@ -1144,7 +1164,7 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
         return constraint(reference_date - timedelta(days=150), reference_date - timedelta(days=60))
 
     if chinese_search(r"一两年前|[两二]三年前|三两年前"):
-        return constraint(add_years(reference_date, -3), add_years(reference_date, -1))
+        return safe_constraint(add_years(reference_date, -3), add_years(reference_date, -1))
 
     rolling_this_adjacent_match = chinese_search(
         r"这(一两|[两二]三|三两|三四|四五|五六|六七|七八|八九|九十)个?(天|日|周|星期|礼拜|月|年)"
@@ -1154,7 +1174,7 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
         end_amount = 3 if amount_text in ("一两", "三两") else parse_chinese_number(amount_text[-1])
         unit = rolling_this_adjacent_match.group(2)
         if end_amount is not None:
-            return constraint(relative_offset_datetime(end_amount, unit, -1), reference_date)
+            return safe_constraint(relative_offset_datetime(end_amount, unit, -1), reference_date)
 
     rolling_this_count_match = chinese_search(rf"这([0-9]+|[{_CHINESE_NUMERAL_CHARS}]+)个?(天|日|周|星期|礼拜|月|年)")
     if rolling_this_count_match:
@@ -1191,7 +1211,7 @@ def extract_chinese_period(query: str, reference_date: datetime) -> DateRange |
         end_amount = 3 if amount_text in ("一两", "三两") else parse_chinese_number(amount_text[-1])
         unit = rolling_past_adjacent_match.group(3)
         if end_amount is not None:
-            return constraint(relative_offset_datetime(end_amount, unit, -1), reference_date)
+            return safe_constraint(relative_offset_datetime(end_amount, unit, -1), reference_date)
 
     rolling_past_few_match = chinese_search(r"(过去|近|最近)几个?(天|日|周|星期|礼拜|月|年)")
     if rolling_past_few_match:

hindsight-api-slim/hindsight_api/engine/consolidation/consolidator.py

@@ -28,6 +28,7 @@ from fnmatch import fnmatchcase
 from itertools import combinations
 from typing import TYPE_CHECKING, Any, Literal
 
+import asyncpg
 from pydantic import BaseModel, field_validator
 
 from ...config import get_config
@@ -102,6 +103,29 @@ class _DedupDecision(BaseModel):
     text: str = ""  # the synthesized merged observation (when action == "merge")
     reason: str = ""
 
+    @field_validator("action", mode="before")
+    @classmethod
+    def _normalize_action(cls, value: object) -> str:
+        if isinstance(value, str):
+            normalized = value.strip().lower()
+            if normalized in {"merge", "keep"}:
+                return normalized
+
+        logger.warning("Invalid consolidation dedup action %r; defaulting to keep", value)
+        return "keep"
+
+
+def _dedup_decision_from_response(raw: Any) -> _DedupDecision:
+    try:
+        if isinstance(raw, _DedupDecision):
+            return raw
+        if isinstance(raw, str):
+            return _DedupDecision.model_validate_json(raw)
+        return _DedupDecision.model_validate(raw)
+    except ValueError as exc:
+        logger.warning("Invalid consolidation dedup response %r; defaulting to keep: %s", raw, exc)
+        return _DedupDecision(action="keep", reason="invalid structured response")
+
 
 _DEDUP_PROMPT = """You reconcile long-term memory observations. A NEW observation is about to be \
 stored, and it is highly similar to an EXISTING one:
@@ -109,9 +133,20 @@ stored, and it is highly similar to an EXISTING one:
 [NEW] {new}
 [EXISTING] {existing}
 
-If they assert the SAME fact (wording aside), respond action="merge" and provide `text`: a single \
-observation that preserves EVERY detail from both. If they differ in ANY important detail — a \
-number/quantity, a named entity or language, a negation, or a condition — respond action="keep"."""
+Respond with ONLY one valid JSON object matching one of these shapes:
+
+For duplicate facts:
+{{"action": "merge", "text": "...", "reason": "..."}}
+
+For distinct facts:
+{{"action": "keep", "text": "", "reason": "..."}}
+
+Do NOT use key=value lines, markdown fences, or any text outside the JSON object.
+
+If they assert the SAME fact (wording aside), set "action" to "merge" and provide "text": a \
+single observation that preserves EVERY detail from both. If they differ in ANY important detail \
+— a number/quantity, a named entity or language, a negation, or a condition — set "action" to \
+"keep" and "text" to an empty string."""
 
 
 def _dedup_active(config: Any) -> bool:
@@ -189,10 +224,12 @@ async def _dedup_adjudicate(
     if best_id is None:
         return _DedupOutcome(best_id=None, merged_text="", should_merge=False)
 
-    decision: _DedupDecision = await dedup_llm_config.call(
-        messages=[{"role": "user", "content": _DEDUP_PROMPT.format(new=anchor_text, existing=best_text)}],
-        response_format=_DedupDecision,
-        scope="consolidation_dedup",
+    decision = _dedup_decision_from_response(
+        await dedup_llm_config.call(
+            messages=[{"role": "user", "content": _DEDUP_PROMPT.format(new=anchor_text, existing=best_text)}],
+            response_format=_DedupDecision,
+            scope="consolidation_dedup",
+        )
     )
     if decision.action != "merge":
         return _DedupOutcome(best_id=best_id, merged_text="", should_merge=False)
@@ -1766,15 +1803,22 @@ async def _append_observation_history(
     history from growing without bound.
     """
     obs_uuid = uuid.UUID(observation_id)
-    await conn.execute(
-        f"""
+    try:
+        await conn.execute(
+            f"""
         INSERT INTO {fq_table("observation_history")} (observation_id, bank_id, content, changed_at)
         VALUES ($1, $2, $3::jsonb, now())
         """,
-        obs_uuid,
-        bank_id,
-        json.dumps(asdict(snapshot)),
-    )
+            obs_uuid,
+            bank_id,
+            json.dumps(asdict(snapshot)),
+        )
+    except asyncpg.exceptions.ForeignKeyViolationError:
+        logger.warning(
+            f"FK violation writing observation_history for {observation_id}: "
+            "observation was removed before history could be written (race with parallel consolidation). Skipping."
+        )
+        return
     if max_entries and max_entries > 0:
         await conn.execute(
             f"""

hindsight-api-slim/hindsight_api/engine/db/ops.py

@@ -18,6 +18,7 @@ and mirrors Django's ``DatabaseOperations`` architecture.
 
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
+from datetime import datetime
 from typing import Any
 
 from .base import DatabaseConnection
@@ -484,6 +485,23 @@ class DataAccessOps(ABC):
 
     # -- Task claiming operations ------------------------------------------
 
+    @abstractmethod
+    async def prune_terminal_operations(
+        self,
+        conn: DatabaseConnection,
+        table: str,
+        cutoff: datetime,
+        *,
+        batch_size: int,
+    ) -> int:
+        """Delete one deterministic batch of terminal operations older than ``cutoff``.
+
+        Implementations must lock candidates without waiting on rows another
+        worker is pruning, never select pending/processing rows, and return the
+        number deleted. The caller provides a transaction around this method.
+        """
+        ...
+
     @abstractmethod
     async def claim_tasks(
         self,

hindsight-api-slim/hindsight_api/engine/db/ops_oracle.py

@@ -13,6 +13,8 @@ from .base import DatabaseConnection
 from .ops import DataAccessOps, TagListingParts
 from .result import DictResultRow as ResultRow
 
+ORACLE_IN_LIST_LIMIT = 1000
+
 
 class OracleOps(DataAccessOps):
     """Oracle-specific data access operations."""
@@ -447,6 +449,14 @@ class OracleOps(DataAccessOps):
                     FROM {ue_table} ue_target
                     WHERE ue_target.entity_id = se.entity_id
                       AND ue_target.unit_id != ALL($1::uuid[])
+                      -- Filter before applying the cap: candidates from other fact
+                      -- types must not consume this entity's bounded fan-out.
+                      AND EXISTS (
+                          SELECT 1
+                          FROM {mu_table} mu_target
+                          WHERE mu_target.id = ue_target.unit_id
+                            AND mu_target.fact_type = $2
+                      )
                     ORDER BY ue_target.unit_id DESC
                     FETCH FIRST {per_entity_limit} ROWS ONLY
                 ) t
@@ -459,7 +469,6 @@ class OracleOps(DataAccessOps):
                        es.score, 'entity' AS source
                 FROM entity_scores es
                 JOIN {mu_table} mu ON mu.id = es.unit_id
-                WHERE mu.fact_type = $2
                 ORDER BY es.score DESC
                 FETCH FIRST $3 ROWS ONLY
             )"""
@@ -824,6 +833,157 @@ class OracleOps(DataAccessOps):
 
     # -- Task claiming operations ------------------------------------------
 
+    async def prune_terminal_operations(
+        self,
+        conn: DatabaseConnection,
+        table: str,
+        cutoff: datetime,
+        *,
+        batch_size: int,
+    ) -> int:
+        # Oracle rejects a row-limited SELECT ... FOR UPDATE (ORA-02014). Pick
+        # the deterministic bounded IDs first, then lock only that candidate
+        # set and re-check eligibility before deleting in the same transaction.
+        # Clamp to Oracle's 1000-expression IN-list limit because the adapter
+        # expands the candidate UUID list into individual bind variables.
+        # Cancelled children cannot complete parent aggregation, so retain the
+        # parent guard only for completed/failed children. Before removing a
+        # cancelled child, preserve its signal by cancelling a pending parent
+        # in this transaction and refreshing the parent's retention window.
+        # Validate metadata before HEXTORAW: CASE makes malformed UUIDs yield
+        # NULL while keeping the indexed RAW parent.operation_id key unwrapped.
+        effective_batch_size = min(batch_size, ORACLE_IN_LIST_LIMIT)
+        candidates = await conn.fetch(
+            f"""
+            SELECT candidate_operation.operation_id
+            FROM {table} candidate_operation
+            WHERE candidate_operation.status IN ('completed', 'failed', 'cancelled')
+              AND candidate_operation.updated_at < $1
+              AND (
+                  candidate_operation.status = 'cancelled'
+                  OR NOT EXISTS (
+                      SELECT 1
+                      FROM {table} parent
+                      WHERE parent.operation_id = CASE
+                          WHEN REGEXP_LIKE(
+                              JSON_VALUE(
+                                  candidate_operation.result_metadata,
+                                  '$.parent_operation_id' RETURNING VARCHAR2(36) NULL ON ERROR
+                              ),
+                              '^[0-9A-Fa-f]{{8}}-[0-9A-Fa-f]{{4}}-[0-9A-Fa-f]{{4}}-[0-9A-Fa-f]{{4}}-[0-9A-Fa-f]{{12}}$'
+                          )
+                          THEN HEXTORAW(REPLACE(
+                              JSON_VALUE(
+                                  candidate_operation.result_metadata,
+                                  '$.parent_operation_id' RETURNING VARCHAR2(36) NULL ON ERROR
+                              ),
+                              '-',
+                              ''
+                          ))
+                          ELSE NULL
+                      END
+                        AND parent.bank_id = candidate_operation.bank_id
+                  )
+              )
+            ORDER BY candidate_operation.updated_at, candidate_operation.operation_id
+            LIMIT $2
+            """,
+            cutoff,
+            effective_batch_size,
+        )
+        if not candidates:
+            return 0
+
+        candidate_ids = [row["operation_id"] for row in candidates]
+        locked = await conn.fetch(
+            f"""
+            SELECT candidate_operation.operation_id
+            FROM {table} candidate_operation
+            WHERE candidate_operation.operation_id = ANY($1)
+              AND candidate_operation.status IN ('completed', 'failed', 'cancelled')
+              AND candidate_operation.updated_at < $2
+              AND (
+                  candidate_operation.status = 'cancelled'
+                  OR NOT EXISTS (
+                      SELECT 1
+                      FROM {table} parent
+                      WHERE parent.operation_id = CASE
+                          WHEN REGEXP_LIKE(
+                              JSON_VALUE(
+                                  candidate_operation.result_metadata,
+                                  '$.parent_operation_id' RETURNING VARCHAR2(36) NULL ON ERROR
+                              ),
+                              '^[0-9A-Fa-f]{{8}}-[0-9A-Fa-f]{{4}}-[0-9A-Fa-f]{{4}}-[0-9A-Fa-f]{{4}}-[0-9A-Fa-f]{{12}}$'
+                          )
+                          THEN HEXTORAW(REPLACE(
+                              JSON_VALUE(
+                                  candidate_operation.result_metadata,
+                                  '$.parent_operation_id' RETURNING VARCHAR2(36) NULL ON ERROR
+                              ),
+                              '-',
+                              ''
+                          ))
+                          ELSE NULL
+                      END
+                        AND parent.bank_id = candidate_operation.bank_id
+                  )
+              )
+            ORDER BY candidate_operation.updated_at, candidate_operation.operation_id
+            FOR UPDATE OF candidate_operation.operation_id SKIP LOCKED
+            """,
+            candidate_ids,
+            cutoff,
+        )
+        if not locked:
+            return 0
+        operation_ids = [row["operation_id"] for row in locked]
+        await conn.execute(
+            f"""
+            UPDATE {table} parent
+            SET status = 'cancelled',
+                updated_at = now(),
+                completed_at = COALESCE(parent.completed_at, now()),
+                error_message = COALESCE(
+                    parent.error_message,
+                    'Cancelled because a child operation was cancelled'
+                )
+            WHERE parent.status = 'pending'
+              AND EXISTS (
+                  SELECT 1
+                  FROM {table} candidate_operation
+                  WHERE candidate_operation.operation_id = ANY($1)
+                    AND candidate_operation.status = 'cancelled'
+                    AND candidate_operation.updated_at < $2
+                    AND candidate_operation.bank_id = parent.bank_id
+                    AND parent.operation_id = CASE
+                        WHEN REGEXP_LIKE(
+                            JSON_VALUE(
+                                candidate_operation.result_metadata,
+                                '$.parent_operation_id' RETURNING VARCHAR2(36) NULL ON ERROR
+                            ),
+                            '^[0-9A-Fa-f]{{8}}-[0-9A-Fa-f]{{4}}-[0-9A-Fa-f]{{4}}-[0-9A-Fa-f]{{4}}-[0-9A-Fa-f]{{12}}$'
+                        )
+                        THEN HEXTORAW(REPLACE(
+                            JSON_VALUE(
+                                candidate_operation.result_metadata,
+                                '$.parent_operation_id' RETURNING VARCHAR2(36) NULL ON ERROR
+                            ),
+                            '-',
+                            ''
+                        ))
+                        ELSE NULL
+                    END
+              )
+            """,
+            operation_ids,
+            cutoff,
+        )
+        await conn.execute(
+            f"DELETE FROM {table} WHERE operation_id = ANY($1)",
+            operation_ids,
+        )
+        return len(operation_ids)
+
     async def _claim_consolidation_tasks(
         self,
         conn,

hindsight-api-slim/hindsight_api/engine/db/ops_postgresql.py

@@ -4,11 +4,40 @@ Uses unnest(), LATERAL, DISTINCT ON, and native array operations for
 efficient batch operations.
 """
 
+from datetime import datetime
+
 from .base import DatabaseConnection
 from .ops import DataAccessOps, TagListingParts
 from .result import ResultRow
 
 
+def pg_search_vector_expr(
+    config,
+    *,
+    text_col: str = "text",
+    context_col: str = "context",
+    signals_col: str = "text_signals",
+) -> str | None:
+    """SQL expression that builds ``search_vector`` for the configured PG text-search backend.
+
+    Single source of truth shared by the batch insert (over the ``input_data``
+    CTE columns) and the curation revert recompute (over a ``memory_units`` row),
+    so the two can never drift. Returns ``None`` for backends that leave
+    ``search_vector`` unpopulated — pgroonga / pg_textsearch / pg_search index the
+    base text columns directly and keep only a dummy column, so there is nothing
+    to build.
+
+    ``text_search_extension_native_language`` is validated as a PG identifier in
+    ``HindsightConfig.validate()``, so embedding it as a SQL literal is safe.
+    """
+    combined = f"COALESCE({text_col}, '') || ' ' || COALESCE({context_col}, '') || ' ' || COALESCE({signals_col}, '')"
+    if config.text_search_extension == "vchord":
+        return f"tokenize({combined}, 'llmlingua2')::bm25_catalog.bm25vector"
+    if config.text_search_extension == "native":
+        return f"to_tsvector('{config.text_search_extension_native_language}'::regconfig, {combined})"
+    return None
+
+
 class PostgreSQLOps(DataAccessOps):
     """PostgreSQL-specific data access operations using unnest and LATERAL."""
 
@@ -93,101 +122,39 @@ class PostgreSQLOps(DataAccessOps):
         config = get_config()
         table = self._get_mu_table()
 
-        if config.text_search_extension == "vchord":
-            query = f"""
-                WITH input_data AS (
-                    SELECT * FROM unnest(
-                        $2::text[], $3::vector[], $4::timestamptz[], $5::timestamptz[], $6::timestamptz[], $7::timestamptz[],
-                        $8::text[], $9::text[], $10::jsonb[], $11::text[], $12::text[], $13::jsonb[], $14::jsonb[], $15::text[]
-                    ) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
-                           context, fact_type, metadata, chunk_id, document_id, tags_json,
-                           observation_scopes_json, text_signals)
-                )
-                INSERT INTO {table} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
-                                     context, fact_type, metadata, chunk_id, document_id, tags,
-                                     observation_scopes, text_signals, search_vector)
-                SELECT
-                    $1,
-                    text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
-                    context, fact_type, metadata, chunk_id, document_id,
-                    COALESCE(
-                        (SELECT array_agg(elem) FROM jsonb_array_elements_text(tags_json) AS elem),
-                        '{{}}'::varchar[]
-                    ),
-                    observation_scopes_json,
-                    text_signals,
-                    tokenize(
-                        COALESCE(text, '') || ' ' || COALESCE(context, '') || ' ' || COALESCE(text_signals, ''),
-                        'llmlingua2'
-                    )::bm25_catalog.bm25vector
-                FROM input_data
-                RETURNING id
-            """
-        elif config.text_search_extension == "native":
-            # search_vector is a regular tsvector column populated here using the
-            # configured native dictionary. It used to be GENERATED ALWAYS with
-            # a hardcoded 'english', which prevented per-deployment language
-            # configuration. text_search_extension_native_language is validated
-            # in HindsightConfig.validate() as a PG identifier, so embedding it
-            # as a SQL literal is safe.
-            query = f"""
-                WITH input_data AS (
-                    SELECT * FROM unnest(
-                        $2::text[], $3::vector[], $4::timestamptz[], $5::timestamptz[], $6::timestamptz[], $7::timestamptz[],
-                        $8::text[], $9::text[], $10::jsonb[], $11::text[], $12::text[], $13::jsonb[], $14::jsonb[], $15::text[]
-                    ) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
-                           context, fact_type, metadata, chunk_id, document_id, tags_json,
-                           observation_scopes_json, text_signals)
-                )
-                INSERT INTO {table} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
-                                     context, fact_type, metadata, chunk_id, document_id, tags,
-                                     observation_scopes, text_signals, search_vector)
-                SELECT
-                    $1,
-                    text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
-                    context, fact_type, metadata, chunk_id, document_id,
-                    COALESCE(
-                        (SELECT array_agg(elem) FROM jsonb_array_elements_text(tags_json) AS elem),
-                        '{{}}'::varchar[]
-                    ),
-                    observation_scopes_json,
-                    text_signals,
-                    to_tsvector(
-                        '{config.text_search_extension_native_language}'::regconfig,
-                        COALESCE(text, '') || ' ' || COALESCE(context, '') || ' ' || COALESCE(text_signals, '')
-                    )
-                FROM input_data
-                RETURNING id
-            """
-        else:
-            # pg_textsearch, pgroonga, and pg_search: search_vector is a dummy
-            # TEXT column; the actual full-text index operates on the base text
-            # columns directly, so we don't populate search_vector at insert time.
-            query = f"""
-                WITH input_data AS (
-                    SELECT * FROM unnest(
-                        $2::text[], $3::vector[], $4::timestamptz[], $5::timestamptz[], $6::timestamptz[], $7::timestamptz[],
-                        $8::text[], $9::text[], $10::jsonb[], $11::text[], $12::text[], $13::jsonb[], $14::jsonb[], $15::text[]
-                    ) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
-                           context, fact_type, metadata, chunk_id, document_id, tags_json,
-                           observation_scopes_json, text_signals)
-                )
-                INSERT INTO {table} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
-                                     context, fact_type, metadata, chunk_id, document_id, tags,
-                                     observation_scopes, text_signals)
-                SELECT
-                    $1,
-                    text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
-                    context, fact_type, metadata, chunk_id, document_id,
-                    COALESCE(
-                        (SELECT array_agg(elem) FROM jsonb_array_elements_text(tags_json) AS elem),
-                        '{{}}'::varchar[]
-                    ),
-                    observation_scopes_json,
-                    text_signals
-                FROM input_data
-                RETURNING id
-            """
+        # search_vector is populated inline for backends that store a real vector
+        # (native tsvector, vchord bm25vector). pgroonga / pg_textsearch / pg_search
+        # index the base text columns directly and keep only a dummy column, so the
+        # expression is None and the column is left out of the insert entirely.
+        # Same expression is reused by curation revert (see pg_search_vector_expr).
+        sv_expr = pg_search_vector_expr(config)
+        sv_insert_col = ", search_vector" if sv_expr else ""
+        sv_select_val = f",\n                    {sv_expr}" if sv_expr else ""
+        query = f"""
+            WITH input_data AS (
+                SELECT * FROM unnest(
+                    $2::text[], $3::vector[], $4::timestamptz[], $5::timestamptz[], $6::timestamptz[], $7::timestamptz[],
+                    $8::text[], $9::text[], $10::jsonb[], $11::text[], $12::text[], $13::jsonb[], $14::jsonb[], $15::text[]
+                ) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
+                       context, fact_type, metadata, chunk_id, document_id, tags_json,
+                       observation_scopes_json, text_signals)
+            )
+            INSERT INTO {table} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
+                                 context, fact_type, metadata, chunk_id, document_id, tags,
+                                 observation_scopes, text_signals{sv_insert_col})
+            SELECT
+                $1,
+                text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
+                context, fact_type, metadata, chunk_id, document_id,
+                COALESCE(
+                    (SELECT array_agg(elem) FROM jsonb_array_elements_text(tags_json) AS elem),
+                    '{{}}'::varchar[]
+                ),
+                observation_scopes_json,
+                text_signals{sv_select_val}
+            FROM input_data
+            RETURNING id
+        """
 
         results = await conn.fetch(
             query,
@@ -541,11 +508,18 @@ class PostgreSQLOps(DataAccessOps):
                     FROM {ue_table} ue_target
                     WHERE ue_target.entity_id = se.entity_id
                       AND ue_target.unit_id != ALL($1::uuid[])
+                      -- Filter before applying the cap: candidates from other fact
+                      -- types must not consume this entity's bounded fan-out.
+                      AND EXISTS (
+                          SELECT 1
+                          FROM {mu_table} mu_target
+                          WHERE mu_target.id = ue_target.unit_id
+                            AND mu_target.fact_type = $2
+                      )
                     ORDER BY ue_target.unit_id DESC
                     LIMIT {per_entity_limit}
                 ) t
                 JOIN {mu_table} mu ON mu.id = t.unit_id
-                WHERE mu.fact_type = $2
                 GROUP BY mu.id
                 ORDER BY score DESC
                 LIMIT $3
@@ -894,6 +868,93 @@ class PostgreSQLOps(DataAccessOps):
 
     # -- Task claiming operations ------------------------------------------
 
+    async def prune_terminal_operations(
+        self,
+        conn: DatabaseConnection,
+        table: str,
+        cutoff: datetime,
+        *,
+        batch_size: int,
+    ) -> int:
+        # Lock only the bounded candidate set. SKIP LOCKED lets multiple
+        # workers prune disjoint batches without waiting or double-deleting.
+        # Cancelled children cannot complete parent aggregation, so retain the
+        # parent guard only for completed/failed children. Before removing a
+        # cancelled child, preserve its signal by cancelling a pending parent
+        # in this transaction and refreshing the parent's retention window.
+        candidates = await conn.fetch(
+            f"""
+            SELECT candidate_operation.operation_id
+            FROM {table} candidate_operation
+            WHERE candidate_operation.status IN ('completed', 'failed', 'cancelled')
+              AND candidate_operation.updated_at < $1
+              AND (
+                  candidate_operation.status = 'cancelled'
+                  OR NOT EXISTS (
+                      SELECT 1
+                      FROM {table} parent
+                      WHERE parent.operation_id = CASE
+                          WHEN candidate_operation.result_metadata->>'parent_operation_id'
+                              ~* '^[0-9a-f]{{8}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{12}}$'
+                          THEN (candidate_operation.result_metadata->>'parent_operation_id')::uuid
+                          ELSE NULL
+                      END
+                        AND parent.bank_id = candidate_operation.bank_id
+                  )
+              )
+            ORDER BY candidate_operation.updated_at, candidate_operation.operation_id
+            LIMIT $2
+            FOR UPDATE OF candidate_operation SKIP LOCKED
+            """,
+            cutoff,
+            batch_size,
+        )
+        if not candidates:
+            return 0
+
+        candidate_ids = [row["operation_id"] for row in candidates]
+        await conn.execute(
+            f"""
+            UPDATE {table} parent
+            SET status = 'cancelled',
+                updated_at = now(),
+                completed_at = COALESCE(parent.completed_at, now()),
+                error_message = COALESCE(
+                    parent.error_message,
+                    'Cancelled because a child operation was cancelled'
+                )
+            WHERE parent.status = 'pending'
+              AND EXISTS (
+                  SELECT 1
+                  FROM {table} candidate_operation
+                  WHERE candidate_operation.operation_id = ANY($1)
+                    AND candidate_operation.status = 'cancelled'
+                    AND candidate_operation.updated_at < $2
+                    AND candidate_operation.bank_id = parent.bank_id
+                    AND parent.operation_id = CASE
+                        WHEN candidate_operation.result_metadata->>'parent_operation_id'
+                            ~* '^[0-9a-f]{{8}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{12}}$'
+                        THEN (candidate_operation.result_metadata->>'parent_operation_id')::uuid
+                        ELSE NULL
+                    END
+              )
+            """,
+            candidate_ids,
+            cutoff,
+        )
+        rows = await conn.fetch(
+            f"""
+            DELETE FROM {table}
+            WHERE operation_id = ANY($1)
+              AND status IN ('completed', 'failed', 'cancelled')
+              AND updated_at < $2
+            RETURNING operation_id
+            """,
+            candidate_ids,
+            cutoff,
+        )
+        return len(rows)
+
     async def _claim_consolidation_tasks(
         self,
         conn,

hindsight-api-slim/hindsight_api/engine/db/oracle.py

@@ -1242,6 +1242,10 @@ class OracleBackend(DatabaseBackend):
     def __init__(self) -> None:
         self._pool: Any = None
         self._oracledb: Any = None
+        # Oracle pooled sessions retain CURRENT_SCHEMA across checkouts. Cache
+        # SESSION_USER so default-schema acquisitions can explicitly reset a
+        # connection that was previously used for a tenant schema.
+        self._default_schema: str | None = None
 
     async def initialize(
         self,
@@ -1294,10 +1298,23 @@ class OracleBackend(DatabaseBackend):
         from ..memory_engine import get_current_schema
 
         schema = get_current_schema()
-        if schema and schema != "public":
-            cursor = conn.cursor()
-            await cursor.execute(f'ALTER SESSION SET CURRENT_SCHEMA = "{schema}"')
-            await cursor.close()
+        cursor = conn.cursor()
+        try:
+            if self._default_schema is None:
+                await cursor.execute("SELECT SYS_CONTEXT('USERENV', 'SESSION_USER') FROM DUAL")
+                row = await cursor.fetchone()
+                if not row or not row[0]:
+                    raise RuntimeError("Oracle did not return SESSION_USER while resetting CURRENT_SCHEMA")
+                self._default_schema = str(row[0])
+
+            target_schema = self._default_schema if not schema or schema == "public" else schema
+            safe_schema = target_schema.replace('"', '""')
+            await cursor.execute(f'ALTER SESSION SET CURRENT_SCHEMA = "{safe_schema}"')
+        finally:
+            # oracledb's AsyncCursor.close() is synchronous (not a coroutine);
+            # awaiting it raises "object NoneType can't be used in 'await'
+            # expression" and aborts every acquire().
+            cursor.close()
 
     @asynccontextmanager
     async def acquire(self) -> AsyncIterator[OracleConnection]:

hindsight-api-slim/hindsight_api/engine/llm_trace.py

@@ -477,6 +477,12 @@ class LLMTraceRecorder:
         if pool is None:
             logger.debug("LLM trace skipped: pool not available")
             return
+        # Guard against the backend existing but its internal asyncpg pool being
+        # None (during/after shutdown — close() calls backend.shutdown() which
+        # sets _pool=None before the backend object itself is dereferenced).
+        if getattr(pool, "_pool", "missing") is None:
+            logger.debug("LLM trace skipped: backend pool not initialized (shutdown in progress?)")
+            return
         try:
             schema = self._schema_getter()
             table = f"{schema}.llm_requests"
@@ -518,7 +524,13 @@ class LLMTraceRecorder:
                     _safe_json(record.metadata, self._max_chars) or "{}",
                 )
         except Exception as e:
-            logger.warning(f"LLM trace write failed for scope={record.scope}: {e}")
+            err_str = str(e)
+            # Downgrade known shutdown-related errors to DEBUG — these are
+            # expected races during daemon close()/restart and are not actionable.
+            if "pool is closing" in err_str or "not initialized" in err_str:
+                logger.debug("LLM trace write skipped (shutdown race) for scope=%s: %s", record.scope, e)
+            else:
+                logger.warning(f"LLM trace write failed for scope={record.scope}: {e}")
 
     async def _flush_pending(self, trace_id: str) -> None:
         """Await this trace's in-flight writes so its rows exist before an UPDATE."""
@@ -582,4 +594,8 @@ class LLMTraceRecorder:
                     json.dumps(patch),
                 )
         except Exception as e:
-            logger.warning(f"LLM trace memory_id attach failed for trace={trace_id}: {e}")
+            err_str = str(e)
+            if "pool is closing" in err_str or "not initialized" in err_str:
+                logger.debug("LLM trace memory_id attach skipped (shutdown race) for trace=%s: %s", trace_id, e)
+            else:
+                logger.warning(f"LLM trace memory_id attach failed for trace={trace_id}: {e}")

hindsight-api-slim/hindsight_api/engine/llm_wrapper.py

@@ -235,6 +235,17 @@ def requires_api_key(provider: str) -> bool:
     return provider.lower() not in _PROVIDERS_WITHOUT_API_KEY
 
 
+def _validate_ollama_num_ctx(value: Any) -> int | None:
+    """Validate a native Ollama context-window override."""
+    if value is None:
+        return None
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise ValueError(f"ollama_num_ctx must be a positive integer, got {value!r}")
+    if value < 1:
+        raise ValueError(f"ollama_num_ctx must be >= 1, got {value}")
+    return value
+
+
 def create_llm_provider(
     provider: str,
     api_key: str,
@@ -254,6 +265,7 @@ def create_llm_provider(
     litellmrouter_config: dict[str, Any] | None = None,
     gemini_service_tier: str | None = None,
     timeout: float | None = None,
+    ollama_num_ctx: int | None = None,
 ) -> Any:  # Returns LLMInterface
     """
     Factory function to create the appropriate LLM provider implementation.
@@ -268,6 +280,8 @@ def create_llm_provider(
         openai_service_tier: OpenAI service tier (for OpenAI provider) - None (default) or "flex" (50% cheaper).
         bedrock_service_tier: Bedrock service tier (for Bedrock provider) - None (default), "flex", "priority", or "reserved".
         gemini_service_tier: Gemini service tier (for Gemini provider) - None (default) or "flex" (50% cheaper).
+        ollama_num_ctx: Native Ollama context window override. None lets Ollama use the
+            model/server default.
         extra_body: Extra request-body params merged into the provider's native
             call. Threaded into OpenAI-compatible, Fireworks, Anthropic, Gemini/
             VertexAI and LiteLLM providers (each merges them in its own parameter
@@ -291,6 +305,8 @@ def create_llm_provider(
     Returns:
         LLMInterface implementation for the specified provider.
     """
+    ollama_num_ctx = _validate_ollama_num_ctx(ollama_num_ctx)
+
     from .providers import (
         AnthropicLLM,
         ClaudeCodeLLM,
@@ -494,6 +510,7 @@ def create_llm_provider(
             groq_service_tier=groq_service_tier,
             openai_service_tier=openai_service_tier,
             extra_body=extra_body,
+            ollama_num_ctx=ollama_num_ctx,
             timeout=timeout,
         )
 
@@ -531,6 +548,7 @@ class LLMProvider:
         max_retries: int | None = None,
         initial_backoff: float | None = None,
         max_backoff: float | None = None,
+        ollama_num_ctx: int | None = None,
     ):
         """
         Initialize LLM provider.
@@ -545,6 +563,8 @@ class LLMProvider:
             openai_service_tier: OpenAI service tier (None or "flex") - from config.
             bedrock_service_tier: Bedrock service tier (None, "flex", "priority", "reserved") - from config.
             gemini_service_tier: Gemini service tier (None or "flex") - from config.
+            ollama_num_ctx: Native Ollama context window override. ``None`` lets Ollama
+                use the model/server default.
             gemini_safety_settings: Safety settings for Gemini/VertexAI providers.
             extra_body: Extra request-body params merged into the provider's native call
                 (OpenAI-compatible, Fireworks, Anthropic, Gemini/VertexAI, LiteLLM).
@@ -598,6 +618,7 @@ class LLMProvider:
         self.openai_service_tier = openai_service_tier
         self.bedrock_service_tier = bedrock_service_tier
         self.gemini_service_tier = gemini_service_tier
+        self.ollama_num_ctx = _validate_ollama_num_ctx(ollama_num_ctx)
         # Gemini safety settings (instance default; can be overridden per-request via context var)
         self.gemini_safety_settings = gemini_safety_settings
         # Gemini prompt caching: when True, retain extraction (and any future
@@ -742,6 +763,7 @@ class LLMProvider:
             gemini_safety_settings=self.gemini_safety_settings,
             prompt_cache_enabled=self.prompt_cache_enabled,
             litellmrouter_config=router_config,
+            ollama_num_ctx=self.ollama_num_ctx,
             timeout=self.timeout,
         )
 
@@ -1260,6 +1282,7 @@ class LLMProvider:
             ENV_LLM_GROQ_SERVICE_TIER,
             ENV_LLM_LITELLMROUTER_CONFIG,
             ENV_LLM_MODEL,
+            ENV_LLM_OLLAMA_NUM_CTX,
             ENV_LLM_OPENAI_SERVICE_TIER,
             ENV_LLM_PROMPT_CACHE_ENABLED,
             ENV_LLM_PROVIDER,
@@ -1270,6 +1293,7 @@ class LLMProvider:
             ENV_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY,
             _get_default_model_for_provider,
             _parse_llm_router_config,
+            _parse_optional_positive_int,
             parse_gemini_service_tier,
         )
 
@@ -1314,6 +1338,7 @@ class LLMProvider:
             ),
             gemini_safety_settings=json.loads(os.getenv(ENV_LLM_GEMINI_SAFETY_SETTINGS, "null")),
             prompt_cache_enabled=prompt_cache_enabled,
+            ollama_num_ctx=_parse_optional_positive_int(ENV_LLM_OLLAMA_NUM_CTX, os.getenv(ENV_LLM_OLLAMA_NUM_CTX)),
             litellmrouter_config=_parse_llm_router_config(ENV_LLM_LITELLMROUTER_CONFIG),
             vertexai_project_id=os.getenv(ENV_LLM_VERTEXAI_PROJECT_ID) or None,
             vertexai_region=os.getenv(ENV_LLM_VERTEXAI_REGION) or None,

hindsight-api-slim/hindsight_api/engine/memory_engine.py

@@ -438,6 +438,7 @@ def _member_to_llm(member: "LLMMemberConfig", config: HindsightConfig, defaults:
         reasoning_effort=member.reasoning_effort or config.llm_reasoning_effort,
         extra_body=member.extra_body,
         default_headers=member.default_headers or config.llm_default_headers,
+        ollama_num_ctx=config.llm_ollama_num_ctx,
         bedrock_service_tier=member.bedrock_service_tier,
         gemini_service_tier=member.gemini_service_tier or config.llm_gemini_service_tier,
         gemini_safety_settings=_get_raw_config().llm_gemini_safety_settings,
@@ -1085,6 +1086,7 @@ class MemoryEngine(MemoryEngineInterface):
             reasoning_effort=config.llm_reasoning_effort,
             extra_body=config.llm_extra_body,
             default_headers=config.llm_default_headers,
+            ollama_num_ctx=config.llm_ollama_num_ctx,
             litellmrouter_config=config.llm_litellmrouter_config,
             bedrock_service_tier=config.llm_bedrock_service_tier,
             gemini_service_tier=config.llm_gemini_service_tier,
@@ -1129,6 +1131,7 @@ class MemoryEngine(MemoryEngineInterface):
             reasoning_effort=config.llm_reasoning_effort,
             extra_body=config.llm_extra_body,
             default_headers=config.llm_default_headers,
+            ollama_num_ctx=config.llm_ollama_num_ctx,
             litellmrouter_config=config.retain_llm_litellmrouter_config or config.llm_litellmrouter_config,
             bedrock_service_tier=config.llm_bedrock_service_tier,
             gemini_service_tier=config.llm_gemini_service_tier,
@@ -1167,6 +1170,7 @@ class MemoryEngine(MemoryEngineInterface):
             reasoning_effort=config.llm_reasoning_effort,
             extra_body=config.llm_extra_body,
             default_headers=config.llm_default_headers,
+            ollama_num_ctx=config.llm_ollama_num_ctx,
             litellmrouter_config=config.reflect_llm_litellmrouter_config or config.llm_litellmrouter_config,
             bedrock_service_tier=config.llm_bedrock_service_tier,
             gemini_service_tier=config.llm_gemini_service_tier,
@@ -1205,6 +1209,7 @@ class MemoryEngine(MemoryEngineInterface):
             reasoning_effort=config.llm_reasoning_effort,
             extra_body=config.llm_extra_body,
             default_headers=config.llm_default_headers,
+            ollama_num_ctx=config.llm_ollama_num_ctx,
             litellmrouter_config=config.consolidation_llm_litellmrouter_config or config.llm_litellmrouter_config,
             bedrock_service_tier=config.llm_bedrock_service_tier,
             gemini_service_tier=config.llm_gemini_service_tier,
@@ -2369,11 +2374,57 @@ class MemoryEngine(MemoryEngineInterface):
 
         Also checks if this is a child operation and updates the parent if all siblings are done.
         Uses a single transaction to avoid race conditions when multiple children complete simultaneously.
+
+        Opt-in escape hatch: when ``HINDSIGHT_API_FAIL_ON_EXTRACTION_ERRORS`` is set and the
+        operation's ``result_metadata`` recorded a non-zero ``extraction_errors_count`` (written by
+        ``_write_retain_outcome_metadata`` before this call), the operation is marked ``failed``
+        instead of ``completed``. This surfaces silently-dropped facts as a hard failure rather
+        than a clean success. Default is off, so existing behavior is unchanged (see issue #2700).
         """
         try:
             backend = await self._get_backend()
             async with acquire_with_retry(backend) as conn:
                 async with conn.transaction():
+                    # Read the accumulated extraction-error count (persisted by
+                    # _write_retain_outcome_metadata) to decide the terminal status.
+                    meta_row = await conn.fetchrow(
+                        f"SELECT result_metadata FROM {fq_table('async_operations')} WHERE operation_id = $1",
+                        uuid.UUID(operation_id),
+                    )
+                    extraction_errors_count = 0
+                    if meta_row is not None:
+                        metadata = conn.parse_json(meta_row["result_metadata"]) or {}
+                        extraction_errors_count = int(metadata.get("extraction_errors_count") or 0)
+
+                    fail_on_errors = get_config().fail_on_extraction_errors
+                    if fail_on_errors and extraction_errors_count > 0:
+                        error_message = (
+                            f"Retain completed with {extraction_errors_count} fact extraction error(s); "
+                            "marked failed because HINDSIGHT_API_FAIL_ON_EXTRACTION_ERRORS is enabled. "
+                            "See result_metadata.extraction_errors_sample for details."
+                        )
+                        row = await conn.fetchrow(
+                            f"""
+                            UPDATE {fq_table("async_operations")}
+                            SET status = 'failed', error_message = $2, updated_at = NOW(), completed_at = NOW()
+                            WHERE operation_id = $1
+                            RETURNING operation_id
+                            """,
+                            uuid.UUID(operation_id),
+                            error_message,
+                        )
+                        if row is None:
+                            logger.info(
+                                f"Operation {operation_id} no longer exists (bank deleted), skipping mark-failed"
+                            )
+                            return
+                        logger.warning(
+                            f"Marked async operation as failed due to {extraction_errors_count} "
+                            f"extraction error(s): {operation_id}"
+                        )
+                        await self._maybe_update_parent_operation(operation_id, conn)
+                        return
+
                     # Mark this operation as completed
                     row = await conn.fetchrow(
                         f"""
@@ -6570,13 +6621,18 @@ class MemoryEngine(MemoryEngineInterface):
                     )
 
                 collist = await self._memory_unit_columns(conn)
-                # The archive is cold storage, never a recall surface, so the schema gives it
-                # no `embedding` column at all (dropped in d4f6a8c2e1b3). The move in/out is
-                # therefore over every memory_units column EXCEPT embedding; on revert the
-                # embedding is recomputed from the unit's text/dates/entities below. This makes
-                # a model switch (which re-dimensions memory_units) structurally unable to trip
-                # a vector-dimension mismatch on the INSERT … SELECT round-trip (#2209).
-                arch_cols = ", ".join(c for c in (s.strip() for s in collist.split(",")) if c != '"embedding"')
+                # The archive is cold storage, never a recall surface and carries no index,
+                # so the schema gives it neither the `embedding` (dropped in d4f6a8c2e1b3)
+                # nor the `search_vector` column (dropped in e7c3a9f1b2d5). Both are
+                # recall-surface columns whose type/shape follows server
+                # config, so the move in/out is over every memory_units column EXCEPT those
+                # two; on revert each is recomputed from the unit's text/dates/entities below.
+                # This makes a model switch (which re-dimensions memory_units) structurally
+                # unable to trip a vector-dimension mismatch (#2209), and a text-search backend
+                # switch unable to trip a search_vector type mismatch (#2503), on the
+                # INSERT … SELECT round-trip.
+                _archive_omitted = ('"embedding"', '"search_vector"')
+                arch_cols = ", ".join(c for c in (s.strip() for s in collist.split(",")) if c not in _archive_omitted)
 
                 # --- Edit fields (live rows only): text / context / dates / fact_type / entities ---
                 doing_edit = any(
@@ -6634,6 +6690,17 @@ class MemoryEngine(MemoryEngineInterface):
                         mentioned_at=live["mentioned_at"],
                         entities=[r["canonical_name"] for r in ent_rows],
                     )
+                    # Keep the stored text-search vector in sync with curated
+                    # text/context edits. Use the incoming parameters here:
+                    # PostgreSQL evaluates UPDATE RHS expressions before the
+                    # sibling SET assignments take effect, so column references
+                    # would see the pre-edit text/context.
+                    from .db.ops_postgresql import pg_search_vector_expr
+
+                    sv_expr = pg_search_vector_expr(get_config(), text_col="$3", context_col="$4")
+                    search_vector_clause = (
+                        f",\n                            search_vector = {sv_expr}" if sv_expr else ""
+                    )
                     await enqueue_relink_victims(conn, bank_id, [memory_id], ops=backend.ops)
                     await conn.execute(
                         f"""
@@ -6641,7 +6708,7 @@ class MemoryEngine(MemoryEngineInterface):
                         SET text = $3, context = $4, fact_type = $5, occurred_start = $6,
                             occurred_end = $7, event_date = $8, embedding = $9::vector,
                             consolidated_at = NULL, consolidation_failed_at = NULL,
-                            edited_at = now(), updated_at = now()
+                            edited_at = now(), updated_at = now(){search_vector_clause}
                         WHERE id = $1 AND bank_id = $2
                         """,
                         str(memory_uuid),
@@ -6695,14 +6762,29 @@ class MemoryEngine(MemoryEngineInterface):
                     arch_row = await conn.fetchrow(
                         f"SELECT entity_ids FROM {arch} WHERE id = $1 AND bank_id = $2", str(memory_uuid), bank_id
                     )
-                    # The archive has no embedding column (see arch_cols above), so the live
-                    # row's embedding defaults to NULL on the way back and is recomputed below
-                    # once entities are restored.
+                    # The archive keeps neither embedding nor search_vector (see arch_cols
+                    # above), so both default to NULL on the way back and are recomputed here:
+                    # the embedding below once entities are restored, the search_vector now
+                    # from the row's own text/context/text_signals.
                     await conn.execute(
                         f"INSERT INTO {mu} ({arch_cols}) SELECT {arch_cols} FROM {arch} WHERE id = $1 AND bank_id = $2",
                         str(memory_uuid),
                         bank_id,
                     )
+                    # Rebuild search_vector using the *current* text-search backend, so the
+                    # reverted unit is keyword-searchable again (more correct than carrying a
+                    # verbatim copy that could be stale/wrong-type if the backend changed while
+                    # the fact sat archived). None = pgroonga/pg_textsearch/pg_search, which
+                    # index base columns directly and leave search_vector empty (#2503).
+                    from .db.ops_postgresql import pg_search_vector_expr
+
+                    sv_expr = pg_search_vector_expr(get_config())
+                    if sv_expr is not None:
+                        await conn.execute(
+                            f"UPDATE {mu} SET search_vector = {sv_expr} WHERE id = $1 AND bank_id = $2",
+                            str(memory_uuid),
+                            bank_id,
+                        )
                     # Re-consolidate from scratch; links are rebuilt by graph maintenance.
                     await conn.execute(
                         f"UPDATE {mu} SET consolidated_at = NULL, consolidation_failed_at = NULL, updated_at = now() "
@@ -7446,7 +7528,7 @@ class MemoryEngine(MemoryEngineInterface):
                 f"""
                 SELECT id, text, event_date, context, fact_type, document_id,
                        mentioned_at, occurred_start, occurred_end, chunk_id, proof_count,
-                       tags, consolidated_at, consolidation_failed_at, edited_at, {curation_cols}
+                       tags, metadata, consolidated_at, consolidation_failed_at, edited_at, {curation_cols}
                 FROM {source_table}
                 {where_clause}
                 ORDER BY mentioned_at DESC NULLS LAST, created_at DESC
@@ -7501,6 +7583,7 @@ class MemoryEngine(MemoryEngineInterface):
                         "chunk_id": row["chunk_id"] if row["chunk_id"] else None,
                         "proof_count": row["proof_count"] if row["proof_count"] is not None else 1,
                         "tags": list(row["tags"]) if row["tags"] else [],
+                        "metadata": conn.parse_json(row["metadata"]) if row["metadata"] is not None else {},
                         "consolidated_at": row["consolidated_at"].isoformat() if row["consolidated_at"] else None,
                         "consolidation_failed_at": (
                             row["consolidation_failed_at"].isoformat() if row["consolidation_failed_at"] else None
@@ -7553,7 +7636,7 @@ class MemoryEngine(MemoryEngineInterface):
             # back to the archive (with its invalidation bookkeeping) on a miss.
             select_cols = (
                 "id, text, context, event_date, occurred_start, occurred_end, "
-                "mentioned_at, fact_type, document_id, chunk_id, tags, source_memory_ids, "
+                "mentioned_at, fact_type, document_id, chunk_id, tags, metadata, source_memory_ids, "
                 "observation_scopes, edited_at"
             )
             row = await conn.fetchrow(
@@ -7597,6 +7680,7 @@ class MemoryEngine(MemoryEngineInterface):
                 "document_id": row["document_id"] if row["document_id"] else None,
                 "chunk_id": str(row["chunk_id"]) if row["chunk_id"] else None,
                 "tags": row["tags"] if row["tags"] else [],
+                "metadata": conn.parse_json(row["metadata"]) if row["metadata"] is not None else {},
                 "observation_scopes": row["observation_scopes"] if row["observation_scopes"] else None,
                 "state": unit_state,
                 "invalidation_reason": row["invalidation_reason"],
@@ -11901,22 +11985,11 @@ class MemoryEngine(MemoryEngineInterface):
         op_uuid = uuid.UUID(operation_id)
 
         async with acquire_with_retry(backend) as conn:
-            row = await conn.fetchrow(
-                f"SELECT bank_id, status FROM {fq_table('async_operations')} WHERE operation_id = $1 AND bank_id = $2",
-                op_uuid,
-                bank_id,
-            )
-
-            if not row:
-                raise ValueError(f"Operation {operation_id} not found for bank {bank_id}")
-
-            if row["status"] not in ("failed", "cancelled"):
-                raise OperationValidationError(
-                    f"Operation {operation_id} cannot be retried: status is '{row['status']}', expected 'failed' or 'cancelled'",
-                    409,
-                )
-
-            await conn.execute(
+            # Make the retry transition a single conditional write. This
+            # coordinates with retention cleanup's row locks: either retry wins
+            # and the row becomes nonterminal, or pruning wins and this call
+            # returns not-found instead of falsely acknowledging a vanished job.
+            updated = await conn.fetchrow(
                 f"""
                 UPDATE {fq_table("async_operations")}
                 SET status = 'pending',
@@ -11928,10 +12001,27 @@ class MemoryEngine(MemoryEngineInterface):
                     retry_count = 0,
                     updated_at = NOW()
                 WHERE operation_id = $1
+                  AND bank_id = $2
+                  AND status IN ('failed', 'cancelled')
+                RETURNING operation_id
                 """,
                 op_uuid,
+                bank_id,
             )
 
+            if updated is None:
+                row = await conn.fetchrow(
+                    f"SELECT status FROM {fq_table('async_operations')} WHERE operation_id = $1 AND bank_id = $2",
+                    op_uuid,
+                    bank_id,
+                )
+                if not row:
+                    raise ValueError(f"Operation {operation_id} not found for bank {bank_id}")
+                raise OperationValidationError(
+                    f"Operation {operation_id} cannot be retried: status is '{row['status']}', expected 'failed' or 'cancelled'",
+                    409,
+                )
+
             return {
                 "success": True,
                 "message": f"Operation {operation_id} queued for retry",

hindsight-api-slim/hindsight_api/engine/parsers/markitdown.py

@@ -186,7 +186,11 @@ class MarkitdownParser(FileParser):
         if Path(filename).suffix.lower() not in _TEXT_EXTENSIONS:
             return None
         try:
-            file_data.decode("utf-8")
+            # file_data may arrive as a non-``bytes`` buffer (e.g. a memoryview or
+            # a native/Rust-backed buffer object) that has no ``.decode``; coerce
+            # through the buffer protocol before the UTF-8 probe. The ``tmp.write``
+            # in the caller already relies only on the same buffer protocol.
+            bytes(file_data).decode("utf-8")
         except UnicodeDecodeError:
             return None
         from markitdown import StreamInfo

hindsight-api-slim/hindsight_api/engine/providers/anthropic_llm.py

@@ -34,6 +34,43 @@ def _usage_from_anthropic_response(response: Any) -> LLMResponseUsage:
     )
 
 
+_EPHEMERAL_CACHE = {"type": "ephemeral"}
+
+
+def _cached_system_blocks(system_prompt: str) -> list[dict[str, Any]]:
+    """Render the system prompt as a block list with a cache_control marker.
+
+    Anthropic prompt caching is a prefix match: marking the (single) system
+    block caches tools + system together. The system prompt is stable per
+    scope — fact extraction reuses it across every chunk, reflect and
+    consolidation keep their stable instructions there — so repeat calls read
+    it at ~10% of the base input price. Markers below the model's minimum
+    cacheable prefix are silently ignored (no write premium), so marking is
+    safe unconditionally. This is the "inline-marker provider" strategy that
+    ``LLMInterface.get_or_create_cached_prefix`` documents for Anthropic.
+    """
+    return [{"type": "text", "text": system_prompt, "cache_control": _EPHEMERAL_CACHE}]
+
+
+def _mark_last_message_for_caching(messages: list[dict[str, Any]]) -> None:
+    """Add a cache_control marker to the final content block, in place.
+
+    Used on the multi-turn (tool-calling) path: the reflect agent loop resends
+    the entire growing conversation each iteration, so this request's
+    end-marker becomes the next iteration's cache read point. Together with
+    the system marker this uses 2 of the 4 allowed breakpoints.
+    """
+    if not messages:
+        return
+    last = messages[-1]
+    content = last.get("content")
+    if isinstance(content, str):
+        if content.strip():  # the API rejects empty text blocks
+            last["content"] = [{"type": "text", "text": content, "cache_control": _EPHEMERAL_CACHE}]
+    elif isinstance(content, list) and content and isinstance(content[-1], dict):
+        content[-1]["cache_control"] = _EPHEMERAL_CACHE
+
+
 class AnthropicLLM(LLMInterface):
     """
     LLM provider using Anthropic's Claude models.
@@ -206,7 +243,9 @@ class AnthropicLLM(LLMInterface):
         }
 
         if system_prompt:
-            call_params["system"] = system_prompt
+            # One-shot calls share only the system prompt with each other, so
+            # that is the sole cache breakpoint on this path.
+            call_params["system"] = _cached_system_blocks(system_prompt)
 
         if use_forced_tool:
             # Single tool whose input_schema IS the response schema; force the model to
@@ -450,6 +489,11 @@ class AnthropicLLM(LLMInterface):
             else:
                 anthropic_messages.append({"role": role, "content": content})
 
+        # Multi-turn tool loop: cache the stable prefix (tools + system) via
+        # the system marker, and the growing conversation via an end-marker
+        # that the next iteration reads back.
+        _mark_last_message_for_caching(anthropic_messages)
+
         call_params: dict[str, Any] = {
             "model": self.model,
             "messages": anthropic_messages,
@@ -457,7 +501,7 @@ class AnthropicLLM(LLMInterface):
             "max_tokens": max_completion_tokens or 4096,
         }
         if system_prompt:
-            call_params["system"] = system_prompt
+            call_params["system"] = _cached_system_blocks(system_prompt)
 
         if self._extra_body:
             call_params["extra_body"] = self._extra_body
@@ -543,6 +587,211 @@ class AnthropicLLM(LLMInterface):
             raise last_exception
         raise RuntimeError("Anthropic tool call failed")
 
+    # ── Message Batches API (50% token discount) ─────────────────────────────
+
+    _BATCH_TOOL_NAME = "structured_response"
+
+    async def supports_batch_api(self) -> bool:
+        """Anthropic supports batch operations via the Message Batches API."""
+        return True
+
+    @staticmethod
+    def _map_batch_status(processing_status: str) -> str:
+        """Map Anthropic ``processing_status`` onto the OpenAI vocabulary.
+
+        The engine's poll loop breaks on "completed" and hard-fails on
+        "failed"/"expired"/"cancelled"; anything else keeps polling. Anthropic
+        batches only end as "ended" (per-request failures surface in the
+        results, mirroring OpenAI's "completed"-with-errors semantics), so
+        "ended" maps to "completed" and the non-terminal states pass through.
+        """
+        return "completed" if processing_status == "ended" else processing_status
+
+    def _translate_batch_body(self, body: dict[str, Any]) -> dict[str, Any]:
+        """Translate one OpenAI-shaped request body into Messages API params.
+
+        Mirrors the conversion rules of ``call()``: system messages fold into
+        the ``system`` param; ``max_completion_tokens`` becomes ``max_tokens``
+        (default 4096); ``temperature`` is dropped (the sync path never sends
+        it either — current Claude models reject non-default sampling params);
+        an OpenAI ``response_format`` json_schema becomes a single forced
+        tool_use tool when strict (native constrained decoding, issue #1002),
+        else the schema is injected into the system prompt.
+        """
+        system_prompt: str | None = None
+        messages: list[dict[str, Any]] = []
+        for msg in body.get("messages", []):
+            role = msg.get("role", "user")
+            content = msg.get("content", "")
+            if role == "system":
+                system_prompt = (system_prompt + "\n\n" + content) if system_prompt else content
+            else:
+                messages.append({"role": role, "content": content})
+
+        params: dict[str, Any] = {
+            "model": body.get("model") or self.model,
+            "messages": messages,
+            "max_tokens": body.get("max_completion_tokens") or 4096,
+        }
+
+        json_schema = (body.get("response_format") or {}).get("json_schema") or {}
+        schema = json_schema.get("schema")
+        if schema is not None:
+            if json_schema.get("strict"):
+                params["tools"] = [
+                    {
+                        "name": self._BATCH_TOOL_NAME,
+                        "description": "Return the structured response.",
+                        "input_schema": schema,
+                    }
+                ]
+                params["tool_choice"] = {"type": "tool", "name": self._BATCH_TOOL_NAME}
+            else:
+                schema_msg = "\n\nYou must respond with valid JSON matching this schema:\n" + json.dumps(
+                    schema, indent=2, ensure_ascii=False
+                )
+                system_prompt = (system_prompt + schema_msg) if system_prompt else schema_msg
+
+        if system_prompt:
+            params["system"] = system_prompt
+
+        # Batch params ARE the raw Messages body, so operator-configured extra
+        # body params merge directly (the sync path routes them through the
+        # SDK's extra_body, which does the same merge server-side).
+        if self._extra_body:
+            params.update(self._extra_body)
+
+        return params
+
+    def _translate_batch_message(self, message: Any) -> dict[str, Any]:
+        """Render an Anthropic Message as the OpenAI response body the engine parses.
+
+        The engine reads ``choices[0].message.content`` (json.loads'ing it when
+        a schema was requested) and sums ``usage`` under the OpenAI key names.
+        Forced-tool responses carry their JSON in the tool_use block's input,
+        so that is re-serialized as the content string.
+        """
+        content = ""
+        tool_input = None
+        for block in message.content:
+            if block.type == "tool_use" and block.name == self._BATCH_TOOL_NAME:
+                tool_input = block.input or {}
+            elif block.type == "text":
+                content += block.text
+        if tool_input is not None:
+            content = json.dumps(tool_input, ensure_ascii=False)
+
+        usage = getattr(message, "usage", None)
+        input_tokens = (usage.input_tokens or 0) if usage else 0
+        output_tokens = (usage.output_tokens or 0) if usage else 0
+
+        return {
+            "choices": [
+                {
+                    "message": {"role": "assistant", "content": content},
+                    "finish_reason": getattr(message, "stop_reason", None),
+                }
+            ],
+            "usage": {
+                "prompt_tokens": input_tokens,
+                "completion_tokens": output_tokens,
+                "total_tokens": input_tokens + output_tokens,
+            },
+        }
+
+    async def submit_batch(
+        self,
+        requests: list[dict[str, Any]],
+        endpoint: str = "/v1/chat/completions",
+        completion_window: str = "24h",
+    ) -> dict[str, Any]:
+        """Submit a batch of requests to the Message Batches API.
+
+        Accepts the engine's OpenAI-JSONL-shaped entries. ``endpoint`` and
+        ``completion_window`` belong to that shared shape and have no Anthropic
+        equivalent (batches always resolve within 24 hours); both are ignored.
+        """
+        batch_requests = [
+            {
+                "custom_id": req["custom_id"],
+                "params": self._translate_batch_body(req.get("body") or {}),
+            }
+            for req in requests
+        ]
+
+        logger.info(f"Submitting Anthropic message batch with {len(batch_requests)} requests")
+        batch = await self._client.messages.batches.create(requests=batch_requests)
+        logger.info(f"Anthropic batch submitted: {batch.id}, status={batch.processing_status}")
+
+        return {
+            "batch_id": batch.id,
+            "status": self._map_batch_status(batch.processing_status),
+            "created_at": batch.created_at,
+            "request_count": len(batch_requests),
+        }
+
+    async def get_batch_status(self, batch_id: str) -> dict[str, Any]:
+        """Get batch status in the shape the engine's poll loop expects."""
+        batch = await self._client.messages.batches.retrieve(batch_id)
+
+        counts = batch.request_counts
+        processing = getattr(counts, "processing", 0) or 0
+        succeeded = getattr(counts, "succeeded", 0) or 0
+        errored = getattr(counts, "errored", 0) or 0
+        canceled = getattr(counts, "canceled", 0) or 0
+        expired = getattr(counts, "expired", 0) or 0
+        resolved = succeeded + errored + canceled + expired
+
+        result: dict[str, Any] = {
+            "batch_id": batch.id,
+            "status": self._map_batch_status(batch.processing_status),
+            "created_at": batch.created_at,
+            "request_counts": {
+                "total": processing + resolved,
+                "completed": resolved,
+                "failed": errored,
+            },
+        }
+
+        ended_at = getattr(batch, "ended_at", None)
+        if ended_at:
+            result["completed_at"] = ended_at
+
+        return result
+
+    async def retrieve_batch_results(self, batch_id: str) -> list[dict[str, Any]]:
+        """Retrieve completed batch results, translated to the OpenAI shape.
+
+        Succeeded entries become ``{"custom_id", "response": {"body": ...}}``;
+        errored/canceled/expired entries become ``{"custom_id", "error": ...}``
+        so the engine's per-result error handling applies unchanged.
+        """
+        batch = await self._client.messages.batches.retrieve(batch_id)
+        if batch.processing_status != "ended":
+            raise ValueError(f"Batch {batch_id} is not completed yet (status: {batch.processing_status})")
+
+        decoder = await self._client.messages.batches.results(batch_id)
+        results: list[dict[str, Any]] = []
+        async for entry in decoder:
+            outcome = entry.result
+            if outcome.type == "succeeded":
+                results.append(
+                    {
+                        "custom_id": entry.custom_id,
+                        "response": {"body": self._translate_batch_message(outcome.message)},
+                    }
+                )
+            else:
+                error = getattr(outcome, "error", None)
+                if error is not None:
+                    detail = f"{getattr(error, 'type', 'error')}: {getattr(error, 'message', error)}"
+                else:
+                    detail = f"batch request {outcome.type}"
+                results.append({"custom_id": entry.custom_id, "error": detail})
+
+        logger.info(f"Retrieved {len(results)} results for Anthropic batch {batch_id}")
+        return results
+
     async def cleanup(self) -> None:
         """Clean up resources (close Anthropic client connections)."""
         if hasattr(self, "_client") and self._client:

hindsight-api-slim/hindsight_api/engine/providers/claude_code_llm.py

@@ -49,6 +49,20 @@ def _get_isolated_claude_env() -> dict[str, str]:
     return _isolated_claude_env
 
 
+def _result_error_detail(message: Any) -> str:
+    """Build an actionable error string from an ``is_error`` ResultMessage.
+
+    The CLI can report a failure with ``is_error=True`` while ``subtype``
+    still reads ``"success"``, putting the real detail in ``result`` (e.g.
+    quota exhaustion: ``You've hit your weekly limit · resets ...`` with
+    ``api_error_status: 429``). The SDK's own fallback exception surfaces
+    only the subtype, producing the misleading "Claude Code returned an
+    error result: success" (issue #2702) — so prefer ``result``.
+    """
+    detail = (message.result or "").strip() or message.subtype or "unknown error"
+    return f"Claude Code reported an error: {detail}"
+
+
 class ClaudeCodeLLM(LLMInterface):
     """
     LLM provider using Claude Code authentication.
@@ -176,6 +190,7 @@ class ClaudeCodeLLM(LLMInterface):
         from claude_agent_sdk import (  # type: ignore[unresolved-import]
             AssistantMessage,
             ClaudeAgentOptions,
+            ResultMessage,
             TextBlock,
             query,
         )
@@ -228,6 +243,11 @@ class ClaudeCodeLLM(LLMInterface):
                         for block in message.content:
                             if isinstance(block, TextBlock):
                                 full_text += block.text
+                    elif isinstance(message, ResultMessage) and message.is_error:
+                        # Surface the CLI's actual error text (e.g. quota
+                        # exhaustion) instead of the SDK's subtype-based
+                        # fallback exception (issue #2702).
+                        raise RuntimeError(_result_error_detail(message))
 
                 # The Claude Agent SDK doesn't report exact counts; stash the same
                 # char/4 estimate the success path traces so a later parse/validate
@@ -393,6 +413,7 @@ class ClaudeCodeLLM(LLMInterface):
             AssistantMessage,
             ClaudeAgentOptions,
             ClaudeSDKClient,
+            ResultMessage,
             SdkMcpTool,
             TextBlock,
             ToolUseBlock,
@@ -532,6 +553,9 @@ class ClaudeCodeLLM(LLMInterface):
 
                     # Receive response
                     async for message in client.receive_response():
+                        if isinstance(message, ResultMessage) and message.is_error:
+                            # Surface the CLI's actual error text (issue #2702).
+                            raise RuntimeError(_result_error_detail(message))
                         if isinstance(message, AssistantMessage):
                             for block in message.content:
                                 if isinstance(block, TextBlock):

hindsight-api-slim/hindsight_api/engine/providers/codex_auth.py

@@ -19,6 +19,7 @@ from __future__ import annotations
 
 import base64
 import binascii
+import contextlib
 import json
 import logging
 import os
@@ -31,6 +32,11 @@ from typing import Any
 
 import httpx
 
+try:
+    import fcntl
+except ImportError:  # pragma: no cover - Windows
+    fcntl = None  # type: ignore[assignment]
+
 logger = logging.getLogger(__name__)
 
 
@@ -58,6 +64,9 @@ _CODEX_TOKEN_REFRESH_SKEW_SECONDS = 60
 _CODEX_TERMINAL_REFRESH_ERROR_CODES = frozenset(
     {"refresh_token_expired", "refresh_token_reused", "refresh_token_invalidated"}
 )
+_CODEX_AUTH_LOCK_TIMEOUT_SECONDS = 20.0
+_CODEX_AUTH_LOCKS_GUARD = threading.Lock()
+_CODEX_AUTH_LOCKS: dict[Path, threading.Lock] = {}
 
 
 def default_codex_auth_file() -> Path:
@@ -76,6 +85,44 @@ def default_codex_auth_file() -> Path:
     return Path.home() / ".codex" / "auth.json"
 
 
+def _path_scoped_lock(auth_file: Path) -> threading.Lock:
+    key = auth_file.expanduser().resolve(strict=False)
+    with _CODEX_AUTH_LOCKS_GUARD:
+        lock = _CODEX_AUTH_LOCKS.get(key)
+        if lock is None:
+            lock = threading.Lock()
+            _CODEX_AUTH_LOCKS[key] = lock
+        return lock
+
+
+@contextlib.contextmanager
+def _codex_auth_lock(auth_file: Path, timeout_seconds: float = _CODEX_AUTH_LOCK_TIMEOUT_SECONDS):
+    """Cross-process advisory lock for one Codex auth store."""
+    with _path_scoped_lock(auth_file):
+        if fcntl is None:  # pragma: no cover - Windows
+            logger.debug("fcntl unavailable; Codex refresh proceeds without a cross-process lock.")
+            yield
+            return
+
+        lock_path = auth_file.with_suffix(".lock")
+        lock_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(lock_path, "a+") as lock_file:
+            deadline = time.monotonic() + max(1.0, timeout_seconds)
+            while True:
+                try:
+                    fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+                    break
+                except (BlockingIOError, OSError):
+                    if time.monotonic() >= deadline:
+                        raise TimeoutError("Timed out waiting for the Codex auth store lock") from None
+                    time.sleep(0.05)
+            try:
+                yield
+            finally:
+                with contextlib.suppress(OSError):
+                    fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
+
+
 class CodexRefreshExpiredError(RuntimeError):
     """Raised when the Codex refresh_token itself is no longer valid.
 
@@ -192,6 +239,34 @@ class CodexAuthManager:
             return None
         return data.get("tokens", {}).get("refresh_token")
 
+    @staticmethod
+    def _load_tokens_from_file(auth_file: Path) -> dict[str, Any] | None:
+        try:
+            with open(auth_file) as f:
+                data = json.load(f)
+        except (OSError, json.JSONDecodeError):
+            return None
+        tokens = data.get("tokens")
+        return tokens if isinstance(tokens, dict) else None
+
+    def _adopt_tokens(self, tokens: dict[str, Any]) -> bool:
+        """Adopt a newer on-disk Codex token set if present."""
+        access_token = tokens.get("access_token")
+        refresh_token = tokens.get("refresh_token")
+        account_id = tokens.get("account_id")
+
+        changed = False
+        if isinstance(access_token, str) and access_token and access_token != self.access_token:
+            self.access_token = access_token
+            changed = True
+        if isinstance(refresh_token, str) and refresh_token and refresh_token != self.refresh_token:
+            self.refresh_token = refresh_token
+            changed = True
+        if isinstance(account_id, str) and account_id and account_id != self.account_id:
+            self.account_id = account_id
+            changed = True
+        return changed
+
     @staticmethod
     def _decode_jwt_exp_unixtime(token: str) -> int | None:
         """Return the JWT ``exp`` claim as a unix timestamp, or None on parse failure.
@@ -225,6 +300,11 @@ class CodexAuthManager:
             return False
         return exp <= int(time.time()) + skew_seconds
 
+    def _token_is_fresh_with_known_expiry(self, skew_seconds: int = _CODEX_TOKEN_REFRESH_SKEW_SECONDS) -> bool:
+        """True only when the cached token has a known expiry outside the skew window."""
+        exp = self._decode_jwt_exp_unixtime(self.access_token)
+        return exp is not None and exp > int(time.time()) + skew_seconds
+
     # ------------------------------------------------------------------
     # Persistence
     # ------------------------------------------------------------------
@@ -339,78 +419,93 @@ class CodexAuthManager:
                 if not self._token_is_stale():
                     return
 
-            if not self.refresh_token:
-                raise RuntimeError(
-                    "Codex access_token is expired but no refresh_token is available. "
-                    "Run 'codex auth login' to re-authenticate."
-                )
+            with _codex_auth_lock(self._auth_file):
+                disk_tokens = self._load_tokens_from_file(self._auth_file)
+                if disk_tokens and self._adopt_tokens(disk_tokens):
+                    if force or self._token_is_fresh_with_known_expiry():
+                        return
 
-            log_reason = f" ({reason})" if reason else ""
-            logger.info(f"Refreshing Codex OAuth access_token{log_reason}")
-
-            request_body = {
-                "client_id": _CODEX_CLIENT_ID,
-                "grant_type": "refresh_token",
-                "refresh_token": self.refresh_token,
-            }
-            try:
-                response = self._http_client.post(
-                    _CODEX_REFRESH_TOKEN_URL,
-                    json=request_body,
-                    headers={"Content-Type": "application/json"},
-                    timeout=30.0,
-                )
-            except httpx.RequestError as e:
-                raise RuntimeError(f"Codex OAuth refresh network error: {type(e).__name__}") from e
-
-            if response.status_code == 401:
-                error_code = self._extract_oauth_error_code(response)
-                if error_code in _CODEX_TERMINAL_REFRESH_ERROR_CODES:
-                    raise CodexRefreshExpiredError(
-                        f"Codex refresh_token is permanently invalid (error.code={error_code}). "
+                if not self.refresh_token:
+                    raise RuntimeError(
+                        "Codex access_token is expired but no refresh_token is available. "
                         "Run 'codex auth login' to re-authenticate."
                     )
-                raise CodexRefreshExpiredError(
-                    f"Codex OAuth refresh returned 401 with unrecognized error code "
-                    f"({error_code or 'none'}). Run 'codex auth login' to re-authenticate."
-                )
 
-            if response.status_code >= 400:
-                raise RuntimeError(f"Codex OAuth refresh failed with HTTP {response.status_code}")
+                log_reason = f" ({reason})" if reason else ""
+                logger.info(f"Refreshing Codex OAuth access_token{log_reason}")
 
-            try:
-                body = response.json()
-            except json.JSONDecodeError as e:
-                raise RuntimeError(f"Codex OAuth refresh returned non-JSON body: {e}") from e
+                request_access_token = self.access_token
+                request_refresh_token = self.refresh_token
+                request_body = {
+                    "client_id": _CODEX_CLIENT_ID,
+                    "grant_type": "refresh_token",
+                    "refresh_token": request_refresh_token,
+                }
+                try:
+                    response = self._http_client.post(
+                        _CODEX_REFRESH_TOKEN_URL,
+                        json=request_body,
+                        headers={"Content-Type": "application/json"},
+                        timeout=30.0,
+                    )
+                except httpx.RequestError as e:
+                    raise RuntimeError(f"Codex OAuth refresh network error: {type(e).__name__}") from e
 
-            new_access = body.get("access_token")
-            if not new_access:
-                raise RuntimeError("Codex OAuth refresh returned no access_token")
+                if response.status_code == 401:
+                    error_code = self._extract_oauth_error_code(response)
+                    disk_tokens = self._load_tokens_from_file(self._auth_file)
+                    if disk_tokens and (
+                        disk_tokens.get("access_token") != request_access_token
+                        or disk_tokens.get("refresh_token") != request_refresh_token
+                    ):
+                        self._adopt_tokens(disk_tokens)
+                        return
+                    if error_code in _CODEX_TERMINAL_REFRESH_ERROR_CODES:
+                        raise CodexRefreshExpiredError(
+                            f"Codex refresh_token is permanently invalid (error.code={error_code}). "
+                            "Run 'codex auth login' to re-authenticate."
+                        )
+                    raise CodexRefreshExpiredError(
+                        f"Codex OAuth refresh returned 401 with unrecognized error code "
+                        f"({error_code or 'none'}). Run 'codex auth login' to re-authenticate."
+                    )
 
-            new_refresh = body.get("refresh_token") or self.refresh_token
-            new_id_token = body.get("id_token")
+                if response.status_code >= 400:
+                    raise RuntimeError(f"Codex OAuth refresh failed with HTTP {response.status_code}")
 
-            # Update in-memory state first so waiters see fresh credentials
-            # immediately, even if disk write fails.
-            self.access_token = new_access
-            self.refresh_token = new_refresh
+                try:
+                    body = response.json()
+                except json.JSONDecodeError as e:
+                    raise RuntimeError(f"Codex OAuth refresh returned non-JSON body: {e}") from e
 
-            persisted: dict[str, Any] = {
-                "access_token": new_access,
-                "refresh_token": new_refresh,
-            }
-            if new_id_token:
-                persisted["id_token"] = new_id_token
+                new_access = body.get("access_token")
+                if not new_access:
+                    raise RuntimeError("Codex OAuth refresh returned no access_token")
 
-            try:
-                self._persist_auth_atomic(persisted)
-            except OSError as e:
-                logger.warning(
-                    f"Codex OAuth refresh succeeded but persisting auth.json failed: {type(e).__name__}. "
-                    "In-memory credentials are up to date; on-disk file is stale."
-                )
+                new_refresh = body.get("refresh_token") or self.refresh_token
+                new_id_token = body.get("id_token")
 
-            logger.info("Codex OAuth access_token refreshed successfully")
+                # Update in-memory state first so waiters see fresh credentials
+                # immediately, even if disk write fails.
+                self.access_token = new_access
+                self.refresh_token = new_refresh
+
+                persisted: dict[str, Any] = {
+                    "access_token": new_access,
+                    "refresh_token": new_refresh,
+                }
+                if new_id_token:
+                    persisted["id_token"] = new_id_token
+
+                try:
+                    self._persist_auth_atomic(persisted)
+                except OSError as e:
+                    logger.warning(
+                        f"Codex OAuth refresh succeeded but persisting auth.json failed: {type(e).__name__}. "
+                        "In-memory credentials are up to date; on-disk file is stale."
+                    )
+
+                logger.info("Codex OAuth access_token refreshed successfully")
 
     def ensure_fresh_token(self) -> None:
         """Proactively refresh the access_token if it is near or past expiry.

hindsight-api-slim/hindsight_api/engine/providers/codex_llm.py

@@ -53,6 +53,59 @@ __all__ = [
 
 logger = logging.getLogger(__name__)
 
+# Newer Codex models are gated on the first-party client identity; the previous
+# browser-shaped User-Agent returned "Model not found" for Luna (#2643).
+# Use a neutral version because Hindsight must not claim a specific Codex release.
+_CODEX_ORIGINATOR = "codex_cli_rs"
+_CODEX_USER_AGENT = "codex_cli_rs/0.0.0 (Hindsight)"
+
+# Name of the single forced function tool used to carry structured output when
+# strict_schema is on. The Codex backend speaks the OpenAI Responses API, so a
+# forced function call gives us constrained decoding straight into the response
+# schema — no prompt-injected schema, no raw json.loads on free-form model text,
+# no invalid-\escape retry storm (issue #2504, same class as #1002 / #2339).
+_STRUCTURED_TOOL_NAME = "structured_response"
+
+# Valid JSON string escape characters (the char that may follow a backslash).
+_VALID_JSON_ESCAPE_CHARS = set('"\\/bfnrtu')
+
+
+def _repair_invalid_json_escapes(text: str) -> str:
+    """Best-effort repair of invalid ``\\escape`` sequences in a JSON string.
+
+    Escape-heavy content (code, serial/CLI commands, Windows paths, regexes)
+    makes weaker models emit backslashes that aren't valid JSON escapes (e.g.
+    ``\\d``, ``\\s``, ``C:\\Users``), so ``json.loads`` fails deterministically
+    and every retry re-fails the same way (issue #2504). This doubles any
+    backslash that isn't part of a valid escape so the payload parses. It is a
+    lenient fallback only — the strict_schema forced-tool path is the real fix.
+    """
+    result: list[str] = []
+    i = 0
+    n = len(text)
+    while i < n:
+        ch = text[i]
+        if ch == "\\" and i + 1 < n:
+            nxt = text[i + 1]
+            if nxt in _VALID_JSON_ESCAPE_CHARS:
+                # Preserve the valid escape (both chars) verbatim.
+                result.append(ch)
+                result.append(nxt)
+                i += 2
+                continue
+            # Invalid escape: escape the lone backslash so JSON parses.
+            result.append("\\\\")
+            i += 1
+            continue
+        if ch == "\\" and i + 1 == n:
+            # Trailing lone backslash — escape it.
+            result.append("\\\\")
+            i += 1
+            continue
+        result.append(ch)
+        i += 1
+    return "".join(result)
+
 
 class CodexLLM(LLMInterface):
     """
@@ -140,6 +193,18 @@ class CodexLLM(LLMInterface):
     def account_id(self) -> str:
         return self._auth_manager.account_id
 
+    def _build_request_headers(self) -> httpx.Headers:
+        return httpx.Headers(
+            {
+                "Authorization": f"Bearer {self.access_token}",
+                "Content-Type": "application/json",
+                "OpenAI-Account-ID": self.account_id,
+                "User-Agent": _CODEX_USER_AGENT,
+                "Origin": "https://chatgpt.com",
+                "originator": _CODEX_ORIGINATOR,
+            }
+        )
+
     @property
     def refresh_token(self) -> str | None:
         return self._auth_manager.refresh_token
@@ -336,7 +401,18 @@ class CodexLLM(LLMInterface):
         strict_schema: bool = False,
         return_usage: bool = False,
     ) -> Any:
-        """Make API call to Codex backend with SSE streaming."""
+        """Make API call to Codex backend with SSE streaming.
+
+        Args:
+            strict_schema: Route structured output through a single forced
+                function tool (constrained decoding) instead of prompt-injecting
+                the schema and parsing free-form text. The Codex backend speaks
+                the OpenAI Responses API, so the forced function call emits the
+                response schema directly as tool arguments — eliminating the
+                invalid-``\\escape`` retry storm (issue #2504). When False, falls
+                back to schema-in-prompt + JSON parse, now hardened with a lenient
+                invalid-escape repair before giving up.
+        """
         start_time = time.time()
 
         # Proactively refresh the OAuth access_token if it's near expiry.
@@ -361,11 +437,22 @@ class CodexLLM(LLMInterface):
             else:
                 user_messages.append(msg)
 
-        # Add JSON schema instruction if response_format is provided
+        # Structured output: prefer a single forced function tool (constrained
+        # decoding) over text-injecting the schema and parsing the reply. The
+        # forced tool guarantees schema-shaped JSON in the tool arguments,
+        # eliminating the invalid-\escape retry storm (issue #2504). When
+        # strict_schema is off we keep the schema-in-prompt + json.loads
+        # fallback (now hardened with a lenient escape repair) for callers that
+        # can't force tools.
+        schema = None
+        use_forced_tool = False
         if response_format is not None and hasattr(response_format, "model_json_schema"):
             schema = response_format.model_json_schema()
-            schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2, ensure_ascii=False)}"
-            system_instruction += schema_msg
+            if strict_schema:
+                use_forced_tool = True
+            else:
+                schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2, ensure_ascii=False)}"
+                system_instruction += schema_msg
 
         # gpt-5.2-codex only supports "detailed" reasoning summary
         reasoning_summary = "detailed" if "5.2" in self.model else self.reasoning_summary
@@ -392,13 +479,21 @@ class CodexLLM(LLMInterface):
             "prompt_cache_key": str(uuid.uuid4()),
         }
 
-        headers = {
-            "Authorization": f"Bearer {self.access_token}",
-            "Content-Type": "application/json",
-            "OpenAI-Account-ID": self.account_id,
-            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
-            "Origin": "https://chatgpt.com",
-        }
+        if use_forced_tool and schema is not None:
+            # Single function tool whose parameters ARE the response schema;
+            # force it via tool_choice so the backend does constrained decoding.
+            payload["tools"] = [
+                {
+                    "type": "function",
+                    "name": _STRUCTURED_TOOL_NAME,
+                    "description": "Return the structured response.",
+                    "parameters": schema,
+                }
+            ]
+            payload["tool_choice"] = {"type": "function", "name": _STRUCTURED_TOOL_NAME}
+            payload["parallel_tool_calls"] = False
+
+        headers = self._build_request_headers()
 
         url = f"{self.base_url}/codex/responses"
 
@@ -412,8 +507,15 @@ class CodexLLM(LLMInterface):
                 response = await self._client.post(url, json=payload, headers=headers, timeout=120.0)
                 response.raise_for_status()
 
-                # Parse SSE stream
-                content = await self._parse_sse_stream(response)
+                # Forced-tool path: read structured output from the function-call
+                # arguments (already a JSON string in a dedicated channel) rather
+                # than from free-form assistant text.
+                if use_forced_tool:
+                    text_content, tool_calls = await self._parse_sse_tool_stream(response)
+                    content = text_content or ""
+                else:
+                    tool_calls = []
+                    content = await self._parse_sse_stream(response)
 
                 # Codex SSE carries no usage block; stash the same char/4 estimate
                 # the success path traces so a later parse/validate failure records
@@ -426,7 +528,28 @@ class CodexLLM(LLMInterface):
                 )
 
                 # Handle structured output
-                if response_format is not None:
+                if use_forced_tool:
+                    tool_input = None
+                    for tc in tool_calls:
+                        if tc.name == _STRUCTURED_TOOL_NAME:
+                            tool_input = tc.arguments if isinstance(tc.arguments, dict) else None
+                            break
+                    if tool_input is None:
+                        # Model ignored the forced tool (rare — e.g. a gateway that
+                        # drops tool_choice). Retry so we don't hard-fail.
+                        logger.warning(
+                            f"Codex forced structured tool missing from response "
+                            f"(attempt {attempt + 1}/{max_retries + 1})"
+                        )
+                        if attempt < max_retries:
+                            backoff = min(initial_backoff * (2**attempt), max_backoff)
+                            await asyncio.sleep(backoff)
+                            attempt += 1
+                            continue
+                        raise RuntimeError("Codex did not return the forced structured_response tool call")
+                    content = json.dumps(tool_input)
+                    result = tool_input if skip_validation else response_format.model_validate(tool_input)
+                elif response_format is not None:
                     # Models may wrap JSON in markdown
                     clean_content = content
                     if "```json" in content:
@@ -437,13 +560,20 @@ class CodexLLM(LLMInterface):
                     try:
                         json_data = json.loads(clean_content)
                     except json.JSONDecodeError as e:
-                        logger.warning(f"Codex JSON parse error (attempt {attempt + 1}/{max_retries + 1}): {e}")
-                        if attempt < max_retries:
-                            backoff = min(initial_backoff * (2**attempt), max_backoff)
-                            await asyncio.sleep(backoff)
-                            attempt += 1
-                            continue
-                        raise
+                        # Escape-heavy content deterministically re-fails every
+                        # retry (issue #2504). Try a lenient invalid-escape repair
+                        # before burning a retry / re-raising.
+                        try:
+                            json_data = json.loads(_repair_invalid_json_escapes(clean_content))
+                            logger.info("Codex JSON parsed after repairing invalid escape sequences")
+                        except json.JSONDecodeError:
+                            logger.warning(f"Codex JSON parse error (attempt {attempt + 1}/{max_retries + 1}): {e}")
+                            if attempt < max_retries:
+                                backoff = min(initial_backoff * (2**attempt), max_backoff)
+                                await asyncio.sleep(backoff)
+                                attempt += 1
+                                continue
+                            raise
 
                     if skip_validation:
                         result = json_data
@@ -725,13 +855,7 @@ class CodexLLM(LLMInterface):
             "prompt_cache_key": str(uuid.uuid4()),
         }
 
-        headers = {
-            "Authorization": f"Bearer {self.access_token}",
-            "Content-Type": "application/json",
-            "OpenAI-Account-ID": self.account_id,
-            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
-            "Origin": "https://chatgpt.com",
-        }
+        headers = self._build_request_headers()
 
         url = f"{self.base_url}/codex/responses"
 
@@ -872,8 +996,13 @@ class CodexLLM(LLMInterface):
                             try:
                                 arguments = json.loads(arguments_str)
                             except json.JSONDecodeError:
-                                logger.warning(f"Failed to parse tool arguments: {arguments_str}")
-                                arguments = {}
+                                # Escape-heavy content can emit invalid \escape
+                                # sequences (issue #2504); repair before giving up.
+                                try:
+                                    arguments = json.loads(_repair_invalid_json_escapes(arguments_str))
+                                except json.JSONDecodeError:
+                                    logger.warning(f"Failed to parse tool arguments: {arguments_str}")
+                                    arguments = {}
 
                             tool_calls.append(
                                 LLMToolCall(

hindsight-api-slim/hindsight_api/engine/providers/gemini_llm.py

@@ -1030,7 +1030,7 @@ class GeminiLLM(LLMInterface):
         Mirrors the synchronous ``call`` path: system messages become
         ``systemInstruction``; a ``response_format`` json_schema forces JSON
         output (``responseMimeType``), appends the schema as a textual hint, and
-        grammar-enforces via ``responseJsonSchema`` when ``strict`` is set.
+        grammar-enforces via ``responseJsonSchema`` whenever a schema is present.
         """
         system_texts: list[str] = []
         contents: list[dict[str, Any]] = []
@@ -1059,8 +1059,13 @@ class GeminiLLM(LLMInterface):
                 system_texts.append(
                     "You must respond with valid JSON matching this schema:\n" + json.dumps(schema, ensure_ascii=False)
                 )
-                if json_schema.get("strict"):
-                    generation_config["responseJsonSchema"] = schema
+                # #2699: Gemini always grammar-enforces structured output via its native
+                # response_schema (``strict`` is an OpenAI concept, meaningless here). Set
+                # the native schema whenever one is present so the batch path mirrors the
+                # interactive path; otherwise batch requests at default config
+                # (HINDSIGHT_API_LLM_STRICT_SCHEMA=False) get only a textual hint and
+                # intermittently emit malformed JSON, losing every fact in the chunk.
+                generation_config["responseJsonSchema"] = schema
 
         request: dict[str, Any] = {"contents": contents}
         if system_texts:

hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py

@@ -47,6 +47,19 @@ logger = logging.getLogger(__name__)
 # Seed applied to every Groq request for deterministic behavior
 DEFAULT_LLM_SEED = 4242
 JSON_MODE_USER_HINT = "Return valid json only."
+DEFAULT_VERIFICATION_MAX_COMPLETION_TOKENS = 512
+
+
+def _validate_ollama_num_ctx(value: Any) -> int | None:
+    """Validate a native Ollama context-window override."""
+    if value is None:
+        return None
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise ValueError(f"ollama_num_ctx must be a positive integer, got {value!r}")
+    if value < 1:
+        raise ValueError(f"ollama_num_ctx must be >= 1, got {value}")
+    return value
+
 
 # Self-hosted OpenAI-compatible servers that advertise tool_choice="required"
 # but silently ignore it: instead of forcing a tool call they return
@@ -67,23 +80,68 @@ class ProviderResponseError(RuntimeError):
         self.retryable = retryable
 
 
+def _is_json(text: str) -> bool:
+    """True if ``text`` parses as a JSON value."""
+    try:
+        json.loads(text)
+    except (json.JSONDecodeError, ValueError):
+        return False
+    return True
+
+
+def _outer_json_span(content: str) -> str | None:
+    """Return the outermost ``{...}`` / ``[...]`` span if it parses as JSON, else None.
+
+    Fallback for responses where fences are partial/absent or the model wrapped
+    the JSON in surrounding prose. Only returned when it is valid JSON so callers
+    never receive a worse candidate than the raw content.
+    """
+    starts = [i for i in (content.find("{"), content.find("[")) if i >= 0]
+    ends = [i for i in (content.rfind("}"), content.rfind("]")) if i >= 0]
+    if not starts or not ends:
+        return None
+    start, end = min(starts), max(ends)
+    if end <= start:
+        return None
+    candidate = content[start : end + 1].strip()
+    return candidate if _is_json(candidate) else None
+
+
 def _strip_code_fences(content: str) -> str:
     """Strip markdown code fences from LLM response if present.
 
     Many LLM providers (MiniMax, some Ollama models, Claude via proxies)
     wrap JSON responses in ```json ... ``` fences even when json_object
-    response format is requested. This strips the fences while preserving
-    the JSON content inside. Returns the original content unchanged if
-    no fences are detected.
+    response format is requested. Fences are detected by line (a closing
+    ``` must sit alone on its line) so triple-backticks *inside* JSON string
+    values do not truncate the payload. When the stripped candidate is not
+    valid JSON (partial fence, prose-wrapped output, truncated response), fall
+    back to the outermost parseable JSON span. Returns the original content
+    unchanged if no better candidate is found.
     """
-    if "```" not in content:
-        return content
-    try:
-        if "```json" in content:
-            return content.split("```json")[1].split("```")[0].strip()
-        return content.split("```")[1].split("```")[0].strip()
-    except (IndexError, ValueError):
-        return content
+    candidate = content
+    if "```" in content:
+        lines = content.split("\n")
+        # Find first line that starts a code fence (``` optionally followed by language)
+        fence_start = next((i for i, line in enumerate(lines) if line.startswith("```")), None)
+        if fence_start is not None:
+            # Find matching closing fence (``` alone or with trailing whitespace)
+            fence_end = next(
+                (j for j in range(fence_start + 1, len(lines)) if lines[j].strip() == "```"),
+                None,
+            )
+            if fence_end is not None:
+                candidate = "\n".join(lines[fence_start + 1 : fence_end]).strip()
+
+    if _is_json(candidate):
+        return candidate
+
+    # Fence stripping did not yield valid JSON — try to recover the outer JSON span.
+    span = _outer_json_span(content)
+    if span is not None:
+        return span
+
+    return candidate
 
 
 # Reasoning/thinking tags emitted by extended-thinking models. Some providers
@@ -435,6 +493,8 @@ class OpenAICompatibleLLM(LLMInterface):
         timeout: float | None = None,
         groq_service_tier: str | None = None,
         extra_body: dict[str, Any] | None = None,
+        *,
+        ollama_num_ctx: int | None = None,
         **kwargs: Any,
     ):
         """
@@ -449,6 +509,8 @@ class OpenAICompatibleLLM(LLMInterface):
             timeout: Request timeout in seconds (uses env var or 120s default).
             groq_service_tier: Groq service tier ("on_demand", "flex", "auto").
             extra_body: Extra body params merged into every API call.
+            ollama_num_ctx: Native Ollama context window override. None lets Ollama use
+                the model/server default.
             **kwargs: Additional provider-specific parameters.
         """
         super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs)
@@ -529,6 +591,7 @@ class OpenAICompatibleLLM(LLMInterface):
         # Service tier configuration (from config, not env vars)
         self.groq_service_tier = groq_service_tier
         self.openai_service_tier = kwargs.get("openai_service_tier")
+        self.ollama_num_ctx = _validate_ollama_num_ctx(ollama_num_ctx)
         # User-configured extra body params (merged into every API call)
         self._config_extra_body = extra_body or {}
 
@@ -571,6 +634,10 @@ class OpenAICompatibleLLM(LLMInterface):
             return True
         return self.provider == "openai" and bool(self.base_url)
 
+    def _verification_max_completion_tokens(self) -> int:
+        """Return the startup verification budget for OpenAI-compatible gateways."""
+        return DEFAULT_VERIFICATION_MAX_COMPLETION_TOKENS
+
     async def verify_connection(self) -> None:
         """
         Verify that the provider is configured correctly by making a simple test call.
@@ -582,7 +649,7 @@ class OpenAICompatibleLLM(LLMInterface):
             logger.info(f"Verifying connection: {self.provider}/{self.model}")
             await self.call(
                 messages=[{"role": "user", "content": "Say 'ok'"}],
-                max_completion_tokens=100,
+                max_completion_tokens=self._verification_max_completion_tokens(),
                 max_retries=2,
                 initial_backoff=0.5,
                 max_backoff=2.0,
@@ -644,6 +711,11 @@ class OpenAICompatibleLLM(LLMInterface):
         # use the widely-supported max_tokens
         return "max_tokens"
 
+    def _apply_provider_extra_body_defaults(self, extra_body: dict[str, Any]) -> None:
+        """Apply provider-specific extra_body defaults while preserving user overrides."""
+        if self.provider == "minimax":
+            extra_body.setdefault("thinking", {"type": "disabled"})
+
     async def call(
         self,
         messages: list[dict[str, str]],
@@ -731,6 +803,7 @@ class OpenAICompatibleLLM(LLMInterface):
 
         # Provider-specific parameters
         extra_body: dict[str, Any] = {**self._config_extra_body}
+        self._apply_provider_extra_body_defaults(extra_body)
         if self.provider == "groq":
             call_params["seed"] = DEFAULT_LLM_SEED
             # Add service_tier if configured
@@ -1149,6 +1222,7 @@ class OpenAICompatibleLLM(LLMInterface):
 
         # Provider-specific parameters
         extra_body: dict[str, Any] = {**self._config_extra_body}
+        self._apply_provider_extra_body_defaults(extra_body)
         if self.provider == "groq":
             call_params["seed"] = DEFAULT_LLM_SEED
         if extra_body:
@@ -1337,9 +1411,10 @@ class OpenAICompatibleLLM(LLMInterface):
 
         # Add optional parameters with optimized defaults for Ollama
         options: dict[str, Any] = {
-            "num_ctx": 16384,  # 16k context window for larger prompts
             "num_batch": 512,  # Optimal batch size for prompt processing
         }
+        if self.ollama_num_ctx is not None:
+            options["num_ctx"] = self.ollama_num_ctx
         if max_completion_tokens:
             options["num_predict"] = max_completion_tokens
         if temperature is not None:

hindsight-api-slim/hindsight_api/engine/reflect/agent.py

@@ -1386,22 +1386,35 @@ async def _execute_tool(
         query = args.get("query")
         if not query:
             return {"error": "search_mental_models requires a query parameter"}
-        max_results = int(args.get("max_results") or 5)
+        max_results, error = _parse_tool_int_arg_or_error(args, "max_results", default=5)
+        if error:
+            return {"error": error}
         return await search_mental_models_fn(query, max_results)
 
     elif tool_name == "search_observations":
         query = args.get("query")
         if not query:
             return {"error": "search_observations requires a query parameter"}
-        max_tokens = max(int(args.get("max_tokens") or 5000), 1000)  # Default 5000, min 1000
+        max_tokens, error = _parse_tool_int_arg_or_error(args, "max_tokens", default=5000, minimum=1000)
+        if error:
+            return {"error": error}
         return await search_observations_fn(query, max_tokens)
 
     elif tool_name == "recall":
         query = args.get("query")
         if not query:
             return {"error": "recall requires a query parameter"}
-        max_tokens = max(int(args.get("max_tokens") or 2048), 1000)  # Default 2048, min 1000
-        max_chunk_tokens = max(int(args.get("max_chunk_tokens") or 1000), 1000)  # Always enabled, min 1000
+        max_tokens, error = _parse_tool_int_arg_or_error(args, "max_tokens", default=2048, minimum=1000)
+        if error:
+            return {"error": error}
+        max_chunk_tokens, error = _parse_tool_int_arg_or_error(
+            args,
+            "max_chunk_tokens",
+            default=1000,
+            minimum=1000,
+        )
+        if error:
+            return {"error": error}
         return await recall_fn(query, max_tokens, max_chunk_tokens)
 
     elif tool_name == "expand":
@@ -1415,23 +1428,63 @@ async def _execute_tool(
         return {"error": f"Unknown tool: {tool_name}"}
 
 
+_NULLISH_TOOL_INT_STRINGS = {"", "none", "null"}
+
+
+def _parse_tool_int_arg(args: dict[str, Any], key: str, *, default: int, minimum: int | None = None) -> int:
+    raw_value = args.get(key)
+    if not raw_value:
+        value = default
+    elif isinstance(raw_value, str) and raw_value.strip().lower() in _NULLISH_TOOL_INT_STRINGS:
+        value = default
+    else:
+        value = int(raw_value)
+    if minimum is None:
+        return value
+    return max(value, minimum)
+
+
+def _parse_tool_int_arg_or_error(
+    args: dict[str, Any],
+    key: str,
+    *,
+    default: int,
+    minimum: int | None = None,
+) -> tuple[int, str | None]:
+    try:
+        return _parse_tool_int_arg(args, key, default=default, minimum=minimum), None
+    except (OverflowError, TypeError, ValueError):
+        return default, f"{key} must be an integer or null-like value"
+
+
+def _summarize_tool_int_arg(args: dict[str, Any], key: str, *, default: int, minimum: int | None = None) -> str:
+    try:
+        return str(_parse_tool_int_arg(args, key, default=default, minimum=minimum))
+    except (OverflowError, TypeError, ValueError):
+        return f"invalid:{args.get(key)!r}"
+
+
+def _summarize_tool_query(args: dict[str, Any]) -> str:
+    query = args.get("query") or ""
+    if not isinstance(query, str):
+        query = str(query)
+    return f"'{query[:30]}...'" if len(query) > 30 else f"'{query}'"
+
+
 def _summarize_input(tool_name: str, args: dict[str, Any]) -> str:
     """Create a summary of tool input for logging, showing all params."""
     if tool_name == "search_mental_models":
-        query = args.get("query", "")
-        query_preview = f"'{query[:30]}...'" if len(query) > 30 else f"'{query}'"
-        max_results = int(args.get("max_results") or 5)
+        query_preview = _summarize_tool_query(args)
+        max_results = _summarize_tool_int_arg(args, "max_results", default=5)
         return f"(query={query_preview}, max_results={max_results})"
     elif tool_name == "search_observations":
-        query = args.get("query", "")
-        query_preview = f"'{query[:30]}...'" if len(query) > 30 else f"'{query}'"
-        max_tokens = max(int(args.get("max_tokens") or 5000), 1000)
+        query_preview = _summarize_tool_query(args)
+        max_tokens = _summarize_tool_int_arg(args, "max_tokens", default=5000, minimum=1000)
         return f"(query={query_preview}, max_tokens={max_tokens})"
     elif tool_name == "recall":
-        query = args.get("query", "")
-        query_preview = f"'{query[:30]}...'" if len(query) > 30 else f"'{query}'"
-        max_tokens = max(int(args.get("max_tokens") or 2048), 1000)
-        max_chunk_tokens = max(int(args.get("max_chunk_tokens") or 1000), 1000)
+        query_preview = _summarize_tool_query(args)
+        max_tokens = _summarize_tool_int_arg(args, "max_tokens", default=2048, minimum=1000)
+        max_chunk_tokens = _summarize_tool_int_arg(args, "max_chunk_tokens", default=1000, minimum=1000)
         return f"(query={query_preview}, max_tokens={max_tokens}, max_chunk_tokens={max_chunk_tokens})"
     elif tool_name == "expand":
         memory_ids = args.get("memory_ids", [])

hindsight-api-slim/hindsight_api/engine/retain/chunk_storage.py

@@ -64,8 +64,53 @@ async def delete_chunks_by_ids(conn, chunk_ids: list[str]) -> None:
     """
     if not chunk_ids:
         return
+
+    # PostgreSQL's FK cascade deletes child memory_links in executor-chosen
+    # order. Concurrent chunk deletes for the same bank can then lock overlapping
+    # memory_links in opposite orders and deadlock. Delete links explicitly in a
+    # total order before deleting chunks so every writer takes row locks the same
+    # way; the FK cascade still handles anything inserted later in this txn.
     await conn.execute(
-        f"DELETE FROM {fq_table('chunks')} WHERE chunk_id = ANY($1::text[])",
+        f"""
+        WITH target_units AS MATERIALIZED (
+            SELECT id
+            FROM {fq_table("memory_units")}
+            WHERE chunk_id = ANY($1::text[])
+        ),
+        ordered_links AS MATERIALIZED (
+            SELECT ml.ctid
+            FROM {fq_table("memory_links")} ml
+            WHERE EXISTS (
+                SELECT 1
+                FROM target_units tu
+                WHERE tu.id = ml.from_unit_id OR tu.id = ml.to_unit_id
+            )
+            ORDER BY
+                LEAST(ml.from_unit_id, ml.to_unit_id),
+                GREATEST(ml.from_unit_id, ml.to_unit_id),
+                ml.link_type,
+                COALESCE(ml.entity_id, '00000000-0000-0000-0000-000000000000'::uuid)
+            FOR UPDATE OF ml
+        )
+        DELETE FROM {fq_table("memory_links")} ml
+        USING ordered_links ol
+        WHERE ml.ctid = ol.ctid
+        """,
+        chunk_ids,
+    )
+    await conn.execute(
+        f"""
+        WITH ordered_chunks AS MATERIALIZED (
+            SELECT chunk_id
+            FROM {fq_table("chunks")}
+            WHERE chunk_id = ANY($1::text[])
+            ORDER BY chunk_id
+            FOR UPDATE
+        )
+        DELETE FROM {fq_table("chunks")} c
+        USING ordered_chunks oc
+        WHERE c.chunk_id = oc.chunk_id
+        """,
         chunk_ids,
     )
 

hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py

@@ -15,7 +15,7 @@ from typing import Any, Literal, cast
 from pydantic import BaseModel, ConfigDict, Field, create_model, field_validator
 
 from ..llm_interface import ProviderRateLimitResetError
-from ..llm_wrapper import LLMConfig, OutputTooLongError, sanitize_llm_output
+from ..llm_wrapper import LLMConfig, OutputTooLongError, parse_llm_json, sanitize_llm_output
 from ..operation_metadata import RetainExtractionErrors
 from ..response_models import TokenUsage
 from .entity_labels import (
@@ -232,6 +232,55 @@ class FactExtractionResponse(BaseModel):
     facts: list[ExtractedFact] = Field(description="List of extracted factual statements")
 
 
+def _split_chunk_for_output_retry(chunk: str) -> tuple[str, str] | None:
+    """Split an oversized extraction chunk without corrupting structured input."""
+    stripped = chunk.strip()
+    if len(stripped) <= 1:
+        return None
+
+    try:
+        parsed = json.loads(stripped)
+    except (TypeError, ValueError, json.JSONDecodeError):
+        parsed = None
+
+    if isinstance(parsed, list):
+        if len(parsed) >= 2:
+            mid = len(parsed) // 2
+            return json.dumps(parsed[:mid]), json.dumps(parsed[mid:])
+
+        if len(parsed) == 1 and isinstance(parsed[0], dict):
+            turn = parsed[0]
+            content = turn.get("content")
+            if isinstance(content, str) and len(content) > 1:
+                cut = len(content) // 2
+                first_turn = dict(turn)
+                second_turn = dict(turn)
+                first_turn["content"] = content[:cut]
+                second_turn["content"] = content[cut:]
+                return json.dumps([first_turn]), json.dumps([second_turn])
+
+        return None
+
+    # Split plain text at the midpoint, preferring sentence boundaries nearby.
+    mid_point = len(stripped) // 2
+    search_range = int(len(stripped) * 0.2)
+    search_start = max(0, mid_point - search_range)
+    search_end = min(len(stripped), mid_point + search_range)
+
+    best_split = mid_point
+    for ending in [". ", "! ", "? ", "\n\n"]:
+        pos = stripped.rfind(ending, search_start, search_end)
+        if pos != -1:
+            best_split = pos + len(ending)
+            break
+
+    first_half = stripped[:best_split].strip()
+    second_half = stripped[best_split:].strip()
+    if not first_half or not second_half or first_half == stripped or second_half == stripped:
+        return None
+    return first_half, second_half
+
+
 class ExtractedFactVerbose(BaseModel):
     """A single extracted fact with verbose field descriptions for detailed extraction."""
 
@@ -1236,6 +1285,15 @@ def _build_request_body(llm_config, config, prompt: str, user_message: str, resp
     return request_body
 
 
+def _coerce_fact_response(response: Any) -> dict[str, Any] | None:
+    """Accept the schema wrapper, or a recoverable top-level facts array."""
+    if isinstance(response, dict):
+        return response
+    if isinstance(response, list) and all(isinstance(item, dict) for item in response):
+        return {"facts": response}
+    return None
+
+
 async def _extract_facts_from_chunk(
     chunk: str,
     chunk_index: int,
@@ -1341,7 +1399,8 @@ async def _extract_facts_from_chunk(
             has_malformed_facts = False
 
             # Handle malformed LLM responses
-            if not isinstance(extraction_response_json, dict):
+            coerced_response_json = _coerce_fact_response(extraction_response_json)
+            if coerced_response_json is None:
                 if attempt < llm_max_retries - 1:
                     logger.warning(
                         f"LLM returned non-dict JSON on attempt {attempt + 1}/{llm_max_retries}: {type(extraction_response_json).__name__}. Retrying..."
@@ -1356,6 +1415,7 @@ async def _extract_facts_from_chunk(
                         f"Fact extraction failed: LLM returned non-dict JSON after {llm_max_retries} attempts "
                         f"({type(extraction_response_json).__name__}). Raw: {str(extraction_response_json)[:500]}"
                     )
+            extraction_response_json = coerced_response_json
 
             raw_facts = extraction_response_json.get("facts", [])
 
@@ -1664,33 +1724,22 @@ async def _extract_facts_with_auto_split(
             metadata=metadata,
         )
     except OutputTooLongError:
-        # Output exceeded token limits - split the chunk in half and retry
+        # Output exceeded token limits - split the chunk and retry. Conversation
+        # chunks are JSON arrays, so preserve array/turn boundaries when possible.
         logger.warning(
             f"Output too long for chunk {chunk_index + 1}/{total_chunks} "
-            f"({len(chunk)} chars). Splitting in half and retrying..."
+            f"({len(chunk)} chars). Splitting and retrying..."
         )
 
-        # Split at the midpoint, preferring sentence boundaries
-        mid_point = len(chunk) // 2
+        split_chunks = _split_chunk_for_output_retry(chunk)
+        if split_chunks is None:
+            logger.warning(
+                f"Cannot make progress splitting chunk {chunk_index + 1}/{total_chunks} "
+                f"({len(chunk)} chars); dropping this sub-chunk."
+            )
+            return [], TokenUsage()
 
-        # Try to find a sentence boundary near the midpoint
-        # Look for ". ", "! ", "? " within 20% of midpoint
-        search_range = int(len(chunk) * 0.2)
-        search_start = max(0, mid_point - search_range)
-        search_end = min(len(chunk), mid_point + search_range)
-
-        sentence_endings = [". ", "! ", "? ", "\n\n"]
-        best_split = mid_point
-
-        for ending in sentence_endings:
-            pos = chunk.rfind(ending, search_start, search_end)
-            if pos != -1:
-                best_split = pos + len(ending)
-                break
-
-        # Split the chunk
-        first_half = chunk[:best_split].strip()
-        second_half = chunk[best_split:].strip()
+        first_half, second_half = split_chunks
 
         logger.info(
             f"Split chunk {chunk_index + 1} into two sub-chunks: {len(first_half)} chars and {len(second_half)} chars"
@@ -2132,7 +2181,10 @@ async def extract_facts_from_contents_batch_api(
         content_str = message.get("content", "{}")
 
         try:
-            extraction_response_json = json.loads(content_str)
+            # #2701: use the lenient parser (strips markdown fences, scrubs
+            # embedded control chars) so recoverable batch responses — e.g.
+            # transient Gemini quirks — aren't dropped along with all their facts.
+            extraction_response_json = parse_llm_json(content_str)
         except json.JSONDecodeError as e:
             message = f"{custom_id}: failed to parse JSON: {e}"
             logger.error(message)
@@ -2144,6 +2196,19 @@ async def extract_facts_from_contents_batch_api(
             )
             continue
 
+        response_type_name = type(extraction_response_json).__name__
+        extraction_response_json = _coerce_fact_response(extraction_response_json)
+        if extraction_response_json is None:
+            message = f"{custom_id}: LLM returned non-dict JSON ({response_type_name})"
+            logger.error(message)
+            extraction_errors.add(message)
+            chunks_metadata.append(
+                ChunkMetadata(
+                    chunk_text=chunk_content, fact_count=0, content_index=content_index, chunk_index=chunk_idx
+                )
+            )
+            continue
+
         # Parse facts (reuse existing logic from _extract_facts_from_chunk)
         raw_facts = extraction_response_json.get("facts", [])
         chunk_facts = []

hindsight-api-slim/hindsight_api/engine/retain/link_creation.py

@@ -74,7 +74,9 @@ async def create_causal_links_batch(
     """
     Create causal links between facts.
 
-    Links facts that have causal relationships (causes, enables, prevents).
+    Retain writes the canonical ``caused_by`` relationship only. The database and
+    retrieval paths also recognize historical causal types so imported and
+    pre-existing memories remain traversable.
 
     Args:
         conn: Database connection
@@ -90,22 +92,7 @@ async def create_causal_links_batch(
     if len(unit_ids) != len(facts):
         raise ValueError(f"Mismatch between unit_ids ({len(unit_ids)}) and facts ({len(facts)})")
 
-    # Extract causal relations in the format expected by link_utils
-    # Format: List of lists, where each inner list is the causal relations for that fact
-    causal_relations_per_fact = []
-    for fact in facts:
-        if fact.causal_relations:
-            # Convert CausalRelation objects to dicts
-            relations_dicts = [
-                {
-                    "relation_type": rel.relation_type,
-                    "target_fact_index": rel.target_fact_index,
-                }
-                for rel in fact.causal_relations
-            ]
-            causal_relations_per_fact.append(relations_dicts)
-        else:
-            causal_relations_per_fact.append([])
+    causal_relations_per_fact = [fact.causal_relations or [] for fact in facts]
 
     link_count = await link_utils.create_causal_links_batch(conn, bank_id, unit_ids, causal_relations_per_fact, ops=ops)
 

hindsight-api-slim/hindsight_api/engine/retain/link_utils.py

@@ -7,12 +7,17 @@ import time
 from datetime import UTC, datetime, timedelta
 
 from ..._vector_index import ann_search_tuning_settings, configured_vector_extension
+from ..db.base import DatabaseConnection
+from ..db.ops import DataAccessOps
 from ..memory_engine import fq_table
+from .types import CausalRelation
 
 logger = logging.getLogger(__name__)
 
 # Sentinel UUID used in the unique index to represent NULL entity_id
 _NIL_ENTITY_UUID = "00000000-0000-0000-0000-000000000000"
+_CANONICAL_CAUSAL_LINK_TYPES = frozenset({"caused_by"})
+_LEGACY_CAUSAL_LINK_TYPES = frozenset({"causes", "enables", "prevents"})
 
 # Maximum number of temporal links to keep per unit (from_unit_id).
 # Retrieval only reads top 10-20 per unit via LATERAL join, so keeping
@@ -771,28 +776,61 @@ async def create_semantic_links_batch(
 
 
 async def create_causal_links_batch(
-    conn,
+    conn: DatabaseConnection,
     bank_id: str,
     unit_ids: list[str],
-    causal_relations_per_fact: list[list[dict]],
-    ops=None,
+    causal_relations_per_fact: list[list[CausalRelation]],
+    ops: DataAccessOps | None = None,
 ) -> int:
-    """
-    Create causal links between facts based on LLM-extracted causal relationships.
+    """Create canonical causal links for the retain pipeline.
 
-    Args:
-        conn: Database connection
-        unit_ids: List of unit IDs (in same order as causal_relations_per_fact)
-        causal_relations_per_fact: List of causal relations for each fact.
-            Each element is a list of dicts with:
-            - target_fact_index: Index into unit_ids for the target fact
-            - relation_type: "caused_by"
+    Retain must only create the backward-looking ``caused_by`` form. Historical
+    types are restored exclusively through ``restore_legacy_causal_links_batch``.
+    """
+    return await _write_causal_links_batch(
+        conn,
+        bank_id,
+        unit_ids,
+        causal_relations_per_fact,
+        _CANONICAL_CAUSAL_LINK_TYPES,
+        ops=ops,
+    )
+
+
+async def restore_legacy_causal_links_batch(
+    conn: DatabaseConnection,
+    bank_id: str,
+    unit_ids: list[str],
+    causal_relations_per_fact: list[list[CausalRelation]],
+    ops: DataAccessOps | None = None,
+) -> int:
+    """Restore historical causal links while importing a transfer archive.
+
+    This is deliberately separate from the retain writer: retrieval continues
+    reading historical types, but only transfer import may create them.
+    """
+    return await _write_causal_links_batch(
+        conn,
+        bank_id,
+        unit_ids,
+        causal_relations_per_fact,
+        _LEGACY_CAUSAL_LINK_TYPES,
+        ops=ops,
+    )
+
+
+async def _write_causal_links_batch(
+    conn: DatabaseConnection,
+    bank_id: str,
+    unit_ids: list[str],
+    causal_relations_per_fact: list[list[CausalRelation]],
+    allowed_relation_types: frozenset[str],
+    ops: DataAccessOps | None = None,
+) -> int:
+    """Write causal links after the caller has selected its allowed taxonomy.
 
     Returns:
         Number of causal links created
-
-    Causal link type:
-    - "caused_by": This fact was caused by the target fact
     """
     if not unit_ids or not causal_relations_per_fact:
         return 0
@@ -809,15 +847,13 @@ async def create_causal_links_batch(
             from_unit_id = unit_ids[fact_idx]
 
             for relation in causal_relations:
-                target_idx = relation["target_fact_index"]
-                relation_type = relation["relation_type"]
+                target_idx = relation.target_fact_index
+                relation_type = relation.relation_type
 
-                # Validate relation_type - only "caused_by" is supported (DB constraint)
-                valid_types = {"caused_by"}
-                if relation_type not in valid_types:
+                if relation_type not in allowed_relation_types:
                     logger.error(
                         f"Invalid relation_type '{relation_type}' (type: {type(relation_type).__name__}) "
-                        f"from fact {fact_idx}. Must be one of: {valid_types}. "
+                        f"from fact {fact_idx}. Must be one of: {allowed_relation_types}. "
                         f"Relation data: {relation}"
                     )
                     continue

hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py

@@ -831,6 +831,12 @@ async def retain_batch(
             first = contents_dicts[0]
             if first.get("context"):
                 existing_content["context"] = first["context"]
+            if first.get("event_date"):
+                existing_content["event_date"] = first["event_date"]
+            if first.get("metadata"):
+                existing_content["metadata"] = first["metadata"]
+            if first.get("observation_scopes") is not None:
+                existing_content["observation_scopes"] = first["observation_scopes"]
             if first.get("tags"):
                 existing_content["tags"] = first["tags"]
             contents_dicts = [existing_content, *contents_dicts]
@@ -852,6 +858,12 @@ async def retain_batch(
                     contents_dicts = [{"content": json.dumps(_merged, ensure_ascii=False)}]
                     if first.get("context"):
                         contents_dicts[0]["context"] = first["context"]
+                    if first.get("event_date"):
+                        contents_dicts[0]["event_date"] = first["event_date"]
+                    if first.get("metadata"):
+                        contents_dicts[0]["metadata"] = first["metadata"]
+                    if first.get("observation_scopes") is not None:
+                        contents_dicts[0]["observation_scopes"] = first["observation_scopes"]
                     if first.get("tags"):
                         contents_dicts[0]["tags"] = first["tags"]
             except (json.JSONDecodeError, ValueError, TypeError):

hindsight-api-slim/hindsight_api/engine/retain/types.py

@@ -96,10 +96,12 @@ class CausalRelation:
     """
     Causal relationship between facts.
 
-    Represents how one fact was caused by another.
+    Retain emits only the backward-looking ``caused_by`` form. Transfer import
+    reuses this structure to restore historical causal types without allowing
+    normal retain writes to create them.
     """
 
-    relation_type: str  # "caused_by"
+    relation_type: str  # ``caused_by`` for retain; legacy types for transfer restore
     target_fact_index: int  # Index of the target fact in the batch
 
 

hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py

@@ -243,12 +243,16 @@ class LinkExpansionRetriever(GraphRetriever):
         }
 
         sorted_ids = sorted(score_map.keys(), key=lambda x: score_map[x], reverse=True)[:budget]
-        rows = [row_map[fact_id] for fact_id in sorted_ids]
 
         results = []
-        for row in rows:
+        for fact_id in sorted_ids:
+            row = row_map[fact_id]
             result = RetrievalResult.from_db_row(dict(row))
-            result.activation = row["score"]
+            # ``activation`` is used to re-sort graph results after fact types are
+            # combined. It must retain the final additive score rather than the
+            # raw score from one signal, which would otherwise discard the other
+            # signals and make the cross-fact-type order disagree with this order.
+            result.activation = score_map[fact_id]
             results.append(result)
 
         # filter_results_by_tags is a no-op when no filter applies (tags falsy and not

hindsight-api-slim/hindsight_api/engine/search/retrieval.py

@@ -15,7 +15,7 @@ from dataclasses import dataclass, field
 from datetime import UTC, datetime
 from typing import TYPE_CHECKING, Any, Optional
 
-from ...config import get_config
+from ...config import DEFAULT_BM25_MAX_QUERY_TERMS, get_config
 from ..db_utils import acquire_with_retry
 from ..memory_engine import fq_table
 from ..sql import create_sql_dialect
@@ -222,7 +222,12 @@ async def retrieve_semantic_bm25_combined(
     # --- BM25 UNION ALL arms (one per fact_type, only when tokens present) ---
     if _include_bm25:
         text_ext = config.text_search_extension
-        bm25_text_param: str = dialect.prepare_bm25_text(tokens, query_text, text_search_extension=text_ext)
+        bm25_text_param: str = dialect.prepare_bm25_text(
+            tokens,
+            query_text,
+            text_search_extension=text_ext,
+            max_query_terms=getattr(config, "bm25_max_query_terms", DEFAULT_BM25_MAX_QUERY_TERMS),
+        )
         for i, ft in enumerate(fact_types):
             arms.append(
                 dialect.build_bm25_arm(
@@ -793,7 +798,7 @@ async def retrieve_all_fact_types_parallel(
                 tc_start,
                 tc_end,
                 budget=thinking_budget,
-                semantic_threshold=min_semantic if min_semantic is not None else 0.1,
+                semantic_threshold=0.1,
                 tags=tags,
                 tags_match=tags_match,
                 tag_groups=tag_groups,

hindsight-api-slim/hindsight_api/engine/sql/base.py

@@ -449,6 +449,7 @@ class SQLDialect(ABC):
         query_text: str,
         *,
         text_search_extension: str = "native",
+        max_query_terms: int | None = None,
     ) -> str:
         """Prepare the text parameter value for BM25 search.
 
@@ -459,6 +460,8 @@ class SQLDialect(ABC):
             tokens: Tokenized query words.
             query_text: Original query text.
             text_search_extension: Full-text search backend variant.
+            max_query_terms: Optional backend-specific token cap. 0 or None
+                leaves query terms uncapped.
 
         Returns:
             Prepared text string to bind as the BM25 text parameter.

hindsight-api-slim/hindsight_api/engine/sql/oracle.py

@@ -303,6 +303,7 @@ class OracleDialect(SQLDialect):
         query_text: str,
         *,
         text_search_extension: str = "native",
+        max_query_terms: int | None = None,
     ) -> str:
         # Oracle Text: filter tokens with special chars, escape reserved words
         # with curly braces (e.g. "about" → "{about}"), and join with OR.

hindsight-api-slim/hindsight_api/engine/sql/postgresql.py

@@ -254,8 +254,11 @@ class PostgreSQLDialect(SQLDialect):
         query_text: str,
         *,
         text_search_extension: str = "native",
+        max_query_terms: int | None = None,
     ) -> str:
         if text_search_extension in ("vchord", "pg_textsearch", "pgroonga", "pg_search"):
             return query_text
+        if max_query_terms is not None and max_query_terms > 0:
+            tokens = tokens[:max_query_terms]
         # native tsvector: join tokens with OR operator
         return " | ".join(tokens)

hindsight-api-slim/hindsight_api/engine/transfer/export.py

@@ -125,8 +125,9 @@ class _LoadedExport:
     unit_index: dict[Any, _UnitLocation] = field(default_factory=dict)
 
 
-# Causal link types that retain persists between facts. Only these travel in the
-# archive; temporal/semantic/entity links are regenerated against the target bank.
+# Retain currently writes only ``caused_by``. The legacy types stay in archives
+# so importing a historical bank preserves its graph; temporal/semantic/entity
+# links are regenerated against the target bank.
 _CAUSAL_LINK_TYPES = ("caused_by", "causes", "enables", "prevents")
 
 # Facts of these types are exported; observations are derived and excluded.

hindsight-api-slim/hindsight_api/engine/transfer/importer.py

@@ -19,7 +19,7 @@ from datetime import UTC, date, datetime
 from typing import Any, Literal
 
 from ..db_utils import acquire_with_retry
-from ..retain import bank_utils, chunk_storage, embedding_processing, fact_storage, orchestrator
+from ..retain import bank_utils, chunk_storage, embedding_processing, fact_storage, link_utils, orchestrator
 from ..retain.types import (
     CausalRelation,
     ChunkMetadata,
@@ -40,6 +40,7 @@ logger = logging.getLogger(__name__)
 
 OnConflict = Literal["skip", "replace", "new-id"]
 _VALID_CONFLICT_MODES: tuple[OnConflict, ...] = ("skip", "replace", "new-id")
+_LEGACY_CAUSAL_LINK_TYPES = frozenset({"causes", "enables", "prevents"})
 
 
 @dataclass
@@ -474,6 +475,7 @@ async def _import_one_document(
     )
 
     extracted_facts = [_to_extracted_fact(fact) for fact in document.facts]
+    legacy_causal_relations = _legacy_causal_relations(document)
 
     processed_facts: list[ProcessedFact] = []
     if extracted_facts:
@@ -545,6 +547,18 @@ async def _import_one_document(
                 ops=ops,
             )
 
+            # Retain writes only ``caused_by``. Restore legacy archive edges
+            # separately so their distinct direction and semantics survive a
+            # transfer without broadening the normal retain write contract.
+            if result_unit_ids and legacy_causal_relations:
+                await link_utils.restore_legacy_causal_links_batch(
+                    conn,
+                    bank_id,
+                    result_unit_ids[0],
+                    legacy_causal_relations,
+                    ops=ops,
+                )
+
         try:
             await entity_resolver.flush_pending_stats()
         except Exception:
@@ -705,6 +719,7 @@ def _to_extracted_fact(fact: TransferFact) -> ExtractedFact:
         causal_relations=[
             CausalRelation(relation_type=rel.relation_type, target_fact_index=rel.target_fact_index)
             for rel in fact.causal_relations
+            if rel.relation_type == "caused_by"
         ],
         content_index=0,
         chunk_index=fact.chunk_index,
@@ -714,3 +729,19 @@ def _to_extracted_fact(fact: TransferFact) -> ExtractedFact:
         tags=list(fact.tags),
         observation_scopes=fact.observation_scopes,
     )
+
+
+def _legacy_causal_relations(document: TransferDocument) -> list[list[CausalRelation]]:
+    """Return legacy archive edges for transfer-only restoration.
+
+    Invalid archive values are excluded. The write helper repeats the explicit
+    compatibility allowlist as a persistence boundary.
+    """
+    return [
+        [
+            CausalRelation(relation_type=relation.relation_type, target_fact_index=relation.target_fact_index)
+            for relation in fact.causal_relations
+            if relation.relation_type in _LEGACY_CAUSAL_LINK_TYPES
+        ]
+        for fact in document.facts
+    ]

hindsight-api-slim/hindsight_api/models.py

@@ -284,6 +284,8 @@ class MemoryLink(Base):
     entity = relationship("Entity", back_populates="memory_links")
 
     __table_args__ = (
+        # Retain writes ``caused_by`` only. Keep the historical causal values
+        # valid so existing rows and transfer archives remain queryable.
         CheckConstraint(
             "link_type IN ('temporal', 'semantic', 'entity', 'causes', 'caused_by', 'enables', 'prevents')",
             name="memory_links_link_type_check",

hindsight-api-slim/hindsight_api/worker/main.py

@@ -197,6 +197,11 @@ def main():
     shared_pool = max(0, config.worker_max_slots - sum(reservations.values()))
     print(f"  Slot reservations: {reservations_str}")
     print(f"  Shared pool: {shared_pool}")
+    if config.operation_retention_days == 0:
+        print("  Operation retention: disabled (terminal rows and payloads are kept)")
+    else:
+        print(f"  Operation retention: {config.operation_retention_days} days (terminal rows, payloads, and metadata)")
+        print(f"  Operation cleanup batch: {config.operation_cleanup_batch_size} rows/schema/cycle")
     print(f"  HTTP server: {args.http_host}:{args.http_port}")
     print()
 
@@ -264,6 +269,8 @@ def main():
             max_slots=config.worker_max_slots,
             slot_reservations=config.worker_slot_reservations,
             consolidation_bank_priority=config.worker_consolidation_bank_priority or None,
+            operation_retention_days=config.operation_retention_days,
+            operation_cleanup_batch_size=config.operation_cleanup_batch_size,
         )
 
         # Create the HTTP app for metrics/health

hindsight-api-slim/hindsight_api/worker/poller.py

@@ -17,6 +17,7 @@ import traceback
 from collections import Counter
 from collections.abc import Awaitable, Callable, Iterable
 from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
 from typing import TYPE_CHECKING, Any
 
 from ..engine.schema import fq_table_explicit as fq_table
@@ -37,6 +38,19 @@ def _metric_operation_label(operation_type: str | None) -> str:
     return operation_type or "unknown"
 
 
+def _updated_row_count(result: Any) -> int:
+    """Extract a row count from backend execute() results."""
+    if isinstance(result, int):
+        return result
+    if isinstance(result, str):
+        try:
+            return int(result.rsplit(" ", 1)[-1])
+        except (TypeError, ValueError):
+            return 0
+    rowcount = getattr(result, "rowcount", None)
+    return rowcount if isinstance(rowcount, int) else 0
+
+
 if TYPE_CHECKING:
     from hindsight_api.engine.db.base import DatabaseBackend, DatabaseConnection
     from hindsight_api.extensions.tenant import TenantExtension
@@ -45,6 +59,7 @@ logger = logging.getLogger(__name__)
 
 # Progress logging interval in seconds
 PROGRESS_LOG_INTERVAL = 30
+OPERATION_CLEANUP_INTERVAL_SECONDS = 60
 
 # Stuck-task stack-dump thresholds (seconds). Each task gets one stack dump
 # per threshold it crosses (5min, 10min, 20min, 40min, 80min...).
@@ -148,6 +163,8 @@ class WorkerPoller:
         max_slots: int = 10,
         slot_reservations: dict[str, int] | None = None,
         consolidation_bank_priority: dict[str, int] | None = None,
+        operation_retention_days: int = 30,
+        operation_cleanup_batch_size: int = 1000,
     ):
         """
         Initialize the worker poller.
@@ -170,7 +187,15 @@ class WorkerPoller:
                 Patterns support ``*`` as wildcard. A bare ``*`` key is the catch-all default.
                 When set, consolidation tasks are claimed in priority tiers rather than
                 pure created_at order. None or empty dict preserves current behavior.
+            operation_retention_days: Days to retain completed, failed, and cancelled
+                operation rows with their payload and metadata. Zero disables cleanup.
+            operation_cleanup_batch_size: Maximum terminal rows deleted per schema
+                during each cleanup cycle.
         """
+        if operation_retention_days < 0:
+            raise ValueError("operation_retention_days must be >= 0")
+        if operation_cleanup_batch_size < 1:
+            raise ValueError("operation_cleanup_batch_size must be >= 1")
         self._backend = backend
         self._worker_id = worker_id
         self._executor = executor
@@ -191,6 +216,9 @@ class WorkerPoller:
         self._consolidation_bank_priority: dict[str, int] | None = (
             consolidation_bank_priority if consolidation_bank_priority else None
         )
+        self._operation_retention_days = operation_retention_days
+        self._operation_cleanup_batch_size = operation_cleanup_batch_size
+        self._last_operation_cleanup_monotonic: float | None = None
         # Cache of which optional PG routines are installed on the server
         # (probed once, memoised for the life of the poller).
         from ..engine.db.optional_routines import OptionalRoutines
@@ -209,6 +237,9 @@ class WorkerPoller:
         # Rotation offset for per-tenant fair claiming. Advances past the last
         # schema we serviced so a busy tenant can't monopolize the poll order.
         self._next_schema_idx: int = 0
+        # Retention cleanup runs outside the claim loop. Keep one task per
+        # poller so maintenance cannot overlap with itself or block slot refill.
+        self._operation_cleanup_task: asyncio.Task[None] | None = None
 
     @staticmethod
     def _normalize_poll_schema(schema: str | None) -> str | None:
@@ -223,6 +254,67 @@ class WorkerPoller:
         # Convert default schema to None for SQL compatibility (no prefix), keep others as-is
         return [self._normalize_poll_schema(t.schema) for t in tenants]
 
+    async def _cleanup_terminal_operations_if_due(self) -> None:
+        """Schedule one cleanup sweep without blocking the task-claiming loop."""
+        if self._operation_retention_days == 0:
+            return
+        if self._operation_cleanup_task is not None and not self._operation_cleanup_task.done():
+            return
+
+        now = time.monotonic()
+        if (
+            self._last_operation_cleanup_monotonic is not None
+            and now - self._last_operation_cleanup_monotonic < OPERATION_CLEANUP_INTERVAL_SECONDS
+        ):
+            return
+        # Advance the guard before scheduling so a failing database cannot turn
+        # the tight poll loop into an unbounded maintenance retry loop.
+        self._last_operation_cleanup_monotonic = now
+        self._operation_cleanup_task = asyncio.create_task(self._cleanup_terminal_operations())
+
+    async def _cleanup_terminal_operations(self) -> None:
+        """Prune one bounded terminal-operation batch from every tenant schema."""
+        try:
+            schemas = await self._get_schemas()
+        except Exception as e:
+            logger.warning(f"Worker {self._worker_id} failed to discover schemas for operation cleanup: {e}")
+            return
+
+        # Oracle resolves unqualified table names from a context-bound session
+        # schema. Bind every iteration before acquiring its connection; on
+        # PostgreSQL this is harmless and fq_table remains explicit.
+        from ..engine.memory_engine import _current_schema
+
+        cutoff = datetime.now(UTC) - timedelta(days=self._operation_retention_days)
+        for schema in schemas:
+            table = fq_table("async_operations", schema)
+            schema_display = f'"{schema}"' if schema else "default"
+            schema_token = _current_schema.set(schema)
+            try:
+                async with self._backend.acquire() as conn:
+                    async with conn.transaction():
+                        deleted = await self._backend.ops.prune_terminal_operations(
+                            conn,
+                            table,
+                            cutoff,
+                            batch_size=self._operation_cleanup_batch_size,
+                        )
+                if deleted:
+                    logger.info(
+                        f"Worker {self._worker_id} pruned {deleted} expired terminal operations "
+                        f"from schema {schema_display}"
+                    )
+            except Exception as e:
+                logger.warning(
+                    f"Worker {self._worker_id} failed to prune terminal operations from schema {schema_display}: {e}"
+                )
+            finally:
+                _current_schema.reset(schema_token)
+
+        # Measure the next interval from completion as well as from the initial
+        # attempt. A slow multi-schema sweep remains bounded to one active task.
+        self._last_operation_cleanup_monotonic = time.monotonic()
+
     async def _scan_active_schemas(self, schemas: list[str | None]) -> set[str | None]:
         """Find which schemas have pending work.
 
@@ -504,17 +596,20 @@ class WorkerPoller:
                 return result
 
     async def _mark_completed(self, operation_id: str, schema: str | None):
-        """Mark a task as completed."""
+        """Mark a processing task as completed, then propagate to parent if needed."""
         table = fq_table("async_operations", schema)
         async with self._backend.acquire() as conn:
-            await conn.execute(
-                f"""
-                UPDATE {table}
-                SET status = 'completed', completed_at = now(), updated_at = now()
-                WHERE operation_id = $1
-                """,
-                operation_id,
-            )
+            async with conn.transaction():
+                result = await conn.execute(
+                    f"""
+                    UPDATE {table}
+                    SET status = 'completed', completed_at = now(), updated_at = now()
+                    WHERE operation_id = $1 AND status = 'processing'
+                    """,
+                    operation_id,
+                )
+                if _updated_row_count(result):
+                    await self._maybe_update_parent_operation(operation_id, schema, conn)
 
     async def _mark_failed(self, operation_id: str, error_message: str, schema: str | None):
         """Mark a task as failed with error message, then propagate to parent if applicable."""
@@ -749,6 +844,7 @@ class WorkerPoller:
                 task.task_dict["_schema"] = task.schema
             await self._executor(task.task_dict)
             logger.debug(f"Task {task.operation_id} execution finished")
+            await self._mark_completed(task.operation_id, task.schema)
             terminal_success = True
         except DeferOperation as e:
             # Deferral is not a terminal outcome — do not record a completion.
@@ -938,6 +1034,13 @@ class WorkerPoller:
                     for task in tasks:
                         await self.execute_task(task)
 
+                # Run maintenance after newly claimed work has started. Keeping
+                # this before the continue/sleep split means a perpetually
+                # non-empty queue cannot starve cleanup, while a large tenant
+                # sweep cannot delay the first available task either.
+                await self._cleanup_terminal_operations_if_due()
+
+                if tasks:
                     # Continue immediately to claim more tasks (if slots available)
                     continue
 
@@ -963,6 +1066,14 @@ class WorkerPoller:
                 # Backoff on error
                 await asyncio.sleep(1)
 
+        cleanup_task = self._operation_cleanup_task
+        if cleanup_task is not None and not cleanup_task.done():
+            cleanup_task.cancel()
+            try:
+                await cleanup_task
+            except asyncio.CancelledError:
+                pass
+
         logger.info(f"Worker {self._worker_id} polling loop stopped")
 
     async def shutdown_graceful(self, timeout: float = 30.0):

hindsight-api-slim/pyproject.toml

@@ -51,7 +51,7 @@ dependencies = [
     "anthropic>=0.40.0",
     "typer>=0.9.0",
     "cohere>=5.0.0",
-    "litellm>=1.83.14", # 1.82.7/1.82.8 had a supply chain compromise (yanked); 1.83.0+ also fixes GHSA-jjhc-v7c2-5hh6 / GHSA-53mr-6c8q-9789 / GHSA-pq44-5pcq-4r5g / GHSA-8cjq-wjmh-q42r
+    "litellm>=1.84.0", # 1.82.7/1.82.8 had a supply chain compromise (yanked); 1.83.0+ also fixes GHSA-jjhc-v7c2-5hh6 / GHSA-53mr-6c8q-9789 / GHSA-pq44-5pcq-4r5g / GHSA-8cjq-wjmh-q42r; 1.84.0 fixes GHSA-4xpc-pv4p-pm3w
     "markitdown[pdf,docx,pptx,xlsx,xls]>=0.1.4", # File to markdown conversion
     "obstore>=0.4.0", # S3/GCS/Azure object storage client (Rust-backed)
     "winloop>=0.1.0; sys_platform == 'win32'",

hindsight-api-slim/tests/test_anthropic_batch.py

@@ -0,0 +1,234 @@
+"""Anthropic Message Batches support for the provider batch interface.
+
+The engine's batch path (retain fact extraction, gated on
+``retain_batch_enabled``) speaks the OpenAI batch wire shape: JSONL entries
+with ``custom_id``/``method``/``url``/``body`` going in, and
+``response.body.choices[0].message.content`` (+ OpenAI-keyed ``usage``) coming
+out. ``AnthropicLLM`` translates both directions onto the Message Batches API,
+which bills all token usage at 50% of standard price.
+
+Translation rules mirror the provider's synchronous ``call()`` path:
+- system messages fold into the ``system`` param;
+- ``max_completion_tokens`` becomes ``max_tokens`` (default 4096);
+- ``temperature`` is dropped (the sync path never sends it either — current
+  Claude models reject non-default sampling params);
+- ``response_format`` with ``strict=True`` becomes a single forced tool_use
+  tool (native constrained decoding, issue #1002); non-strict injects the
+  schema into the system prompt and expects JSON text back.
+"""
+
+import json
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+pytestmark = pytest.mark.asyncio
+
+
+def _make_provider():
+    with patch("anthropic.AsyncAnthropic") as mock_client_cls:
+        mock_client_cls.return_value = MagicMock()
+        from hindsight_api.engine.providers.anthropic_llm import AnthropicLLM
+
+        provider = AnthropicLLM(
+            provider="anthropic",
+            api_key="fake-key",
+            base_url="",
+            model="claude-sonnet-5",
+        )
+    provider._client = MagicMock()
+    return provider
+
+
+_SCHEMA = {
+    "type": "object",
+    "properties": {"facts": {"type": "array", "items": {"type": "string"}}},
+    "required": ["facts"],
+}
+
+
+def _openai_request(custom_id: str, *, strict: bool = True, temperature: float | None = 0.1) -> dict:
+    body = {
+        "model": "claude-sonnet-5",
+        "messages": [
+            {"role": "system", "content": "Extract facts."},
+            {"role": "user", "content": f"Text for {custom_id}"},
+        ],
+        "max_completion_tokens": 2000,
+        "response_format": {
+            "type": "json_schema",
+            "json_schema": {"name": "facts", "schema": _SCHEMA, "strict": strict},
+        },
+    }
+    if temperature is not None:
+        body["temperature"] = temperature
+    return {"custom_id": custom_id, "method": "POST", "url": "/v1/chat/completions", "body": body}
+
+
+def _batch(status: str = "in_progress", **counts) -> SimpleNamespace:
+    defaults = {"processing": 0, "succeeded": 0, "errored": 0, "canceled": 0, "expired": 0}
+    defaults.update(counts)
+    return SimpleNamespace(
+        id="msgbatch_test1",
+        processing_status=status,
+        created_at="2026-07-08T00:00:00Z",
+        ended_at="2026-07-08T00:30:00Z" if status == "ended" else None,
+        request_counts=SimpleNamespace(**defaults),
+    )
+
+
+class _AsyncIter:
+    def __init__(self, items):
+        self._items = list(items)
+
+    def __aiter__(self):
+        self._iter = iter(self._items)
+        return self
+
+    async def __anext__(self):
+        try:
+            return next(self._iter)
+        except StopIteration:
+            raise StopAsyncIteration from None
+
+
+def _succeeded_entry(custom_id: str, tool_input: dict) -> SimpleNamespace:
+    block = SimpleNamespace(type="tool_use", name="structured_response", input=tool_input, text=None)
+    message = SimpleNamespace(
+        content=[block],
+        usage=SimpleNamespace(input_tokens=100, output_tokens=40, cache_read_input_tokens=0),
+        stop_reason="tool_use",
+    )
+    return SimpleNamespace(custom_id=custom_id, result=SimpleNamespace(type="succeeded", message=message))
+
+
+def _errored_entry(custom_id: str) -> SimpleNamespace:
+    error = SimpleNamespace(type="invalid_request", message="bad request")
+    return SimpleNamespace(custom_id=custom_id, result=SimpleNamespace(type="errored", error=error))
+
+
+async def test_supports_batch_api():
+    provider = _make_provider()
+    assert await provider.supports_batch_api() is True
+
+
+async def test_submit_batch_translates_openai_requests():
+    provider = _make_provider()
+    provider._client.messages.batches.create = AsyncMock(return_value=_batch("in_progress", processing=2))
+
+    requests = [_openai_request("chunk_0"), _openai_request("chunk_1")]
+    metadata = await provider.submit_batch(requests)
+
+    provider._client.messages.batches.create.assert_awaited_once()
+    submitted = provider._client.messages.batches.create.await_args.kwargs["requests"]
+    assert [r["custom_id"] for r in submitted] == ["chunk_0", "chunk_1"]
+
+    params = submitted[0]["params"]
+    assert params["model"] == "claude-sonnet-5"
+    # System message folded into the system param, not left in messages.
+    assert "Extract facts." in params["system"]
+    assert all(m["role"] != "system" for m in params["messages"])
+    assert params["messages"] == [{"role": "user", "content": "Text for chunk_0"}]
+    assert params["max_tokens"] == 2000
+    # temperature is dropped, mirroring the sync call() path.
+    assert "temperature" not in params
+    # strict=True → forced tool_use (native constrained decoding).
+    assert params["tools"][0]["input_schema"] == _SCHEMA
+    assert params["tool_choice"] == {"type": "tool", "name": "structured_response"}
+
+    assert metadata["batch_id"] == "msgbatch_test1"
+    assert metadata["status"] == "in_progress"
+    assert metadata["request_count"] == 2
+
+
+async def test_submit_batch_non_strict_schema_injects_into_system():
+    provider = _make_provider()
+    provider._client.messages.batches.create = AsyncMock(return_value=_batch("in_progress", processing=1))
+
+    await provider.submit_batch([_openai_request("chunk_0", strict=False)])
+
+    params = provider._client.messages.batches.create.await_args.kwargs["requests"][0]["params"]
+    assert "tools" not in params
+    assert "tool_choice" not in params
+    # Schema is injected into the system prompt for JSON-text output.
+    assert "facts" in params["system"]
+    assert "valid JSON" in params["system"]
+
+
+async def test_get_batch_status_in_progress():
+    provider = _make_provider()
+    provider._client.messages.batches.retrieve = AsyncMock(
+        return_value=_batch("in_progress", processing=3, succeeded=1)
+    )
+
+    status = await provider.get_batch_status("msgbatch_test1")
+
+    assert status["batch_id"] == "msgbatch_test1"
+    assert status["status"] == "in_progress"
+    assert status["request_counts"]["total"] == 4
+    assert status["request_counts"]["completed"] == 1
+
+
+async def test_get_batch_status_ended_maps_to_completed():
+    """The engine's poll loop breaks on the OpenAI-vocabulary status 'completed'."""
+    provider = _make_provider()
+    provider._client.messages.batches.retrieve = AsyncMock(return_value=_batch("ended", succeeded=3, errored=1))
+
+    status = await provider.get_batch_status("msgbatch_test1")
+
+    assert status["status"] == "completed"
+    assert status["request_counts"]["total"] == 4
+    assert status["request_counts"]["completed"] == 4
+    assert status["request_counts"]["failed"] == 1
+    assert status["completed_at"] == "2026-07-08T00:30:00Z"
+
+
+async def test_retrieve_batch_results_translates_to_openai_shape():
+    provider = _make_provider()
+    provider._client.messages.batches.retrieve = AsyncMock(return_value=_batch("ended", succeeded=1, errored=1))
+    entries = [
+        _succeeded_entry("chunk_0", {"facts": ["Alice is an engineer."]}),
+        _errored_entry("chunk_1"),
+    ]
+    provider._client.messages.batches.results = AsyncMock(return_value=_AsyncIter(entries))
+
+    results = await provider.retrieve_batch_results("msgbatch_test1")
+
+    by_id = {r["custom_id"]: r for r in results}
+    ok = by_id["chunk_0"]
+    body = ok["response"]["body"]
+    # The engine reads choices[0].message.content and json.loads() it.
+    assert json.loads(body["choices"][0]["message"]["content"]) == {"facts": ["Alice is an engineer."]}
+    # Usage arrives under the OpenAI key names the engine sums.
+    assert body["usage"] == {"prompt_tokens": 100, "completion_tokens": 40, "total_tokens": 140}
+
+    failed = by_id["chunk_1"]
+    assert failed["error"]
+    assert "response" not in failed
+
+
+async def test_retrieve_batch_results_text_content_passthrough():
+    """Non-strict requests come back as text blocks; concatenate them as content."""
+    provider = _make_provider()
+    provider._client.messages.batches.retrieve = AsyncMock(return_value=_batch("ended", succeeded=1))
+    text_block = SimpleNamespace(type="text", text='{"facts": []}')
+    message = SimpleNamespace(
+        content=[text_block],
+        usage=SimpleNamespace(input_tokens=10, output_tokens=5, cache_read_input_tokens=0),
+        stop_reason="end_turn",
+    )
+    entry = SimpleNamespace(custom_id="chunk_0", result=SimpleNamespace(type="succeeded", message=message))
+    provider._client.messages.batches.results = AsyncMock(return_value=_AsyncIter([entry]))
+
+    results = await provider.retrieve_batch_results("msgbatch_test1")
+
+    assert results[0]["response"]["body"]["choices"][0]["message"]["content"] == '{"facts": []}'
+
+
+async def test_retrieve_batch_results_raises_when_not_ended():
+    provider = _make_provider()
+    provider._client.messages.batches.retrieve = AsyncMock(return_value=_batch("in_progress", processing=2))
+
+    with pytest.raises(ValueError, match="not completed"):
+        await provider.retrieve_batch_results("msgbatch_test1")

hindsight-api-slim/tests/test_anthropic_prompt_caching.py

@@ -0,0 +1,181 @@
+"""Anthropic prompt caching via inline cache_control markers.
+
+``LLMInterface.get_or_create_cached_prefix`` documents Anthropic as an
+"inline-marker provider": rather than returning an explicit cache handle, the
+provider marks the reusable prefix inside ``call`` / ``call_with_tools`` with
+``cache_control`` breakpoints. Cache reads bill at ~10% of the base input
+price; a marker below the model's minimum cacheable prefix is silently
+ignored by the API (no premium), so marking is safe unconditionally.
+
+Two breakpoints (of the 4 allowed):
+- the system prompt, in both entry points — it is stable per scope (fact
+  extraction reuses it across every chunk; reflect/consolidation put their
+  stable instructions there), so tools+system cache across calls;
+- the last message content block, in ``call_with_tools`` only — the reflect
+  agent loop resends the whole growing conversation each iteration, so each
+  request's end-marker becomes the next iteration's cache read point.
+"""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from pydantic import BaseModel
+
+pytestmark = pytest.mark.asyncio
+
+EPHEMERAL = {"type": "ephemeral"}
+
+
+def _make_provider():
+    with patch("anthropic.AsyncAnthropic") as mock_client_cls:
+        mock_client_cls.return_value = MagicMock()
+        from hindsight_api.engine.providers.anthropic_llm import AnthropicLLM
+
+        provider = AnthropicLLM(
+            provider="anthropic",
+            api_key="fake-key",
+            base_url="",
+            model="claude-sonnet-5",
+        )
+    provider._client = MagicMock()
+    return provider
+
+
+def _text_response(text: str = "ok"):
+    block = MagicMock()
+    block.type = "text"
+    block.text = text
+    resp = MagicMock()
+    resp.content = [block]
+    resp.usage = MagicMock(input_tokens=10, output_tokens=2, cache_read_input_tokens=0)
+    resp.stop_reason = "end_turn"
+    return resp
+
+
+def _tool_response():
+    resp = MagicMock()
+    resp.content = []
+    resp.usage = MagicMock(input_tokens=10, output_tokens=2, cache_read_input_tokens=0)
+    resp.stop_reason = "end_turn"
+    return resp
+
+
+class _Out(BaseModel):
+    facts: list[str]
+
+
+async def test_call_marks_system_prompt_for_caching():
+    provider = _make_provider()
+    provider._client.messages.create = AsyncMock(return_value=_text_response())
+
+    with patch("hindsight_api.engine.providers.anthropic_llm.get_metrics_collector"):
+        await provider.call(
+            messages=[
+                {"role": "system", "content": "Stable extraction instructions."},
+                {"role": "user", "content": "Chunk text."},
+            ],
+            scope="test",
+            max_retries=0,
+        )
+
+    params = provider._client.messages.create.await_args.kwargs
+    assert params["system"] == [{"type": "text", "text": "Stable extraction instructions.", "cache_control": EPHEMERAL}]
+    # User messages are untouched in call() — one-shot calls share no
+    # conversation prefix with each other, only the system prompt.
+    assert params["messages"] == [{"role": "user", "content": "Chunk text."}]
+
+
+async def test_call_non_strict_schema_lands_inside_cached_system_block():
+    """Schema injection happens before marking, so the marked block includes it."""
+    provider = _make_provider()
+    provider._client.messages.create = AsyncMock(return_value=_text_response('{"facts": []}'))
+
+    with patch("hindsight_api.engine.providers.anthropic_llm.get_metrics_collector"):
+        await provider.call(
+            messages=[
+                {"role": "system", "content": "Extract."},
+                {"role": "user", "content": "Text."},
+            ],
+            response_format=_Out,
+            scope="test",
+            max_retries=0,
+        )
+
+    params = provider._client.messages.create.await_args.kwargs
+    assert len(params["system"]) == 1
+    system_block = params["system"][0]
+    assert system_block["cache_control"] == EPHEMERAL
+    assert "Extract." in system_block["text"]
+    assert "valid JSON" in system_block["text"]
+
+
+async def test_call_without_system_prompt_sends_no_system_param():
+    provider = _make_provider()
+    provider._client.messages.create = AsyncMock(return_value=_text_response())
+
+    with patch("hindsight_api.engine.providers.anthropic_llm.get_metrics_collector"):
+        await provider.call(
+            messages=[{"role": "user", "content": "hi"}],
+            scope="test",
+            max_retries=0,
+        )
+
+    assert "system" not in provider._client.messages.create.await_args.kwargs
+
+
+async def test_call_with_tools_marks_system_and_last_message():
+    provider = _make_provider()
+    provider._client.messages.create = AsyncMock(return_value=_tool_response())
+
+    with patch("hindsight_api.engine.providers.anthropic_llm.get_metrics_collector"):
+        await provider.call_with_tools(
+            messages=[
+                {"role": "system", "content": "Reflect agent instructions."},
+                {"role": "user", "content": "Question?"},
+                {"role": "assistant", "content": "Working on it."},
+                {"role": "user", "content": "Latest turn."},
+            ],
+            tools=[{"function": {"name": "recall", "description": "d", "parameters": {"type": "object"}}}],
+            max_retries=0,
+        )
+
+    params = provider._client.messages.create.await_args.kwargs
+    assert params["system"] == [{"type": "text", "text": "Reflect agent instructions.", "cache_control": EPHEMERAL}]
+
+    messages = params["messages"]
+    # Earlier messages carry no markers — only the final block gets one, so
+    # the next iteration of the agent loop reads the whole prefix from cache.
+    assert messages[0] == {"role": "user", "content": "Question?"}
+    assert messages[1] == {"role": "assistant", "content": "Working on it."}
+    assert messages[2]["content"] == [{"type": "text", "text": "Latest turn.", "cache_control": EPHEMERAL}]
+
+
+async def test_call_with_tools_marks_last_block_of_tool_result_message():
+    """Tool-result turns arrive as block lists; the marker goes on the last block."""
+    provider = _make_provider()
+    provider._client.messages.create = AsyncMock(return_value=_tool_response())
+
+    with patch("hindsight_api.engine.providers.anthropic_llm.get_metrics_collector"):
+        await provider.call_with_tools(
+            messages=[
+                {"role": "user", "content": "Question?"},
+                {
+                    "role": "assistant",
+                    "tool_calls": [
+                        {"id": "t1", "function": {"name": "recall", "arguments": "{}"}},
+                        {"id": "t2", "function": {"name": "recall", "arguments": "{}"}},
+                    ],
+                },
+                {"role": "tool", "tool_call_id": "t1", "content": "result one"},
+                {"role": "tool", "tool_call_id": "t2", "content": "result two"},
+            ],
+            tools=[{"function": {"name": "recall", "description": "d", "parameters": {"type": "object"}}}],
+            max_retries=0,
+        )
+
+    messages = provider._client.messages.create.await_args.kwargs["messages"]
+    last_blocks = messages[-1]["content"]
+    assert last_blocks[-1]["type"] == "tool_result"
+    assert last_blocks[-1]["cache_control"] == EPHEMERAL
+    # The earlier tool-result message is unmarked.
+    assert all("cache_control" not in block for block in messages[-2]["content"])

hindsight-api-slim/tests/test_anthropic_structured_tool_use.py

@@ -107,5 +107,8 @@ async def test_non_strict_keeps_text_injection_fallback():
         )
     kwargs = provider._client.messages.create.call_args.kwargs
     assert "tools" not in kwargs  # no forced tool when not strict
-    assert "valid JSON matching this schema" in (kwargs.get("system") or "")
+    # system is a cache_control-marked block list; the schema text-injection
+    # lands inside the (single) block.
+    system_text = "".join(block["text"] for block in (kwargs.get("system") or []))
+    assert "valid JSON matching this schema" in system_text
     assert isinstance(result, _Decision)

hindsight-api-slim/tests/test_async_batch_retain.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import json
+import os
 import uuid
 
 import pytest
@@ -515,6 +516,105 @@ async def test_retain_outcome_metadata_records_zero_counts(memory, request_conte
     assert "extraction_errors_sample" not in parent["result_metadata"]
 
 
+async def _seed_retain_op_with_errors(pool, bank_id: str, error_count: int) -> uuid.UUID:
+    """Insert a pending retain operation whose outcome metadata records extraction errors."""
+    operation_id = uuid.uuid4()
+    await pool.execute(
+        """
+        INSERT INTO async_operations (operation_id, bank_id, operation_type, result_metadata, status)
+        VALUES ($1, $2, $3, $4, $5)
+        """,
+        operation_id,
+        bank_id,
+        "retain",
+        json.dumps(
+            {
+                "unit_ids_count": 3,
+                "extraction_errors_count": error_count,
+                "extraction_errors_sample": ["chunk 2 failed to parse"],
+            }
+        ),
+        "pending",
+    )
+    return operation_id
+
+
+async def _op_row(pool, operation_id: uuid.UUID):
+    return await pool.fetchrow(
+        "SELECT status, error_message FROM async_operations WHERE operation_id = $1",
+        operation_id,
+    )
+
+
+@pytest.mark.asyncio
+async def test_completion_marks_failed_when_flag_on_and_errors_present(memory):
+    """With the escape hatch on, a retain that dropped facts ends 'failed' (issue #2700)."""
+    from hindsight_api.config import ENV_FAIL_ON_EXTRACTION_ERRORS, clear_config_cache
+
+    bank_id = "test_fail_on_extraction_errors_on"
+    pool = await memory._get_pool()
+    await _ensure_bank(pool, bank_id)
+    operation_id = await _seed_retain_op_with_errors(pool, bank_id, error_count=2)
+
+    os.environ[ENV_FAIL_ON_EXTRACTION_ERRORS] = "true"
+    clear_config_cache()
+    try:
+        await memory._mark_operation_completed(str(operation_id))
+    finally:
+        del os.environ[ENV_FAIL_ON_EXTRACTION_ERRORS]
+        clear_config_cache()
+
+    row = await _op_row(pool, operation_id)
+    assert row["status"] == "failed"
+    assert row["error_message"] is not None
+    assert "2" in row["error_message"]
+    assert "extraction error" in row["error_message"].lower()
+
+
+@pytest.mark.asyncio
+async def test_completion_stays_completed_when_flag_off(memory):
+    """Default behavior is preserved: extraction errors still complete the operation."""
+    from hindsight_api.config import ENV_FAIL_ON_EXTRACTION_ERRORS, clear_config_cache
+
+    bank_id = "test_fail_on_extraction_errors_off"
+    pool = await memory._get_pool()
+    await _ensure_bank(pool, bank_id)
+    operation_id = await _seed_retain_op_with_errors(pool, bank_id, error_count=2)
+
+    os.environ.pop(ENV_FAIL_ON_EXTRACTION_ERRORS, None)
+    clear_config_cache()
+    try:
+        await memory._mark_operation_completed(str(operation_id))
+    finally:
+        clear_config_cache()
+
+    row = await _op_row(pool, operation_id)
+    assert row["status"] == "completed"
+    assert row["error_message"] is None
+
+
+@pytest.mark.asyncio
+async def test_completion_completed_when_flag_on_but_no_errors(memory):
+    """The flag only fails operations that actually accumulated extraction errors."""
+    from hindsight_api.config import ENV_FAIL_ON_EXTRACTION_ERRORS, clear_config_cache
+
+    bank_id = "test_fail_on_extraction_errors_none"
+    pool = await memory._get_pool()
+    await _ensure_bank(pool, bank_id)
+    operation_id = await _seed_retain_op_with_errors(pool, bank_id, error_count=0)
+
+    os.environ[ENV_FAIL_ON_EXTRACTION_ERRORS] = "true"
+    clear_config_cache()
+    try:
+        await memory._mark_operation_completed(str(operation_id))
+    finally:
+        del os.environ[ENV_FAIL_ON_EXTRACTION_ERRORS]
+        clear_config_cache()
+
+    row = await _op_row(pool, operation_id)
+    assert row["status"] == "completed"
+
+
 @pytest.mark.asyncio
 async def test_retain_records_user_provided_document_ids(memory, request_context):
     """User-supplied document_ids land in child op result_metadata.document_ids."""

hindsight-api-slim/tests/test_bank_templates.py

@@ -5,6 +5,7 @@ import pytest_asyncio
 import httpx
 from datetime import datetime
 from hindsight_api.api import create_app
+from hindsight_api.api.http import BankTemplateManifest, validate_bank_template
 
 
 @pytest_asyncio.fixture
@@ -81,6 +82,17 @@ class TestImportValidation:
         assert set(data["mental_models_created"]) == {"test-model-one", "test-model-two"}
         assert set(data["directives_created"]) == {"Be concise", "Use examples"}
 
+    def test_verbatim_extraction_mode_is_valid(self):
+        """verbatim is a valid retain extraction mode in bank manifests."""
+        manifest = BankTemplateManifest.model_validate(
+            {
+                "version": "1",
+                "bank": {"retain_extraction_mode": "verbatim"},
+            }
+        )
+
+        assert validate_bank_template(manifest) == []
+
     @pytest.mark.asyncio
     async def test_import_invalid_version(self, api_client, bank_id):
         """Reject manifest with unsupported version."""

hindsight-api-slim/tests/test_batch_api.py

@@ -8,18 +8,15 @@ Tests cover:
 - Worker recovery on restart
 """
 
-import asyncio
 import json
 import logging
 import uuid
 from datetime import datetime, timezone
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock
 
 import pytest
 
-from hindsight_api import RequestContext
 from hindsight_api.config import HindsightConfig
-from hindsight_api.engine.llm_wrapper import create_llm_provider
 from hindsight_api.engine.retain.fact_extraction import (
     RetainContent,
     extract_facts_from_contents,
@@ -202,6 +199,224 @@ async def test_batch_api_normal_flow(mock_llm_config, test_contents, hindsight_c
             pass
 
 
+@pytest.mark.asyncio
+async def test_batch_api_accepts_top_level_fact_list(mock_llm_config, test_contents, hindsight_config):
+    """Batch extraction accepts a recoverable top-level facts array."""
+    batch_id = "batch_top_level_facts"
+    mock_llm_config._provider_impl.supports_batch_api = AsyncMock(return_value=True)
+    mock_llm_config._provider_impl.submit_batch = AsyncMock(
+        return_value={
+            "batch_id": batch_id,
+            "status": "validating",
+            "request_counts": {"total": 1, "completed": 0, "failed": 0},
+        }
+    )
+    mock_llm_config._provider_impl.get_batch_status = AsyncMock(
+        return_value={"status": "completed", "request_counts": {"total": 1, "completed": 1, "failed": 0}}
+    )
+    mock_llm_config._provider_impl.retrieve_batch_results = AsyncMock(
+        return_value=[
+            {
+                "custom_id": "chunk_0",
+                "response": {
+                    "body": {
+                        "choices": [
+                            {
+                                "message": {
+                                    "content": json.dumps(
+                                        [
+                                            {
+                                                "what": "Alice is a senior software engineer at TechCorp",
+                                                "when": "present",
+                                                "where": "TechCorp",
+                                                "who": "Alice",
+                                                "why": "Professional background information",
+                                                "fact_type": "world",
+                                                "fact_kind": "conversation",
+                                            }
+                                        ]
+                                    )
+                                }
+                            }
+                        ],
+                        "usage": {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150},
+                    }
+                },
+            }
+        ]
+    )
+
+    facts, chunks, usage = await extract_facts_from_contents_batch_api(
+        contents=[test_contents[0]],
+        llm_config=mock_llm_config,
+        agent_name="test_agent",
+        config=hindsight_config,
+        pool=None,
+        operation_id=None,
+        schema=None,
+    )
+
+    assert len(facts) == 1
+    assert "Alice" in facts[0].fact_text
+    assert len(chunks) == 1
+    assert chunks[0].fact_count == 1
+    assert usage.total_tokens == 150
+
+
+@pytest.mark.asyncio
+async def test_batch_api_rejects_top_level_non_fact_list(mock_llm_config, test_contents, hindsight_config):
+    """Batch extraction records malformed top-level lists instead of crashing."""
+    batch_id = "batch_malformed_list"
+    mock_llm_config._provider_impl.supports_batch_api = AsyncMock(return_value=True)
+    mock_llm_config._provider_impl.submit_batch = AsyncMock(return_value={"batch_id": batch_id})
+    mock_llm_config._provider_impl.get_batch_status = AsyncMock(
+        return_value={"status": "completed", "request_counts": {"total": 1, "completed": 1, "failed": 0}}
+    )
+    mock_llm_config._provider_impl.retrieve_batch_results = AsyncMock(
+        return_value=[
+            {
+                "custom_id": "chunk_0",
+                "response": {
+                    "body": {
+                        "choices": [{"message": {"content": json.dumps(["not a fact dict"])}}],
+                        "usage": {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150},
+                    }
+                },
+            }
+        ]
+    )
+
+    facts, chunks, usage = await extract_facts_from_contents_batch_api(
+        contents=[test_contents[0]],
+        llm_config=mock_llm_config,
+        agent_name="test_agent",
+        config=hindsight_config,
+        pool=None,
+        operation_id=None,
+        schema=None,
+    )
+
+    assert facts == []
+    assert len(chunks) == 1
+    assert chunks[0].fact_count == 0
+    assert usage.total_tokens == 0
+
+
+@pytest.mark.asyncio
+async def test_batch_api_recovers_fenced_and_control_char_json(mock_llm_config, test_contents, hindsight_config):
+    """#2701: batch content that bare json.loads can't parse but parse_llm_json can
+    (markdown code fences + an embedded raw control character, e.g. a transient
+    Gemini quirk) must still yield facts instead of dropping the whole chunk."""
+    batch_id = "batch_recoverable_json"
+
+    # Valid facts JSON, but wrapped in ```json fences AND containing a raw
+    # control character (\x01) inside a string value. Bare json.loads fails on
+    # both; parse_llm_json strips the fences and scrubs the control char.
+    inner_json = json.dumps(
+        {
+            "facts": [
+                {
+                    "what": "Alice is a senior software engineer at TechCorp",
+                    "when": "present",
+                    "where": "TechCorp",
+                    "who": "Alice",
+                    "why": "Professional background\x01information",
+                    "fact_type": "world",
+                    "fact_kind": "conversation",
+                }
+            ]
+        }
+    )
+    unparseable_content = f"```json\n{inner_json}\n```"
+
+    # Sanity: the raw content is NOT parseable by the bare parser.
+    with pytest.raises(json.JSONDecodeError):
+        json.loads(unparseable_content)
+
+    mock_llm_config._provider_impl.supports_batch_api = AsyncMock(return_value=True)
+    mock_llm_config._provider_impl.submit_batch = AsyncMock(return_value={"batch_id": batch_id})
+    mock_llm_config._provider_impl.get_batch_status = AsyncMock(
+        return_value={"status": "completed", "request_counts": {"total": 1, "completed": 1, "failed": 0}}
+    )
+    mock_llm_config._provider_impl.retrieve_batch_results = AsyncMock(
+        return_value=[
+            {
+                "custom_id": "chunk_0",
+                "response": {
+                    "body": {
+                        "choices": [{"message": {"content": unparseable_content}}],
+                        "usage": {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150},
+                    }
+                },
+            }
+        ]
+    )
+
+    facts, chunks, usage = await extract_facts_from_contents_batch_api(
+        contents=[test_contents[0]],
+        llm_config=mock_llm_config,
+        agent_name="test_agent",
+        config=hindsight_config,
+        pool=None,
+        operation_id=None,
+        schema=None,
+    )
+
+    # The facts are recovered rather than lost.
+    assert len(facts) == 1
+    assert "Alice" in facts[0].fact_text
+    assert len(chunks) == 1
+    assert chunks[0].fact_count == 1
+    assert usage.total_tokens == 150
+
+
+@pytest.mark.asyncio
+async def test_batch_api_unparseable_json_still_records_error(mock_llm_config, test_contents, hindsight_config):
+    """#2701: genuinely unparseable content (not recoverable by parse_llm_json)
+    must preserve the existing behavior — record the error, fact_count=0, no crash."""
+    batch_id = "batch_unparseable_json"
+
+    # Not JSON at all, and not recoverable by fence-stripping or control-char scrubbing.
+    unparseable_content = "this is not json {{{ ["
+
+    with pytest.raises(json.JSONDecodeError):
+        json.loads(unparseable_content)
+
+    mock_llm_config._provider_impl.supports_batch_api = AsyncMock(return_value=True)
+    mock_llm_config._provider_impl.submit_batch = AsyncMock(return_value={"batch_id": batch_id})
+    mock_llm_config._provider_impl.get_batch_status = AsyncMock(
+        return_value={"status": "completed", "request_counts": {"total": 1, "completed": 1, "failed": 0}}
+    )
+    mock_llm_config._provider_impl.retrieve_batch_results = AsyncMock(
+        return_value=[
+            {
+                "custom_id": "chunk_0",
+                "response": {
+                    "body": {
+                        "choices": [{"message": {"content": unparseable_content}}],
+                        "usage": {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150},
+                    }
+                },
+            }
+        ]
+    )
+
+    facts, chunks, usage = await extract_facts_from_contents_batch_api(
+        contents=[test_contents[0]],
+        llm_config=mock_llm_config,
+        agent_name="test_agent",
+        config=hindsight_config,
+        pool=None,
+        operation_id=None,
+        schema=None,
+    )
+
+    assert facts == []
+    assert len(chunks) == 1
+    assert chunks[0].fact_count == 0
+    assert usage.total_tokens == 0
+
+
 @pytest.mark.asyncio
 async def test_batch_api_crash_recovery(mock_llm_config, test_contents, hindsight_config, memory, request_context):
     """Test crash recovery: resume polling from existing batch_id."""

hindsight-api-slim/tests/test_causal_link_taxonomy.py

@@ -0,0 +1,22 @@
+"""Regression tests for the causal link taxonomy used by retain."""
+
+import pytest
+from pydantic import ValidationError
+
+from hindsight_api.engine.retain.fact_extraction import CausalRelation, FactCausalRelation
+
+
+@pytest.mark.parametrize("relation_type", ["causes", "enables", "prevents"])
+def test_retain_causal_models_reject_legacy_relation_types(relation_type: str) -> None:
+    """New retain output is canonical even though storage reads legacy links."""
+    with pytest.raises(ValidationError):
+        CausalRelation(target_fact_index=0, relation_type=relation_type)
+
+    with pytest.raises(ValidationError):
+        FactCausalRelation(target_index=0, relation_type=relation_type)
+
+
+def test_retain_causal_models_accept_caused_by() -> None:
+    """The canonical causal relationship remains valid in both extraction schemas."""
+    assert CausalRelation(target_fact_index=0, relation_type="caused_by").relation_type == "caused_by"
+    assert FactCausalRelation(target_index=0, relation_type="caused_by").relation_type == "caused_by"

hindsight-api-slim/tests/test_causal_relations.py

@@ -63,9 +63,7 @@ class TestCausalRelationsValidation:
                     assert rel.target_fact_index >= 0, (
                         f"Fact {i} has negative causal relation index: {rel.target_fact_index}"
                     )
-                    assert rel.relation_type in ["caused_by", "enabled_by", "prevented_by"], (
-                        f"Invalid relation_type: {rel.relation_type}"
-                    )
+                    assert rel.relation_type == "caused_by", f"Invalid relation_type: {rel.relation_type}"
 
     @pytest.mark.asyncio
     async def test_first_fact_has_no_causal_relations(self):
@@ -196,10 +194,10 @@ class TestCausalRelationsValidation:
             )
 
     @pytest.mark.asyncio
-    async def test_relation_types_are_backward_looking(self):
+    async def test_relation_types_use_the_canonical_form(self):
         """
-        Test that all relation types describe how the current fact
-        relates to a previous fact (caused_by, enabled_by, prevented_by).
+        Test that all extracted relation types use the canonical backward-looking
+        ``caused_by`` form.
         """
         text = """
         Alice learned Python programming.
@@ -220,12 +218,9 @@ class TestCausalRelationsValidation:
             config=_get_raw_config(),
         )
 
-        # Verify relation types are all backward-looking
-        valid_types = {"caused_by", "enabled_by", "prevented_by"}
-
         for i, fact in enumerate(facts):
             if fact.causal_relations:
                 for rel in fact.causal_relations:
-                    assert rel.relation_type in valid_types, (
-                        f"Invalid relation_type '{rel.relation_type}'. Must be one of: {valid_types}"
+                    assert rel.relation_type == "caused_by", (
+                        f"Invalid relation_type '{rel.relation_type}'. Must be 'caused_by'"
                     )

hindsight-api-slim/tests/test_causal_relationships.py

@@ -85,11 +85,10 @@ After searching for weeks, I finally found a cheaper apartment in Brooklyn.
             f"Got {len(all_causal_relations)}: {all_causal_relations}"
         )
 
-        # Verify relation types are valid (passive only - facts reference PREVIOUS facts)
-        valid_types = {"caused_by", "enabled_by", "prevented_by"}
+        # Retain writes the single canonical passive form for previous facts.
         for rel in all_causal_relations:
-            assert rel["relation_type"] in valid_types, (
-                f"Invalid relation_type '{rel['relation_type']}'. Must be one of {valid_types}"
+            assert rel["relation_type"] == "caused_by", (
+                f"Invalid relation_type '{rel['relation_type']}'. Must be 'caused_by'"
             )
 
     @pytest.mark.asyncio
@@ -165,10 +164,9 @@ Machine learning fascinated me so much that I changed my career to data science.
                     )
 
     @pytest.mark.asyncio
-    async def test_bidirectional_causal_relationships(self):
+    async def test_causal_relationships_use_backward_references(self):
         """
-        Test that bidirectional causal relationships (causes and caused_by)
-        are handled correctly.
+        Test that causal relationships are represented as backward references.
         """
         text = """
 My promotion at work caused me to move to New York.

hindsight-api-slim/tests/test_chunk_storage_delete_ordering.py

@@ -0,0 +1,50 @@
+"""Regression coverage for deterministic chunk deletion ordering."""
+
+import pytest
+
+from hindsight_api.engine.retain import chunk_storage
+
+
+class RecordingConn:
+    def __init__(self) -> None:
+        self.calls: list[tuple[str, tuple[object, ...]]] = []
+
+    async def execute(self, sql: str, *args: object) -> None:
+        self.calls.append((sql, args))
+
+
+@pytest.mark.asyncio
+async def test_delete_chunks_by_ids_predeletes_links_before_chunks():
+    conn = RecordingConn()
+    chunk_ids = ["chunk-b", "chunk-a"]
+
+    await chunk_storage.delete_chunks_by_ids(conn, chunk_ids)
+
+    assert len(conn.calls) == 2
+    link_sql, link_args = conn.calls[0]
+    chunk_sql, chunk_args = conn.calls[1]
+
+    assert link_args == (chunk_ids,)
+    assert chunk_args == (chunk_ids,)
+
+    assert "DELETE FROM" in link_sql
+    assert "memory_links" in link_sql
+    assert "target_units AS MATERIALIZED" in link_sql
+    assert "ordered_links AS MATERIALIZED" in link_sql
+    assert "ORDER BY" in link_sql
+    assert "FOR UPDATE OF ml" in link_sql
+
+    assert "DELETE FROM" in chunk_sql
+    assert "chunks" in chunk_sql
+    assert "ordered_chunks AS MATERIALIZED" in chunk_sql
+    assert "ORDER BY chunk_id" in chunk_sql
+    assert "FOR UPDATE" in chunk_sql
+
+
+@pytest.mark.asyncio
+async def test_delete_chunks_by_ids_noops_without_chunks():
+    conn = RecordingConn()
+
+    await chunk_storage.delete_chunks_by_ids(conn, [])
+
+    assert conn.calls == []

hindsight-api-slim/tests/test_claude_code_llm_error_result.py

@@ -0,0 +1,199 @@
+"""Regression test for surfacing the CLI's real error text (issue #2702).
+
+The Claude Code CLI can report a failure with ``is_error=True`` while
+``subtype`` still reads ``"success"``, putting the actual detail in
+``result`` — e.g. quota exhaustion:
+
+    {"type":"result","subtype":"success","is_error":true,
+     "api_error_status":429,
+     "result":"You've hit your weekly limit · resets Jul 18, 12pm (UTC)"}
+
+The Agent SDK's fallback exception is built from ``errors`` (empty here)
+or ``subtype``, producing the misleading "Claude Code returned an error
+result: success". These tests assert that both provider call paths inspect
+the ResultMessage directly and raise with the CLI's actual error text.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import pytest
+
+QUOTA_ERROR_TEXT = "You've hit your weekly limit · resets Jul 18, 12pm (UTC)"
+
+
+@dataclass
+class _FakeOptions:
+    """Stand-in for ClaudeAgentOptions; captures kwargs without importing SDK."""
+
+    system_prompt: str | None = None
+    max_turns: int | None = None
+    allowed_tools: list[str] = field(default_factory=list)
+    tools: list[str] = field(default_factory=list)
+    env: dict[str, str] = field(default_factory=dict)
+    mcp_servers: dict[str, Any] = field(default_factory=dict)
+
+
+class _FakeAssistantMessage:
+    def __init__(self, content: list[Any]) -> None:
+        self.content = content
+
+
+class _FakeTextBlock:
+    def __init__(self, text: str) -> None:
+        self.text = text
+
+
+class _FakeResultMessage:
+    def __init__(self, subtype: str, is_error: bool, result: str | None) -> None:
+        self.subtype = subtype
+        self.is_error = is_error
+        self.result = result
+
+
+def _instantiate_provider():
+    from hindsight_api.engine.providers.claude_code_llm import ClaudeCodeLLM
+
+    return ClaudeCodeLLM(
+        provider="claude-code",
+        api_key="",
+        base_url="",
+        model="claude-haiku-4-5",
+        reasoning_effort="low",
+    )
+
+
+@pytest.mark.asyncio
+async def test_call_raises_with_result_text_on_error_result(monkeypatch):
+    """call() must surface ResultMessage.result, not the 'success' subtype."""
+    import claude_agent_sdk
+
+    async def fake_query(prompt: str, options: _FakeOptions):
+        yield _FakeResultMessage(subtype="success", is_error=True, result=QUOTA_ERROR_TEXT)
+
+    monkeypatch.setattr(claude_agent_sdk, "ClaudeAgentOptions", _FakeOptions)
+    monkeypatch.setattr(claude_agent_sdk, "AssistantMessage", _FakeAssistantMessage)
+    monkeypatch.setattr(claude_agent_sdk, "TextBlock", _FakeTextBlock)
+    monkeypatch.setattr(claude_agent_sdk, "ResultMessage", _FakeResultMessage)
+    monkeypatch.setattr(claude_agent_sdk, "query", fake_query)
+
+    provider = _instantiate_provider()
+    with pytest.raises(RuntimeError) as excinfo:
+        await provider.call(
+            messages=[{"role": "user", "content": "hi"}],
+            max_retries=0,
+            scope="test",
+        )
+
+    assert QUOTA_ERROR_TEXT in str(excinfo.value)
+    assert "error result: success" not in str(excinfo.value)
+
+
+@pytest.mark.asyncio
+async def test_call_falls_back_to_subtype_when_result_empty(monkeypatch):
+    """With no result text, the subtype is still better than nothing."""
+    import claude_agent_sdk
+
+    async def fake_query(prompt: str, options: _FakeOptions):
+        yield _FakeResultMessage(subtype="error_max_turns", is_error=True, result=None)
+
+    monkeypatch.setattr(claude_agent_sdk, "ClaudeAgentOptions", _FakeOptions)
+    monkeypatch.setattr(claude_agent_sdk, "AssistantMessage", _FakeAssistantMessage)
+    monkeypatch.setattr(claude_agent_sdk, "TextBlock", _FakeTextBlock)
+    monkeypatch.setattr(claude_agent_sdk, "ResultMessage", _FakeResultMessage)
+    monkeypatch.setattr(claude_agent_sdk, "query", fake_query)
+
+    provider = _instantiate_provider()
+    with pytest.raises(RuntimeError, match="error_max_turns"):
+        await provider.call(
+            messages=[{"role": "user", "content": "hi"}],
+            max_retries=0,
+            scope="test",
+        )
+
+
+@pytest.mark.asyncio
+async def test_call_ignores_non_error_result_message(monkeypatch):
+    """A normal is_error=False ResultMessage must not affect the response."""
+    import claude_agent_sdk
+
+    async def fake_query(prompt: str, options: _FakeOptions):
+        yield _FakeAssistantMessage(content=[_FakeTextBlock(text="ok")])
+        yield _FakeResultMessage(subtype="success", is_error=False, result="ok")
+
+    monkeypatch.setattr(claude_agent_sdk, "ClaudeAgentOptions", _FakeOptions)
+    monkeypatch.setattr(claude_agent_sdk, "AssistantMessage", _FakeAssistantMessage)
+    monkeypatch.setattr(claude_agent_sdk, "TextBlock", _FakeTextBlock)
+    monkeypatch.setattr(claude_agent_sdk, "ResultMessage", _FakeResultMessage)
+    monkeypatch.setattr(claude_agent_sdk, "query", fake_query)
+
+    provider = _instantiate_provider()
+    result = await provider.call(
+        messages=[{"role": "user", "content": "hi"}],
+        max_retries=0,
+        scope="test",
+    )
+
+    assert result == "ok"
+
+
+@pytest.mark.asyncio
+async def test_call_with_tools_raises_with_result_text_on_error_result(monkeypatch):
+    """call_with_tools() must surface ResultMessage.result the same way."""
+    import claude_agent_sdk
+
+    class _FakeClient:
+        def __init__(self, options: _FakeOptions) -> None:
+            self.options = options
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, exc_type, exc, tb):
+            return False
+
+        async def query(self, prompt: str) -> None:
+            return None
+
+        async def receive_response(self):
+            yield _FakeResultMessage(subtype="success", is_error=True, result=QUOTA_ERROR_TEXT)
+
+    @dataclass
+    class _FakeSdkMcpTool:
+        name: str
+        description: str
+        input_schema: dict[str, Any]
+        handler: Any
+
+    def fake_create_sdk_mcp_server(name: str, version: str, tools=None):
+        return {"name": name, "version": version, "tools": tools}
+
+    monkeypatch.setattr(claude_agent_sdk, "ClaudeAgentOptions", _FakeOptions)
+    monkeypatch.setattr(claude_agent_sdk, "AssistantMessage", _FakeAssistantMessage)
+    monkeypatch.setattr(claude_agent_sdk, "TextBlock", _FakeTextBlock)
+    monkeypatch.setattr(claude_agent_sdk, "ResultMessage", _FakeResultMessage)
+    monkeypatch.setattr(claude_agent_sdk, "ToolUseBlock", type("ToolUseBlock", (), {}))
+    monkeypatch.setattr(claude_agent_sdk, "ClaudeSDKClient", _FakeClient)
+    monkeypatch.setattr(claude_agent_sdk, "SdkMcpTool", _FakeSdkMcpTool)
+    monkeypatch.setattr(claude_agent_sdk, "create_sdk_mcp_server", fake_create_sdk_mcp_server)
+
+    provider = _instantiate_provider()
+    with pytest.raises(RuntimeError) as excinfo:
+        await provider.call_with_tools(
+            messages=[{"role": "user", "content": "hi"}],
+            tools=[
+                {
+                    "function": {
+                        "name": "noop",
+                        "description": "no-op",
+                        "parameters": {"type": "object", "properties": {}},
+                    }
+                }
+            ],
+            max_retries=0,
+            scope="test",
+        )
+
+    assert QUOTA_ERROR_TEXT in str(excinfo.value)

hindsight-api-slim/tests/test_codex_oauth_refresh.py

@@ -24,10 +24,11 @@ from __future__ import annotations
 import asyncio
 import base64
 import json
-import os
 import stat
 import sys
+import threading
 import time
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -38,6 +39,7 @@ import pytest
 from hindsight_api.engine.providers.codex_llm import (
     _CODEX_CLIENT_ID,
     _CODEX_REFRESH_TOKEN_URL,
+    CodexAuthManager,
     CodexLLM,
     CodexRefreshExpiredError,
 )
@@ -396,6 +398,63 @@ async def test_concurrent_ensure_fresh_token_calls_produce_one_refresh(tmp_path:
     assert call_count == 1, f"expected 1 network refresh under contention, got {call_count}"
 
 
+def test_sibling_auth_manager_adopts_rotated_codex_credentials(tmp_path: Path):
+    """A stale sibling manager should adopt auth.json rotation before reusing the old RT."""
+    expired = _make_jwt(int(time.time()) - 60)
+    new_access = _make_jwt(int(time.time()) + 3600)
+    auth_file = _make_codex_auth_file(tmp_path, expired, refresh_token="rt-old")
+
+    first = CodexAuthManager.from_file(auth_file)
+    sibling = CodexAuthManager.from_file(auth_file)
+
+    refresh_resp = _refresh_response(200, {"access_token": new_access, "refresh_token": "rt-new"})
+    with patch.object(first._http_client, "post", return_value=refresh_resp):
+        first.refresh_tokens(reason="test")
+
+    def unexpected_post(*args, **kwargs):
+        raise AssertionError("stale sibling should not call refresh endpoint with old refresh_token")
+
+    with patch.object(sibling._http_client, "post", new=unexpected_post):
+        sibling.refresh_tokens(reason="test", force=True)
+
+    assert sibling.access_token == new_access
+    assert sibling.refresh_token == "rt-new"
+
+
+def test_parallel_auth_managers_share_one_refresh_for_same_auth_file(tmp_path: Path):
+    """Separate managers in one process should single-flight per canonical auth path."""
+    expired = _make_jwt(int(time.time()) - 60)
+    new_access = _make_jwt(int(time.time()) + 3600)
+    auth_file = _make_codex_auth_file(tmp_path, expired, refresh_token="rt-old")
+
+    managers = [CodexAuthManager.from_file(auth_file), CodexAuthManager.from_file(auth_file)]
+    call_count = 0
+    call_count_lock = threading.Lock()
+
+    def fake_post(*args, **kwargs):
+        nonlocal call_count
+        with call_count_lock:
+            call_count += 1
+        time.sleep(0.02)
+        return _refresh_response(200, {"access_token": new_access, "refresh_token": "rt-new"})
+
+    patches = [patch.object(manager._http_client, "post", new=fake_post) for manager in managers]
+    for patcher in patches:
+        patcher.start()
+    try:
+        with ThreadPoolExecutor(max_workers=2) as executor:
+            futures = [executor.submit(manager.refresh_tokens, "test") for manager in managers]
+            for future in futures:
+                future.result()
+    finally:
+        for patcher in patches:
+            patcher.stop()
+
+    assert call_count == 1
+    assert [manager.access_token for manager in managers] == [new_access, new_access]
+    assert [manager.refresh_token for manager in managers] == ["rt-new", "rt-new"]
+
+
 # ---------------------------------------------------------------------------
 # Reactive 401 retry on the request path
 # ---------------------------------------------------------------------------
@@ -422,6 +481,7 @@ async def test_call_reactively_refreshes_on_401_and_retries(tmp_path: Path):
     refresh_resp = _refresh_response(200, {"access_token": new_access, "refresh_token": "rt-new"})
 
     call_count = {"refresh": 0, "post": 0}
+    sent_headers: list[httpx.Headers] = []
 
     # Sync mock for the auth manager's HTTP client (used for token refresh).
     def fake_refresh_post(*args, **kwargs):
@@ -431,6 +491,7 @@ async def test_call_reactively_refreshes_on_401_and_retries(tmp_path: Path):
     # Async mock for the LLM's HTTP client (used for backend calls).
     async def fake_backend_post(url, **kwargs):
         call_count["post"] += 1
+        sent_headers.append(httpx.Headers(kwargs["headers"]))
         if call_count["post"] == 1:
             raise httpx.HTTPStatusError("401", request=MagicMock(), response=fail_response)
         return success_resp
@@ -451,6 +512,10 @@ async def test_call_reactively_refreshes_on_401_and_retries(tmp_path: Path):
     assert call_count["refresh"] == 1
     assert call_count["post"] == 2  # one 401, one success after refresh
     assert llm.access_token == new_access
+    assert sent_headers[0]["Authorization"] == f"Bearer {fresh}"
+    assert sent_headers[1]["Authorization"] == f"Bearer {new_access}"
+    for header_name in ("Content-Type", "OpenAI-Account-ID", "User-Agent", "Origin", "originator"):
+        assert sent_headers[1][header_name] == sent_headers[0][header_name]
 
 
 @pytest.mark.asyncio

hindsight-api-slim/tests/test_codex_request_headers.py

@@ -0,0 +1,63 @@
+"""Regression tests for Codex request identity headers."""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import httpx
+import pytest
+
+from hindsight_api.engine.providers.codex_llm import CodexLLM
+
+
+def build_llm() -> CodexLLM:
+    with (
+        patch.object(CodexLLM, "_load_codex_auth", return_value=("token", "account")),
+        patch.object(CodexLLM, "_load_codex_refresh_token", return_value=None),
+    ):
+        return CodexLLM(
+            provider="openai-codex",
+            api_key="ignored",
+            base_url="https://chatgpt.com/backend-api",
+            model="gpt-5.6-luna",
+        )
+
+
+def assert_codex_request_identity(headers: httpx.Headers) -> None:
+    assert headers["originator"] == "codex_cli_rs"
+    assert headers["User-Agent"] == "codex_cli_rs/0.0.0 (Hindsight)"
+
+
+@pytest.mark.asyncio
+async def test_call_sends_codex_request_identity() -> None:
+    llm = build_llm()
+    response = MagicMock()
+    response.raise_for_status.return_value = None
+
+    with (
+        patch.object(llm._client, "post", new_callable=AsyncMock) as mock_post,
+        patch.object(llm, "_parse_sse_stream", new_callable=AsyncMock, return_value="ok"),
+    ):
+        mock_post.return_value = response
+        await llm.call(messages=[{"role": "user", "content": "hello"}], max_retries=0)
+
+    assert_codex_request_identity(mock_post.call_args.kwargs["headers"])
+
+
+@pytest.mark.asyncio
+async def test_call_with_tools_sends_codex_request_identity() -> None:
+    llm = build_llm()
+    response = MagicMock()
+    response.status_code = 200
+    response.raise_for_status.return_value = None
+
+    with (
+        patch.object(llm._client, "post", new_callable=AsyncMock) as mock_post,
+        patch.object(llm, "_parse_sse_tool_stream", new_callable=AsyncMock, return_value=(None, [])),
+    ):
+        mock_post.return_value = response
+        await llm.call_with_tools(
+            messages=[{"role": "user", "content": "hello"}],
+            tools=[],
+            max_retries=0,
+        )
+
+    assert_codex_request_identity(mock_post.call_args.kwargs["headers"])

hindsight-api-slim/tests/test_codex_strict_schema.py

@@ -0,0 +1,188 @@
+"""
+Regression tests for Codex structured output (issue #2504).
+
+Before the fix, ``CodexLLM.call(strict_schema=True)`` was a dead no-op: structured
+output always went through prompt-injected schema + raw ``json.loads`` on the
+model's free-form text. Escape-heavy content (code, serial/CLI commands, Windows
+paths, regexes) makes weaker models emit invalid ``\\escape`` sequences, so every
+parse attempt fails and retain/consolidation burn all retries and fail.
+
+The fix:
+- ``strict_schema=True`` routes structured output through a single forced function
+  tool (constrained decoding into the response schema).
+- The non-strict fallback now repairs invalid ``\\escape`` sequences before giving up.
+"""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from pydantic import BaseModel
+
+from hindsight_api.engine.providers.codex_llm import (
+    CodexLLM,
+    _repair_invalid_json_escapes,
+)
+from hindsight_api.engine.response_models import LLMToolCall
+
+
+class _Fact(BaseModel):
+    fact: str
+
+
+def build_llm() -> CodexLLM:
+    with patch.object(CodexLLM, "_load_codex_auth", return_value=("token", "account")):
+        return CodexLLM(
+            provider="openai-codex",
+            api_key="ignored",
+            base_url="https://chatgpt.com/backend-api",
+            model="gpt-5.4-mini",
+        )
+
+
+# ---------------------------------------------------------------------------
+# _repair_invalid_json_escapes — pure unit tests
+# ---------------------------------------------------------------------------
+
+
+def test_repair_fixes_invalid_escape_in_json():
+    # `\d` and `\s` are not valid JSON escapes; raw json.loads fails.
+    broken = r'{"fact": "regex \d+\s matches digits"}'
+    import json
+
+    with pytest.raises(json.JSONDecodeError):
+        json.loads(broken)
+    repaired = _repair_invalid_json_escapes(broken)
+    assert json.loads(repaired) == {"fact": r"regex \d+\s matches digits"}
+
+
+def test_repair_preserves_valid_escapes():
+    import json
+
+    valid = r'{"fact": "line1\nline2\ttab \"quoted\" \\backslash é"}'
+    # Already valid — repair must not corrupt it.
+    assert json.loads(_repair_invalid_json_escapes(valid)) == json.loads(valid)
+
+
+def test_repair_handles_windows_paths():
+    import json
+
+    # Uses path segments whose first char isn't a valid JSON escape letter
+    # (b/f/n/r/t/u), where the repair is unambiguous.
+    broken = r'{"path": "C:\Windows\System32\app.exe"}'
+    assert json.loads(_repair_invalid_json_escapes(broken)) == {"path": r"C:\Windows\System32\app.exe"}
+
+
+def test_repair_handles_trailing_backslash():
+    # A lone trailing backslash must be escaped, not dropped.
+    assert _repair_invalid_json_escapes("abc\\") == "abc\\\\"
+
+
+# ---------------------------------------------------------------------------
+# strict_schema forced-tool path
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_strict_schema_uses_forced_function_tool():
+    llm = build_llm()
+    response = MagicMock()
+    response.raise_for_status.return_value = None
+    tool_call = LLMToolCall(id="call-1", name="structured_response", arguments={"fact": "the sky is blue"})
+
+    with patch.object(llm._client, "post", new_callable=AsyncMock) as mock_post:
+        mock_post.return_value = response
+        with patch.object(llm, "_parse_sse_tool_stream", new_callable=AsyncMock) as mock_parse:
+            mock_parse.return_value = (None, [tool_call])
+            result = await llm.call(
+                messages=[{"role": "user", "content": "The sky is blue"}],
+                response_format=_Fact,
+                strict_schema=True,
+                max_retries=0,
+            )
+        sent_payload = mock_post.call_args.kwargs["json"]
+        sent_headers = mock_post.call_args.kwargs["headers"]
+
+    # Forced tool wired into the request payload.
+    assert sent_payload["tool_choice"] == {"type": "function", "name": "structured_response"}
+    assert len(sent_payload["tools"]) == 1
+    assert sent_payload["tools"][0]["name"] == "structured_response"
+    assert sent_payload["parallel_tool_calls"] is False
+    assert sent_headers["originator"] == "codex_cli_rs"
+    assert sent_headers["User-Agent"] == "codex_cli_rs/0.0.0 (Hindsight)"
+    # No prompt-injected schema in the instructions.
+    assert "You must respond with valid JSON" not in sent_payload["instructions"]
+
+    assert isinstance(result, _Fact)
+    assert result.fact == "the sky is blue"
+
+
+@pytest.mark.asyncio
+async def test_strict_schema_skip_validation_returns_dict():
+    llm = build_llm()
+    response = MagicMock()
+    response.raise_for_status.return_value = None
+    tool_call = LLMToolCall(id="c", name="structured_response", arguments={"fact": "x"})
+
+    with patch.object(llm._client, "post", new_callable=AsyncMock) as mock_post:
+        mock_post.return_value = response
+        with patch.object(llm, "_parse_sse_tool_stream", new_callable=AsyncMock) as mock_parse:
+            mock_parse.return_value = (None, [tool_call])
+            result = await llm.call(
+                messages=[{"role": "user", "content": "hi"}],
+                response_format=_Fact,
+                strict_schema=True,
+                skip_validation=True,
+                max_retries=0,
+            )
+
+    assert result == {"fact": "x"}
+
+
+@pytest.mark.asyncio
+async def test_strict_schema_retries_when_forced_tool_missing():
+    llm = build_llm()
+    response = MagicMock()
+    response.raise_for_status.return_value = None
+
+    with patch.object(llm._client, "post", new_callable=AsyncMock) as mock_post:
+        mock_post.return_value = response
+        # Model returns no tool call at all — should raise after retries exhausted.
+        with patch.object(llm, "_parse_sse_tool_stream", new_callable=AsyncMock) as mock_parse:
+            mock_parse.return_value = ("some prose", [])
+            with pytest.raises(RuntimeError, match="structured_response"):
+                await llm.call(
+                    messages=[{"role": "user", "content": "hi"}],
+                    response_format=_Fact,
+                    strict_schema=True,
+                    max_retries=0,
+                )
+
+
+# ---------------------------------------------------------------------------
+# Non-strict fallback: escape repair keeps the retry storm from happening
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_non_strict_repairs_invalid_escapes_without_retrying():
+    llm = build_llm()
+    response = MagicMock()
+    response.raise_for_status.return_value = None
+    # Escape-heavy content the model would emit as invalid JSON.
+    escape_heavy = r'{"fact": "run rig-control \d serial \s command"}'
+
+    with patch.object(llm._client, "post", new_callable=AsyncMock) as mock_post:
+        mock_post.return_value = response
+        with patch.object(llm, "_parse_sse_stream", new_callable=AsyncMock) as mock_parse:
+            mock_parse.return_value = escape_heavy
+            result = await llm.call(
+                messages=[{"role": "user", "content": "coding transcript"}],
+                response_format=_Fact,
+                strict_schema=False,
+                max_retries=3,
+            )
+
+    # Parsed on the first attempt (no retry storm): the SSE stream was read once.
+    assert mock_post.await_count == 1
+    assert isinstance(result, _Fact)
+    assert result.fact == r"run rig-control \d serial \s command"

hindsight-api-slim/tests/test_config_validation.py

@@ -122,6 +122,77 @@ def test_retain_structured_chunk_size_reads_from_env():
     assert config.retain_structured_chunk_size == 9000
 
 
+def test_fail_on_extraction_errors_defaults_to_false(monkeypatch):
+    """Silent-success behavior is preserved by default (issue #2700)."""
+    from hindsight_api.config import ENV_FAIL_ON_EXTRACTION_ERRORS, HindsightConfig
+
+    monkeypatch.delenv(ENV_FAIL_ON_EXTRACTION_ERRORS, raising=False)
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    config = HindsightConfig.from_env()
+
+    assert config.fail_on_extraction_errors is False
+
+
+def test_fail_on_extraction_errors_reads_true_from_env(monkeypatch):
+    """The opt-in escape hatch parses truthy values from the environment."""
+    from hindsight_api.config import ENV_FAIL_ON_EXTRACTION_ERRORS, HindsightConfig
+
+    monkeypatch.setenv(ENV_FAIL_ON_EXTRACTION_ERRORS, "true")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    config = HindsightConfig.from_env()
+
+    assert config.fail_on_extraction_errors is True
+
+
+def test_llm_ollama_num_ctx_defaults_to_none(monkeypatch):
+    """Unset Ollama num_ctx override lets Ollama use its model/server default."""
+    from hindsight_api.config import ENV_LLM_OLLAMA_NUM_CTX, HindsightConfig
+
+    monkeypatch.delenv(ENV_LLM_OLLAMA_NUM_CTX, raising=False)
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    config = HindsightConfig.from_env()
+
+    assert config.llm_ollama_num_ctx is None
+
+
+def test_llm_ollama_num_ctx_keeps_direct_construction_default():
+    """Direct HindsightConfig construction should not require the new field."""
+    from dataclasses import fields
+
+    from hindsight_api.config import HindsightConfig
+
+    config_field = next(item for item in fields(HindsightConfig) if item.name == "llm_ollama_num_ctx")
+
+    assert config_field.default is None
+    assert config_field.kw_only
+
+
+def test_llm_ollama_num_ctx_reads_positive_int(monkeypatch):
+    """The native Ollama context override is parsed as a positive integer."""
+    from hindsight_api.config import ENV_LLM_OLLAMA_NUM_CTX, HindsightConfig
+
+    monkeypatch.setenv(ENV_LLM_OLLAMA_NUM_CTX, "65536")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    config = HindsightConfig.from_env()
+
+    assert config.llm_ollama_num_ctx == 65536
+
+
+def test_llm_ollama_num_ctx_rejects_non_positive_values(monkeypatch):
+    """Zero would be accepted by neither Ollama nor downstream range logic."""
+    from hindsight_api.config import ENV_LLM_OLLAMA_NUM_CTX, HindsightConfig
+
+    monkeypatch.setenv(ENV_LLM_OLLAMA_NUM_CTX, "0")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    with pytest.raises(ValueError, match=ENV_LLM_OLLAMA_NUM_CTX):
+        HindsightConfig.from_env()
+
+
 def test_retain_structured_chunk_size_can_be_less_than_chunk_size():
     """Structured-chunk cap can be smaller than the retain chunk target."""
     from hindsight_api.config import HindsightConfig
@@ -706,3 +777,61 @@ def test_gemini_service_tier_empty_env_is_unset(monkeypatch):
 
     config = HindsightConfig.from_env()
     assert config.llm_gemini_service_tier is None
+
+
+def test_operation_retention_defaults(monkeypatch):
+    from hindsight_api.config import (
+        ENV_OPERATION_CLEANUP_BATCH_SIZE,
+        ENV_OPERATION_RETENTION_DAYS,
+        HindsightConfig,
+    )
+
+    monkeypatch.delenv(ENV_OPERATION_RETENTION_DAYS, raising=False)
+    monkeypatch.delenv(ENV_OPERATION_CLEANUP_BATCH_SIZE, raising=False)
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    config = HindsightConfig.from_env()
+
+    assert config.operation_retention_days == 30
+    assert config.operation_cleanup_batch_size == 1000
+    assert "operation_retention_days" in HindsightConfig.get_static_fields()
+    assert "operation_cleanup_batch_size" in HindsightConfig.get_static_fields()
+
+
+def test_operation_retention_env_overrides(monkeypatch):
+    from hindsight_api.config import (
+        ENV_OPERATION_CLEANUP_BATCH_SIZE,
+        ENV_OPERATION_RETENTION_DAYS,
+        HindsightConfig,
+    )
+
+    monkeypatch.setenv(ENV_OPERATION_RETENTION_DAYS, "0")
+    monkeypatch.setenv(ENV_OPERATION_CLEANUP_BATCH_SIZE, "37")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    config = HindsightConfig.from_env()
+
+    assert config.operation_retention_days == 0
+    assert config.operation_cleanup_batch_size == 37
+
+
+@pytest.mark.parametrize("raw", ["-1", "not-an-int"])
+def test_operation_retention_rejects_invalid_values(monkeypatch, raw):
+    from hindsight_api.config import ENV_OPERATION_RETENTION_DAYS, HindsightConfig
+
+    monkeypatch.setenv(ENV_OPERATION_RETENTION_DAYS, raw)
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    with pytest.raises(ValueError, match=ENV_OPERATION_RETENTION_DAYS):
+        HindsightConfig.from_env()
+
+
+@pytest.mark.parametrize("raw", ["0", "-1", "not-an-int"])
+def test_operation_cleanup_batch_size_requires_positive_integer(monkeypatch, raw):
+    from hindsight_api.config import ENV_OPERATION_CLEANUP_BATCH_SIZE, HindsightConfig
+
+    monkeypatch.setenv(ENV_OPERATION_CLEANUP_BATCH_SIZE, raw)
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    with pytest.raises(ValueError, match=ENV_OPERATION_CLEANUP_BATCH_SIZE):
+        HindsightConfig.from_env()

hindsight-api-slim/tests/test_consolidation_dedup.py

@@ -5,13 +5,18 @@ guard the fix in CI — unlike the real-LLM integration test, which only trigger
 the path stochastically.
 """
 
+import logging
 import types
 import uuid
 from dataclasses import dataclass
 from unittest.mock import AsyncMock, patch
 
+import pytest
+
 from hindsight_api.engine.consolidation.consolidator import (
+    _DEDUP_PROMPT,
     _dedup_active,
+    _dedup_decision_from_response,
     _dedup_reconcile_create,
     _dedup_reconcile_update,
     _DedupDecision,
@@ -124,7 +129,7 @@ async def test_dedup_no_twin_above_threshold_returns_none() -> None:
 
 async def test_dedup_llm_keep_does_not_merge() -> None:
     kwargs, conn, llm = _ctx()
-    llm.call.return_value = _DedupDecision(action="keep", reason="different language")
+    llm.call.return_value = '{"action": "keep", "text": "", "reason": "different language"}'
     with _patch_embed(), _patch_probe([_obs("Uzbek content on YouTube is described as very rich.", 0.98)]):
         result = await _dedup_reconcile_create(**kwargs)
     assert result is None
@@ -142,10 +147,91 @@ async def test_dedup_llm_missing_action_defaults_to_keep() -> None:
     conn.execute.assert_not_called()  # missing action is a conservative no-merge
 
 
+def test_dedup_decision_accepts_exact_valid_actions() -> None:
+    assert _DedupDecision(action="merge").action == "merge"
+    assert _DedupDecision(action="keep").action == "keep"
+
+
+def test_dedup_decision_invalid_action_defaults_to_keep(caplog) -> None:
+    with caplog.at_level(logging.WARNING):
+        decision = _DedupDecision(action="need_input", reason="model asked for more context")
+
+    assert decision.action == "keep"
+    assert "need_input" in caplog.text
+    assert "defaulting to keep" in caplog.text
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        # Case / whitespace variants of the CORRECT verdict are recovered via
+        # normalize, not discarded — a genuine merge must not become a missed merge.
+        ("Merge", "merge"),
+        (" MERGE ", "merge"),
+        ("keep\n", "keep"),
+        ("KEEP", "keep"),
+        # Unrecognized / non-str values still degrade to keep (unchanged fail-safe;
+        # the warning path is covered by the dedicated tests below).
+        ("await", "keep"),
+        ("unknown", "keep"),
+        (None, "keep"),
+        (123, "keep"),
+    ],
+)
+def test_dedup_decision_normalizes_action_case_and_whitespace(raw: object, expected: str) -> None:
+    assert _DedupDecision(action=raw).action == expected
+
+
+def test_dedup_decision_non_scalar_action_defaults_to_keep(caplog) -> None:
+    with caplog.at_level(logging.WARNING):
+        list_decision = _DedupDecision(action=[])
+        dict_decision = _DedupDecision(action={"value": "merge"})
+
+    assert list_decision.action == "keep"
+    assert dict_decision.action == "keep"
+    assert "defaulting to keep" in caplog.text
+
+
+def test_dedup_decision_accepts_raw_json_and_dict_responses() -> None:
+    raw_merge = '{"action": "merge", "text": "Merged observation.", "reason": "same fact"}'
+    raw_keep = {"action": "keep", "text": "", "reason": "different fact"}
+
+    merge_decision = _dedup_decision_from_response(raw_merge)
+    keep_decision = _dedup_decision_from_response(raw_keep)
+
+    assert merge_decision.action == "merge"
+    assert merge_decision.text == "Merged observation."
+    assert keep_decision.action == "keep"
+    assert keep_decision.text == ""
+
+
+def test_dedup_decision_legacy_raw_text_defaults_to_keep(caplog) -> None:
+    with caplog.at_level(logging.WARNING):
+        decision = _dedup_decision_from_response('action="merge" text="Merged observation."')
+
+    assert decision.action == "keep"
+    assert decision.reason == "invalid structured response"
+    assert "Invalid consolidation dedup response" in caplog.text
+
+
+def test_dedup_prompt_contract_requests_json_not_key_value() -> None:
+    prompt = _DEDUP_PROMPT.format(new="The agent checked health at 14:07.", existing="Health was checked.")
+
+    assert '{"action": "merge", "text": "...", "reason": "..."}' in prompt
+    assert '{"action": "keep", "text": "", "reason": "..."}' in prompt
+    assert '"text" to an empty string' in prompt
+    assert "Do NOT use key=value" in prompt
+    assert 'respond action="merge"' not in prompt
+    assert "{new}" not in prompt
+    assert "{existing}" not in prompt
+
+
 async def test_dedup_llm_merge_folds_into_twin() -> None:
     kwargs, conn, llm = _ctx()
     kwargs["create_source_ids"] = [uuid.uuid4(), uuid.uuid4()]
-    llm.call.return_value = _DedupDecision(action="merge", text="Uzbek content on YouTube is very rich.")
+    llm.call.return_value = (
+        '{"action": "merge", "text": "Uzbek content on YouTube is very rich.", "reason": "same fact"}'
+    )
     with _patch_embed(), _patch_probe([_obs("Uzbek content on YouTube is described as very rich.", 0.99)]):
         result = await _dedup_reconcile_create(**kwargs)
     assert result == _TWIN_ID  # merged into the twin; caller skips the CREATE

hindsight-api-slim/tests/test_curation_http.py

@@ -78,6 +78,39 @@ async def test_patch_invalidate_and_revert_over_http(api_client, memory):
     await memory.delete_bank(bank_id, request_context=RequestContext())
 
 
+@pytest.mark.asyncio
+async def test_patch_clears_occurred_dates_with_explicit_null(api_client, memory):
+    bank_id = f"curation-http-clear-dates-{uuid.uuid4().hex[:8]}"
+    mem_id = await _insert_fact(memory, bank_id, "Release v1.2 happened on Monday.")
+    pool = await memory._get_pool()
+    async with pool.acquire() as conn:
+        await conn.execute(
+            """
+            UPDATE memory_units
+            SET occurred_start = '2024-01-15T10:30:00Z',
+                occurred_end = '2024-01-15T11:00:00Z'
+            WHERE id = $1
+            """,
+            uuid.UUID(mem_id),
+        )
+
+    resp = await api_client.patch(
+        f"/v1/default/banks/{bank_id}/memories/{mem_id}",
+        json={"occurred_start": None, "occurred_end": None},
+    )
+
+    assert resp.status_code == 200, resp.text
+    assert resp.json()["occurred_start"] is None
+    assert resp.json()["occurred_end"] is None
+
+    resp = await api_client.get(f"/v1/default/banks/{bank_id}/memories/{mem_id}")
+    assert resp.status_code == 200
+    assert resp.json()["occurred_start"] is None
+    assert resp.json()["occurred_end"] is None
+
+    await memory.delete_bank(bank_id, request_context=RequestContext())
+
+
 @pytest.mark.asyncio
 async def test_patch_not_found_returns_404(api_client, memory):
     bank_id = f"curation-http-404-{uuid.uuid4().hex[:8]}"

hindsight-api-slim/tests/test_db_abstraction.py

@@ -4,8 +4,7 @@ Unit tests that verify the abstraction interfaces work correctly
 without requiring a live database connection.
 """
 
-import json
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import AsyncMock, patch
 
 import pytest
 
@@ -592,6 +591,37 @@ class TestConfig:
         assert DEFAULT_DATABASE_BACKEND == "postgresql"
 
 
+# ---------------------------------------------------------------------------
+# Entity expansion CTE tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("ops_module", "ops_class", "limit_clause"),
+    [
+        ("hindsight_api.engine.db.ops_postgresql", "PostgreSQLOps", "LIMIT 7"),
+        ("hindsight_api.engine.db.ops_oracle", "OracleOps", "FETCH FIRST 7 ROWS ONLY"),
+    ],
+)
+def test_entity_expansion_filters_fact_type_before_per_entity_cap(
+    ops_module: str, ops_class: str, limit_clause: str
+) -> None:
+    """The cap is per entity *and target fact type*, preventing mixed types from
+    exhausting a target type's candidate budget before the outer query sees it.
+    """
+    from importlib import import_module
+
+    ops = getattr(import_module(ops_module), ops_class)()
+    cte = ops.build_entity_expansion_cte("memory_units", "unit_entities", 7)
+
+    lateral_start = cte.index("CROSS JOIN LATERAL")
+    lateral_end = cte.index(") t", lateral_start)
+    lateral_query = cte[lateral_start:lateral_end]
+
+    assert "mu_target.fact_type = $2" in lateral_query
+    assert lateral_query.index("mu_target.fact_type = $2") < lateral_query.index(limit_clause)
+
+
 # ---------------------------------------------------------------------------
 # OracleOps unit tests (mock DatabaseConnection, no live DB)
 # ---------------------------------------------------------------------------
@@ -748,6 +778,85 @@ class TestOracleOpsInsertFactsBatch:
         assert rows_data[0][13] == []
 
 
+# ---------------------------------------------------------------------------
+# PostgreSQL search_vector handling (insert). Since the curation archive drops
+# search_vector (#2503), the insert is the single place it is populated, and
+# pg_search_vector_expr is its one source of truth (shared with revert recompute).
+# ---------------------------------------------------------------------------
+
+
+class TestPostgreSQLSearchVector:
+    @staticmethod
+    def _cfg(ext: str, lang: str = "english"):
+        from types import SimpleNamespace
+
+        return SimpleNamespace(text_search_extension=ext, text_search_extension_native_language=lang)
+
+    @pytest.mark.parametrize(
+        "ext,needle",
+        [
+            ("native", "to_tsvector('english'::regconfig,"),
+            ("vchord", "::bm25_catalog.bm25vector"),
+        ],
+    )
+    def test_expr_builds_vector_for_vector_backends(self, ext, needle):
+        from hindsight_api.engine.db.ops_postgresql import pg_search_vector_expr
+
+        expr = pg_search_vector_expr(self._cfg(ext))
+        assert expr is not None and needle in expr
+        # Always built from the same three carried columns.
+        assert "COALESCE(text, '')" in expr and "COALESCE(text_signals, '')" in expr
+
+    @pytest.mark.parametrize("ext", ["pgroonga", "pg_textsearch", "pg_search"])
+    def test_expr_none_for_base_column_backends(self, ext):
+        from hindsight_api.engine.db.ops_postgresql import pg_search_vector_expr
+
+        # These index the base text columns directly; search_vector stays empty.
+        assert pg_search_vector_expr(self._cfg(ext)) is None
+
+    def test_expr_accepts_custom_column_refs(self):
+        from hindsight_api.engine.db.ops_postgresql import pg_search_vector_expr
+
+        expr = pg_search_vector_expr(self._cfg("native"), text_col="mu.text", context_col="mu.context")
+        assert "COALESCE(mu.text, '')" in expr and "COALESCE(mu.context, '')" in expr
+
+    async def _insert_query(self, ext: str) -> str:
+        from hindsight_api.engine.db.ops_postgresql import PostgreSQLOps
+
+        conn = AsyncMock(spec=DatabaseConnection)
+        conn.fetch = AsyncMock(return_value=[{"id": "00000000-0000-0000-0000-000000000001"}])
+        batch = dict(
+            bank_id="b",
+            fact_texts=["t"],
+            embeddings=["[0.1]"],
+            event_dates=[None],
+            occurred_starts=[None],
+            occurred_ends=[None],
+            mentioned_ats=[None],
+            contexts=["c"],
+            fact_types=["world"],
+            metadata_jsons=["{}"],
+            chunk_ids=[None],
+            document_ids=[None],
+            tags_list=[""],
+            observation_scopes_list=[None],
+            text_signals_list=[None],
+        )
+        with patch("hindsight_api.config.get_config", return_value=self._cfg(ext)):
+            await PostgreSQLOps().insert_facts_batch(conn=conn, **batch)
+        return conn.fetch.call_args.args[0]
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("ext", ["native", "vchord"])
+    async def test_insert_includes_search_vector_column(self, ext):
+        assert "search_vector" in await self._insert_query(ext)
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("ext", ["pgroonga", "pg_textsearch", "pg_search"])
+    async def test_insert_omits_search_vector_column(self, ext):
+        assert "search_vector" not in await self._insert_query(ext)
+
+
 # ---------------------------------------------------------------------------
 # normalize_schema tests
 # ---------------------------------------------------------------------------
@@ -769,3 +878,95 @@ class TestNormalizeSchema:
         assert backend.normalize_schema("public") is None
         assert backend.normalize_schema("tenant_abc") == "tenant_abc"
         assert backend.normalize_schema(None) is None
+
+
+# ---------------------------------------------------------------------------
+# OracleBackend._set_session_schema regression
+# ---------------------------------------------------------------------------
+
+
+class TestOracleSetSessionSchema:
+    """Regression coverage for _set_session_schema (no live Oracle required)."""
+
+    @pytest.mark.asyncio
+    async def test_does_not_await_synchronous_cursor_close(self):
+        """A non-public schema is applied without awaiting the sync cursor.close().
+
+        oracledb's AsyncCursor.close() is synchronous (returns None), so
+        ``await cursor.close()`` raised "object NoneType can't be used in
+        'await' expression" on every acquire() under a non-public schema —
+        breaking the DB health check and all memory operations on Oracle.
+        Reproduced with a fake cursor whose close() is synchronous, exactly
+        like oracledb: this test fails (TypeError) against the buggy code and
+        passes once the erroneous await is removed.
+        """
+        from hindsight_api.engine import memory_engine
+        from hindsight_api.engine.db.oracle import OracleBackend
+
+        executed: list[str] = []
+        closed = {"count": 0}
+
+        class _FakeAsyncCursor:
+            async def execute(self, sql: str) -> None:
+                executed.append(sql)
+
+            async def fetchone(self):
+                # SESSION_USER lookup used to cache the connection's default schema.
+                return ("APP_USER",)
+
+            def close(self) -> None:  # synchronous, like oracledb.AsyncCursor.close
+                closed["count"] += 1
+
+        class _FakeConn:
+            def cursor(self) -> "_FakeAsyncCursor":
+                return _FakeAsyncCursor()
+
+        backend = OracleBackend()
+        token = memory_engine._current_schema.set("TENANT_X")
+        try:
+            await backend._set_session_schema(_FakeConn())
+        finally:
+            memory_engine._current_schema.reset(token)
+
+        assert closed["count"] == 1
+        assert any('ALTER SESSION SET CURRENT_SCHEMA = "TENANT_X"' in s for s in executed)
+
+    @pytest.mark.asyncio
+    async def test_public_schema_resets_to_default_schema(self):
+        """The default ``public`` schema resets a pooled Oracle session to its default.
+
+        Oracle pooled connections retain ``CURRENT_SCHEMA`` across checkouts, so a
+        connection previously used for a tenant schema would still point at that
+        tenant unless the ``public`` acquisition explicitly resets it (#2708). The
+        reset applies ``ALTER SESSION SET CURRENT_SCHEMA`` to the cached SESSION_USER,
+        and the synchronous ``cursor.close()`` is not awaited.
+        """
+        from hindsight_api.engine import memory_engine
+        from hindsight_api.engine.db.oracle import OracleBackend
+
+        executed: list[str] = []
+        closed = {"count": 0}
+
+        class _FakeAsyncCursor:
+            async def execute(self, sql: str) -> None:
+                executed.append(sql)
+
+            async def fetchone(self):
+                return ("APP_USER",)
+
+            def close(self) -> None:  # synchronous, like oracledb.AsyncCursor.close
+                closed["count"] += 1
+
+        class _FakeConn:
+            def cursor(self) -> "_FakeAsyncCursor":
+                return _FakeAsyncCursor()
+
+        backend = OracleBackend()
+        token = memory_engine._current_schema.set("public")
+        try:
+            await backend._set_session_schema(_FakeConn())
+        finally:
+            memory_engine._current_schema.reset(token)
+
+        assert closed["count"] == 1
+        assert any('ALTER SESSION SET CURRENT_SCHEMA = "APP_USER"' in s for s in executed)

hindsight-api-slim/tests/test_delta_retain_duplicates.py

@@ -10,13 +10,11 @@ Verifies that:
 
 import asyncio
 import logging
-import os
 from datetime import datetime, timezone
 
 import pytest
 import pytest_asyncio
 
-from hindsight_api import RequestContext
 from hindsight_api.engine.task_backend import SyncTaskBackend
 
 logger = logging.getLogger(__name__)
@@ -396,3 +394,74 @@ async def test_concurrent_upserts_no_duplicates(memory_no_llm, request_context):
 
     finally:
         await memory_no_llm.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_append_mode_preserves_document_metadata_projection(memory_no_llm, request_context):
+    """Append retains should keep item metadata visible through document APIs."""
+    bank_id = f"test_append_metadata_{_ts()}"
+    document_id = "append-metadata-doc"
+
+    try:
+        await memory_no_llm.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {
+                    "content": "first turn from the agent session",
+                    "context": "agent conversation",
+                    "document_id": document_id,
+                    "metadata": {
+                        "source": "hermes",
+                        "platform": "weixin",
+                        "session_id": document_id,
+                        "turn_index": "1",
+                    },
+                    "tags": ["source:hermes", "scope:local-agent", f"session:{document_id}"],
+                    "update_mode": "append",
+                }
+            ],
+            request_context=request_context,
+        )
+
+        await memory_no_llm.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {
+                    "content": "second turn from the agent session",
+                    "context": "agent conversation",
+                    "document_id": document_id,
+                    "metadata": {
+                        "source": "hermes",
+                        "platform": "weixin",
+                        "session_id": document_id,
+                        "turn_index": "2",
+                    },
+                    "tags": ["source:hermes", "scope:local-agent", f"session:{document_id}"],
+                    "update_mode": "append",
+                }
+            ],
+            request_context=request_context,
+        )
+
+        doc = await memory_no_llm.get_document(document_id, bank_id, request_context=request_context)
+        assert doc is not None
+        assert doc["document_metadata"] == {
+            "source": "hermes",
+            "platform": "weixin",
+            "session_id": document_id,
+            "turn_index": "2",
+        }
+        assert doc["retain_params"]["metadata"] == doc["document_metadata"]
+        assert doc["retain_params"]["context"] == "agent conversation"
+
+        listed = await memory_no_llm.list_documents(
+            bank_id,
+            tags=["source:hermes"],
+            request_context=request_context,
+        )
+        listed_doc = next(item for item in listed["items"] if item["id"] == document_id)
+        assert listed_doc["document_metadata"] == doc["document_metadata"]
+        assert listed_doc["retain_params"]["metadata"] == doc["document_metadata"]
+
+    finally:
+        await memory_no_llm.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_document_transfer.py

@@ -567,6 +567,57 @@ async def test_full_roundtrip_integrity(memory, request_context):
         await memory.delete_bank(dst, request_context=request_context)
 
 
+@pytest.mark.asyncio
+async def test_transfer_preserves_legacy_causal_links(memory, request_context):
+    """Legacy causal edges survive export/import without becoming retain inputs."""
+    src = _unique_bank("transfer_legacy_causal_src")
+    dst = _unique_bank("transfer_legacy_causal_dst")
+    legacy_types = ("causes", "enables", "prevents")
+    try:
+        await _retain(
+            memory,
+            src,
+            "Alice completed the design. Bob began implementation after the design.",
+            request_context,
+            "doc-legacy-causal",
+        )
+        units = await memory.list_memory_units(src, fact_type="world", request_context=request_context)
+        assert len(units["items"]) >= 2
+        from_unit_id = uuid.UUID(str(units["items"][0]["id"]))
+        to_unit_id = uuid.UUID(str(units["items"][1]["id"]))
+        from_text = units["items"][0]["text"]
+        to_text = units["items"][1]["text"]
+
+        backend = await memory._get_backend()
+        async with acquire_with_retry(backend) as conn:
+            await conn.executemany(
+                f"INSERT INTO {fq_table('memory_links')} "
+                "(from_unit_id, to_unit_id, link_type, entity_id, bank_id, weight) "
+                "VALUES ($1, $2, $3, NULL, $4, 1.0)",
+                [(from_unit_id, to_unit_id, link_type, src) for link_type in legacy_types],
+            )
+
+        archive = await memory.export_documents_async(src, request_context)
+        await _import(memory, dst, archive, request_context)
+
+        async with acquire_with_retry(backend) as conn:
+            imported_types = await conn.fetch(
+                f"SELECT ml.link_type, source.text AS source_text, target.text AS target_text "
+                f"FROM {fq_table('memory_links')} ml "
+                f"JOIN {fq_table('memory_units')} source ON source.id = ml.from_unit_id "
+                f"JOIN {fq_table('memory_units')} target ON target.id = ml.to_unit_id "
+                "WHERE ml.bank_id = $1 AND ml.link_type = ANY($2)",
+                dst,
+                list(legacy_types),
+            )
+        assert {(row["link_type"], row["source_text"], row["target_text"]) for row in imported_types} == {
+            (link_type, from_text, to_text) for link_type in legacy_types
+        }
+    finally:
+        await memory.delete_bank(src, request_context=request_context)
+        await memory.delete_bank(dst, request_context=request_context)
+
+
 @pytest.mark.asyncio
 async def test_export_import_observations(memory, request_context):
     """With include_observations, observations transfer and their sources re-link."""

hindsight-api-slim/tests/test_fact_extraction_retry.py

@@ -9,12 +9,87 @@ BaseException'), which happened when last_error was only set in the
 BadRequestError handler and not for non-dict JSON responses.
 """
 
+import json
 from datetime import datetime, timezone
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
 
+def test_output_retry_split_preserves_conversation_array_boundaries():
+    """OutputTooLong retry splitting must keep conversation chunks valid JSON arrays."""
+    from hindsight_api.engine.retain.fact_extraction import _split_chunk_for_output_retry
+
+    turns = [
+        {"role": "user", "content": "alpha"},
+        {"role": "assistant", "content": "bravo"},
+        {"role": "user", "content": "charlie"},
+        {"role": "assistant", "content": "delta"},
+    ]
+
+    split = _split_chunk_for_output_retry(json.dumps(turns))
+
+    assert split is not None
+    first, second = split
+    assert json.loads(first) == turns[:2]
+    assert json.loads(second) == turns[2:]
+
+
+def test_output_retry_split_divides_single_oversized_turn_content():
+    """A lone oversized conversation turn is split inside content and rewrapped."""
+    from hindsight_api.engine.retain.fact_extraction import _split_chunk_for_output_retry
+
+    turn = {"role": "user", "content": "abcdefghijklmnopqrstuvwxyz", "name": "casey"}
+
+    split = _split_chunk_for_output_retry(json.dumps([turn]))
+
+    assert split is not None
+    first, second = split
+    first_turn = json.loads(first)[0]
+    second_turn = json.loads(second)[0]
+    assert first_turn["role"] == "user"
+    assert second_turn["role"] == "user"
+    assert first_turn["name"] == "casey"
+    assert second_turn["name"] == "casey"
+    assert first_turn["content"] + second_turn["content"] == turn["content"]
+
+
+def test_output_retry_split_returns_none_when_no_progress_possible():
+    """Pathological tiny chunks should be dropped instead of recursively retried."""
+    from hindsight_api.engine.retain.fact_extraction import _split_chunk_for_output_retry
+
+    assert _split_chunk_for_output_retry("x") is None
+    assert _split_chunk_for_output_retry(json.dumps([{"role": "user", "content": ""}])) is None
+
+
+@pytest.mark.asyncio
+async def test_output_too_long_drops_unsplittable_subchunk_without_recursing():
+    """If a chunk cannot be reduced further, auto-split exits gracefully."""
+    from hindsight_api.engine.llm_wrapper import OutputTooLongError
+    from hindsight_api.engine.retain.fact_extraction import _extract_facts_with_auto_split
+
+    config = _make_config(llm_max_retries=1)
+    llm_config = _make_llm_config(mock_response={})
+
+    with patch(
+        "hindsight_api.engine.retain.fact_extraction._extract_facts_from_chunk",
+        side_effect=OutputTooLongError("too long"),
+    ) as extract:
+        facts, usage = await _extract_facts_with_auto_split(
+            chunk="x",
+            chunk_index=0,
+            total_chunks=1,
+            event_date=datetime(2023, 1, 1, tzinfo=timezone.utc),
+            context="",
+            llm_config=llm_config,
+            config=config,
+            agent_name="agent",
+        )
+
+    assert facts == []
+    assert extract.call_count == 1
+
+
 def _make_config(llm_max_retries: int = 3, retain_llm_max_retries: int | None = None):
     """Build a minimal HindsightConfig for fact extraction tests."""
     from hindsight_api.config import HindsightConfig
@@ -60,8 +135,8 @@ async def test_non_dict_json_all_retries_raises():
 
     config = _make_config(llm_max_retries=3, retain_llm_max_retries=None)
 
-    # Mock: always returns a list (non-dict), which is invalid
-    llm_config = _make_llm_config(mock_response=[{"invalid": "response"}])
+    # Mock: always returns a list containing a non-dict item, which is invalid.
+    llm_config = _make_llm_config(mock_response=["invalid response"])
 
     with patch(
         "hindsight_api.engine.retain.fact_extraction._build_extraction_prompt_and_schema",
@@ -82,6 +157,50 @@ async def test_non_dict_json_all_retries_raises():
     assert llm_config.call.call_count == 3
 
 
+@pytest.mark.asyncio
+async def test_top_level_fact_list_is_accepted_without_retry():
+    """
+    Some lax-JSON models return the facts array directly instead of wrapping it
+    in {"facts": [...]}. A top-level list of dict-shaped facts is recoverable
+    and should not burn retries.
+    """
+    from hindsight_api.engine.retain.fact_extraction import _extract_facts_from_chunk
+
+    config = _make_config(llm_max_retries=3, retain_llm_max_retries=None)
+    llm_config = _make_llm_config(
+        mock_response=[
+            {
+                "what": "Alice visited Paris",
+                "when": "2023",
+                "where": "Paris",
+                "who": "Alice",
+                "why": "vacation",
+                "fact_type": "world",
+                "fact_kind": "conversation",
+            }
+        ]
+    )
+
+    with patch(
+        "hindsight_api.engine.retain.fact_extraction._build_extraction_prompt_and_schema",
+        return_value=("system prompt", MagicMock()),
+    ):
+        facts, _usage = await _extract_facts_from_chunk(
+            chunk="Alice visited Paris in 2023.",
+            chunk_index=0,
+            total_chunks=1,
+            event_date=datetime(2023, 1, 1, tzinfo=timezone.utc),
+            context="travel notes",
+            llm_config=llm_config,
+            config=config,
+            agent_name="test-agent",
+        )
+
+    assert llm_config.call.call_count == 1
+    assert len(facts) == 1
+    assert "Alice visited Paris" in facts[0].fact
+
+
 @pytest.mark.asyncio
 async def test_non_dict_json_with_default_max_retries_raises():
     """

hindsight-api-slim/tests/test_gemini_batch.py

@@ -103,18 +103,22 @@ def test_body_translation_maps_roles_and_generation_config():
     assert gc["temperature"] == 0.1
     assert gc["maxOutputTokens"] == 2048
     assert gc["responseMimeType"] == "application/json"
-    # strict=True -> grammar-enforced via responseJsonSchema
+    # grammar-enforced via responseJsonSchema
     assert gc["responseJsonSchema"] == {"type": "object", "properties": {"facts": {"type": "array"}}}
     # schema is also appended as a textual hint (mirrors the sync call path)
     assert "valid JSON matching this schema" in req["systemInstruction"]["parts"][0]["text"]
 
 
-def test_body_translation_omits_response_json_schema_when_not_strict():
+def test_body_translation_grammar_enforces_schema_without_strict():
+    # #2699: Gemini always grammar-enforces via its native response_schema, so the
+    # batch path must set responseJsonSchema even when strict is absent/False
+    # (the interactive path already does). Otherwise batch requests at default
+    # config (HINDSIGHT_API_LLM_STRICT_SCHEMA=False) only get a textual schema hint
+    # and intermittently emit malformed JSON, dropping every fact in the chunk.
     req = GeminiLLM._openai_body_to_gemini_request(_openai_request("c", strict=False)["body"])
     gc = req["generationConfig"]
-    # Non-strict still forces JSON output, but does not grammar-enforce the schema
     assert gc["responseMimeType"] == "application/json"
-    assert "responseJsonSchema" not in gc
+    assert gc["responseJsonSchema"] == {"type": "object", "properties": {"facts": {"type": "array"}}}
 
 
 def test_assistant_role_maps_to_model():

hindsight-api-slim/tests/test_link_expansion_scoring.py

@@ -0,0 +1,61 @@
+"""Regression tests for Link Expansion's final graph score."""
+
+from contextlib import asynccontextmanager
+import math
+from types import SimpleNamespace
+
+import pytest
+
+from hindsight_api.engine.search import link_expansion_retrieval
+from hindsight_api.engine.search.link_expansion_retrieval import LinkExpansionRetriever
+from hindsight_api.engine.search.types import RetrievalResult
+
+
+def _row(fact_id: str, score: float, fact_type: str) -> dict[str, str | float]:
+    """Create the subset of an expansion query row needed by RetrievalResult."""
+    return {"id": fact_id, "text": fact_id, "fact_type": fact_type, "score": score}
+
+
+@pytest.mark.asyncio
+async def test_activation_preserves_additive_score_across_fact_types(monkeypatch):
+    """Graph merge order must match Link Expansion's additive per-type score."""
+    retriever = LinkExpansionRetriever()
+
+    @asynccontextmanager
+    async def fake_acquire_with_retry(_pool):
+        yield object()
+
+    async def fake_expand_combined(_conn, _seed_ids, fact_type, _budget, *, ops):
+        if fact_type == "world":
+            # Convergent semantic and causal signals make this fact's total
+            # score higher, despite its raw entity count being only 1.
+            return [_row("a", 1.0, fact_type)], [_row("a", 0.9, fact_type)], [_row("a", 0.3, fact_type)]
+        return [_row("b", 2.0, fact_type)], [_row("b", 0.7, fact_type)], []
+
+    monkeypatch.setattr(link_expansion_retrieval, "acquire_with_retry", fake_acquire_with_retry)
+    monkeypatch.setattr(retriever, "_expand_combined", fake_expand_combined)
+    pool = SimpleNamespace(ops=object())
+
+    world_results, _ = await retriever.retrieve(
+        pool,
+        query_embedding_str="unused",
+        bank_id="bank",
+        fact_type="world",
+        budget=2,
+        semantic_seeds=[RetrievalResult(id="seed-world", text="seed", fact_type="world")],
+    )
+    experience_results, _ = await retriever.retrieve(
+        pool,
+        query_embedding_str="unused",
+        bank_id="bank",
+        fact_type="experience",
+        budget=2,
+        semantic_seeds=[RetrievalResult(id="seed-experience", text="seed", fact_type="experience")],
+    )
+
+    combined = world_results + experience_results
+    combined.sort(key=lambda result: result.activation or 0.0, reverse=True)
+
+    assert [result.id for result in combined] == ["a", "b"]
+    assert world_results[0].activation == pytest.approx(math.tanh(0.5) + 0.9 + 0.3)
+    assert experience_results[0].activation == pytest.approx(math.tanh(1.0) + 0.7)

hindsight-api-slim/tests/test_llm_strict_schema.py

@@ -182,7 +182,7 @@ def test_batch_request_body_strict_follows_config(strict):
     from hindsight_api.engine.retain.fact_extraction import _build_request_body
 
     llm_config = SimpleNamespace(model="gpt-4o-mini", provider="openai", _provider_impl=SimpleNamespace())
-    config = SimpleNamespace(retain_max_completion_tokens=None, llm_strict_schema=strict)
+    config = SimpleNamespace(retain_max_completion_tokens=None, llm_strict_schema=strict, llm_temperature_retain=None)
     # provider != "openai" service-tier branch skipped via _provider_impl without attr
     llm_config._provider_impl.openai_service_tier = None
 

hindsight-api-slim/tests/test_llm_trace.py

@@ -9,6 +9,7 @@ success/error paths, and the HTTP read API (list / stats / tokens).
 
 import asyncio
 import json
+import logging
 from datetime import datetime, timezone
 from types import SimpleNamespace
 from unittest.mock import AsyncMock
@@ -799,3 +800,129 @@ async def test_real_llm_retain_and_consolidation_traced(memory_real_llm):
         assert by_op["consolidation"][0]["metadata"].get("source_memory_ids"), (
             "consolidation trace missing source_memory_ids"
         )
+
+
+# ── recorder: shutdown / pre-init race conditions ─────────────────────────────
+
+
+class _UninitializedBackend:
+    """Mimics a DB backend before initialize() or after shutdown(): the object
+    exists but the internal asyncpg pool is None, so acquiring raises."""
+
+    _pool: object | None = None
+
+    async def acquire(self):
+        raise RuntimeError("PostgreSQLBackend is not initialized. Call initialize() first.")
+
+
+class _ClosingBackend:
+    """Mimics a backend whose pool is mid-shutdown: the pool object exists but
+    asyncpg raises InterfaceError('pool is closing') on acquire."""
+
+    _pool = object()  # not None, passes the getattr guard
+
+    async def acquire(self):
+        raise Exception("pool is closing")
+
+
+class _UnexpectedErrorBackend:
+    """Mimics a backend with an unexpected (non-shutdown) error."""
+
+    _pool = object()
+
+    async def acquire(self):
+        raise RuntimeError("connection refused: some other error")
+
+
+def _make_record(scope: str = "verification") -> LLMRequestRecord:
+    return LLMRequestRecord(
+        provider="test",
+        model="test-model",
+        scope=scope,
+        status="success",
+        started_at=datetime.now(timezone.utc),
+        ended_at=datetime.now(timezone.utc),
+    )
+
+
+@pytest.mark.asyncio
+async def test_safe_write_pool_none_skips_quietly(caplog):
+    """pool_getter returning None should skip at debug, never warn."""
+    recorder = LLMTraceRecorder(
+        pool_getter=lambda: None,
+        schema_getter=lambda: "public",
+        enabled=True,
+        allowed_scopes=[],
+    )
+    with caplog.at_level(logging.DEBUG, logger="hindsight_api.engine.llm_trace"):
+        await recorder._safe_write(_make_record())
+    warnings = [r for r in caplog.records if r.levelno >= logging.WARNING]
+    assert not warnings, f"expected no warning for pool=None, got: {[r.message for r in warnings]}"
+
+
+@pytest.mark.asyncio
+async def test_safe_write_backend_pool_none_skips_quietly(caplog):
+    """Backend exists but its internal _pool is None (post-shutdown) — should
+    skip at debug via the getattr guard, never warn."""
+    recorder = LLMTraceRecorder(
+        pool_getter=lambda: _UninitializedBackend(),
+        schema_getter=lambda: "public",
+        enabled=True,
+        allowed_scopes=[],
+    )
+    with caplog.at_level(logging.DEBUG, logger="hindsight_api.engine.llm_trace"):
+        await recorder._safe_write(_make_record())
+    warnings = [r for r in caplog.records if r.levelno >= logging.WARNING]
+    assert not warnings, f"expected no warning for backend._pool=None, got: {[r.message for r in warnings]}"
+    assert any("not initialized" in r.message or "pool not" in r.message for r in caplog.records)
+
+
+@pytest.mark.asyncio
+async def test_safe_write_pool_closing_downgrades_to_debug(caplog):
+    """asyncpg InterfaceError('pool is closing') during acquire should be
+    downgraded to debug, not warned."""
+    recorder = LLMTraceRecorder(
+        pool_getter=lambda: _ClosingBackend(),
+        schema_getter=lambda: "public",
+        enabled=True,
+        allowed_scopes=[],
+    )
+    with caplog.at_level(logging.DEBUG, logger="hindsight_api.engine.llm_trace"):
+        await recorder._safe_write(_make_record())
+    warnings = [r for r in caplog.records if r.levelno >= logging.WARNING]
+    assert not warnings, f"expected no warning for pool-is-closing, got: {[r.message for r in warnings]}"
+    assert any("shutdown race" in r.message for r in caplog.records)
+
+
+@pytest.mark.asyncio
+async def test_safe_write_unexpected_error_still_warns(caplog):
+    """Non-shutdown errors should still produce a WARNING."""
+    recorder = LLMTraceRecorder(
+        pool_getter=lambda: _UnexpectedErrorBackend(),
+        schema_getter=lambda: "public",
+        enabled=True,
+        allowed_scopes=[],
+    )
+    with caplog.at_level(logging.DEBUG, logger="hindsight_api.engine.llm_trace"):
+        await recorder._safe_write(_make_record())
+    warnings = [r for r in caplog.records if r.levelno >= logging.WARNING]
+    assert len(warnings) == 1, f"expected exactly 1 warning for unexpected error, got: {[r.message for r in warnings]}"
+
+
+@pytest.mark.asyncio
+async def test_attach_memory_ids_pool_none_skips_quietly(caplog):
+    """_attach_memory_ids with _pool=None should skip at debug, never warn."""
+    recorder = LLMTraceRecorder(
+        pool_getter=lambda: _UninitializedBackend(),
+        schema_getter=lambda: "public",
+        enabled=True,
+        allowed_scopes=[],
+    )
+    with caplog.at_level(logging.DEBUG, logger="hindsight_api.engine.llm_trace"):
+        await recorder._attach_memory_ids(
+            bank_id="test-bank",
+            trace_id="test-trace",
+            patch={"memory_ids": ["m1"]},
+        )
+    warnings = [r for r in caplog.records if r.levelno >= logging.WARNING]
+    assert not warnings, f"expected no warning for attach with _pool=None, got: {[r.message for r in warnings]}"

hindsight-api-slim/tests/test_llm_wrapper.py

@@ -1,6 +1,199 @@
 import pytest
 
-from hindsight_api.engine.llm_wrapper import sanitize_llm_output
+from hindsight_api.engine.llm_wrapper import create_llm_provider, sanitize_llm_output
+
+
+def test_create_llm_provider_preserves_positional_timeout_compatibility():
+    """The new Ollama knob must not steal the old positional timeout slot."""
+    impl = create_llm_provider(
+        "ollama",
+        "",
+        "",
+        "llama3.2",
+        "low",
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        False,
+        None,
+        None,
+        7.5,
+    )
+
+    assert impl.timeout == 7.5
+    assert impl.ollama_num_ctx is None
+
+
+def test_llm_provider_preserves_positional_timeout_compatibility():
+    """The new Ollama knob must not steal the old positional timeout slot."""
+    from hindsight_api.engine.llm_wrapper import LLMProvider
+
+    provider = LLMProvider(
+        "ollama",
+        "",
+        "",
+        "llama3.2",
+        "low",
+        None,
+        None,
+        None,
+        None,
+        False,
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        7.5,
+    )
+
+    assert provider.timeout == 7.5
+    assert provider.ollama_num_ctx is None
+
+
+def test_llm_provider_threads_ollama_num_ctx_to_provider_impl():
+    """LLMProvider carries the native Ollama context override to the implementation."""
+    from hindsight_api.engine.llm_wrapper import LLMProvider
+
+    provider = LLMProvider(
+        provider="ollama",
+        api_key="",
+        base_url="",
+        model="llama3.2",
+        ollama_num_ctx=65536,
+    )
+
+    assert provider.ollama_num_ctx == 65536
+    assert provider._provider_impl.ollama_num_ctx == 65536
+
+
+@pytest.mark.parametrize("bad_value", [0, -1, 1.5, "65536", True])
+def test_llm_provider_rejects_invalid_ollama_num_ctx(bad_value):
+    """Direct callers should fail before sending invalid Ollama request options."""
+    from hindsight_api.engine.llm_wrapper import LLMProvider
+
+    with pytest.raises(ValueError, match="ollama_num_ctx"):
+        LLMProvider(
+            provider="ollama",
+            api_key="",
+            base_url="",
+            model="llama3.2",
+            ollama_num_ctx=bad_value,
+        )
+
+
+@pytest.mark.parametrize("bad_value", [0, -1, 1.5, "65536", True])
+def test_openai_compatible_llm_rejects_invalid_ollama_num_ctx(bad_value):
+    """The provider implementation also validates direct construction."""
+    from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
+
+    with pytest.raises(ValueError, match="ollama_num_ctx"):
+        OpenAICompatibleLLM(
+            provider="ollama",
+            api_key="",
+            base_url="",
+            model="llama3.2",
+            ollama_num_ctx=bad_value,
+        )
+
+
+def test_llm_provider_from_env_reads_ollama_num_ctx(monkeypatch):
+    """Direct env construction uses the same optional positive-int parser."""
+    from hindsight_api.config import ENV_LLM_OLLAMA_NUM_CTX, clear_config_cache
+    from hindsight_api.engine.llm_wrapper import LLMProvider
+
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "ollama")
+    monkeypatch.setenv(ENV_LLM_OLLAMA_NUM_CTX, "32768")
+    clear_config_cache()
+
+    provider = LLMProvider.from_env()
+
+    assert provider.ollama_num_ctx == 32768
+    assert provider._provider_impl.ollama_num_ctx == 32768
+    clear_config_cache()
+
+
+@pytest.mark.asyncio
+async def test_native_ollama_omits_num_ctx_unless_configured(monkeypatch):
+    """Native Ollama calls should not override the model context window by default."""
+    from pydantic import BaseModel
+
+    from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
+
+    class Answer(BaseModel):
+        ok: bool
+
+    class FakeResponse:
+        def raise_for_status(self):
+            return None
+
+        def json(self):
+            return {"message": {"content": '{"ok": true}'}}
+
+    calls = []
+
+    class FakeAsyncClient:
+        def __init__(self, **kwargs):
+            self.kwargs = kwargs
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, exc_type, exc, tb):
+            return None
+
+        async def post(self, url, json, headers):
+            calls.append({"url": url, "json": json, "headers": headers})
+            return FakeResponse()
+
+    monkeypatch.setattr("hindsight_api.engine.providers.openai_compatible_llm.httpx.AsyncClient", FakeAsyncClient)
+
+    default_provider = OpenAICompatibleLLM(
+        provider="ollama",
+        api_key="",
+        base_url="",
+        model="llama3.2",
+    )
+    await default_provider._call_ollama_native(
+        messages=[{"role": "user", "content": "ping"}],
+        response_format=Answer,
+        max_completion_tokens=None,
+        temperature=None,
+        max_retries=0,
+        initial_backoff=1,
+        max_backoff=1,
+        skip_validation=False,
+    )
+
+    configured_provider = OpenAICompatibleLLM(
+        provider="ollama",
+        api_key="",
+        base_url="",
+        model="llama3.2",
+        ollama_num_ctx=65536,
+    )
+    await configured_provider._call_ollama_native(
+        messages=[{"role": "user", "content": "ping"}],
+        response_format=Answer,
+        max_completion_tokens=None,
+        temperature=None,
+        max_retries=0,
+        initial_backoff=1,
+        max_backoff=1,
+        skip_validation=False,
+    )
+
+    assert "num_ctx" not in calls[0]["json"]["options"]
+    assert calls[0]["json"]["options"]["num_batch"] == 512
+    assert calls[1]["json"]["options"]["num_ctx"] == 65536
 
 
 @pytest.mark.parametrize(

hindsight-api-slim/tests/test_maintenance_routines.py

@@ -182,3 +182,45 @@ async def test_schemas_with_expired_rows(memory: MemoryEngine):
         # Disabled retention (days <= 0): always empty.
         disabled = await conn.fetch("SELECT * FROM public.schemas_with_expired_rows('audit_log', 'started_at', 0)")
         assert len(disabled) == 0
+
+
+def _load_all_schema_migration():
+    """Import the #2638 all-schema-run install migration by path."""
+    path = (
+        Path(__file__).resolve().parent.parent
+        / "hindsight_api/alembic/versions/f2a4b6c8d0e2_maintenance_routines_all_schema_runs.py"
+    )
+    spec = importlib.util.spec_from_file_location("_maint_routines_all_schema", path)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_routines_install_on_non_public_schema_run(monkeypatch):
+    """Regression for #2638: a single-tenant deploy migrated into a non-``public``
+    schema must still create the shared ``public.*`` routines.
+
+    The runtime migrates only the configured schema, so ``target_schema`` is
+    never falsy/``public`` and the earlier public-only-gated migrations skipped
+    creation, leaving the maintenance loop logging ``function public.… does not
+    exist`` forever. ``f2a4b6c8d0e2`` installs unconditionally (guarded by a
+    transaction-scoped advisory lock), so ``_pg_upgrade`` issues the
+    ``CREATE OR REPLACE`` regardless of ``target_schema``.
+    """
+    migration = _load_all_schema_migration()
+
+    executed: list[str] = []
+    monkeypatch.setattr(migration.op, "execute", lambda sql: executed.append(str(sql)))
+
+    migration._pg_upgrade()
+
+    joined = "\n".join(executed)
+    # Serialized against concurrent per-schema migration processes.
+    assert "pg_advisory_xact_lock" in joined
+    # Both routines (re)created in public — this is what a non-public run failed to do.
+    assert "CREATE OR REPLACE FUNCTION public.banks_needing_consolidation" in joined
+    assert "CREATE OR REPLACE FUNCTION public.schemas_with_expired_rows" in joined
+    # And crucially: no target_schema gate — the module must not reintroduce the
+    # public-only guard (``_should_install_public_routines``) that caused #2638.
+    assert not hasattr(migration, "_should_install_public_routines")

hindsight-api-slim/tests/test_markitdown_parser.py

@@ -63,3 +63,23 @@ def test_utf8_stream_info_skips_non_utf8_text():
     latin1 = "café".encode("latin-1")  # 0xe9, invalid as standalone UTF-8
 
     assert MarkitdownParser._utf8_stream_info(latin1, "a.txt") is None
+
+
+def test_utf8_stream_info_accepts_non_bytes_buffer():
+    """file_data may arrive as a buffer-protocol object that is not a Python
+    ``bytes`` (e.g. a memoryview or a native/Rust-backed buffer) and therefore
+    has no ``.decode``. The UTF-8 probe must coerce via ``bytes()`` instead of
+    assuming concrete ``bytes``, else every text file fails to parse with
+    ``'...' object has no attribute 'decode'``.
+    """
+    # memoryview is a buffer-protocol object with no ``.decode`` and, unlike a
+    # PEP 688 ``__buffer__`` class, ``bytes(memoryview)`` works on every
+    # supported Python version — a portable stand-in for the native buffer the
+    # storage layer returns.
+    buf = memoryview("über".encode("utf-8"))
+    assert not hasattr(buf, "decode")  # precondition: would hit the original AttributeError
+
+    info = MarkitdownParser._utf8_stream_info(buf, "a.txt")
+
+    assert info is not None
+    assert info.charset == "utf-8"

hindsight-api-slim/tests/test_memory_curation.py

@@ -6,6 +6,7 @@ These tests cover the move semantics, lossless revert (incl. entity
 associations), edit, the guards, listing, and recall exclusion.
 """
 
+import json
 import uuid
 from unittest.mock import AsyncMock, patch
 
@@ -21,21 +22,29 @@ from hindsight_api.engine.retain import embedding_processing
 
 
 async def _insert_memory(
-    conn, memory: MemoryEngine, bank_id: str, text: str, fact_type: str = "experience"
+    conn,
+    memory: MemoryEngine,
+    bank_id: str,
+    text: str,
+    fact_type: str = "experience",
+    metadata: dict | None = None,
 ) -> uuid.UUID:
     """Insert a live memory unit with a real embedding, bypassing the LLM pipeline."""
     mem_id = uuid.uuid4()
     emb = await embedding_processing.generate_embeddings_batch(memory.embeddings, [text])
     await conn.execute(
         """
-        INSERT INTO memory_units (id, bank_id, text, fact_type, embedding, event_date, created_at, updated_at, consolidated_at)
-        VALUES ($1, $2, $3, $4, $5::vector, NOW(), NOW(), NOW(), NOW())
+        INSERT INTO memory_units (
+            id, bank_id, text, fact_type, embedding, event_date, metadata, created_at, updated_at, consolidated_at
+        )
+        VALUES ($1, $2, $3, $4, $5::vector, NOW(), $6::jsonb, NOW(), NOW(), NOW())
         """,
         mem_id,
         bank_id,
         text,
         fact_type,
         str(emb[0]),
+        json.dumps(metadata or {}),
     )
     return mem_id
 
@@ -101,11 +110,12 @@ async def _archive_row(conn, mem_id: uuid.UUID) -> dict | None:
     return dict(row) if row else None
 
 
-async def _archive_has_embedding_column(conn) -> bool:
+async def _archive_has_column(conn, column: str) -> bool:
     return bool(
         await conn.fetchval(
             "SELECT 1 FROM information_schema.columns "
-            "WHERE table_name = 'invalidated_memory_units' AND column_name = 'embedding'"
+            "WHERE table_name = 'invalidated_memory_units' AND column_name = $1",
+            column,
         )
     )
 
@@ -174,9 +184,12 @@ class TestInvalidate:
             arch = await _archive_row(conn, m1)
             assert arch is not None, "row must be in the archive"
             assert arch["invalidation_reason"] == "decommissioned"
-            assert not await _archive_has_embedding_column(conn), (
+            assert not await _archive_has_column(conn, "embedding"), (
                 "archive is cold storage; the schema drops the embedding column (#2209)"
             )
+            assert not await _archive_has_column(conn, "search_vector"), (
+                "archive is cold storage with no index; the schema drops search_vector (#2503)"
+            )
             assert await _link_count(conn, m1) == 0, "links cascade-pruned on move"
             assert str(obs_id) not in await _obs_ids(conn, bank_id), "derived observation removed"
             assert await _consolidated_at(conn, m2) is None, "surviving source reset for re-consolidation"
@@ -216,6 +229,10 @@ class TestInvalidate:
             assert e1 in await _entity_ids_for(conn, m1), "entity associations restored on revert"
             reverted_emb = await conn.fetchval("SELECT embedding FROM memory_units WHERE id = $1", m1)
             assert reverted_emb is not None, "embedding recomputed on revert (archive keeps none)"
+            # Native backend (test default) stores a real tsvector; it must be rebuilt on
+            # revert so the reverted fact is keyword-searchable again (archive keeps none, #2503).
+            reverted_sv = await conn.fetchval("SELECT search_vector FROM memory_units WHERE id = $1", m1)
+            assert reverted_sv is not None, "search_vector recomputed on revert (archive keeps none)"
 
         await memory.delete_bank(bank_id, request_context=request_context)
 
@@ -261,6 +278,10 @@ class TestEdit:
         pool = await memory._get_pool()
         async with pool.acquire() as conn:
             m1 = await _insert_memory(conn, memory, bank_id, "The assistant visited Paris in 2023.")
+            await conn.execute(
+                "UPDATE memory_units SET search_vector = to_tsvector('english'::regconfig, text) WHERE id = $1",
+                m1,
+            )
             obs_id = await _insert_observation(conn, bank_id, "The assistant went to Paris.", [m1])
 
         with (
@@ -279,9 +300,17 @@ class TestEdit:
         assert result["state"] == "valid"
         async with pool.acquire() as conn:
             assert await _in_live(conn, m1), "edited row stays live"
-            row = dict(await conn.fetchrow("SELECT text, consolidated_at FROM memory_units WHERE id = $1", m1))
+            row = dict(
+                await conn.fetchrow(
+                    "SELECT text, consolidated_at, search_vector::text AS search_vector "
+                    "FROM memory_units WHERE id = $1",
+                    m1,
+                )
+            )
             assert row["text"] == "The user visited Paris in 2023."
             assert row["consolidated_at"] is None, "edited memory re-consolidates"
+            assert "'assist'" not in row["search_vector"], "old text must not stay in native FTS search_vector"
+            assert "'user'" in row["search_vector"], "new text must refresh native FTS search_vector"
             assert str(obs_id) not in await _obs_ids(conn, bank_id), "stale observation re-derived"
 
         await memory.delete_bank(bank_id, request_context=request_context)
@@ -472,6 +501,47 @@ class TestGuardsAndListing:
 
         await memory.delete_bank(bank_id, request_context=request_context)
 
+    @pytest.mark.asyncio
+    async def test_list_and_get_memory_units_include_metadata(
+        self, memory: MemoryEngine, request_context: RequestContext
+    ):
+        bank_id = f"test-curation-metadata-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(memory, bank_id, request_context)
+        metadata = {"source": "slack", "channel": "engineering", "thread_id": "T123"}
+
+        pool = await memory._get_pool()
+        async with pool.acquire() as conn:
+            mem_id = await _insert_memory(conn, memory, bank_id, "Fact with metadata.", metadata=metadata)
+
+        live = (await memory.list_memory_units(bank_id, request_context=request_context))["items"]
+        live_item = next(item for item in live if item["id"] == str(mem_id))
+        assert live_item["metadata"] == metadata
+
+        detail = await memory.get_memory_unit(bank_id, str(mem_id), request_context=request_context)
+        assert detail is not None
+        assert detail["metadata"] == metadata
+
+        with (
+            patch.object(memory, "submit_async_consolidation", new=AsyncMock()),
+            patch.object(memory, "submit_async_graph_maintenance", new=AsyncMock()),
+        ):
+            await memory.update_memory_unit(
+                bank_id, str(mem_id), state="invalidated", reason="stale", request_context=request_context
+            )
+
+        invalid = (await memory.list_memory_units(bank_id, state="invalidated", request_context=request_context))[
+            "items"
+        ]
+        assert invalid[0]["id"] == str(mem_id)
+        assert invalid[0]["metadata"] == metadata
+
+        invalid_detail = await memory.get_memory_unit(bank_id, str(mem_id), request_context=request_context)
+        assert invalid_detail is not None
+        assert invalid_detail["state"] == "invalidated"
+        assert invalid_detail["metadata"] == metadata
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
     @pytest.mark.asyncio
     async def test_list_filters_by_document(self, memory: MemoryEngine, request_context: RequestContext):
         bank_id = f"test-curation-doc-{uuid.uuid4().hex[:8]}"

hindsight-api-slim/tests/test_minimax_thinking_body.py

@@ -0,0 +1,99 @@
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
+
+
+def _make_minimax(extra_body=None) -> OpenAICompatibleLLM:
+    return OpenAICompatibleLLM(
+        provider="minimax",
+        api_key="test-key",
+        base_url="",
+        model="MiniMax-M3",
+        extra_body=extra_body,
+    )
+
+
+def _text_response(content: str = "ok"):
+    return SimpleNamespace(
+        error=None,
+        usage=None,
+        choices=[
+            SimpleNamespace(
+                finish_reason="stop",
+                message=SimpleNamespace(content=content, tool_calls=None, refusal=None),
+            )
+        ],
+    )
+
+
+def _tool_response():
+    tool_call = SimpleNamespace(
+        id="call_minimax_123",
+        function=SimpleNamespace(name="recall", arguments='{"query": "Project Rin"}'),
+    )
+    return SimpleNamespace(
+        error=None,
+        usage=None,
+        choices=[
+            SimpleNamespace(
+                finish_reason="tool_calls",
+                message=SimpleNamespace(content=None, tool_calls=[tool_call], refusal=None),
+            )
+        ],
+    )
+
+
+@pytest.mark.asyncio
+async def test_minimax_call_disables_thinking_by_default():
+    llm = _make_minimax()
+    llm._client.chat.completions.create = AsyncMock(return_value=_text_response())
+
+    with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector"):
+        await llm.call(messages=[{"role": "user", "content": "hi"}], max_retries=0)
+
+    assert llm._client.chat.completions.create.call_args.kwargs["extra_body"] == {"thinking": {"type": "disabled"}}
+
+
+@pytest.mark.asyncio
+async def test_minimax_call_preserves_configured_thinking_extra_body():
+    llm = _make_minimax(extra_body={"thinking": {"type": "enabled"}, "reasoning_split": True})
+    llm._client.chat.completions.create = AsyncMock(return_value=_text_response())
+
+    with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector"):
+        await llm.call(messages=[{"role": "user", "content": "hi"}], max_retries=0)
+
+    assert llm._client.chat.completions.create.call_args.kwargs["extra_body"] == {
+        "thinking": {"type": "enabled"},
+        "reasoning_split": True,
+    }
+
+
+@pytest.mark.asyncio
+async def test_minimax_tool_call_disables_thinking_by_default():
+    llm = _make_minimax()
+    llm._client.chat.completions.create = AsyncMock(return_value=_tool_response())
+
+    with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector"):
+        await llm.call_with_tools(
+            messages=[{"role": "user", "content": "Search memory."}],
+            tools=[
+                {
+                    "type": "function",
+                    "function": {
+                        "name": "recall",
+                        "description": "Recall memories",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {"query": {"type": "string"}},
+                            "required": ["query"],
+                        },
+                    },
+                }
+            ],
+            max_retries=0,
+        )
+
+    assert llm._client.chat.completions.create.call_args.kwargs["extra_body"] == {"thinking": {"type": "disabled"}}

hindsight-api-slim/tests/test_multilingual_bm25.py

@@ -10,12 +10,18 @@ Covers:
 from __future__ import annotations
 
 from pathlib import Path
+from types import SimpleNamespace
 from unittest.mock import MagicMock
 
+import pytest
+
 from hindsight_api.engine.consolidation.prompts import build_batch_consolidation_prompt
 from hindsight_api.engine.prompt_utils import output_language_directive
 from hindsight_api.engine.reflect.prompts import build_final_system_prompt
 from hindsight_api.engine.retain.fact_extraction import _build_extraction_prompt_and_schema
+from hindsight_api.engine.search import retrieval as retrieval_mod
+from hindsight_api.engine.search.retrieval import tokenize_query
+from hindsight_api.engine.sql.postgresql import PostgreSQLDialect
 
 
 def _baseline_config() -> MagicMock:
@@ -165,3 +171,75 @@ def test_configurable_bm25_language_migration_chains_off_head():
     src = target.read_text()
     assert 'revision: str = "p4q5r6s7t8u9"' in src
     assert 'down_revision: str | Sequence[str] | None = "86f7a033d372"' in src
+
+
+# ---------------------------------------------------------------------------
+# BM25 query term cap
+# ---------------------------------------------------------------------------
+
+
+def test_postgresql_native_bm25_caps_raw_terms_preserving_order():
+    query = "Alpha beta alpha, gamma delta beta epsilon"
+    tokens = tokenize_query(query)
+
+    assert PostgreSQLDialect().prepare_bm25_text(tokens, query, max_query_terms=3) == "alpha | beta | alpha"
+
+
+def test_postgresql_native_bm25_zero_cap_keeps_existing_unlimited_behavior():
+    query = "Alpha beta alpha"
+    tokens = tokenize_query(query)
+
+    assert PostgreSQLDialect().prepare_bm25_text(tokens, query, max_query_terms=0) == "alpha | beta | alpha"
+
+
+def test_postgresql_extension_bm25_keeps_raw_query_text():
+    query = "Alpha beta alpha, gamma delta beta epsilon"
+    tokens = tokenize_query(query)
+
+    assert (
+        PostgreSQLDialect().prepare_bm25_text(tokens, query, text_search_extension="vchord", max_query_terms=3) == query
+    )
+
+
+@pytest.mark.asyncio
+async def test_combined_retrieval_uses_default_bm25_cap_for_legacy_config(monkeypatch):
+    class FakeDialect:
+        max_query_terms: int | None = None
+
+        def build_semantic_arm(self, **kwargs):
+            return "SELECT 'semantic' AS source"
+
+        def build_bm25_arm(self, **kwargs):
+            return "SELECT 'bm25' AS source"
+
+        def prepare_bm25_text(self, tokens, query_text, *, text_search_extension="native", max_query_terms=None):
+            self.max_query_terms = max_query_terms
+            return " | ".join(tokens)
+
+    class FakeConn:
+        backend_type = "postgresql"
+
+        async def fetch(self, query, *params):
+            return []
+
+    fake_dialect = FakeDialect()
+    legacy_config = SimpleNamespace(
+        semantic_min_similarity=0.0,
+        bm25_min_score=0.0,
+        text_search_extension="native",
+        text_search_extension_native_language="english",
+    )
+    monkeypatch.setattr(retrieval_mod, "get_config", lambda: legacy_config)
+    monkeypatch.setattr(retrieval_mod, "create_sql_dialect", lambda backend: fake_dialect)
+
+    result = await retrieval_mod.retrieve_semantic_bm25_combined(
+        FakeConn(),
+        "[0.0]",
+        "alpha beta",
+        "bank-1",
+        ["observation"],
+        5,
+    )
+
+    assert result == {"observation": ([], [])}
+    assert fake_dialect.max_query_terms == 0

hindsight-api-slim/tests/test_observation_history_fk_race.py

@@ -0,0 +1,48 @@
+"""Regression: observation_history write for a since-deleted observation is skipped.
+
+Under parallel (or same-batch delete-then-update) consolidation, one write path
+can remove an observation from ``memory_units`` before another writes its
+``observation_history`` snapshot. The INSERT then trips
+``observation_history_observation_id_fkey``. Because consolidation runs in
+autocommit (no enclosing transaction), catching the FK violation and skipping
+the best-effort history row is safe — the connection stays usable and the
+current observation state remains the source of truth.
+
+Regression for #2597 / #2506: before the fix this raised
+``asyncpg.ForeignKeyViolationError`` and failed the whole consolidation task.
+"""
+
+import uuid
+
+import pytest
+
+from hindsight_api.engine.consolidation.consolidator import (
+    _append_observation_history,
+    _ObservationHistorySnapshot,
+)
+from hindsight_api.engine.db_utils import acquire_with_retry
+
+
+@pytest.mark.asyncio
+async def test_append_history_for_missing_observation_is_skipped(memory, request_context):
+    """A history write targeting an absent observation is skipped, not fatal."""
+    bank_id = f"test-obs-history-fk-{uuid.uuid4().hex[:8]}"
+    await memory.get_bank_profile(bank_id, request_context=request_context)
+
+    snapshot = _ObservationHistorySnapshot(
+        previous_text="old",
+        previous_tags=[],
+        previous_occurred_start=None,
+        previous_occurred_end=None,
+        previous_mentioned_at=None,
+        new_source_memory_ids=[],
+    )
+    # Never inserted into memory_units, so the FK target is absent.
+    missing_observation_id = str(uuid.uuid4())
+
+    pool = await memory._get_pool()
+    async with acquire_with_retry(pool) as conn:
+        # Pre-fix this raised asyncpg.ForeignKeyViolationError; the fix skips it.
+        await _append_observation_history(conn, bank_id, missing_observation_id, snapshot, max_entries=10)
+        # Autocommit: the failed INSERT did not poison the connection.
+        assert await conn.fetchval("SELECT 1") == 1

hindsight-api-slim/tests/test_openai_compatible_response_hardening.py

@@ -1,10 +1,13 @@
 from types import SimpleNamespace
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import AsyncMock, patch
 
 import pytest
 from pydantic import BaseModel
 
-from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM, ProviderResponseError
+from hindsight_api.engine.providers.openai_compatible_llm import (
+    OpenAICompatibleLLM,
+    ProviderResponseError,
+)
 
 
 class SimpleJsonResponse(BaseModel):
@@ -70,6 +73,42 @@ async def test_json_object_call_strips_gemma_thought_tags_before_parsing():
     assert result.ok is True
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("model", ["qwen/qwen3.6-35b-a3b", "openai/gpt-oss-120b"])
+async def test_openrouter_verification_uses_larger_reasoning_safe_budget(model: str):
+    llm = OpenAICompatibleLLM(
+        provider="openrouter",
+        api_key="test-key",
+        base_url="",
+        model=model,
+    )
+    create = AsyncMock(return_value=_response(content="ok"))
+    llm._client.chat.completions.create = create
+
+    with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector"):
+        await llm.verify_connection()
+
+    sent = create.call_args.kwargs
+    assert sent["model"] == model
+    assert sent["messages"] == [{"role": "user", "content": "Say 'ok'"}]
+    assert sent["max_tokens"] == 512
+    assert "max_completion_tokens" not in sent
+
+
+@pytest.mark.asyncio
+async def test_verification_uses_larger_budget_for_other_compatible_gateways():
+    llm = _llm()
+    create = AsyncMock(return_value=_response(content="ok"))
+    llm._client.chat.completions.create = create
+
+    with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector"):
+        await llm.verify_connection()
+
+    sent = create.call_args.kwargs
+    assert sent["max_tokens"] == 512
+    assert "max_completion_tokens" not in sent
+
+
 @pytest.mark.asyncio
 async def test_error_payload_with_no_choices_raises_clear_provider_error_without_retry():
     llm = _llm()

hindsight-api-slim/tests/test_operation_retention_migration.py

@@ -0,0 +1,32 @@
+from pathlib import Path
+
+MIGRATION = (
+    Path(__file__).resolve().parent.parent
+    / "hindsight_api"
+    / "alembic"
+    / "versions"
+    / "a8c1e4f7b0d3_add_operation_retention_indexes.py"
+)
+
+
+def test_operation_retention_migration_follows_current_head_and_covers_both_dialects():
+    source = MIGRATION.read_text()
+
+    assert 'down_revision: str | Sequence[str] | None = "f2a4b6c8d0e2"' in source
+    assert "def _pg_upgrade()" in source
+    assert "def _oracle_upgrade()" in source
+    assert "run_for_dialect(pg=_pg_upgrade, oracle=_oracle_upgrade)" in source
+
+
+def test_operation_retention_migration_adds_cleanup_and_newest_first_indexes_with_downgrade():
+    source = MIGRATION.read_text()
+
+    assert "idx_async_operations_terminal_cleanup" in source
+    assert "(updated_at, operation_id)" in source
+    assert "WHERE status IN ('completed', 'failed', 'cancelled')" in source
+    assert "(updated_at, operation_id, status)" in source
+    assert "idx_async_operations_bank_created_desc" in source
+    assert "(bank_id, created_at DESC)" in source
+    assert "def _pg_downgrade()" in source
+    assert "def _oracle_downgrade()" in source
+    assert source.count("DROP INDEX") >= 4

hindsight-api-slim/tests/test_operation_status.py

@@ -8,6 +8,7 @@ Regression tests:
 - Retry now accepts both 'failed' and 'cancelled' operations.
 """
 
+import asyncio
 import uuid
 from datetime import datetime
 
@@ -206,6 +207,58 @@ async def test_retry_cancelled_operation(api_client, memory, test_bank_id):
     assert response.json()["status"] == "pending"
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("terminal_status", ["failed", "cancelled"])
+async def test_fresh_terminal_operation_keeps_payload_and_remains_retryable(
+    api_client, memory, test_bank_id, terminal_status
+):
+    """Retry depends on the original task payload remaining present throughout retention."""
+    pool = memory._pool
+    await _ensure_bank(pool, test_bank_id)
+    op_id = await _insert_operation(pool, test_bank_id, terminal_status)
+
+    raw_before = await pool.fetchrow(
+        "SELECT status, task_payload FROM async_operations WHERE operation_id = $1",
+        uuid.UUID(op_id),
+    )
+    assert raw_before["status"] == terminal_status
+    assert raw_before["task_payload"] is not None
+
+    response = await api_client.post(f"/v1/default/banks/{test_bank_id}/operations/{op_id}/retry")
+    assert response.status_code == 200
+
+    raw_after = await pool.fetchrow(
+        "SELECT status, task_payload FROM async_operations WHERE operation_id = $1",
+        uuid.UUID(op_id),
+    )
+    assert raw_after["status"] == "pending"
+    assert raw_after["task_payload"] is not None
+
+
+@pytest.mark.asyncio
+async def test_retry_does_not_acknowledge_operation_deleted_by_cleanup(api_client, memory, test_bank_id):
+    """A pruning winner must turn a concurrent retry into 404, never false 200."""
+    pool = memory._pool
+    await _ensure_bank(pool, test_bank_id)
+    op_id = await _insert_operation(pool, test_bank_id, "failed")
+    op_uuid = uuid.UUID(op_id)
+
+    async with pool.acquire() as conn:
+        async with conn.transaction():
+            await conn.fetchrow(
+                "SELECT operation_id FROM async_operations WHERE operation_id = $1 FOR UPDATE",
+                op_uuid,
+            )
+            retry_task = asyncio.create_task(
+                api_client.post(f"/v1/default/banks/{test_bank_id}/operations/{op_id}/retry")
+            )
+            await asyncio.sleep(0)
+            await conn.execute("DELETE FROM async_operations WHERE operation_id = $1", op_uuid)
+
+    response = await retry_task
+    assert response.status_code == 404
+
+
 @pytest.mark.asyncio
 async def test_retry_rejects_non_retriable_statuses(api_client, memory, test_bank_id):
     """POST /operations/{id}/retry should reject pending, processing, and completed operations."""

hindsight-api-slim/tests/test_query_analyzer.py

@@ -649,6 +649,71 @@ def test_query_analyzer_chinese_rolling_windows(query_analyzer, query, start, en
     assert analysis.temporal_constraint.end_date.date() == end.date()
 
 
+def test_query_analyzer_chinese_rolling_year_underflow_returns_no_constraint(query_analyzer):
+    """Impossible Chinese rolling windows should not block retrieval."""
+    reference_date = datetime(1, 1, 15, 12, 0, 0)
+
+    analysis = query_analyzer.analyze("过去一年做了什么", reference_date)
+
+    assert analysis.temporal_constraint is None
+
+
+@pytest.mark.parametrize(
+    ("query", "reference_date"),
+    [
+        ("去年今天做了什么", datetime(1, 1, 15, 12, 0, 0)),
+        ("大前年今天做了什么", datetime(1, 1, 15, 12, 0, 0)),
+        ("去年昨天做了什么", datetime(1, 1, 15, 12, 0, 0)),
+        ("昨晚做了什么", datetime(1, 1, 1, 12, 0, 0)),
+        ("前晚做了什么", datetime(1, 1, 1, 12, 0, 0)),
+    ],
+)
+def test_query_analyzer_chinese_fixed_day_underflow_returns_no_constraint(
+    query_analyzer,
+    query,
+    reference_date,
+):
+    """Impossible Chinese fixed-day shifts should not block retrieval."""
+    analysis = query_analyzer.analyze(query, reference_date)
+
+    assert analysis.temporal_constraint is None
+
+
+def test_query_analyzer_chinese_rolling_year_low_safe_boundary_still_extracts(query_analyzer):
+    """Valid low-year Chinese rolling windows should keep their constraint."""
+    reference_date = datetime(2, 1, 15, 12, 0, 0)
+
+    analysis = query_analyzer.analyze("过去一年做了什么", reference_date)
+
+    assert analysis.temporal_constraint is not None
+    assert analysis.temporal_constraint.start_date.date() == datetime(1, 1, 15).date()
+    assert analysis.temporal_constraint.end_date.date() == datetime(2, 1, 15).date()
+
+
+def test_query_analyzer_chinese_fixed_day_low_safe_boundary_still_extracts(query_analyzer):
+    """Valid low-year Chinese fixed-day shifts should keep their constraint."""
+    reference_date = datetime(2, 1, 15, 12, 0, 0)
+
+    analysis = query_analyzer.analyze("去年今天做了什么", reference_date)
+
+    assert analysis.temporal_constraint is not None
+    assert analysis.temporal_constraint.start_date.date() == datetime(1, 1, 15).date()
+    assert analysis.temporal_constraint.end_date.date() == datetime(1, 1, 15).date()
+
+
+def test_query_analyzer_period_valueerror_still_surfaces(query_analyzer, monkeypatch):
+    """Only impossible Chinese date shifts degrade to no constraint."""
+    from hindsight_api.engine import query_analyzer as query_analyzer_module
+
+    def fail_extract_period(query, reference_date):
+        raise ValueError("synthetic extraction bug")
+
+    monkeypatch.setattr(query_analyzer_module, "extract_period", fail_extract_period)
+
+    with pytest.raises(ValueError, match="synthetic extraction bug"):
+        query_analyzer.analyze("past week", datetime(2025, 1, 15, 12, 0, 0))
+
+
 @pytest.mark.parametrize("query", ["三两天前提到的菜是什么"])
 def test_query_analyzer_chinese_exact_relative_boundaries(query_analyzer, query):
     """Test malformed Chinese numerals are not truncated into exact relative rules."""

hindsight-api-slim/tests/test_recall_config.py

@@ -65,12 +65,14 @@ class TestRecallConfigFields:
     """Hierarchical config fields for internal recall."""
 
     def test_fields_exist_on_dataclass(self):
-        from hindsight_api.config import HindsightConfig
+        from hindsight_api.config import DEFAULT_BM25_MAX_QUERY_TERMS, HindsightConfig
 
         names = {f.name for f in dataclasses.fields(HindsightConfig)}
         assert "recall_include_chunks" in names
         assert "recall_max_tokens" in names
         assert "recall_chunks_max_tokens" in names
+        assert "bm25_max_query_terms" in names
+        assert HindsightConfig.__dataclass_fields__["bm25_max_query_terms"].default == DEFAULT_BM25_MAX_QUERY_TERMS
 
     def test_fields_are_configurable(self):
         from hindsight_api.config import HindsightConfig
@@ -82,6 +84,7 @@ class TestRecallConfigFields:
 
     def test_default_values(self):
         from hindsight_api.config import (
+            DEFAULT_BM25_MAX_QUERY_TERMS,
             DEFAULT_RECALL_CHUNKS_MAX_TOKENS,
             DEFAULT_RECALL_INCLUDE_CHUNKS,
             DEFAULT_RECALL_MAX_TOKENS,
@@ -90,9 +93,11 @@ class TestRecallConfigFields:
         assert DEFAULT_RECALL_INCLUDE_CHUNKS is True
         assert DEFAULT_RECALL_MAX_TOKENS == 2048
         assert DEFAULT_RECALL_CHUNKS_MAX_TOKENS == 1000
+        assert DEFAULT_BM25_MAX_QUERY_TERMS == 0
 
     def test_env_var_constants(self):
         from hindsight_api.config import (
+            ENV_BM25_MAX_QUERY_TERMS,
             ENV_RECALL_CHUNKS_MAX_TOKENS,
             ENV_RECALL_INCLUDE_CHUNKS,
             ENV_RECALL_MAX_TOKENS,
@@ -101,6 +106,7 @@ class TestRecallConfigFields:
         assert ENV_RECALL_INCLUDE_CHUNKS == "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS"
         assert ENV_RECALL_MAX_TOKENS == "HINDSIGHT_API_RECALL_MAX_TOKENS"
         assert ENV_RECALL_CHUNKS_MAX_TOKENS == "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS"
+        assert ENV_BM25_MAX_QUERY_TERMS == "HINDSIGHT_API_BM25_MAX_QUERY_TERMS"
 
     @patch.dict(
         "os.environ",
@@ -108,6 +114,7 @@ class TestRecallConfigFields:
             "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS": "false",
             "HINDSIGHT_API_RECALL_MAX_TOKENS": "777",
             "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS": "333",
+            "HINDSIGHT_API_BM25_MAX_QUERY_TERMS": "24",
         },
     )
     def test_from_env_reads_overrides(self):
@@ -117,6 +124,14 @@ class TestRecallConfigFields:
         assert config.recall_include_chunks is False
         assert config.recall_max_tokens == 777
         assert config.recall_chunks_max_tokens == 333
+        assert config.bm25_max_query_terms == 24
+
+    @patch.dict("os.environ", {"HINDSIGHT_API_BM25_MAX_QUERY_TERMS": "-1"})
+    def test_from_env_rejects_negative_bm25_max_query_terms(self):
+        from hindsight_api.config import HindsightConfig
+
+        with pytest.raises(ValueError, match="HINDSIGHT_API_BM25_MAX_QUERY_TERMS must be >= 0"):
+            HindsightConfig.from_env()
 
 
 class TestMentalModelTriggerRecallFields:

hindsight-api-slim/tests/test_reflect_tools.py

@@ -5,6 +5,7 @@ import uuid
 
 import pytest
 
+from hindsight_api.engine.reflect.agent import _execute_tool, _summarize_input
 from hindsight_api.engine.reflect.tools import _document_metadata_from_retain_params, tool_expand
 
 
@@ -118,3 +119,203 @@ def test_document_metadata_from_retain_params_accepts_json_strings() -> None:
 def test_document_metadata_from_retain_params_ignores_invalid_values(retain_params) -> None:
     """Malformed retain_params should not break reflect expansion."""
     assert _document_metadata_from_retain_params(retain_params) is None
+
+
+async def _unexpected_tool_call(*_args):
+    raise AssertionError("unexpected tool callback")
+
+
+@pytest.mark.asyncio
+async def test_execute_tool_treats_string_none_max_tokens_as_default() -> None:
+    """Some providers emit JSON null as the string "None" in tool calls."""
+    captured: dict[str, object] = {}
+
+    async def search_observations(query: str, max_tokens: int) -> dict[str, object]:
+        captured["query"] = query
+        captured["max_tokens"] = max_tokens
+        return {"observations": []}
+
+    result = await _execute_tool(
+        "search_observations",
+        {"query": "deployment failures", "max_tokens": "None"},
+        _unexpected_tool_call,
+        search_observations,
+        _unexpected_tool_call,
+        _unexpected_tool_call,
+    )
+
+    assert result == {"observations": []}
+    assert captured == {"query": "deployment failures", "max_tokens": 5000}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("bad_limit", ["bogus", float("inf")])
+async def test_execute_tool_returns_error_for_invalid_integer_limit(bad_limit) -> None:
+    """Malformed integer limits should be a tool error, not an exception."""
+    result = await _execute_tool(
+        "search_observations",
+        {"query": "deployment failures", "max_tokens": bad_limit},
+        _unexpected_tool_call,
+        _unexpected_tool_call,
+        _unexpected_tool_call,
+        _unexpected_tool_call,
+    )
+
+    assert result == {"error": "max_tokens must be an integer or null-like value"}
+
+
+@pytest.mark.asyncio
+async def test_execute_tool_preserves_search_mental_models_max_results_values() -> None:
+    """Only token limits have minimums; max_results keeps the prior pass-through behavior."""
+    captured: dict[str, object] = {}
+
+    async def search_mental_models(query: str, max_results: int) -> dict[str, object]:
+        captured["query"] = query
+        captured["max_results"] = max_results
+        return {"mental_models": []}
+
+    result = await _execute_tool(
+        "search_mental_models",
+        {"query": "deployment failures", "max_results": -1},
+        search_mental_models,
+        _unexpected_tool_call,
+        _unexpected_tool_call,
+        _unexpected_tool_call,
+    )
+
+    assert result == {"mental_models": []}
+    assert captured == {"query": "deployment failures", "max_results": -1}
+
+
+@pytest.mark.asyncio
+async def test_execute_tool_preserves_falsey_values_as_default_sentinel() -> None:
+    """The old `or default` behavior treated falsey limit values as omitted."""
+    captured: dict[str, object] = {}
+
+    async def search_mental_models(query: str, max_results: int) -> dict[str, object]:
+        captured["mental_model_max_results"] = max_results
+        return {"mental_models": []}
+
+    async def search_observations(query: str, max_tokens: int) -> dict[str, object]:
+        captured["observation_max_tokens"] = max_tokens
+        return {"observations": []}
+
+    async def recall(query: str, max_tokens: int, max_chunk_tokens: int) -> dict[str, object]:
+        captured["recall_max_tokens"] = max_tokens
+        captured["recall_max_chunk_tokens"] = max_chunk_tokens
+        return {"memories": []}
+
+    await _execute_tool(
+        "search_mental_models",
+        {"query": "deployment failures", "max_results": 0},
+        search_mental_models,
+        search_observations,
+        recall,
+        _unexpected_tool_call,
+    )
+    await _execute_tool(
+        "search_observations",
+        {"query": "deployment failures", "max_tokens": 0},
+        search_mental_models,
+        search_observations,
+        recall,
+        _unexpected_tool_call,
+    )
+    await _execute_tool(
+        "recall",
+        {"query": "deployment failures", "max_tokens": 0, "max_chunk_tokens": 0},
+        search_mental_models,
+        search_observations,
+        recall,
+        _unexpected_tool_call,
+    )
+    await _execute_tool(
+        "search_observations",
+        {"query": "deployment failures", "max_tokens": False},
+        search_mental_models,
+        search_observations,
+        recall,
+        _unexpected_tool_call,
+    )
+    await _execute_tool(
+        "search_observations",
+        {"query": "deployment failures", "max_tokens": []},
+        search_mental_models,
+        search_observations,
+        recall,
+        _unexpected_tool_call,
+    )
+    await _execute_tool(
+        "search_observations",
+        {"query": "deployment failures", "max_tokens": {}},
+        search_mental_models,
+        search_observations,
+        recall,
+        _unexpected_tool_call,
+    )
+
+    assert captured == {
+        "mental_model_max_results": 5,
+        "observation_max_tokens": 5000,
+        "recall_max_tokens": 2048,
+        "recall_max_chunk_tokens": 1000,
+    }
+
+
+@pytest.mark.asyncio
+async def test_execute_tool_treats_null_like_recall_limits_as_defaults() -> None:
+    """Null-like string limits should not crash reflect tool execution."""
+    captured: dict[str, object] = {}
+
+    async def recall(query: str, max_tokens: int, max_chunk_tokens: int) -> dict[str, object]:
+        captured["query"] = query
+        captured["max_tokens"] = max_tokens
+        captured["max_chunk_tokens"] = max_chunk_tokens
+        return {"memories": []}
+
+    result = await _execute_tool(
+        "recall",
+        {"query": "incident notes", "max_tokens": "null", "max_chunk_tokens": ""},
+        _unexpected_tool_call,
+        _unexpected_tool_call,
+        recall,
+        _unexpected_tool_call,
+    )
+
+    assert result == {"memories": []}
+    assert captured == {"query": "incident notes", "max_tokens": 2048, "max_chunk_tokens": 1000}
+
+
+def test_summarize_input_never_raises_for_invalid_tool_limit_strings() -> None:
+    """Trace logging must not turn a recoverable tool error into HTTP 500."""
+    assert (
+        _summarize_input(
+            "search_observations",
+            {"query": "deployment failures", "max_tokens": "None"},
+        )
+        == "(query='deployment failures', max_tokens=5000)"
+    )
+
+    assert (
+        _summarize_input(
+            "recall",
+            {"query": "deployment failures", "max_tokens": "bogus", "max_chunk_tokens": "null"},
+        )
+        == "(query='deployment failures', max_tokens=invalid:'bogus', max_chunk_tokens=1000)"
+    )
+
+    assert (
+        _summarize_input(
+            "search_observations",
+            {"query": "deployment failures", "max_tokens": float("inf")},
+        )
+        == "(query='deployment failures', max_tokens=invalid:inf)"
+    )
+
+    assert (
+        _summarize_input(
+            "search_observations",
+            {"query": None, "max_tokens": "None"},
+        )
+        == "(query='', max_tokens=5000)"
+    )

hindsight-api-slim/tests/test_retain.py

@@ -1923,7 +1923,7 @@ async def test_causal_links_creation(memory, request_context):
     """
     Test that causal links are created between facts with causal relationships.
 
-    Causal links connect facts where one causes, enables, or prevents another.
+    Retain represents causal links with the canonical ``caused_by`` type.
     Note: This depends on LLM extracting causal relationships, which may be non-deterministic.
     """
     bank_id = f"test_causal_links_{datetime.now(timezone.utc).timestamp()}"
@@ -1955,7 +1955,7 @@ async def test_causal_links_creation(memory, request_context):
                 SELECT from_unit_id, to_unit_id, link_type, weight
                 FROM memory_links
                 WHERE from_unit_id::text = ANY($1)
-                  AND link_type IN ('causes', 'caused_by', 'enables', 'prevents')
+                  AND link_type = 'caused_by'
                 ORDER BY link_type, weight DESC
                 """,
                 unit_ids,
@@ -1974,9 +1974,7 @@ async def test_causal_links_creation(memory, request_context):
                     logger.info(
                         f"  Link: {from_id[:8]}... -> {to_id[:8]}... ({link_type}, weight: {link['weight']:.2f})"
                     )
-                    assert link["link_type"] in ["causes", "caused_by", "enables", "prevents"], (
-                        f"Causal link type must be valid, got '{link['link_type']}'"
-                    )
+                    assert link["link_type"] == "caused_by"
                     assert 0.0 <= link["weight"] <= 1.0, "Weight should be between 0 and 1"
 
                 logger.info("Causal links created successfully:")

hindsight-api-slim/tests/test_strip_code_fences.py

@@ -1,6 +1,6 @@
 """Tests for _strip_code_fences helper in OpenAI-compatible LLM provider."""
 
-import pytest
+import json
 
 from hindsight_api.engine.providers.openai_compatible_llm import _strip_code_fences
 
@@ -36,6 +36,22 @@ class TestStripCodeFences:
         result = _strip_code_fences(content)
         assert '{"facts": []}' in result
 
+    def test_inner_backticks_preserved(self):
+        """Inner triple-backticks inside a JSON string value must not truncate the JSON.
+
+        Regression for the fact-extraction case where an extracted fact describes
+        code-fence behavior, so the JSON payload itself contains a literal
+        ```` ```json ```` — the old split-based stripper matched that inner
+        occurrence and cut the JSON mid-string.
+        """
+        import json
+
+        content = '```json\n{"facts": [{"what": "the model wraps output in ```json fences"}]}\n```'
+        result = _strip_code_fences(content)
+        assert result == '{"facts": [{"what": "the model wraps output in ```json fences"}]}'
+        parsed = json.loads(result)
+        assert parsed["facts"][0]["what"] == "the model wraps output in ```json fences"
+
     def test_no_fences_no_change(self):
         """Content without any backticks passes through."""
         content = "Just some text without fences"
@@ -53,12 +69,25 @@ class TestStripCodeFences:
         assert '"line2"' in result
         assert "```" not in result
 
-    def test_malformed_fence_returns_original(self):
-        """Malformed fences (missing closing) return something parseable."""
+    def test_missing_closing_fence_recovers_json(self):
+        """A fence with no closing ``` still recovers the JSON via the outer-span fallback."""
         content = '```json\n{"facts": []}'
         result = _strip_code_fences(content)
-        # Should attempt to strip and return best effort
+        assert json.loads(result) == {"facts": []}
+
+    def test_prose_wrapped_json_recovered(self):
+        """JSON surrounded by prose (no usable fence) is recovered by the fallback."""
+        content = 'Sure! Here is the result:\n{"facts": [{"what": "x"}]}\nLet me know if that helps.'
+        result = _strip_code_fences(content)
+        assert json.loads(result) == {"facts": [{"what": "x"}]}
+
+    def test_non_json_fence_left_for_retry(self):
+        """A fenced block that is not JSON yields no valid candidate; content is returned unchanged."""
+        content = "```\nnot json at all\n```"
+        result = _strip_code_fences(content)
+        # No parseable JSON anywhere -> caller sees the stripped body (still a str), never crashes.
         assert isinstance(result, str)
+        assert "not json at all" in result
 
     def test_minimax_style_response(self):
         """Real-world MiniMax response format."""

hindsight-api-slim/tests/test_temporal_recall_selection.py

@@ -10,10 +10,12 @@ to a full scan + disk-spilling sort while dropping the most relevant in-window m
 These are pure mechanics (no LLM), so they assert directly.
 """
 
+from contextlib import asynccontextmanager
 from datetime import UTC, datetime, timedelta
 
 import pytest
 
+import hindsight_api.engine.search.retrieval as retrieval_module
 from hindsight_api.engine.search.retrieval import _select_with_temporal_coverage, retrieve_temporal_combined
 from hindsight_api.engine.task_backend import fq_table
 
@@ -167,3 +169,47 @@ async def test_temporal_recall_covers_window_range(memory):
     assert {apr, jul, octo} <= ids
     selected_months = {r.mentioned_at.month for r in results["world"] if r.mentioned_at}
     assert len(selected_months) >= 3
+
+
+@pytest.mark.asyncio
+async def test_min_semantic_does_not_tighten_temporal_seed_threshold(monkeypatch):
+    """min_scores.semantic filters the semantic arm, not temporal entry-point seeds."""
+    start = datetime(2025, 1, 1, tzinfo=UTC)
+    end = datetime(2025, 2, 1, tzinfo=UTC)
+    temporal_thresholds: list[float] = []
+
+    @asynccontextmanager
+    async def fake_acquire_with_retry(pool):
+        yield object()
+
+    async def fake_semantic_bm25_combined(*args, **kwargs):
+        return {"world": ([], [])}
+
+    async def fake_temporal_combined(*args, **kwargs):
+        temporal_thresholds.append(kwargs["semantic_threshold"])
+        return {"world": []}
+
+    class FakeGraphRetriever:
+        async def retrieve(self, **kwargs):
+            return [], None
+
+    monkeypatch.setattr(retrieval_module, "acquire_with_retry", fake_acquire_with_retry)
+    monkeypatch.setattr(retrieval_module, "retrieve_semantic_bm25_combined", fake_semantic_bm25_combined)
+    monkeypatch.setattr(retrieval_module, "retrieve_temporal_combined", fake_temporal_combined)
+    monkeypatch.setattr(
+        "hindsight_api.engine.search.temporal_extraction.extract_temporal_constraint",
+        lambda *args, **kwargs: (start, end),
+    )
+
+    await retrieval_module.retrieve_all_fact_types_parallel(
+        object(),
+        query_text="what happened in January?",
+        query_embedding_str=_QUERY,
+        bank_id="test_temporal_min_semantic_decoupling",
+        fact_types=["world"],
+        thinking_budget=10,
+        graph_retriever=FakeGraphRetriever(),
+        min_semantic=0.5,
+    )
+
+    assert temporal_thresholds == [0.1]

hindsight-api-slim/tests/test_worker.py

@@ -13,6 +13,9 @@ Tests cover:
 import asyncio
 import json
 import uuid
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
@@ -92,12 +95,13 @@ class TestWorkerOperationMetrics:
 
         poller = WorkerPoller(backend=MagicMock(), worker_id="w-test", executor=executor)
         # Stub terminal-state handlers so _execute_task_inner never touches the DB.
+        poller._mark_completed = AsyncMock()
         poller._mark_failed = AsyncMock()
         poller._defer_operation = AsyncMock()
         poller._schedule_retry = AsyncMock()
         return poller
 
-    async def _run(self, executor, task_type="batch_retain"):
+    async def _run_with_poller(self, executor, task_type="batch_retain"):
         from hindsight_api.worker.poller import ClaimedTask
 
         poller = self._make_poller(executor)
@@ -109,6 +113,10 @@ class TestWorkerOperationMetrics:
         collector = MagicMock()
         with patch("hindsight_api.worker.poller.get_metrics_collector", return_value=collector):
             await poller._execute_task_inner(task)
+        return collector, poller, task
+
+    async def _run(self, executor, task_type="batch_retain"):
+        collector, _poller, _task = await self._run_with_poller(executor, task_type=task_type)
         return collector
 
     @pytest.mark.asyncio
@@ -123,7 +131,8 @@ class TestWorkerOperationMetrics:
         hindsight_async_operations{status="failed"} gauge, which reads each
         operation's final DB status.
         """
-        collector = await self._run(AsyncMock())  # executor returns normally
+        collector, poller, task = await self._run_with_poller(AsyncMock())  # executor returns normally
+        poller._mark_completed.assert_awaited_once_with(task.operation_id, task.schema)
         collector.record_operation_result.assert_called_once()
         call = collector.record_operation_result.call_args
         assert call.args[0] == "retain"  # batch_retain normalised
@@ -164,6 +173,69 @@ class TestWorkerOperationMetrics:
         collector.record_operation_result.assert_not_called()
 
 
+class _AsyncContext:
+    def __init__(self, value):
+        self.value = value
+
+    async def __aenter__(self):
+        return self.value
+
+    async def __aexit__(self, exc_type, exc, tb):
+        return False
+
+
+class _CompletionConn:
+    def __init__(self, execute_result):
+        self.execute_result = execute_result
+        self.execute_calls = []
+
+    def transaction(self):
+        return _AsyncContext(self)
+
+    async def execute(self, query, *args):
+        self.execute_calls.append((query, args))
+        return self.execute_result
+
+
+class _CompletionBackend:
+    def __init__(self, conn):
+        self.conn = conn
+
+    def acquire(self):
+        return _AsyncContext(self.conn)
+
+
+class TestWorkerMarkCompleted:
+    def _make_poller(self, execute_result):
+        from hindsight_api.worker import WorkerPoller
+
+        conn = _CompletionConn(execute_result)
+        poller = WorkerPoller(backend=_CompletionBackend(conn), worker_id="w-test", executor=AsyncMock())
+        poller._maybe_update_parent_operation = AsyncMock()
+        return poller, conn
+
+    @pytest.mark.asyncio
+    async def test_mark_completed_updates_only_processing_rows_and_updates_parent(self):
+        poller, conn = self._make_poller("UPDATE 1")
+
+        await poller._mark_completed("op-1", schema=None)
+
+        query, args = conn.execute_calls[0]
+        assert "status = 'completed'" in query
+        assert "WHERE operation_id = $1 AND status = 'processing'" in query
+        assert args == ("op-1",)
+        poller._maybe_update_parent_operation.assert_awaited_once_with("op-1", None, conn)
+
+    @pytest.mark.asyncio
+    async def test_mark_completed_does_not_overwrite_terminal_rows(self):
+        poller, conn = self._make_poller("UPDATE 0")
+
+        await poller._mark_completed("op-1", schema=None)
+
+        assert conn.execute_calls
+        poller._maybe_update_parent_operation.assert_not_awaited()
+
+
 def test_all_operation_types_have_slot_reservation_config():
     """Every operation_type used in memory_engine must be listed in
     WORKER_SLOT_RESERVATION_TYPES so it can be reserved via env var.
@@ -3747,3 +3819,713 @@ class TestConsolidationBankPriority:
             high_pending_op,
         )
         assert row["status"] == "pending"
+
+
+class TestTerminalOperationRetention:
+    """Terminal operation rows remain intact until bounded worker cleanup expires them."""
+
+    @staticmethod
+    async def _insert_operation(
+        pool,
+        *,
+        operation_id: uuid.UUID,
+        bank_id: str,
+        status: str,
+        updated_at: datetime,
+        marker: str,
+    ) -> None:
+        await pool.execute(
+            """
+            INSERT INTO async_operations
+                (operation_id, bank_id, operation_type, status, task_payload, result_metadata,
+                 created_at, updated_at)
+            VALUES ($1, $2, 'retain', $3, $4::jsonb, $5::jsonb, $6, $6)
+            """,
+            operation_id,
+            bank_id,
+            status,
+            json.dumps({"marker": marker, "contents": ["payload must survive until expiry"]}),
+            json.dumps({"marker": marker, "debug": "metadata must share the same TTL"}),
+            updated_at,
+        )
+
+    @pytest.mark.asyncio
+    async def test_prune_terminal_operations_is_bounded_oldest_first_and_preserves_live_rows(
+        self, pool, backend, clean_operations
+    ):
+        bank_id = f"test-worker-retention-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(pool, bank_id)
+        # Keep this cutoff far outside normal test data so the schema-wide
+        # cleanup method cannot consume another concurrently running test's rows.
+        cutoff = datetime(2000, 1, 1, tzinfo=UTC)
+
+        id_prefix = uuid.uuid4().int & ~0xFFFF
+        oldest_id = uuid.UUID(int=id_prefix + 1)
+        tied_first_id = uuid.UUID(int=id_prefix + 2)
+        tied_second_id = uuid.UUID(int=id_prefix + 3)
+        newer_expired_id = uuid.UUID(int=id_prefix + 4)
+        fresh_failed_id = uuid.UUID(int=id_prefix + 5)
+        fresh_cancelled_id = uuid.UUID(int=id_prefix + 6)
+        old_pending_id = uuid.UUID(int=id_prefix + 7)
+        old_processing_id = uuid.UUID(int=id_prefix + 8)
+
+        rows = [
+            (oldest_id, "completed", cutoff - timedelta(days=4), "expired-completed"),
+            (tied_first_id, "failed", cutoff - timedelta(days=3), "expired-failed"),
+            (tied_second_id, "cancelled", cutoff - timedelta(days=3), "expired-cancelled"),
+            (newer_expired_id, "completed", cutoff - timedelta(days=2), "expired-newer"),
+            (fresh_failed_id, "failed", cutoff + timedelta(days=1), "fresh-failed"),
+            (fresh_cancelled_id, "cancelled", cutoff + timedelta(days=1), "fresh-cancelled"),
+            (old_pending_id, "pending", cutoff - timedelta(days=10), "old-pending"),
+            (old_processing_id, "processing", cutoff - timedelta(days=10), "old-processing"),
+        ]
+        for operation_id, status, updated_at, marker in rows:
+            await self._insert_operation(
+                pool,
+                operation_id=operation_id,
+                bank_id=bank_id,
+                status=status,
+                updated_at=updated_at,
+                marker=marker,
+            )
+
+        raw_before = await pool.fetch(
+            """
+            SELECT operation_id, task_payload->>'marker' AS payload_marker,
+                   result_metadata->>'marker' AS metadata_marker
+            FROM async_operations
+            WHERE bank_id = $1
+            """,
+            bank_id,
+        )
+        assert {row["payload_marker"] for row in raw_before} == {marker for *_, marker in rows}
+        assert {row["metadata_marker"] for row in raw_before} == {marker for *_, marker in rows}
+
+        async with backend.acquire() as conn:
+            async with conn.transaction():
+                deleted = await backend.ops.prune_terminal_operations(
+                    conn,
+                    "async_operations",
+                    cutoff,
+                    batch_size=2,
+                )
+        assert deleted == 2
+
+        raw_after_first_batch = await pool.fetch(
+            """
+            SELECT operation_id, status, task_payload->>'marker' AS payload_marker,
+                   result_metadata->>'marker' AS metadata_marker
+            FROM async_operations
+            WHERE bank_id = $1
+            ORDER BY operation_id
+            """,
+            bank_id,
+        )
+        remaining_ids = {row["operation_id"] for row in raw_after_first_batch}
+        assert oldest_id not in remaining_ids
+        assert tied_first_id not in remaining_ids
+        assert tied_second_id in remaining_ids
+        assert newer_expired_id in remaining_ids
+
+        async with backend.acquire() as conn:
+            async with conn.transaction():
+                deleted = await backend.ops.prune_terminal_operations(
+                    conn,
+                    "async_operations",
+                    cutoff,
+                    batch_size=100,
+                )
+        assert deleted == 2
+
+        raw_final = await pool.fetch(
+            """
+            SELECT operation_id, status, task_payload->>'marker' AS payload_marker,
+                   result_metadata->>'marker' AS metadata_marker
+            FROM async_operations
+            WHERE bank_id = $1
+            ORDER BY operation_id
+            """,
+            bank_id,
+        )
+        assert {(row["status"], row["payload_marker"], row["metadata_marker"]) for row in raw_final} == {
+            ("failed", "fresh-failed", "fresh-failed"),
+            ("cancelled", "fresh-cancelled", "fresh-cancelled"),
+            ("pending", "old-pending", "old-pending"),
+            ("processing", "old-processing", "old-processing"),
+        }
+
+    @pytest.mark.asyncio
+    async def test_concurrent_pruning_is_safe_and_idempotent(self, pool, backend, clean_operations):
+        bank_id = f"test-worker-retention-concurrent-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(pool, bank_id)
+        cutoff = datetime(2000, 1, 1, tzinfo=UTC)
+        operation_ids = [uuid.uuid4() for _ in range(25)]
+        for index, operation_id in enumerate(operation_ids):
+            await self._insert_operation(
+                pool,
+                operation_id=operation_id,
+                bank_id=bank_id,
+                status=("completed", "failed", "cancelled")[index % 3],
+                updated_at=cutoff - timedelta(minutes=25 - index),
+                marker=f"expired-{index}",
+            )
+
+        async def prune(batch_size: int) -> int:
+            async with backend.acquire() as conn:
+                async with conn.transaction():
+                    return await backend.ops.prune_terminal_operations(
+                        conn,
+                        "async_operations",
+                        cutoff,
+                        batch_size=batch_size,
+                    )
+
+        first_counts = await asyncio.gather(prune(10), prune(10))
+        assert sum(first_counts) == 20
+        assert await pool.fetchval("SELECT COUNT(*) FROM async_operations WHERE bank_id = $1", bank_id) == 5
+        second_counts = await asyncio.gather(prune(10), prune(10))
+        assert sum(second_counts) == 5
+        assert await pool.fetchval("SELECT COUNT(*) FROM async_operations WHERE bank_id = $1", bank_id) == 0
+        assert await prune(10) == 0
+
+    @pytest.mark.asyncio
+    async def test_pruning_cancelled_child_cancels_parent_before_deletion_and_releases_siblings_later(
+        self, pool, backend, clean_operations
+    ):
+        bank_id = f"test-worker-retention-parent-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(pool, bank_id)
+        cutoff = datetime(2000, 1, 1, tzinfo=UTC)
+        parent_id = uuid.uuid4()
+        failed_child_id = uuid.uuid4()
+        completed_child_id = uuid.uuid4()
+        cancelled_child_id = uuid.uuid4()
+        standalone_id = uuid.uuid4()
+
+        await pool.execute(
+            """
+            INSERT INTO async_operations
+                (operation_id, bank_id, operation_type, status, result_metadata, created_at, updated_at)
+            VALUES ($1, $2, 'batch_retain', 'pending', '{}'::jsonb, $3, $3)
+            """,
+            parent_id,
+            bank_id,
+            cutoff - timedelta(days=5),
+        )
+        for child_id, status in (
+            (failed_child_id, "failed"),
+            (completed_child_id, "completed"),
+            (cancelled_child_id, "cancelled"),
+        ):
+            await pool.execute(
+                """
+                INSERT INTO async_operations
+                    (operation_id, bank_id, operation_type, status, task_payload, result_metadata,
+                     created_at, updated_at)
+                VALUES ($1, $2, 'retain', $3, '{}'::jsonb, $4::jsonb, $5, $5)
+                """,
+                child_id,
+                bank_id,
+                status,
+                json.dumps({"parent_operation_id": str(parent_id)}),
+                cutoff - timedelta(days=4),
+            )
+        await self._insert_operation(
+            pool,
+            operation_id=standalone_id,
+            bank_id=bank_id,
+            status="completed",
+            updated_at=cutoff - timedelta(days=3),
+            marker="standalone",
+        )
+
+        cleanup_started_at = await pool.fetchval("SELECT now()")
+        async with backend.acquire() as conn:
+            async with conn.transaction():
+                deleted = await backend.ops.prune_terminal_operations(conn, "async_operations", cutoff, batch_size=100)
+
+        assert deleted == 2
+        parent = await pool.fetchrow(
+            """
+            SELECT status, updated_at, completed_at, error_message
+            FROM async_operations
+            WHERE operation_id = $1
+            """,
+            parent_id,
+        )
+        assert parent["status"] == "cancelled"
+        assert parent["updated_at"] >= cleanup_started_at
+        assert parent["completed_at"] >= cleanup_started_at
+        assert parent["error_message"] == "Cancelled because a child operation was cancelled"
+        assert (
+            await pool.fetchval("SELECT COUNT(*) FROM async_operations WHERE operation_id = $1", failed_child_id) == 1
+        )
+        assert (
+            await pool.fetchval("SELECT COUNT(*) FROM async_operations WHERE operation_id = $1", completed_child_id)
+            == 1
+        )
+        assert (
+            await pool.fetchval("SELECT COUNT(*) FROM async_operations WHERE operation_id = $1", cancelled_child_id)
+            == 0
+        )
+        assert await pool.fetchval("SELECT COUNT(*) FROM async_operations WHERE operation_id = $1", standalone_id) == 0
+
+        await pool.execute(
+            "UPDATE async_operations SET updated_at = $2 WHERE operation_id = $1",
+            parent_id,
+            cutoff - timedelta(days=1),
+        )
+        async with backend.acquire() as conn:
+            async with conn.transaction():
+                deleted = await backend.ops.prune_terminal_operations(conn, "async_operations", cutoff, batch_size=100)
+
+        assert deleted == 1
+        assert await pool.fetchval("SELECT COUNT(*) FROM async_operations WHERE operation_id = $1", parent_id) == 0
+        assert (
+            await pool.fetchval("SELECT COUNT(*) FROM async_operations WHERE operation_id = $1", failed_child_id) == 1
+        )
+        assert (
+            await pool.fetchval("SELECT COUNT(*) FROM async_operations WHERE operation_id = $1", completed_child_id)
+            == 1
+        )
+
+        async with backend.acquire() as conn:
+            async with conn.transaction():
+                deleted = await backend.ops.prune_terminal_operations(conn, "async_operations", cutoff, batch_size=100)
+
+        assert deleted == 2
+        assert (
+            await pool.fetchval("SELECT COUNT(*) FROM async_operations WHERE operation_id = $1", failed_child_id) == 0
+        )
+        assert (
+            await pool.fetchval("SELECT COUNT(*) FROM async_operations WHERE operation_id = $1", completed_child_id)
+            == 0
+        )
+
+    @pytest.mark.asyncio
+    async def test_postgresql_pruning_reconciles_pending_same_bank_parent_before_child_delete(self):
+        from hindsight_api.engine.db.ops_postgresql import PostgreSQLOps
+
+        operation_id = uuid.uuid4()
+        calls = []
+        conn = MagicMock()
+
+        async def fetch(query, *args):
+            calls.append(("fetch", query))
+            return [{"operation_id": operation_id}]
+
+        async def execute(query, *args):
+            calls.append(("execute", query))
+
+        conn.fetch = AsyncMock(side_effect=fetch)
+        conn.execute = AsyncMock(side_effect=execute)
+
+        deleted = await PostgreSQLOps().prune_terminal_operations(
+            conn,
+            "async_operations",
+            datetime(2000, 1, 1, tzinfo=UTC),
+            batch_size=100,
+        )
+
+        assert deleted == 1
+        candidate_query = " ".join(conn.fetch.await_args_list[0].args[0].split())
+        reconciliation_query = " ".join(conn.execute.await_args.args[0].split())
+        delete_query = " ".join(conn.fetch.await_args_list[1].args[0].split())
+        assert "candidate_operation.status = 'cancelled' OR NOT EXISTS" in candidate_query
+        assert "parent.operation_id = CASE" in candidate_query
+        assert "UPDATE async_operations parent SET status = 'cancelled'" in reconciliation_query
+        assert "parent.status = 'pending'" in reconciliation_query
+        assert "candidate_operation.bank_id = parent.bank_id" in reconciliation_query
+        assert "candidate_operation.status = 'cancelled'" in reconciliation_query
+        assert "parent.operation_id = CASE" in reconciliation_query
+        assert "~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'" in reconciliation_query
+        assert "completed_at = COALESCE(parent.completed_at, now())" in reconciliation_query
+        assert "error_message = COALESCE(" in reconciliation_query
+        assert "DELETE FROM async_operations" in delete_query
+        assert [kind for kind, _ in calls] == ["fetch", "execute", "fetch"]
+
+    @pytest.mark.asyncio
+    async def test_oracle_pruning_clamps_candidate_batch_to_in_list_limit(self):
+        from hindsight_api.engine.db.ops_oracle import ORACLE_IN_LIST_LIMIT, OracleOps
+
+        candidate_ids = [uuid.uuid4() for _ in range(ORACLE_IN_LIST_LIMIT)]
+        rows = [{"operation_id": operation_id, "result_metadata": {}} for operation_id in candidate_ids]
+        conn = MagicMock()
+        conn.fetch = AsyncMock(side_effect=[rows, rows])
+        conn.execute = AsyncMock()
+
+        deleted = await OracleOps().prune_terminal_operations(
+            conn,
+            "async_operations",
+            datetime(2000, 1, 1, tzinfo=UTC),
+            batch_size=ORACLE_IN_LIST_LIMIT + 500,
+        )
+
+        assert deleted == ORACLE_IN_LIST_LIMIT
+        assert conn.fetch.await_args_list[0].args[-1] == ORACLE_IN_LIST_LIMIT
+        assert len(conn.fetch.await_args_list[1].args[1]) == ORACLE_IN_LIST_LIMIT
+        assert conn.execute.await_count == 2
+        assert len(conn.execute.await_args_list[0].args[1]) == ORACLE_IN_LIST_LIMIT
+        assert len(conn.execute.await_args_list[1].args[1]) == ORACLE_IN_LIST_LIMIT
+
+    @pytest.mark.asyncio
+    async def test_oracle_pruning_reconciles_pending_same_bank_parent_before_child_delete(self):
+        from hindsight_api.engine.db.ops_oracle import OracleOps
+        from hindsight_api.engine.db.oracle import _rewrite_pg_to_oracle
+
+        standalone_id = uuid.uuid4()
+        candidates = [{"operation_id": standalone_id}]
+        calls = []
+        conn = MagicMock()
+
+        async def fetch(query, *args):
+            calls.append(("fetch", query))
+            if len(conn.fetch.await_args_list) == 1:
+                return candidates
+            return [{"operation_id": standalone_id}]
+
+        async def execute(query, *args):
+            calls.append(("execute", query))
+
+        conn.fetch = AsyncMock(side_effect=fetch)
+        conn.execute = AsyncMock(side_effect=execute)
+
+        deleted = await OracleOps().prune_terminal_operations(
+            conn,
+            "async_operations",
+            datetime(2000, 1, 1, tzinfo=UTC),
+            batch_size=100,
+        )
+
+        assert deleted == 1
+        candidate_query = conn.fetch.await_args_list[0].args[0]
+        lock_query = conn.fetch.await_args_list[1].args[0]
+        reconciliation_query = conn.execute.await_args_list[0].args[0]
+        delete_query = conn.execute.await_args_list[1].args[0]
+        safe_parent_lookup = (
+            "parent.operation_id = CASE WHEN REGEXP_LIKE( JSON_VALUE( "
+            "candidate_operation.result_metadata, '$.parent_operation_id' RETURNING VARCHAR2(36) NULL ON ERROR ), "
+            "'^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$' ) "
+            "THEN HEXTORAW(REPLACE( JSON_VALUE( candidate_operation.result_metadata, "
+            "'$.parent_operation_id' RETURNING VARCHAR2(36) NULL ON ERROR ), '-', '' )) ELSE NULL END"
+        )
+        for query in (candidate_query, lock_query):
+            compact_query = " ".join(query.split())
+            assert "candidate_operation.status = 'cancelled' OR NOT EXISTS" in compact_query
+            assert safe_parent_lookup in compact_query
+            assert "RAWTOHEX" not in compact_query
+        compact_reconciliation_query = " ".join(reconciliation_query.split())
+        assert "UPDATE async_operations parent SET status = 'cancelled'" in compact_reconciliation_query
+        assert "parent.status = 'pending'" in compact_reconciliation_query
+        assert "candidate_operation.bank_id = parent.bank_id" in compact_reconciliation_query
+        assert "candidate_operation.status = 'cancelled'" in compact_reconciliation_query
+        assert safe_parent_lookup in compact_reconciliation_query
+        assert "completed_at = COALESCE(parent.completed_at, now())" in compact_reconciliation_query
+        assert "error_message = COALESCE(" in compact_reconciliation_query
+        assert "RAWTOHEX" not in compact_reconciliation_query
+        candidate_oracle_query = _rewrite_pg_to_oracle(candidate_query).query
+        lock_oracle_query = _rewrite_pg_to_oracle(lock_query).query
+        reconciliation_oracle_query = _rewrite_pg_to_oracle(reconciliation_query).query
+        assert "LIMIT" not in candidate_oracle_query
+        assert "FETCH FIRST :2 ROWS ONLY" in candidate_oracle_query
+        for query in (candidate_oracle_query, lock_oracle_query):
+            compact_query = " ".join(query.split())
+            assert "candidate_operation.status = 'cancelled' OR NOT EXISTS" in compact_query
+            assert safe_parent_lookup in compact_query
+            assert "RAWTOHEX" not in compact_query
+        compact_reconciliation_oracle_query = " ".join(reconciliation_oracle_query.split())
+        assert "parent.status = 'pending'" in compact_reconciliation_oracle_query
+        assert "candidate_operation.bank_id = parent.bank_id" in compact_reconciliation_oracle_query
+        assert safe_parent_lookup in compact_reconciliation_oracle_query
+        assert "SYSTIMESTAMP" in compact_reconciliation_oracle_query
+        assert "RAWTOHEX" not in compact_reconciliation_oracle_query
+        assert "FOR UPDATE OF candidate_operation.operation_id SKIP LOCKED" in lock_oracle_query
+        assert conn.fetch.await_args_list[1].args[1] == [standalone_id]
+        assert conn.execute.await_args_list[0].args[1] == [standalone_id]
+        assert conn.execute.await_args_list[1].args[1] == [standalone_id]
+        assert "DELETE FROM async_operations" in delete_query
+        assert [kind for kind, _ in calls] == ["fetch", "fetch", "execute", "execute"]
+
+    @pytest.mark.asyncio
+    async def test_oracle_pruning_filters_parent_blocked_children_before_batch_limit(self):
+        from hindsight_api.engine.db.ops_oracle import OracleOps
+
+        parent_ids = [uuid.uuid4(), uuid.uuid4()]
+        child_ids = [uuid.uuid4(), uuid.uuid4()]
+        eligible_id = uuid.uuid4()
+        blocked_candidates = [
+            {
+                "operation_id": child_id,
+                "result_metadata": {"parent_operation_id": str(parent_id)},
+            }
+            for child_id, parent_id in zip(child_ids, parent_ids, strict=True)
+        ]
+
+        conn = MagicMock()
+
+        async def fetch(query, *args):
+            if "LIMIT $2" in query:
+                if "NOT EXISTS" in query:
+                    return [{"operation_id": eligible_id}]
+                return blocked_candidates
+            if "operation_id = ANY($1)" in query:
+                return [{"operation_id": operation_id} for operation_id in args[0]]
+            raise AssertionError(f"Unexpected query: {query}")
+
+        conn.fetch = AsyncMock(side_effect=fetch)
+        conn.execute = AsyncMock()
+
+        deleted = await OracleOps().prune_terminal_operations(
+            conn,
+            "async_operations",
+            datetime(2000, 1, 1, tzinfo=UTC),
+            batch_size=2,
+        )
+
+        assert deleted == 1
+        candidate_query = conn.fetch.await_args_list[0].args[0]
+        assert candidate_query.index("NOT EXISTS") < candidate_query.index("LIMIT $2")
+        assert conn.execute.await_count == 2
+        assert conn.execute.await_args_list[0].args[1] == [eligible_id]
+        assert conn.execute.await_args_list[1].args[1] == [eligible_id]
+
+    @pytest.mark.asyncio
+    async def test_oracle_backend_resets_default_schema_on_pooled_connection(self):
+        from hindsight_api.engine.db.oracle import OracleBackend
+        from hindsight_api.engine.memory_engine import _current_schema
+
+        backend = OracleBackend()
+        backend._default_schema = "APP_USER"
+        cursor = MagicMock()
+        cursor.execute = AsyncMock()
+        conn = MagicMock()
+        conn.cursor.return_value = cursor
+
+        tenant_token = _current_schema.set("TENANT_A")
+        try:
+            await backend._set_session_schema(conn)
+        finally:
+            _current_schema.reset(tenant_token)
+        cursor.execute.assert_awaited_with('ALTER SESSION SET CURRENT_SCHEMA = "TENANT_A"')
+
+        cursor.execute.reset_mock()
+        default_token = _current_schema.set(None)
+        try:
+            await backend._set_session_schema(conn)
+        finally:
+            _current_schema.reset(default_token)
+        cursor.execute.assert_awaited_once_with('ALTER SESSION SET CURRENT_SCHEMA = "APP_USER"')
+        assert cursor.close.call_count == 2
+
+    @pytest.mark.asyncio
+    async def test_oracle_backend_discovers_session_user_before_first_schema_switch(self):
+        from hindsight_api.engine.db.oracle import OracleBackend
+        from hindsight_api.engine.memory_engine import _current_schema
+
+        backend = OracleBackend()
+        cursor = MagicMock()
+        cursor.execute = AsyncMock()
+        cursor.fetchone = AsyncMock(return_value=("APP_USER",))
+        conn = MagicMock()
+        conn.cursor.return_value = cursor
+
+        schema_token = _current_schema.set(None)
+        try:
+            await backend._set_session_schema(conn)
+        finally:
+            _current_schema.reset(schema_token)
+
+        assert [call.args[0] for call in cursor.execute.await_args_list] == [
+            "SELECT SYS_CONTEXT('USERENV', 'SESSION_USER') FROM DUAL",
+            'ALTER SESSION SET CURRENT_SCHEMA = "APP_USER"',
+        ]
+        assert backend._default_schema == "APP_USER"
+
+
+class TestWorkerOperationCleanupScheduling:
+    @staticmethod
+    def _make_backend(prune_side_effect=None):
+        conn = MagicMock()
+
+        @asynccontextmanager
+        async def transaction():
+            yield conn
+
+        conn.transaction = transaction
+        backend = MagicMock()
+        backend.ops.prune_terminal_operations = AsyncMock(side_effect=prune_side_effect, return_value=0)
+
+        @asynccontextmanager
+        async def acquire():
+            yield conn
+
+        backend.acquire = acquire
+        return backend
+
+    @staticmethod
+    def _tenant_extension(*schemas: str):
+        extension = MagicMock()
+        extension.list_tenants = AsyncMock(return_value=[SimpleNamespace(schema=schema) for schema in schemas])
+        return extension
+
+    @pytest.mark.asyncio
+    async def test_disabled_retention_does_not_discover_schemas_or_touch_db(self):
+        from hindsight_api.worker import WorkerPoller
+
+        backend = self._make_backend()
+        tenant_extension = self._tenant_extension("public", "tenant_a")
+        poller = WorkerPoller(
+            backend=backend,
+            worker_id="cleanup-disabled",
+            executor=AsyncMock(),
+            tenant_extension=tenant_extension,
+            operation_retention_days=0,
+            operation_cleanup_batch_size=1000,
+        )
+
+        await poller._cleanup_terminal_operations_if_due()
+
+        tenant_extension.list_tenants.assert_not_awaited()
+        backend.ops.prune_terminal_operations.assert_not_awaited()
+
+    @pytest.mark.asyncio
+    async def test_cleanup_binds_each_tenant_schema_before_acquiring_connection(self):
+        from hindsight_api.engine.memory_engine import _current_schema
+        from hindsight_api.worker import WorkerPoller
+
+        seen_schemas: list[str | None] = []
+        backend = self._make_backend()
+        original_acquire = backend.acquire
+
+        @asynccontextmanager
+        async def recording_acquire():
+            seen_schemas.append(_current_schema.get())
+            async with original_acquire() as conn:
+                yield conn
+
+        backend.acquire = recording_acquire
+        poller = WorkerPoller(
+            backend=backend,
+            worker_id="cleanup-schema-context",
+            executor=AsyncMock(),
+            tenant_extension=self._tenant_extension("public", "tenant_a", "tenant_b"),
+            operation_retention_days=30,
+            operation_cleanup_batch_size=1000,
+        )
+
+        await poller._cleanup_terminal_operations_if_due()
+        cleanup_task = poller._operation_cleanup_task
+        assert cleanup_task is not None
+        await cleanup_task
+
+        assert seen_schemas == [None, "tenant_a", "tenant_b"]
+
+    @pytest.mark.asyncio
+    async def test_cleanup_visits_every_schema_and_isolates_one_schema_failure(self, caplog):
+        from hindsight_api.worker import WorkerPoller
+
+        async def prune(_conn, table, _cutoff, *, batch_size):
+            assert batch_size == 17
+            if "tenant_bad" in table:
+                raise RuntimeError("tenant cleanup failed")
+            return 1
+
+        backend = self._make_backend(prune)
+        poller = WorkerPoller(
+            backend=backend,
+            worker_id="cleanup-tenants",
+            executor=AsyncMock(),
+            tenant_extension=self._tenant_extension("public", "tenant_bad", "tenant_good"),
+            operation_retention_days=30,
+            operation_cleanup_batch_size=17,
+        )
+
+        await poller._cleanup_terminal_operations_if_due()
+        cleanup_task = poller._operation_cleanup_task
+        assert cleanup_task is not None
+        await cleanup_task
+
+        tables = [call.args[1] for call in backend.ops.prune_terminal_operations.await_args_list]
+        assert tables == ["async_operations", '"tenant_bad".async_operations', '"tenant_good".async_operations']
+        assert "tenant cleanup failed" in caplog.text
+
+    @pytest.mark.asyncio
+    async def test_monotonic_due_guard_limits_cleanup_to_once_per_minute(self):
+        from hindsight_api.worker import WorkerPoller
+
+        backend = self._make_backend()
+        poller = WorkerPoller(
+            backend=backend,
+            worker_id="cleanup-guard",
+            executor=AsyncMock(),
+            tenant_extension=self._tenant_extension("public"),
+            operation_retention_days=30,
+            operation_cleanup_batch_size=1000,
+        )
+
+        await poller._cleanup_terminal_operations_if_due()
+        first_task = poller._operation_cleanup_task
+        assert first_task is not None
+        await first_task
+        await poller._cleanup_terminal_operations_if_due()
+
+        assert poller._operation_cleanup_task is first_task
+        assert backend.ops.prune_terminal_operations.await_count == 1
+
+    @pytest.mark.asyncio
+    async def test_slow_cleanup_measures_next_interval_from_completion(self):
+        from hindsight_api.worker import WorkerPoller
+
+        backend = self._make_backend()
+        poller = WorkerPoller(
+            backend=backend,
+            worker_id="cleanup-slow-cycle",
+            executor=AsyncMock(),
+            tenant_extension=self._tenant_extension("public"),
+            operation_retention_days=30,
+            operation_cleanup_batch_size=1000,
+        )
+
+        await poller._cleanup_terminal_operations_if_due()
+        cleanup_task = poller._operation_cleanup_task
+        assert cleanup_task is not None
+        await cleanup_task
+        completed_at = poller._last_operation_cleanup_monotonic
+        await poller._cleanup_terminal_operations_if_due()
+
+        assert poller._last_operation_cleanup_monotonic == completed_at
+        assert backend.ops.prune_terminal_operations.await_count == 1
+
+    @pytest.mark.asyncio
+    async def test_cleanup_runs_in_background_without_blocking_task_claiming(self):
+        from hindsight_api.worker import WorkerPoller
+
+        cleanup_started = asyncio.Event()
+        release_cleanup = asyncio.Event()
+
+        async def prune(_conn, _table, _cutoff, *, batch_size):
+            assert batch_size == 1000
+            cleanup_started.set()
+            await release_cleanup.wait()
+            return 0
+
+        poller = WorkerPoller(
+            backend=self._make_backend(prune),
+            worker_id="cleanup-background",
+            executor=AsyncMock(),
+            tenant_extension=self._tenant_extension("public"),
+            operation_retention_days=30,
+            operation_cleanup_batch_size=1000,
+        )
+
+        await asyncio.wait_for(poller._cleanup_terminal_operations_if_due(), timeout=0.1)
+        await asyncio.wait_for(cleanup_started.wait(), timeout=0.1)
+        assert poller._operation_cleanup_task is not None
+        assert not poller._operation_cleanup_task.done()
+
+        poller.claim_batch = AsyncMock(return_value=[])
+        await asyncio.wait_for(poller.claim_batch(), timeout=0.1)
+
+        release_cleanup.set()
+        cleanup_task = poller._operation_cleanup_task
+        assert cleanup_task is not None
+        await cleanup_task

hindsight-cli/src/api.rs

@@ -10,6 +10,26 @@ use serde::{Deserialize, Serialize};
 use serde_json;
 use std::collections::HashMap;
 
+const DEFAULT_CLI_USER_AGENT: &str = concat!("hindsight-cli/", env!("CARGO_PKG_VERSION"));
+
+fn default_headers(api_key: Option<&str>) -> Result<reqwest::header::HeaderMap> {
+    let mut headers = reqwest::header::HeaderMap::new();
+    headers.insert(
+        reqwest::header::USER_AGENT,
+        reqwest::header::HeaderValue::from_static(DEFAULT_CLI_USER_AGENT),
+    );
+
+    if let Some(key) = api_key {
+        let auth_value = format!("Bearer {}", key);
+        headers.insert(
+            reqwest::header::AUTHORIZATION,
+            reqwest::header::HeaderValue::from_str(&auth_value)?,
+        );
+    }
+
+    Ok(headers)
+}
+
 /// Convert a progenitor client error into an anyhow error that includes the
 /// HTTP response body. Without this, errors render as
 /// "Unexpected Response: Response { ... }" with no body, hiding validation
@@ -112,15 +132,7 @@ impl ApiClient {
         let mut client_builder =
             reqwest::Client::builder().timeout(std::time::Duration::from_secs(120));
 
-        if let Some(key) = api_key {
-            let mut headers = reqwest::header::HeaderMap::new();
-            let auth_value = format!("Bearer {}", key);
-            headers.insert(
-                reqwest::header::AUTHORIZATION,
-                reqwest::header::HeaderValue::from_str(&auth_value)?,
-            );
-            client_builder = client_builder.default_headers(headers);
-        }
+        client_builder = client_builder.default_headers(default_headers(api_key.as_deref())?);
 
         let http_client = client_builder.build()?;
 
@@ -1309,6 +1321,31 @@ pub use types::{
 mod tests {
     use super::*;
 
+    #[test]
+    fn default_headers_set_cli_user_agent_without_api_key() {
+        let headers = default_headers(None).unwrap();
+
+        assert_eq!(
+            headers.get(reqwest::header::USER_AGENT).unwrap(),
+            DEFAULT_CLI_USER_AGENT,
+        );
+        assert!(!headers.contains_key(reqwest::header::AUTHORIZATION));
+    }
+
+    #[test]
+    fn default_headers_keep_authorization_with_cli_user_agent() {
+        let headers = default_headers(Some("hsk_test")).unwrap();
+
+        assert_eq!(
+            headers.get(reqwest::header::USER_AGENT).unwrap(),
+            DEFAULT_CLI_USER_AGENT,
+        );
+        assert_eq!(
+            headers.get(reqwest::header::AUTHORIZATION).unwrap(),
+            "Bearer hsk_test",
+        );
+    }
+
     #[test]
     fn test_operation_deserialize() {
         let json = r#"{

hindsight-clients/go/api/openapi.yaml

@@ -2286,8 +2286,9 @@ paths:
       - Operations
     get:
       description: "Get the status of a specific async operation. Returns 'pending',\
-        \ 'completed', or 'failed'. Completed operations are removed from storage,\
-        \ so 'completed' means the operation finished successfully."
+        \ 'processing', 'completed', 'failed', or 'cancelled'. Completed operations\
+        \ remain queryable with their payload for the configured retention window\
+        \ and are pruned afterward."
       operationId: get_operation_status
       parameters:
       - explode: false
@@ -6339,6 +6340,9 @@ components:
           date: 2024-01-15T10:30:00Z
           entities: "Alice (PERSON), Google (ORGANIZATION)"
           id: 550e8400-e29b-41d4-a716-446655440000
+          metadata:
+            channel: engineering
+            source: slack
           text: Alice works at Google on the AI team
           type: world
         limit: 100

hindsight-clients/go/api_operations.go

@@ -176,7 +176,7 @@ func (r ApiGetOperationStatusRequest) Execute() (*OperationStatusResponse, *http
 /*
 GetOperationStatus Get operation status
 
-Get the status of a specific async operation. Returns 'pending', 'completed', or 'failed'. Completed operations are removed from storage, so 'completed' means the operation finished successfully.
+Get the status of a specific async operation. Returns 'pending', 'processing', 'completed', 'failed', or 'cancelled'. Completed operations remain queryable with their payload for the configured retention window and are pruned afterward.
 
  @param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
  @param bankId

hindsight-clients/python/hindsight_client/hindsight_client.py

@@ -592,7 +592,7 @@ class Hindsight:
             disposition_empathy: Deprecated. Use update_bank_config(disposition_empathy=...) instead.
             disposition: Deprecated. Use update_bank_config(disposition_skepticism=...) instead.
             retain_mission: Steers what gets extracted during retain(). Injected alongside built-in rules.
-            retain_extraction_mode: Fact extraction mode: 'concise' (default), 'verbose', or 'custom'.
+            retain_extraction_mode: Fact extraction mode: 'concise' (default), 'verbose', 'custom', 'verbatim', or 'chunks'.
             retain_custom_instructions: Custom extraction prompt (only active when mode is 'custom').
             retain_chunk_size: Target maximum characters for each content chunk during retain.
             retain_structured_chunk_size: Maximum characters for a single JSONL line or conversation
@@ -731,7 +731,7 @@ class Hindsight:
             disposition_empathy: Deprecated. Use update_bank_config(disposition_empathy=...) instead.
             disposition: Deprecated. Use update_bank_config(disposition_skepticism=...) instead.
             retain_mission: Steers what gets extracted during retain(). Injected alongside built-in rules.
-            retain_extraction_mode: Fact extraction mode: 'concise' (default), 'verbose', or 'custom'.
+            retain_extraction_mode: Fact extraction mode: 'concise' (default), 'verbose', 'custom', 'verbatim', or 'chunks'.
             retain_custom_instructions: Custom extraction prompt (only active when mode is 'custom').
             retain_chunk_size: Target maximum characters for each content chunk during retain.
             retain_structured_chunk_size: Maximum characters for a single JSONL line or conversation

hindsight-clients/python/hindsight_client_api/api/operations_api.py

@@ -357,7 +357,7 @@ class OperationsApi:
     ) -> OperationStatusResponse:
         """Get operation status
 
-        Get the status of a specific async operation. Returns 'pending', 'completed', or 'failed'. Completed operations are removed from storage, so 'completed' means the operation finished successfully.
+        Get the status of a specific async operation. Returns 'pending', 'processing', 'completed', 'failed', or 'cancelled'. Completed operations remain queryable with their payload for the configured retention window and are pruned afterward.
 
         :param bank_id: (required)
         :type bank_id: str
@@ -437,7 +437,7 @@ class OperationsApi:
     ) -> ApiResponse[OperationStatusResponse]:
         """Get operation status
 
-        Get the status of a specific async operation. Returns 'pending', 'completed', or 'failed'. Completed operations are removed from storage, so 'completed' means the operation finished successfully.
+        Get the status of a specific async operation. Returns 'pending', 'processing', 'completed', 'failed', or 'cancelled'. Completed operations remain queryable with their payload for the configured retention window and are pruned afterward.
 
         :param bank_id: (required)
         :type bank_id: str
@@ -517,7 +517,7 @@ class OperationsApi:
     ) -> RESTResponseType:
         """Get operation status
 
-        Get the status of a specific async operation. Returns 'pending', 'completed', or 'failed'. Completed operations are removed from storage, so 'completed' means the operation finished successfully.
+        Get the status of a specific async operation. Returns 'pending', 'processing', 'completed', 'failed', or 'cancelled'. Completed operations remain queryable with their payload for the configured retention window and are pruned afterward.
 
         :param bank_id: (required)
         :type bank_id: str

hindsight-clients/typescript/generated/sdk.gen.ts

@@ -877,7 +877,7 @@ export const cancelOperation = <ThrowOnError extends boolean = false>(
 /**
  * Get operation status
  *
- * Get the status of a specific async operation. Returns 'pending', 'completed', or 'failed'. Completed operations are removed from storage, so 'completed' means the operation finished successfully.
+ * Get the status of a specific async operation. Returns 'pending', 'processing', 'completed', 'failed', or 'cancelled'. Completed operations remain queryable with their payload for the configured retention window and are pruned afterward.
  */
 export const getOperationStatus = <ThrowOnError extends boolean = false>(
   options: Options<GetOperationStatusData, ThrowOnError>

hindsight-clients/typescript/generated/types.gen.ts

@@ -453,7 +453,7 @@ export type BankTemplateConfig = {
   /**
    * Retain Extraction Mode
    *
-   * Fact extraction mode: 'concise' (default), 'verbose', or 'custom'
+   * Fact extraction mode: 'concise' (default), 'verbose', 'custom', 'verbatim', or 'chunks'
    */
   retain_extraction_mode?: string | null;
   /**
@@ -1080,7 +1080,7 @@ export type CreateBankRequest = {
   /**
    * Retain Extraction Mode
    *
-   * Fact extraction mode: 'concise' (default), 'verbose', or 'custom'.
+   * Fact extraction mode: 'concise' (default), 'verbose', 'custom', 'verbatim', or 'chunks'.
    */
   retain_extraction_mode?: string | null;
   /**

hindsight-clients/typescript/src/index.ts

@@ -500,7 +500,7 @@ export class HindsightClient {
       dispositionEmpathy?: number;
       /** Steers what gets extracted during retain(). Injected alongside built-in rules. */
       retainMission?: string;
-      /** Fact extraction mode: 'concise' (default), 'verbose', or 'custom'. */
+      /** Fact extraction mode: 'concise' (default), 'verbose', 'custom', 'verbatim', or 'chunks'. */
       retainExtractionMode?: string;
       /** Custom extraction prompt (only active when retainExtractionMode is 'custom'). */
       retainCustomInstructions?: string;

hindsight-control-plane/package.json

@@ -11,6 +11,7 @@
     "public"
   ],
   "scripts": {
+    "prebuild": "npm run build -w @vectorize-io/hindsight-client",
     "dev": "next dev --turbopack -p ${PORT:-9999}",
     "build": "NODE_ENV=production next build && npm run build:standalone",
     "build:standalone": "rm -rf standalone && SERVER_JS=$(find .next/standalone -path '*/node_modules' -prune -o -name 'server.js' -print | head -1) && test -n \"$SERVER_JS\" || (echo 'Error: server.js not found in .next/standalone - standalone build failed' && exit 1) && STANDALONE_ROOT=$(dirname \"$SERVER_JS\") && cp -r \"$STANDALONE_ROOT\" standalone && cp -r .next/standalone/node_modules standalone/node_modules && mkdir -p standalone/.next && cp -r .next/static standalone/.next/static && mkdir -p standalone/public && (cp -r public/* standalone/public/ 2>/dev/null || true)",
@@ -38,14 +39,12 @@
     "@radix-ui/react-label": "^2.1.8",
     "@radix-ui/react-popover": "^1.1.15",
     "@radix-ui/react-select": "^2.2.6",
-    "@radix-ui/react-slider": "^1.3.6",
     "@radix-ui/react-slot": "^1.2.4",
     "@radix-ui/react-switch": "^1.2.6",
     "@radix-ui/react-tabs": "^1.1.13",
     "@radix-ui/react-visually-hidden": "^1.2.5",
     "@tailwindcss/postcss": "^4.1.17",
     "@tailwindcss/typography": "^0.5.19",
-    "@types/cytoscape": "^3.21.9",
     "@types/node": "^24.10.0",
     "@types/react": "^19.2.2",
     "@types/react-dom": "^19.2.2",
@@ -55,8 +54,6 @@
     "cmdk": "^1.1.1",
     "cron-parser": "^5.6.1",
     "cronstrue": "^3.21.0",
-    "cytoscape": "^3.33.1",
-    "cytoscape-fcose": "^2.2.0",
     "eslint": "^9.39.1",
     "eslint-config-next": "^16.0.1",
     "lucide-react": "^0.553.0",

hindsight-control-plane/src/app/api/banks/[bankId]/mental-models/route.ts

@@ -8,6 +8,8 @@ export async function GET(request: Request, { params }: { params: Promise<{ bank
     const { searchParams } = new URL(request.url);
     const tags = searchParams.getAll("tags");
     const tagsMatch = searchParams.get("tags_match");
+    const limit = searchParams.get("limit");
+    const offset = searchParams.get("offset");
 
     if (!bankId) {
       return NextResponse.json(
@@ -26,6 +28,12 @@ export async function GET(request: Request, { params }: { params: Promise<{ bank
     if (tagsMatch) {
       queryParams.append("tags_match", tagsMatch);
     }
+    if (limit) {
+      queryParams.append("limit", limit);
+    }
+    if (offset) {
+      queryParams.append("offset", offset);
+    }
 
     const url = dataplaneBankUrl(
       bankId,

hindsight-control-plane/src/app/api/documents/route.ts

@@ -17,13 +17,14 @@ export async function GET(request: NextRequest) {
     );
   }
 
+  const q = searchParams.get("q") || undefined;
   const limit = searchParams.get("limit") ? Number(searchParams.get("limit")) : undefined;
   const offset = searchParams.get("offset") ? Number(searchParams.get("offset")) : undefined;
 
   const response = await sdk.listDocuments({
     client: lowLevelClient,
     path: { bank_id: bankId },
-    query: { limit, offset },
+    query: { q, limit, offset },
   });
   return respondWithSdk(response, "Failed to fetch documents", { request });
 }

hindsight-control-plane/src/components/bank-selector.tsx

@@ -505,7 +505,11 @@ function BankSelectorInner() {
               aria-expanded={open}
               className="w-[250px] justify-between font-bold border-2 border-primary hover:bg-accent"
             >
-              <span className="truncate">{currentBank || tNavBank("select")}</span>
+              <span className="truncate">
+                {bankInfos.find((b) => b.bank_id === currentBank)?.name ||
+                  currentBank ||
+                  tNavBank("select")}
+              </span>
               <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
             </Button>
           </PopoverTrigger>
@@ -555,8 +559,11 @@ function BankSelectorInner() {
                               isSelected ? "opacity-100" : "opacity-0"
                             )}
                           />
-                          <span className="truncate flex-1 font-medium" title={bank.bank_id}>
-                            {bank.bank_id}
+                          <span
+                            className="truncate flex-1 font-medium"
+                            title={bank.name || bank.bank_id}
+                          >
+                            {bank.name || bank.bank_id}
                           </span>
                           <button
                             type="button"

hindsight-control-plane/src/components/constellation.tsx

@@ -4,7 +4,7 @@ import { useRef, useEffect, useCallback, useMemo, useState } from "react";
 import type { CSSProperties } from "react";
 import { useTranslations } from "next-intl";
 import { prepare, layout, prepareWithSegments, layoutWithLines } from "@chenglou/pretext";
-import type { GraphData, GraphNode, GraphLink } from "./graph-2d";
+import type { GraphData, GraphNode, GraphLink } from "./graph-data";
 
 // ============================================================================
 // Types
@@ -21,6 +21,12 @@ interface PreparedNode {
   /** Color derived from link count (heat gradient) */
   heatColor: string;
   linkCount: number;
+  /**
+   * Per-node phase in [0, 2π), derived from the id hash. Desynchronizes the
+   * ambient drift + pulse so the field breathes organically instead of in
+   * lockstep. Precomputed here so the animation loop stays trig-only.
+   */
+  phase: number;
 }
 
 // ============================================================================
@@ -372,6 +378,7 @@ export function Constellation({
         // so the grouping reads at a glance; otherwise it keeps the heat gradient.
         heatColor: centroid ? color : heat,
         linkCount: lc,
+        phase: ((Math.abs(seed) % 1000) / 1000) * Math.PI * 2,
       };
     });
 
@@ -448,15 +455,24 @@ export function Constellation({
     ctx.fillStyle = bg;
     ctx.fillRect(0, 0, W, H);
 
-    // Screen positions
+    // Ambient-motion clock (seconds).
+    const time = (typeof performance !== "undefined" ? performance.now() : 0) / 1000;
+    // Drift amplitude in world units — nodes slowly wander around their home
+    // position so the whole field visibly breathes.
+    const DRIFT_AMP = 16;
+
+    // Screen positions (with a slow per-node ambient drift baked in, so links —
+    // which read straight from screenX/screenY below — follow for free).
     const screenX = new Float32Array(preparedNodes.length);
     const screenY = new Float32Array(preparedNodes.length);
     const visible = new Uint8Array(preparedNodes.length);
 
     for (let i = 0; i < preparedNodes.length; i++) {
       const n = preparedNodes[i];
-      const sx = cx + n.wx * zoom;
-      const sy = cy + n.wy * zoom;
+      const driftX = DRIFT_AMP * Math.sin(time * 0.6 + n.phase);
+      const driftY = DRIFT_AMP * Math.cos(time * 0.5 + n.phase * 1.3);
+      const sx = cx + (n.wx + driftX) * zoom;
+      const sy = cy + (n.wy + driftY) * zoom;
       screenX[i] = sx;
       screenY[i] = sy;
       visible[i] = sx > -margin && sx < W + margin && sy > -margin && sy < H + margin ? 1 : 0;
@@ -508,11 +524,39 @@ export function Constellation({
         ctx.moveTo(ax, ay);
         ctx.quadraticCurveTo(midX, midY, bx, by);
         ctx.stroke();
+
+        // A small bead of light travels the curve from the hovered node outward,
+        // so connections read as live signal paths rather than static lines.
+        {
+          // Phase-offset per link so beads don't march in lockstep. Travel runs
+          // from the hovered node (u=0) toward its neighbor (u=1).
+          const fromHovered = link.a === hoverIndex;
+          const raw = (time * 0.22 + (li % 13) / 13) % 1;
+          const u = fromHovered ? raw : 1 - raw;
+          const iu = 1 - u;
+          // Point on the quadratic Bézier at parameter u.
+          const px = iu * iu * ax + 2 * iu * u * midX + u * u * bx;
+          const py = iu * iu * ay + 2 * iu * u * midY + u * u * by;
+          ctx.globalAlpha = 0.9 * (0.4 + 0.6 * Math.sin(u * Math.PI)); // fade at the ends
+          ctx.fillStyle = link.color;
+          ctx.shadowColor = link.color;
+          ctx.shadowBlur = 6;
+          ctx.beginPath();
+          ctx.arc(px, py, 2, 0, Math.PI * 2);
+          ctx.fill();
+          ctx.shadowBlur = 0;
+          // Restore the stroke state the loop's next iteration expects.
+          ctx.globalAlpha = 0.5;
+          ctx.lineWidth = 1.5;
+        }
         linksDrawn++;
       }
       ctx.globalAlpha = 1;
     } else {
-      const baseAlpha = 0.06 + Math.min(zoom * 0.04, 0.1);
+      // Faint, slow breathing across the whole web so idle links feel alive
+      // without flickering (one global sine, not per-link — stays calm).
+      const shimmer = 1 + 0.18 * Math.sin(time * 0.6);
+      const baseAlpha = (0.06 + Math.min(zoom * 0.04, 0.1)) * shimmer;
       ctx.lineWidth = 0.4;
 
       for (const link of linksWithIndices) {
@@ -661,11 +705,18 @@ export function Constellation({
       // Size varies slightly by link count — subtle range like star magnitudes.
       // When nodeSizeFn is provided (e.g. entities view), it overrides linkCount
       // sizing so dots can scale by an external weight like co-occurrence count.
-      const baseR = nodeSizeFn ? nodeSizeFn(n.node) : 2.5 + Math.min(n.linkCount * 0.15, 2.5);
+      const rawR = nodeSizeFn ? nodeSizeFn(n.node) : 2.5 + Math.min(n.linkCount * 0.15, 2.5);
+      // Gentle pulse — each dot "breathes" in size, out of phase with its
+      // neighbors, so the field twinkles like a living star map.
+      const pulse = 1 + 0.13 * Math.sin(time * 1.05 + n.phase);
+      const baseR = rawR * pulse;
       const r = Math.max(1.5, baseR * Math.min(zoom, 2));
 
-      // Opacity varies — fewer links = dimmer, more links = brighter
-      const baseAlpha = 0.45 + Math.min(n.linkCount * 0.03, 0.5);
+      // Opacity varies — fewer links = dimmer, more links = brighter. A brightness
+      // twinkle (offset from the size pulse) makes even tiny dots read as alive,
+      // where a radius pulse alone would be imperceptible.
+      const twinkleAlpha = 0.82 + 0.18 * Math.sin(time * 1.4 + n.phase * 2.1);
+      const baseAlpha = (0.45 + Math.min(n.linkCount * 0.03, 0.5)) * twinkleAlpha;
 
       // Dot — star-like: heat-gradient color, varied size & opacity
       ctx.beginPath();
@@ -679,12 +730,14 @@ export function Constellation({
       }
       ctx.fill();
 
-      // Soft glow halo for brighter stars (high link count)
+      // Soft glow halo for brighter stars (high link count) — the halo twinkles
+      // a little (out of phase with the dot's pulse) so hubs feel radiant.
       if (n.linkCount > 3 && !isHovered && hoverIndex < 0) {
+        const twinkle = 1 + 0.25 * Math.sin(time * 0.9 + n.phase * 1.7);
         ctx.beginPath();
         ctx.arc(sx, sy, r * 2, 0, Math.PI * 2);
         ctx.fillStyle = n.heatColor;
-        ctx.globalAlpha = 0.06 + Math.min(n.linkCount * 0.005, 0.08);
+        ctx.globalAlpha = (0.06 + Math.min(n.linkCount * 0.005, 0.08)) * twinkle;
         ctx.fill();
       }
 
@@ -1119,9 +1172,17 @@ export function Constellation({
     canvas.addEventListener("mouseup", handleMouseUp);
     canvas.addEventListener("mouseleave", handleMouseLeave);
 
+    // The canvas also changes size when its container reflows (e.g. the side
+    // panel opening/closing) with no window "resize" event. Observe the element
+    // so the backing store is re-measured — otherwise CSS stretches the old
+    // bitmap and the text/dots look squeezed.
+    const ro = typeof ResizeObserver !== "undefined" ? new ResizeObserver(() => resize()) : null;
+    ro?.observe(canvas);
+
     return () => {
       cancelAnimationFrame(animRef.current);
       window.removeEventListener("resize", handleResize);
+      ro?.disconnect();
       canvas.removeEventListener("wheel", handleWheel);
       canvas.removeEventListener("mousemove", handleMouseMove);
       canvas.removeEventListener("mousedown", handleMouseDown);

hindsight-control-plane/src/components/data-view.tsx

@@ -16,11 +16,9 @@ import {
   ChevronsRight,
   Settings2,
   Eye,
-  EyeOff,
   RefreshCw,
   CheckCircle,
   Clock,
-  Network,
   List,
   Search,
   Layers,
@@ -33,8 +31,6 @@ import {
   TableHeader,
   TableRow,
 } from "@/components/ui/table";
-import { Label } from "@/components/ui/label";
-import { Slider } from "@/components/ui/slider";
 import { Switch } from "@/components/ui/switch";
 import {
   Select,
@@ -43,16 +39,15 @@ import {
   SelectTrigger,
   SelectValue,
 } from "@/components/ui/select";
-import { MemoryDetailPanel } from "./memory-detail-panel";
 import { MemoryDetailModal } from "./memory-detail-modal";
-import { Graph2D, convertHindsightGraphData, GraphNode } from "./graph-2d";
+import { convertHindsightGraphData, GraphNode } from "./graph-data";
 import { Constellation } from "./constellation";
 import { TagFilterInput } from "./tag-filter-input";
 import { ObservationScopeFilter, ObservationScope } from "./observation-scope-filter";
 import { ScatterChart, Plus, FileText } from "lucide-react";
 
 type FactType = "world" | "experience" | "observation";
-type ViewMode = "graph" | "table" | "timeline" | "constellation";
+type ViewMode = "table" | "timeline" | "constellation";
 
 // Categorical palette for coloring observation scopes (exact tag sets) when
 // "Group by scope" clusters the constellation. Distinct, reasonably separable hues.
@@ -105,7 +100,6 @@ export function DataView({
   const [scopes, setScopes] = useState<ObservationScope[]>([]);
   const [selectedScope, setSelectedScope] = useState<string[] | null>(null);
   const [currentPage, setCurrentPage] = useState(1);
-  const [selectedGraphNode, setSelectedGraphNode] = useState<any>(null);
   const [modalMemoryId, setModalMemoryId] = useState<string | null>(null);
   // Table view: toggle between live facts (graph-fed) and invalidated facts (archive).
   const [showInvalidated, setShowInvalidated] = useState(false);
@@ -132,10 +126,7 @@ export function DataView({
     last_consolidated_at: string | null;
   } | null>(null);
 
-  // Graph controls state
-  const [showLabels, setShowLabels] = useState(true);
-  const [maxNodes, setMaxNodes] = useState<number | undefined>(undefined);
-  const [showControlPanel, setShowControlPanel] = useState(true);
+  // Constellation controls state
   const [visibleLinkTypes, setVisibleLinkTypes] = useState<Set<string>>(
     new Set(["semantic", "temporal", "entity", "causal"])
   );
@@ -152,17 +143,6 @@ export function DataView({
     });
   };
 
-  // Esc key handler to deselect graph node
-  useEffect(() => {
-    const handleKeyDown = (e: KeyboardEvent) => {
-      if (e.key === "Escape" && selectedGraphNode) {
-        setSelectedGraphNode(null);
-      }
-    };
-    window.addEventListener("keydown", handleKeyDown);
-    return () => window.removeEventListener("keydown", handleKeyDown);
-  }, [selectedGraphNode]);
-
   // `silent` skips the loading spinner — used by the background consolidation
   // poll so the view refreshes in place without flashing.
   const loadData = async (
@@ -230,6 +210,8 @@ export function DataView({
     if (showInvalidated) return invalidatedRows;
     return data?.table_rows ?? [];
   }, [data, showInvalidated, invalidatedRows]);
+  const hasActiveMemoryFilters =
+    searchQuery.trim().length > 0 || tagFilters.length > 0 || selectedScope !== null;
 
   // Helper to get normalized link type
   const getLinkTypeCategory = (type: string | undefined): string => {
@@ -239,7 +221,7 @@ export function DataView({
     return "semantic";
   };
 
-  // Convert data for Graph2D (graph data is already filtered server-side)
+  // Convert data for the constellation (graph data is already filtered server-side)
   const graph2DData = useMemo(() => {
     if (!data) return { nodes: [], links: [] };
     const fullData = convertHindsightGraphData(data);
@@ -253,44 +235,11 @@ export function DataView({
     return { nodes: fullData.nodes, links };
   }, [data, visibleLinkTypes]);
 
-  // Calculate link stats for display
-  const linkStats = useMemo(() => {
-    let semantic = 0,
-      temporal = 0,
-      entity = 0,
-      causal = 0,
-      total = 0;
-    const otherTypes: Record<string, number> = {};
-    graph2DData.links.forEach((l) => {
-      total++;
-      const type = l.type || "unknown";
-      if (type === "semantic") semantic++;
-      else if (type === "temporal") temporal++;
-      else if (type === "entity") entity++;
-      else if (
-        type === "causes" ||
-        type === "caused_by" ||
-        type === "enables" ||
-        type === "prevents"
-      )
-        causal++;
-      else {
-        otherTypes[type] = (otherTypes[type] || 0) + 1;
-      }
-    });
-    return { semantic, temporal, entity, causal, total, otherTypes };
-  }, [graph2DData]);
-
   // Handle node click in graph - show in panel
-  const handleGraphNodeClick = useCallback(
-    (node: GraphNode) => {
-      const nodeData = data?.table_rows?.find((row: any) => row.id === node.id);
-      if (nodeData) {
-        setSelectedGraphNode(nodeData);
-      }
-    },
-    [data]
-  );
+  const handleGraphNodeClick = useCallback((node: GraphNode) => {
+    // Open the memory dialog for the clicked node (same dialog the table/timeline use).
+    setModalMemoryId(node.id);
+  }, []);
 
   // Memoized color functions to prevent graph re-initialization
   // Uses brand colors: primary blue (#0074d9), teal (#009296), amber for entity, purple for causal
@@ -499,29 +448,26 @@ export function DataView({
   // restarts when consolidation starts/stops, not on every tick.
   const isConsolidating =
     factType === "observation" && (consolidationStatus?.pending_consolidation ?? 0) > 0;
+  // The tick goes through a ref so each fire sees the CURRENT filters; the
+  // interval itself still only restarts when consolidation starts/stops. Without
+  // this, the closure captures the filters from arming time, and a tag/scope
+  // selected mid-consolidation is clobbered ~4s later by a refetch using the
+  // stale (usually empty) filter — the same stale-closure guard as
+  // bank-profile-view's polling.
+  const pollTickRef = useRef<() => void>(() => {});
   useEffect(() => {
-    if (!isConsolidating || !currentBank) return;
-    const id = setInterval(() => {
+    pollTickRef.current = () => {
       const { tags, match } = resolveTagQuery();
       loadData(undefined, searchQuery || undefined, tags, match, true);
       loadScopes();
-    }, 4000);
+    };
+  });
+  useEffect(() => {
+    if (!isConsolidating || !currentBank) return;
+    const id = setInterval(() => pollTickRef.current(), 4000);
     return () => clearInterval(id);
   }, [isConsolidating, currentBank]);
 
-  // Enforce 50 node limit to prevent UI instability, default to 20 or max whichever is smaller
-  useEffect(() => {
-    if (data && maxNodes === undefined) {
-      if (graph2DData.nodes.length > 50) {
-        // Always set maxNodes to 20 when we have >50 nodes (never leave as undefined)
-        setMaxNodes(20);
-      } else if (graph2DData.nodes.length > 20) {
-        setMaxNodes(20);
-      }
-      // If ≤20 nodes, leave maxNodes undefined to show all
-    }
-  }, [data, graph2DData.nodes.length, maxNodes]);
-
   return (
     <div>
       {loading && !data ? (
@@ -529,7 +475,7 @@ export function DataView({
           <RefreshCw className="w-8 h-8 mx-auto mb-3 text-muted-foreground animate-spin" />
           <p className="text-muted-foreground">{t("loadingMemories")}</p>
         </div>
-      ) : data && data.total_units === 0 ? (
+      ) : data && data.total_units === 0 && !hasActiveMemoryFilters ? (
         <div className="text-center py-20">
           <FileText className="w-10 h-10 mx-auto mb-4 text-muted-foreground/50" />
           <h3 className="text-base font-medium text-foreground mb-1">{t("noMemoriesYet")}</h3>
@@ -644,7 +590,7 @@ export function DataView({
                   </Button>
                 )}
                 <div className="text-sm text-muted-foreground">
-                  {searchQuery || tagFilters.length > 0 ? (
+                  {hasActiveMemoryFilters ? (
                     t("matchingMemories", { count: filteredTableRows.length })
                   ) : data.table_rows?.length < data.total_units ? (
                     <span>
@@ -656,11 +602,8 @@ export function DataView({
                         onClick={() => {
                           const newLimit = Math.min(data.total_units, fetchLimit + 1000);
                           setFetchLimit(newLimit);
-                          loadData(
-                            newLimit,
-                            searchQuery || undefined,
-                            tagFilters.length > 0 ? tagFilters : undefined
-                          );
+                          const { tags, match } = resolveTagQuery();
+                          loadData(newLimit, searchQuery || undefined, tags, match);
                         }}
                         className="ml-2 text-primary hover:underline"
                       >
@@ -705,11 +648,10 @@ export function DataView({
                         {t("pendingCount", { count: consolidationStatus.pending_consolidation })}
                         <button
                           onClick={() =>
-                            loadData(
-                              fetchLimit,
-                              searchQuery || undefined,
-                              tagFilters.length > 0 ? tagFilters : undefined
-                            )
+                            (() => {
+                              const { tags, match } = resolveTagQuery();
+                              loadData(fetchLimit, searchQuery || undefined, tags, match);
+                            })()
                           }
                           disabled={loading}
                           className="ml-0.5 opacity-70 hover:opacity-100 disabled:opacity-40 transition-opacity"
@@ -734,17 +676,6 @@ export function DataView({
                   <ScatterChart className="w-4 h-4" />
                   {t("constellation")}
                 </button>
-                <button
-                  onClick={() => setViewMode("graph")}
-                  className={`px-3 py-1.5 rounded-md text-sm font-medium transition-all flex items-center gap-1.5 ${
-                    viewMode === "graph"
-                      ? "bg-background text-foreground shadow-sm"
-                      : "text-muted-foreground hover:text-foreground"
-                  }`}
-                >
-                  <Network className="w-4 h-4" />
-                  {t("graph")}
-                </button>
                 <button
                   onClick={() => setViewMode("table")}
                   className={`px-3 py-1.5 rounded-md text-sm font-medium transition-all flex items-center gap-1.5 ${
@@ -771,244 +702,76 @@ export function DataView({
             </div>
           )}
 
-          {!compactMode && viewMode === "graph" && (
-            <div className="flex gap-0">
-              {/* Graph */}
-              <div className="flex-1 min-w-0">
-                <Graph2D
-                  data={graph2DData}
-                  height={700}
-                  showLabels={showLabels}
-                  onNodeClick={handleGraphNodeClick}
-                  maxNodes={maxNodes}
-                  nodeColorFn={nodeColorFn}
-                  linkColorFn={linkColorFn}
-                />
-              </div>
-
-              {/* Right Toggle Button */}
-              <button
-                onClick={() => setShowControlPanel(!showControlPanel)}
-                className="flex-shrink-0 w-5 h-[700px] bg-transparent hover:bg-muted/50 flex items-center justify-center transition-colors"
-                title={showControlPanel ? t("hidePanel") : t("showPanel")}
-              >
-                {showControlPanel ? (
-                  <ChevronRight className="w-3 h-3 text-muted-foreground/60" />
-                ) : (
-                  <ChevronLeft className="w-3 h-3 text-muted-foreground/60" />
-                )}
-              </button>
-
-              {/* Right Panel - Legend/Controls OR Memory Details */}
-              <div
-                className={`${showControlPanel ? "w-80" : "w-0"} transition-all duration-300 overflow-hidden flex-shrink-0`}
-              >
-                <div className="w-80 h-[700px] bg-card border-l border-border overflow-y-auto">
-                  {selectedGraphNode ? (
-                    /* Memory Detail View */
-                    <MemoryDetailPanel
-                      memory={selectedGraphNode}
-                      onClose={() => setSelectedGraphNode(null)}
-                      inPanel
-                      bankId={currentBank || undefined}
-                    />
-                  ) : (
-                    /* Legend & Controls View */
-                    <div className="p-4 space-y-5">
-                      {/* Legend & Stats */}
-                      <div>
-                        <h3 className="text-sm font-semibold mb-3 text-foreground">
-                          {t("graphTitle")}
-                        </h3>
-                        <div className="space-y-2">
-                          {/* Nodes */}
-                          <div className="flex items-center justify-between text-sm">
-                            <div className="flex items-center gap-2">
-                              <div
-                                className="w-3 h-3 rounded-full"
-                                style={{ backgroundColor: "#0074d9" }}
-                              />
-                              <span className="text-foreground">{t("nodes")}</span>
-                            </div>
-                            <span className="font-mono text-foreground">
-                              {Math.min(
-                                maxNodes ?? graph2DData.nodes.length,
-                                graph2DData.nodes.length
-                              )}
-                              /{graph2DData.nodes.length}
-                            </span>
-                          </div>
-
-                          <div className="text-xs font-medium text-muted-foreground mt-2 mb-1">
-                            {t("linksWithCount", { count: linkStats.total })}{" "}
-                            <span className="text-muted-foreground/60">{t("clickToFilter")}</span>
-                          </div>
-                          <button
-                            onClick={() => toggleLinkType("semantic")}
-                            className={`w-full flex items-center justify-between text-sm px-2 py-1 rounded transition-all ${
-                              visibleLinkTypes.has("semantic")
-                                ? "hover:bg-muted"
-                                : "opacity-40 hover:opacity-60"
-                            }`}
-                          >
-                            <div className="flex items-center gap-2">
-                              <div className="w-4 h-0.5 bg-[#0074d9]" />
-                              <span className="text-foreground">{t("semantic")}</span>
-                            </div>
-                            <span
-                              className={`font-mono ${linkStats.semantic === 0 ? "text-destructive" : "text-foreground"}`}
-                            >
-                              {linkStats.semantic}
-                            </span>
-                          </button>
-                          <button
-                            onClick={() => toggleLinkType("temporal")}
-                            className={`w-full flex items-center justify-between text-sm px-2 py-1 rounded transition-all ${
-                              visibleLinkTypes.has("temporal")
-                                ? "hover:bg-muted"
-                                : "opacity-40 hover:opacity-60"
-                            }`}
-                          >
-                            <div className="flex items-center gap-2">
-                              <div className="w-4 h-0.5 bg-[#009296]" />
-                              <span className="text-foreground">{t("temporal")}</span>
-                            </div>
-                            <span
-                              className={`font-mono ${linkStats.temporal === 0 ? "text-destructive" : "text-foreground"}`}
-                            >
-                              {linkStats.temporal}
-                            </span>
-                          </button>
-                          <button
-                            onClick={() => toggleLinkType("entity")}
-                            className={`w-full flex items-center justify-between text-sm px-2 py-1 rounded transition-all ${
-                              visibleLinkTypes.has("entity")
-                                ? "hover:bg-muted"
-                                : "opacity-40 hover:opacity-60"
-                            }`}
-                          >
-                            <div className="flex items-center gap-2">
-                              <div className="w-4 h-0.5 bg-[#f59e0b]" />
-                              <span className="text-foreground">{t("entity")}</span>
-                            </div>
-                            <span className="font-mono text-foreground">{linkStats.entity}</span>
-                          </button>
-                          <button
-                            onClick={() => toggleLinkType("causal")}
-                            className={`w-full flex items-center justify-between text-sm px-2 py-1 rounded transition-all ${
-                              visibleLinkTypes.has("causal")
-                                ? "hover:bg-muted"
-                                : "opacity-40 hover:opacity-60"
-                            }`}
-                          >
-                            <div className="flex items-center gap-2">
-                              <div className="w-4 h-0.5 bg-[#8b5cf6]" />
-                              <span className="text-foreground">{t("causal")}</span>
-                            </div>
-                            <span
-                              className={`font-mono ${linkStats.causal === 0 ? "text-muted-foreground" : "text-foreground"}`}
-                            >
-                              {linkStats.causal}
-                            </span>
-                          </button>
-                          {Object.entries(linkStats.otherTypes || {}).map(([type, count]) => (
-                            <div key={type} className="flex items-center justify-between text-sm">
-                              <span className="text-muted-foreground capitalize ml-6">{type}</span>
-                              <span className="font-mono text-muted-foreground">
-                                {count as number}
-                              </span>
-                            </div>
-                          ))}
-                        </div>
-                      </div>
-
-                      <div className="border-t border-border" />
-
-                      {/* Controls Section */}
-                      <div>
-                        <h3 className="text-sm font-semibold mb-3 text-foreground">
-                          {t("displayTitle")}
-                        </h3>
-                        <div className="space-y-4">
-                          <div className="flex items-center justify-between">
-                            <Label htmlFor="show-labels" className="text-sm text-foreground">
-                              {t("showLabels")}
-                            </Label>
-                            <Switch
-                              id="show-labels"
-                              checked={showLabels}
-                              onCheckedChange={setShowLabels}
-                            />
-                          </div>
-                        </div>
-                      </div>
-
-                      <div className="border-t border-border" />
-
-                      {/* Limits Section */}
-                      <div>
-                        <h3 className="text-sm font-semibold mb-3 text-foreground">
-                          {t("performanceTitle")}
-                        </h3>
-                        <div className="space-y-4">
-                          <div>
-                            <div className="flex items-center justify-between mb-2">
-                              <Label className="text-sm text-foreground">{t("maxNodes")}</Label>
-                              <span className="text-xs text-muted-foreground">
-                                {graph2DData.nodes.length > 50
-                                  ? `${maxNodes ?? 50} / ${graph2DData.nodes.length}`
-                                  : `${maxNodes ?? "All"} / ${graph2DData.nodes.length}`}
-                              </span>
-                            </div>
-                            <Slider
-                              value={[
-                                graph2DData.nodes.length > 50
-                                  ? maxNodes || 20
-                                  : maxNodes || Math.min(graph2DData.nodes.length, 20),
-                              ]}
-                              min={10}
-                              max={Math.min(Math.max(graph2DData.nodes.length, 10), 50)}
-                              step={10}
-                              onValueChange={([v]) => {
-                                const effectiveMax = Math.min(graph2DData.nodes.length, 50);
-                                // If we have >50 nodes, never allow "All" (undefined), cap at 50
-                                if (graph2DData.nodes.length > 50) {
-                                  setMaxNodes(v);
-                                } else {
-                                  // Original behavior for ≤50 nodes: allow "All" when slider reaches max
-                                  setMaxNodes(v >= effectiveMax ? undefined : v);
-                                }
-                              }}
-                              className="w-full"
-                            />
-                          </div>
-                          <p className="text-xs text-muted-foreground">
-                            {t("allLinksVisible")}
-                            {graph2DData.nodes.length > 50 && (
-                              <span className="block text-amber-600 dark:text-amber-400 mt-1">
-                                {t("limitedTo50Nodes", { count: graph2DData.nodes.length })}
-                              </span>
-                            )}
-                          </p>
-                        </div>
-                      </div>
-
-                      <div className="border-t border-border" />
-
-                      {/* Hint */}
-                      <div className="text-xs text-muted-foreground/60 text-center pt-2">
-                        {t("clickNodeForDetails")}
-                      </div>
+          {(compactMode || viewMode === "constellation") && (
+            <div className="space-y-3">
+              {/* Constellation controls — moved out of the old side panel to sit
+                  inline above the graph, next to the view toggle / filters. */}
+              {!compactMode && (
+                <div className="flex flex-wrap items-center gap-x-6 gap-y-2">
+                  {factType === "observation" && (
+                    <div className="flex items-center gap-2">
+                      <Layers className="w-3.5 h-3.5 text-muted-foreground" />
+                      <span className="text-xs font-medium text-muted-foreground">
+                        {t("groupByScope")}
+                      </span>
+                      <Switch checked={groupByScope} onCheckedChange={setGroupByScope} />
                     </div>
                   )}
+                  {!(factType === "observation" && groupByScope) && (
+                    <div className="flex items-center gap-2">
+                      <span className="text-xs font-medium text-muted-foreground">
+                        {t("colorBy")}
+                      </span>
+                      <Select
+                        value={recencyBasis}
+                        onValueChange={(v) => setRecencyBasis(v as RecencyBasis)}
+                      >
+                        <SelectTrigger className="h-8 w-44 text-xs">
+                          <SelectValue />
+                        </SelectTrigger>
+                        <SelectContent>
+                          <SelectItem value="mentioned_at">{t("mentioned")}</SelectItem>
+                          <SelectItem value="occurred_start">{t("occurredStart")}</SelectItem>
+                          <SelectItem value="occurred_end">{t("occurredEnd")}</SelectItem>
+                        </SelectContent>
+                      </Select>
+                    </div>
+                  )}
+                  <div className="flex items-center gap-3">
+                    <span className="text-xs font-medium text-muted-foreground">
+                      {t("linkTypes")}
+                    </span>
+                    {Object.entries({
+                      semantic: "#0074d9",
+                      temporal: "#009296",
+                      entity: "#f59e0b",
+                      causal: "#8b5cf6",
+                    }).map(([type, color]) => (
+                      <button
+                        key={type}
+                        type="button"
+                        className="flex items-center gap-1.5"
+                        onClick={() => toggleLinkType(type)}
+                      >
+                        <span
+                          className="w-3 h-3 rounded-full"
+                          style={{
+                            backgroundColor: color,
+                            opacity: visibleLinkTypes.has(type) ? 1 : 0.2,
+                          }}
+                        />
+                        <span
+                          className={`text-xs capitalize ${visibleLinkTypes.has(type) ? "text-foreground" : "text-muted-foreground line-through"}`}
+                        >
+                          {type}
+                        </span>
+                      </button>
+                    ))}
+                  </div>
                 </div>
-              </div>
-            </div>
-          )}
+              )}
 
-          {(compactMode || viewMode === "constellation") && (
-            <div className="flex gap-0">
-              <div className="flex-1 min-w-0 border border-border rounded-lg overflow-hidden">
+              <div className="border border-border rounded-lg overflow-hidden">
                 <Constellation
                   key={compactMode ? "compact" : "full"}
                   data={graph2DData}
@@ -1049,119 +812,6 @@ export function DataView({
                   }
                 />
               </div>
-
-              {/* Right Toggle Button + Panel (hidden in compact mode) */}
-              {!compactMode && (
-                <>
-                  <button
-                    onClick={() => setShowControlPanel(!showControlPanel)}
-                    className="flex-shrink-0 w-5 h-[700px] bg-transparent hover:bg-muted/50 flex items-center justify-center transition-colors"
-                    title={showControlPanel ? t("hidePanel") : t("showPanel")}
-                  >
-                    {showControlPanel ? (
-                      <ChevronRight className="w-3 h-3 text-muted-foreground" />
-                    ) : (
-                      <ChevronLeft className="w-3 h-3 text-muted-foreground" />
-                    )}
-                  </button>
-
-                  {/* Right Panel — reuse the same panel as graph view */}
-                  {showControlPanel && (
-                    <div className="w-72 flex-shrink-0 border border-border rounded-lg bg-muted/20 overflow-y-auto h-[700px]">
-                      {selectedGraphNode ? (
-                        <MemoryDetailPanel
-                          memory={selectedGraphNode}
-                          onClose={() => setSelectedGraphNode(null)}
-                          inPanel
-                          bankId={currentBank || undefined}
-                        />
-                      ) : (
-                        <div className="p-4 space-y-4">
-                          <h3 className="text-sm font-semibold text-foreground">
-                            {t("constellationViewTitle")}
-                          </h3>
-                          <p className="text-xs text-muted-foreground">
-                            {t("constellationViewDescription")}
-                          </p>
-                          {factType === "observation" && (
-                            <div className="flex items-center justify-between gap-2 pt-2">
-                              <div className="flex items-center gap-1.5">
-                                <Layers className="w-3.5 h-3.5 text-muted-foreground" />
-                                <h4 className="text-xs font-medium text-muted-foreground">
-                                  {t("groupByScope")}
-                                </h4>
-                              </div>
-                              <Switch checked={groupByScope} onCheckedChange={setGroupByScope} />
-                            </div>
-                          )}
-                          {!(factType === "observation" && groupByScope) && (
-                            <div className="space-y-2 pt-2">
-                              <h4 className="text-xs font-medium text-muted-foreground">
-                                {t("colorBy")}
-                              </h4>
-                              <Select
-                                value={recencyBasis}
-                                onValueChange={(v) => setRecencyBasis(v as RecencyBasis)}
-                              >
-                                <SelectTrigger className="h-8 w-full text-xs">
-                                  <SelectValue />
-                                </SelectTrigger>
-                                <SelectContent>
-                                  <SelectItem value="mentioned_at">{t("mentioned")}</SelectItem>
-                                  <SelectItem value="occurred_start">
-                                    {t("occurredStart")}
-                                  </SelectItem>
-                                  <SelectItem value="occurred_end">{t("occurredEnd")}</SelectItem>
-                                </SelectContent>
-                              </Select>
-                            </div>
-                          )}
-                          <div className="space-y-2 pt-2">
-                            <h4 className="text-xs font-medium text-muted-foreground">
-                              {t("linkTypes")}
-                            </h4>
-                            {Object.entries({
-                              semantic: "#0074d9",
-                              temporal: "#009296",
-                              entity: "#f59e0b",
-                              causal: "#8b5cf6",
-                            }).map(([type, color]) => (
-                              <div
-                                key={type}
-                                className="flex items-center gap-2 cursor-pointer"
-                                onClick={() => toggleLinkType(type)}
-                              >
-                                <div
-                                  className="w-3 h-3 rounded-full"
-                                  style={{
-                                    backgroundColor: color,
-                                    opacity: visibleLinkTypes.has(type) ? 1 : 0.2,
-                                  }}
-                                />
-                                <span
-                                  className={`text-xs capitalize ${visibleLinkTypes.has(type) ? "text-foreground" : "text-muted-foreground line-through"}`}
-                                >
-                                  {type}
-                                </span>
-                              </div>
-                            ))}
-                          </div>
-                          <div className="text-xs text-muted-foreground space-y-1 pt-2">
-                            <div>
-                              {t("nodes")}:{" "}
-                              <span className="text-foreground">{graph2DData.nodes.length}</span>
-                            </div>
-                            <div>
-                              {t("links")}:{" "}
-                              <span className="text-foreground">{graph2DData.links.length}</span>
-                            </div>
-                          </div>
-                        </div>
-                      )}
-                    </div>
-                  )}
-                </>
-              )}
             </div>
           )}
 
@@ -1396,9 +1046,7 @@ export function DataView({
                     })()
                   ) : (
                     <div className="text-center py-12 text-muted-foreground">
-                      {data.table_rows?.length > 0
-                        ? t("noMemoriesMatchFilter")
-                        : t("noMemoriesFound")}
+                      {hasActiveMemoryFilters ? t("noMemoriesMatchFilter") : t("noMemoriesFound")}
                     </div>
                   )}
                 </div>

hindsight-control-plane/src/components/entities-view.tsx

@@ -22,7 +22,7 @@ import {
   TableRow,
 } from "@/components/ui/table";
 import { Constellation } from "./constellation";
-import { convertHindsightGraphData, GraphNode } from "./graph-2d";
+import { convertHindsightGraphData, GraphNode } from "./graph-data";
 
 type EntityGraphResponse = Awaited<ReturnType<typeof client.getEntityGraph>>;
 

hindsight-control-plane/src/components/graph-2d.tsx

@@ -1,726 +0,0 @@
-"use client";
-
-import { useRef, useEffect, useState, useMemo } from "react";
-import { useTranslations } from "next-intl";
-import cytoscape from "cytoscape";
-
-import fcose from "cytoscape-fcose";
-
-// Register the fcose extension
-cytoscape.use(fcose);
-
-// Hook to detect dark mode
-function useIsDarkMode() {
-  const [isDark, setIsDark] = useState(false);
-
-  useEffect(() => {
-    const checkDark = () => {
-      setIsDark(document.documentElement.classList.contains("dark"));
-    };
-
-    checkDark();
-
-    // Watch for theme changes
-    const observer = new MutationObserver(checkDark);
-    observer.observe(document.documentElement, { attributes: true, attributeFilter: ["class"] });
-
-    return () => observer.disconnect();
-  }, []);
-
-  return isDark;
-}
-
-// ============================================================================
-// Types & Interfaces
-// ============================================================================
-
-export interface GraphNode {
-  id: string;
-  label?: string;
-  color?: string;
-  size?: number;
-  group?: string;
-  metadata?: Record<string, any>;
-}
-
-export interface GraphLink {
-  source: string;
-  target: string;
-  color?: string;
-  width?: number;
-  type?: string;
-  entity?: string;
-  weight?: number;
-  metadata?: Record<string, any>;
-}
-
-export interface GraphData {
-  nodes: GraphNode[];
-  links: GraphLink[];
-}
-
-export interface Graph2DProps {
-  data: GraphData;
-  height?: number;
-  showLabels?: boolean;
-  onNodeClick?: (node: GraphNode) => void;
-  onNodeHover?: (node: GraphNode | null) => void;
-  nodeColorFn?: (node: GraphNode) => string;
-  nodeSizeFn?: (node: GraphNode) => number;
-  linkColorFn?: (link: GraphLink) => string;
-  linkWidthFn?: (link: GraphLink) => number;
-  maxNodes?: number;
-}
-
-// ============================================================================
-// Default Values
-// ============================================================================
-
-// Brand colors
-const BRAND_PRIMARY = "#0074d9";
-const LINK_SEMANTIC = "#0074d9"; // Primary blue for semantic
-
-const DEFAULT_NODE_COLOR = BRAND_PRIMARY;
-const DEFAULT_LINK_COLOR = LINK_SEMANTIC;
-const DEFAULT_LINK_WIDTH = 1;
-
-// ============================================================================
-// Component
-// ============================================================================
-
-export function Graph2D({
-  data,
-  height = 600,
-  showLabels = true,
-  onNodeClick,
-  onNodeHover,
-  nodeColorFn,
-  nodeSizeFn,
-  linkColorFn,
-  linkWidthFn,
-  maxNodes,
-}: Graph2DProps) {
-  const t = useTranslations("graph2d");
-  const [containerDiv, setContainerDiv] = useState<HTMLDivElement | null>(null);
-  const cyRef = useRef<any>(null);
-  const isInitializingRef = useRef(false);
-  const lastDataSignatureRef = useRef<string>("");
-  const [_hoveredNode, setHoveredNode] = useState<GraphNode | null>(null);
-  const [hoveredLink, setHoveredLink] = useState<GraphLink | null>(null);
-  const [linkTooltipPos, setLinkTooltipPos] = useState<{ x: number; y: number } | null>(null);
-  const [isLoading, setIsLoading] = useState(true);
-  const [isMounted, setIsMounted] = useState(false);
-  const [isFocusMode, setIsFocusMode] = useState(false);
-  const isDarkMode = useIsDarkMode();
-
-  // Use refs to store callbacks and data to prevent re-renders from resetting the graph
-  const onNodeClickRef = useRef(onNodeClick);
-  const onNodeHoverRef = useRef(onNodeHover);
-  const fullDataRef = useRef(data);
-  const nodeColorFnRef = useRef(nodeColorFn);
-  const linkColorFnRef = useRef(linkColorFn);
-  const isFocusModeRef = useRef(isFocusMode);
-  onNodeClickRef.current = onNodeClick;
-  onNodeHoverRef.current = onNodeHover;
-  fullDataRef.current = data;
-  nodeColorFnRef.current = nodeColorFn;
-  linkColorFnRef.current = linkColorFn;
-  isFocusModeRef.current = isFocusMode;
-
-  // Transform and limit data - only limit nodes, show ALL links between visible nodes
-  const graphData = useMemo(() => {
-    let nodes = [...data.nodes];
-
-    // Limit nodes if needed
-    if (maxNodes && nodes.length > maxNodes) {
-      nodes = nodes.slice(0, maxNodes);
-    }
-
-    // Show ALL links between visible nodes (no random link limiting)
-    const nodeIds = new Set(nodes.map((n) => n.id));
-    const links = data.links.filter((l) => nodeIds.has(l.source) && nodeIds.has(l.target));
-
-    return { nodes, links };
-  }, [data, maxNodes]);
-
-  // Track mounting state
-  useEffect(() => {
-    setIsMounted(true);
-    return () => setIsMounted(false);
-  }, []);
-
-  // Convert to Cytoscape format
-  const cyElements = useMemo(() => {
-    // Calculate node importance based on connections
-    const nodeConnections = new Map<string, number>();
-    graphData.links.forEach((link) => {
-      nodeConnections.set(link.source, (nodeConnections.get(link.source) || 0) + 1);
-      nodeConnections.set(link.target, (nodeConnections.get(link.target) || 0) + 1);
-    });
-
-    const nodes = graphData.nodes.map((node) => {
-      const connections = nodeConnections.get(node.id) || 0;
-      const dynamicSize = nodeSizeFn
-        ? nodeSizeFn(node)
-        : Math.max(16, Math.min(40, 16 + connections * 4)); // Smaller, more subtle sizing
-
-      return {
-        data: {
-          id: node.id,
-          label: node.label || node.id.substring(0, 8),
-          color: nodeColorFn ? nodeColorFn(node) : node.color || DEFAULT_NODE_COLOR,
-          size: node.size || dynamicSize,
-          originalNode: node,
-          connections: connections,
-        },
-      };
-    });
-
-    const edges = graphData.links.map((link, idx) => ({
-      data: {
-        id: `edge-${idx}`,
-        source: link.source,
-        target: link.target,
-        color: linkColorFn ? linkColorFn(link) : link.color || DEFAULT_LINK_COLOR,
-        width: linkWidthFn ? linkWidthFn(link) : link.width || DEFAULT_LINK_WIDTH,
-        type: link.type,
-        entity: link.entity,
-        weight: link.weight,
-        originalLink: link,
-      },
-    }));
-
-    return [...nodes, ...edges];
-  }, [graphData, nodeColorFn, nodeSizeFn, linkColorFn, linkWidthFn]);
-
-  // Create data signature to prevent double initialization
-  const dataSignature = useMemo(() => {
-    return JSON.stringify({
-      nodeCount: graphData.nodes.length,
-      linkCount: graphData.links.length,
-      nodeIds: graphData.nodes
-        .map((n) => n.id)
-        .sort()
-        .join(","),
-      showLabels,
-      isDarkMode,
-      maxNodes,
-    });
-  }, [graphData.nodes, graphData.links, showLabels, isDarkMode, maxNodes]);
-
-  // Initialize Cytoscape
-  useEffect(() => {
-    let isCancelled = false;
-
-    // Small delay to ensure container is mounted
-    const timeout = setTimeout(() => {
-      if (isCancelled || !isMounted || !containerDiv || isInitializingRef.current) return;
-
-      // Check if data has actually changed to prevent double initialization
-      if (lastDataSignatureRef.current === dataSignature) {
-        console.log("Data signature unchanged, skipping graph initialization");
-        return;
-      }
-
-      // Additional validation - check if element has dimensions
-      const rect = containerDiv.getBoundingClientRect();
-      if (rect.width === 0 || rect.height === 0) {
-        console.warn("Container has no dimensions, skipping cytoscape initialization");
-        setIsLoading(false);
-        return;
-      }
-
-      // Handle empty data case
-      if (cyElements.length === 0) {
-        setIsLoading(false);
-        return;
-      }
-
-      // Check if we already have a graph with the same data
-      if (cyRef.current && !cyRef.current.destroyed()) {
-        const currentNodes = cyRef.current.nodes().length;
-        const currentEdges = cyRef.current.edges().length;
-        const newNodes = cyElements.filter((el) => !(el.data as any).source).length;
-        const newEdges = cyElements.filter((el) => (el.data as any).source).length;
-
-        // If the element counts are the same, just update styles and skip reinitialization
-        if (currentNodes === newNodes && currentEdges === newEdges) {
-          console.log("Graph already initialized with same data, skipping reinitialization");
-          setIsLoading(false);
-          return;
-        }
-
-        // Clean up existing graph before creating new one
-        console.log("Data changed, destroying existing graph");
-        cyRef.current.destroy();
-        cyRef.current = null;
-      }
-
-      setIsLoading(true);
-      isInitializingRef.current = true;
-
-      // Theme-aware colors
-      const textColor = isDarkMode ? "#ffffff" : "#1f2937";
-      const textBgColor = isDarkMode ? "rgba(0,0,0,0.8)" : "rgba(255,255,255,0.9)";
-
-      try {
-        console.log("Initializing cytoscape with container:", containerDiv);
-        console.log("Elements count:", cyElements.length);
-        console.log("Sample elements:", cyElements.slice(0, 2));
-
-        // Try minimal initialization first
-        const cy = cytoscape({
-          container: containerDiv,
-          elements: [],
-          // Disable edge selection to prevent gray border on click
-          selectionType: "single",
-          userZoomingEnabled: true,
-          userPanningEnabled: true,
-          boxSelectionEnabled: false,
-          // Disable automatic layout on initialization
-          layout: { name: "preset" },
-          style: [
-            {
-              selector: "node",
-              style: {
-                "background-color": "data(color)",
-                width: "data(size)",
-                height: "data(size)",
-                label: showLabels ? "data(label)" : "",
-                color: textColor,
-                "text-valign": "bottom",
-                "text-halign": "center",
-                "font-size": "8px",
-                "font-weight": 500,
-                "text-margin-y": 3,
-                "text-wrap": "wrap",
-                "text-max-width": "80px",
-                "text-background-color": textBgColor,
-                "text-background-opacity": 0.9,
-                "text-background-padding": "2px",
-                "text-background-shape": "roundrectangle",
-                "border-width": 1,
-                "border-color": isDarkMode ? "#ffffff20" : "#00000020",
-                "border-opacity": 0.3,
-              },
-            },
-            {
-              selector: "node:selected",
-              style: {
-                "border-width": 3,
-                "border-color": "#0074d9",
-                "border-opacity": 1,
-              },
-            },
-            {
-              selector: "edge",
-              style: {
-                width: "data(width)",
-                "line-color": "data(color)",
-                "target-arrow-color": "data(color)",
-                "target-arrow-shape": "triangle",
-                "target-arrow-size": 6,
-                "curve-style": "bezier",
-                opacity: isDarkMode ? 0.6 : 0.7,
-              },
-            },
-            // Focus mode styles
-            {
-              selector: ".dimmed",
-              style: {
-                opacity: 0.2,
-              },
-            },
-            {
-              selector: ".focused",
-              style: {
-                "border-width": 4,
-                "border-color": "#ff6b35",
-                "border-opacity": 1,
-                "z-index": 999,
-              },
-            },
-            {
-              selector: ".connected",
-              style: {
-                "border-width": 2,
-                "border-color": "#0074d9",
-                "border-opacity": 0.8,
-                opacity: 1,
-              },
-            },
-            {
-              selector: "edge.connection",
-              style: {
-                width: 2,
-                opacity: 1,
-                "z-index": 100,
-              },
-            },
-            {
-              selector: "edge.connection:hover",
-              style: {
-                width: 3,
-                opacity: 1,
-                "z-index": 200,
-              },
-            },
-            // Disable edge selection styling
-            {
-              selector: "edge:selected",
-              style: {
-                "overlay-opacity": 0,
-                "overlay-color": "transparent",
-                "overlay-padding": 0,
-              },
-            },
-          ],
-        });
-
-        cyRef.current = cy;
-
-        console.log("Cytoscape initialized successfully");
-
-        // Add elements after initialization
-        if (cyElements.length > 0) {
-          console.log("Adding elements to cytoscape");
-          cy.add(cyElements);
-          cy.layout({
-            name: "fcose",
-            quality: "default",
-            randomize: false,
-            animate: true,
-            animationDuration: 1500,
-            // Separation settings - increase to spread nodes more
-            nodeSeparation: 200,
-            idealEdgeLength: () => 250,
-            edgeElasticity: () => 0.05,
-            nestingFactor: 0.05,
-            gravity: 0.05, // Reduced gravity spreads nodes more
-            numIter: 2500,
-            // Overlap prevention
-            nodeOverlap: 30,
-            avoidOverlap: true,
-            nodeDimensionsIncludeLabels: true,
-            // Layout bounds - reduce padding to use more space
-            padding: 20,
-            boundingBox: undefined,
-            // Tiling - increase spacing between disconnected components
-            tile: true,
-            tilingPaddingVertical: 30,
-            tilingPaddingHorizontal: 30,
-            // Force more spread
-            uniformNodeDimensions: false,
-            packComponents: false, // Don't pack components tightly
-          }).run();
-
-          // Fit to viewport
-          cy.fit();
-        }
-
-        // Add basic interactions
-        cy.on("tap", "node", (evt: any) => {
-          const node = evt.target as cytoscape.NodeSingular;
-          const originalNode = node.data("originalNode") as GraphNode;
-          if (onNodeClickRef.current && originalNode) {
-            onNodeClickRef.current(originalNode);
-          }
-        });
-
-        cy.on("mouseover", "node", (evt: any) => {
-          const node = evt.target as cytoscape.NodeSingular;
-          const originalNode = node.data("originalNode") as GraphNode;
-          setHoveredNode(originalNode);
-          if (onNodeHoverRef.current && originalNode) {
-            onNodeHoverRef.current(originalNode);
-          }
-          if (containerDiv) containerDiv.style.cursor = "pointer";
-        });
-
-        cy.on("mouseout", "node", () => {
-          setHoveredNode(null);
-          if (onNodeHoverRef.current) {
-            onNodeHoverRef.current(null);
-          }
-          if (containerDiv) containerDiv.style.cursor = "default";
-        });
-
-        // Edge hover handlers - only work in focus mode and on highlighted edges
-        cy.on("mouseover", "edge", (evt: any) => {
-          const edge = evt.target;
-
-          // Only allow interaction if we're in focus mode and edge is highlighted
-          if (!isFocusModeRef.current || !edge.hasClass("connection")) {
-            return;
-          }
-
-          const originalLink = edge.data("originalLink") as GraphLink;
-          if (originalLink) {
-            setHoveredLink(originalLink);
-            // Get position for tooltip
-            const renderedPos = edge.renderedMidpoint();
-            setLinkTooltipPos({ x: renderedPos.x, y: renderedPos.y });
-          }
-        });
-
-        cy.on("mouseout", "edge", (evt: any) => {
-          const edge = evt.target;
-
-          // Only clear hover state if we were actually hovering a highlighted edge
-          if (!isFocusModeRef.current || !edge.hasClass("connection")) {
-            return;
-          }
-
-          setHoveredLink(null);
-          setLinkTooltipPos(null);
-        });
-
-        // Prevent edge selection to avoid gray border on click
-        cy.on("select", "edge", (evt: any) => {
-          evt.target.unselect();
-        });
-
-        // Double-click to focus on node and its connections
-        cy.on("dblclick", "node", (evt: any) => {
-          const focusedNode = evt.target as cytoscape.NodeSingular;
-          const focusedNodeId = focusedNode.id();
-
-          console.log("Double-clicked node:", focusedNodeId);
-
-          // Enter focus mode
-          setIsFocusMode(true);
-
-          // Clear any existing focus classes
-          cy.elements().removeClass("dimmed focused connected connection");
-
-          // Get all connected nodes and edges
-          const connectedElements = focusedNode.neighborhood();
-          const connectedNodes = connectedElements.nodes();
-          const connectedEdges = connectedElements.edges();
-
-          // Apply styling classes
-          cy.elements().addClass("dimmed"); // Dim everything first
-          focusedNode.removeClass("dimmed").addClass("focused"); // Highlight the focused node
-          connectedNodes.removeClass("dimmed").addClass("connected"); // Highlight connected nodes
-          connectedEdges.removeClass("dimmed").addClass("connection"); // Highlight connecting edges
-
-          // Create a collection of all relevant elements for positioning
-          const relevantElements = focusedNode.union(connectedElements);
-
-          // Reorient the graph to focus on this subgraph
-          cy.animate(
-            {
-              fit: {
-                eles: relevantElements,
-                padding: 100,
-              },
-              center: {
-                eles: focusedNode,
-              },
-            },
-            {
-              duration: 800,
-              easing: "ease-out-cubic",
-            }
-          );
-        });
-
-        // Click on background to reset focus
-        cy.on("tap", (evt: any) => {
-          if (evt.target === cy) {
-            console.log("Clicked background - resetting focus");
-
-            // Exit focus mode
-            setIsFocusMode(false);
-
-            // Remove all focus classes
-            cy.elements().removeClass("dimmed focused connected connection");
-
-            // Zoom out to show all elements
-            cy.animate(
-              {
-                fit: {
-                  eles: cy.elements(),
-                  padding: 50,
-                },
-              },
-              {
-                duration: 600,
-                easing: "ease-out",
-              }
-            );
-          }
-        });
-
-        setIsLoading(false);
-        isInitializingRef.current = false;
-        lastDataSignatureRef.current = dataSignature;
-      } catch (error) {
-        console.error("Error initializing cytoscape:", error);
-        setIsLoading(false);
-        isInitializingRef.current = false;
-      }
-    }, 100); // 100ms delay
-
-    return () => {
-      isCancelled = true;
-      clearTimeout(timeout);
-      isInitializingRef.current = false;
-      if (cyRef.current) {
-        cyRef.current.destroy();
-        cyRef.current = null;
-      }
-    };
-  }, [dataSignature, isMounted, containerDiv]);
-
-  // Handle resize
-  useEffect(() => {
-    const handleResize = () => {
-      if (cyRef.current) {
-        cyRef.current.resize();
-        cyRef.current.fit(undefined, 80);
-      }
-    };
-
-    window.addEventListener("resize", handleResize);
-    return () => window.removeEventListener("resize", handleResize);
-  }, []);
-
-  return (
-    <div
-      className="relative w-full rounded-lg overflow-hidden border border-border"
-      style={{ height }}
-    >
-      {/* Loading state */}
-      {isLoading && (
-        <div className="absolute inset-0 flex items-center justify-center bg-background z-10">
-          <div className="text-center">
-            <div className="animate-spin rounded-full h-8 w-8 border-b-2 border-primary mx-auto mb-4" />
-            <p className="text-sm text-muted-foreground">{t("loading")}</p>
-          </div>
-        </div>
-      )}
-
-      {/* Cytoscape container */}
-      {isMounted && (
-        <div
-          ref={setContainerDiv}
-          className="w-full h-full"
-          style={{
-            backgroundImage: isDarkMode
-              ? "radial-gradient(circle at 1px 1px, rgba(255,255,255,0.08) 1px, transparent 0)"
-              : "radial-gradient(circle at 1px 1px, rgba(0,0,0,0.06) 1px, transparent 0)",
-            backgroundSize: "20px 20px",
-            backgroundColor: isDarkMode ? "#0f1419" : "#f8fafc",
-          }}
-        />
-      )}
-
-      {/* Empty state */}
-      {!isLoading && graphData.nodes.length === 0 && (
-        <div className="absolute inset-0 flex items-center justify-center">
-          <div className="text-center">
-            <p className="text-muted-foreground">{t("emptyState")}</p>
-          </div>
-        </div>
-      )}
-
-      {/* Link hover tooltip */}
-      {hoveredLink && linkTooltipPos && (
-        <div
-          className="absolute z-30 pointer-events-none"
-          style={{
-            left: linkTooltipPos.x,
-            top: linkTooltipPos.y,
-            transform: "translate(-50%, -100%) translateY(-8px)",
-          }}
-        >
-          <div
-            className={`px-3 py-2 rounded-lg shadow-lg text-sm ${
-              isDarkMode
-                ? "bg-gray-800 text-white"
-                : "bg-white text-gray-900 border border-gray-200"
-            }`}
-          >
-            <div className="font-medium capitalize mb-1">
-              {(() => {
-                const type = hoveredLink.type || "semantic";
-                if (["causes", "caused_by", "enables", "prevents"].includes(type)) {
-                  return t("linkTypeCausal", { type: type.replace("_", " ") });
-                }
-                return t("linkTypeGeneric", { type });
-              })()}
-            </div>
-            {hoveredLink.entity && (
-              <div className="text-xs opacity-80">
-                {t("linkTooltipEntity")} <span className="font-medium">{hoveredLink.entity}</span>
-              </div>
-            )}
-            {hoveredLink.weight !== undefined && (
-              <div className="text-xs opacity-80">
-                {t("linkTooltipWeight")}{" "}
-                <span className="font-medium">{hoveredLink.weight.toFixed(3)}</span>
-              </div>
-            )}
-          </div>
-        </div>
-      )}
-
-      {/* Controls hint */}
-      <div className="absolute bottom-4 right-4 text-xs text-muted-foreground/60 z-20">
-        {t("controlsHint")}
-      </div>
-    </div>
-  );
-}
-
-// ============================================================================
-// Utility Functions
-// ============================================================================
-
-export function convertHindsightGraphData(hindsightData: {
-  nodes?: Array<{ data: { id: string; label?: string; color?: string } }>;
-  edges?: Array<{
-    data: {
-      source: string;
-      target: string;
-      color?: string;
-      lineStyle?: string;
-      linkType?: string;
-      entityName?: string;
-      weight?: number;
-      similarity?: number;
-    };
-  }>;
-  table_rows?: Array<{ id: string; text: string; entities?: string; context?: string }>;
-}): GraphData {
-  const nodes: GraphNode[] = (hindsightData.nodes || []).map((n) => {
-    const tableRow = hindsightData.table_rows?.find((r) => r.id === n.data.id);
-    // Use memory text as label, truncated to ~40 chars
-    let label = n.data.label;
-    if (!label && tableRow?.text) {
-      label = tableRow.text.length > 40 ? tableRow.text.substring(0, 40) + "..." : tableRow.text;
-    }
-    if (!label) {
-      label = n.data.id.substring(0, 8);
-    }
-    return {
-      id: n.data.id,
-      label,
-      color: n.data.color,
-      metadata: tableRow,
-    };
-  });
-
-  const links: GraphLink[] = (hindsightData.edges || []).map((e) => ({
-    source: e.data.source,
-    target: e.data.target,
-    color: e.data.color,
-    // Use linkType directly from API, fallback to lineStyle check, default to semantic
-    type: e.data.linkType || (e.data.lineStyle === "dashed" ? "temporal" : "semantic"),
-    entity: e.data.entityName, // API returns entityName
-    weight: e.data.weight ?? e.data.similarity,
-  }));
-
-  return { nodes, links };
-}