blog: update opencode cover image

Co-Authored-By: Claude Haiku 4.5 <[email protected]>

.github/workflows/test.yml

@@ -49,6 +49,7 @@ jobs:
       integrations-opencode: ${{ steps.filter.outputs.integrations-opencode }}
       integrations-cloudflare-oauth-proxy: ${{ steps.filter.outputs.integrations-cloudflare-oauth-proxy }}
       integrations-lockfiles: ${{ steps.filter.outputs.integrations-lockfiles }}
+      integrations-openai-agents: ${{ steps.filter.outputs.integrations-openai-agents }}
       dev: ${{ steps.filter.outputs.dev }}
       ci: ${{ steps.filter.outputs.ci }}
       # Secrets are available for internal PRs, pull_request_review, and workflow_dispatch.
@@ -133,6 +134,8 @@ jobs:
             - 'hindsight-integrations/*/package-lock.json'
             - 'hindsight-integrations/*/package.json'
             - 'scripts/check-integration-lockfiles.sh'
+          integrations-openai-agents:
+            - 'hindsight-integrations/openai-agents/**'
           dev:
             - 'hindsight-dev/**'
           ci:
@@ -2043,6 +2046,43 @@ jobs:
       working-directory: ./hindsight-integrations/llamaindex
       run: uv run pytest tests -v
 
+  test-openai-agents-integration:
+    needs: [detect-changes]
+    if: >-
+      github.event_name != 'pull_request_review' &&
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-openai-agents == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version-file: ".python-version"
+
+    - name: Build openai-agents integration
+      working-directory: ./hindsight-integrations/openai-agents
+      run: uv build
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/openai-agents
+      run: uv sync --frozen
+
+    - name: Run tests
+      working-directory: ./hindsight-integrations/openai-agents
+      run: uv run pytest tests -v
+
   test-pip-slim:
     needs: [detect-changes]
     if: >-
@@ -2547,6 +2587,9 @@ jobs:
     - name: Run generate-openapi
       run: ./scripts/generate-openapi.sh
 
+    - name: Run generate-bank-template-schema
+      run: ./scripts/generate-bank-template-schema.sh
+
     - name: Run generate-clients
       run: ./scripts/generate-clients.sh
 
@@ -2566,6 +2609,7 @@ jobs:
           echo ""
           echo "Please run the following commands locally and commit the changes:"
           echo "  ./scripts/generate-openapi.sh"
+          echo "  ./scripts/generate-bank-template-schema.sh"
           echo "  ./scripts/generate-clients.sh"
           echo "  ./scripts/generate-docs-skill.sh"
           echo "  ./scripts/hooks/lint.sh"

.prettierrc.json

@@ -0,0 +1,7 @@
+{
+  "semi": true,
+  "singleQuote": false,
+  "tabWidth": 2,
+  "trailingComma": "es5",
+  "printWidth": 100
+}

helm/hindsight/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: hindsight
 description: Hindsight helm chart
 type: application
-version: 0.5.1
-appVersion: "0.5.1"
+version: 0.5.3
+appVersion: "0.5.3"
 keywords:
   - ai
   - memory

hindsight-all-npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vectorize-io/hindsight-all",
-  "version": "0.5.1",
+  "version": "0.5.3",
   "description": "Node.js programmatic lifecycle manager for Hindsight — embeds a local hindsight daemon in a Node application. Pair with @vectorize-io/hindsight-client for memory operations.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

hindsight-all-slim/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hindsight-all-slim"
-version = "0.5.1"
+version = "0.5.3"
 description = "Hindsight: Agent Memory That Works Like Human Memory - Slim All-in-One Bundle"
 readme = "README.md"
 requires-python = ">=3.11"

hindsight-all/hindsight/embedded.py

@@ -71,7 +71,7 @@ class HindsightEmbedded:
         llm_model: Model name to use
         llm_base_url: Optional custom base URL for LLM API
         database_url: Optional database URL override (default: profile-specific pg0)
-        idle_timeout: Seconds before daemon auto-exits when idle (default: 300)
+        idle_timeout: Seconds before daemon auto-exits when idle (default: 0, disabled)
         log_level: Daemon log level (default: "info")
         ui: Whether to start the control plane web UI alongside the daemon (default: False)
         ui_port: Port for the UI. Defaults to daemon_port + 10000.
@@ -86,7 +86,7 @@ class HindsightEmbedded:
         llm_model: str = "openai/gpt-oss-120b",
         llm_base_url: Optional[str] = None,
         database_url: Optional[str] = None,
-        idle_timeout: int = 300,
+        idle_timeout: int = 0,
         log_level: str = "info",
         ui: bool = False,
         ui_port: Optional[int] = None,
@@ -102,7 +102,7 @@ class HindsightEmbedded:
             llm_model: Model name to use
             llm_base_url: Optional custom base URL for LLM API
             database_url: Optional database URL override
-            idle_timeout: Seconds before daemon auto-exits when idle
+            idle_timeout: Seconds before daemon auto-exits when idle (0 = disabled)
             log_level: Daemon log level
             ui: Whether to start the control plane web UI alongside the daemon
             ui_port: Port for the UI (defaults to daemon_port + 10000)

hindsight-all/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-all"
-version = "0.5.1"
+version = "0.5.3"
 description = "Hindsight: Agent Memory That Works Like Human Memory - All-in-One Bundle"
 readme = "README.md"
 requires-python = ">=3.11"

hindsight-api-slim/hindsight_api/__init__.py

@@ -46,4 +46,4 @@ __all__ = [
     "RemoteTEICrossEncoder",
     "LLMConfig",
 ]
-__version__ = "0.5.1"
+__version__ = "0.5.3"

hindsight-api-slim/hindsight_api/admin/cli.py

@@ -375,6 +375,140 @@ def decommission_worker(
         typer.echo(f"No tasks found for worker '{worker_id}'")
 
 
+async def _decommission_all_workers(db_url: str, schema: str = "public") -> list[dict[str, Any]]:
+    """Release all processing tasks from all workers, setting them back to pending status."""
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+
+    conn = await asyncpg.connect(resolved_url)
+    try:
+        table = _fq_table("async_operations", schema)
+        rows = await conn.fetch(
+            f"""
+            UPDATE {table}
+            SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
+            WHERE status = 'processing'
+            RETURNING operation_id, worker_id, operation_type
+            """,
+        )
+        return [dict(r) for r in rows]
+    finally:
+        await conn.close()
+
+
+@app.command(name="decommission-workers")
+def decommission_workers(
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema"),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"),
+):
+    """Release all processing tasks from all workers (sets status back to pending).
+
+    Use this command to recover from situations where one or more workers have crashed
+    or been removed without graceful shutdown. All tasks currently in 'processing' status
+    will be released back to the queue regardless of which worker owns them.
+    """
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    if not yes:
+        typer.confirm(
+            "This will release ALL processing tasks from ALL workers back to pending. Continue?",
+            abort=True,
+        )
+
+    typer.echo(f"Decommissioning all workers (schema: {schema})...")
+
+    released = asyncio.run(_decommission_all_workers(config.database_url, schema))
+
+    if released:
+        # Group by worker_id for summary
+        by_worker: dict[str, int] = {}
+        for row in released:
+            wid = row["worker_id"] or "unknown"
+            by_worker[wid] = by_worker.get(wid, 0) + 1
+
+        typer.echo(f"Released {len(released)} task(s):")
+        for wid, count in by_worker.items():
+            typer.echo(f"  {wid}: {count} task(s)")
+    else:
+        typer.echo("No processing tasks found")
+
+
+async def _worker_status(db_url: str, schema: str = "public") -> list[dict[str, Any]]:
+    """Get all processing tasks grouped by worker with their last update time."""
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+
+    conn = await asyncpg.connect(resolved_url)
+    try:
+        table = _fq_table("async_operations", schema)
+        rows = await conn.fetch(
+            f"""
+            SELECT worker_id, operation_id, operation_type, bank_id,
+                   claimed_at, updated_at,
+                   now() - claimed_at AS running_for,
+                   now() - updated_at AS last_update_ago
+            FROM {table}
+            WHERE status = 'processing'
+            ORDER BY worker_id, claimed_at
+            """,
+        )
+        return [dict(r) for r in rows]
+    finally:
+        await conn.close()
+
+
+@app.command(name="worker-status")
+def worker_status(
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema"),
+):
+    """Show all currently processing tasks grouped by worker.
+
+    Displays each worker's active tasks with operation type, bank, how long
+    the task has been running, and when it was last updated. Useful for
+    identifying dead workers with orphaned tasks.
+    """
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    rows = asyncio.run(_worker_status(config.database_url, schema))
+
+    if not rows:
+        typer.echo("No processing tasks found")
+        return
+
+    # Group by worker_id
+    by_worker: dict[str, list[dict[str, Any]]] = {}
+    for row in rows:
+        wid = row["worker_id"] or "unknown"
+        by_worker.setdefault(wid, []).append(row)
+
+    typer.echo(f"Processing tasks across {len(by_worker)} worker(s):\n")
+    for wid, tasks in by_worker.items():
+        typer.echo(f"Worker: {wid} ({len(tasks)} task(s))")
+        for task in tasks:
+            op_id = str(task["operation_id"])[:8]
+            running_for = task["running_for"]
+            last_update = task["last_update_ago"]
+            typer.echo(
+                f"  {op_id}  {task['operation_type']:<20s} bank={task['bank_id']}"
+                f"  running={running_for}  last_update={last_update} ago"
+            )
+        typer.echo("")
+
+
 def main():
     app()
 

hindsight-api-slim/hindsight_api/alembic/versions/2eee35aa3cfc_case_insensitive_entities_trgm_index.py

@@ -4,8 +4,8 @@ The previous GIN trigram index on canonical_name was case-sensitive, causing
 "Alice" and "alice" to have different trigram sets. This recreates it on
 LOWER(canonical_name) so the % operator matches case-insensitively.
 
-Revision ID: d6e7f8a9b0c1
-Revises: c5d6e7f8a9b0
+Revision ID: 2eee35aa3cfc
+Revises: d6e7f8a9b0c1
 Create Date: 2026-03-31
 """
 
@@ -13,8 +13,8 @@ from collections.abc import Sequence
 
 from alembic import context, op
 
-revision: str = "d6e7f8a9b0c1"
-down_revision: str | Sequence[str] | None = "c5d6e7f8a9b0"
+revision: str = "2eee35aa3cfc"
+down_revision: str | Sequence[str] | None = "d6e7f8a9b0c1"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 

hindsight-api-slim/hindsight_api/alembic/versions/a2v3w4x5y6z7_add_last_refreshed_source_query.py

@@ -0,0 +1,38 @@
+"""Add last_refreshed_source_query column to mental_models
+
+Revision ID: a2v3w4x5y6z7
+Revises: z1u2v3w4x5y6
+Create Date: 2026-04-15
+
+Tracks the source_query that was used during the most recent refresh.
+Used by delta-mode refresh to detect when the query has changed: if it has,
+delta mode falls back to a full regeneration because the surgical-edit
+assumption (same topic, new facts) no longer holds.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "a2v3w4x5y6z7"
+down_revision: str | Sequence[str] | None = "z1u2v3w4x5y6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD COLUMN IF NOT EXISTS last_refreshed_source_query TEXT
+    """)
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS last_refreshed_source_query")

hindsight-api-slim/hindsight_api/alembic/versions/a4b5c6d7e8f9_fix_per_bank_vector_index_type.py

@@ -1,7 +1,7 @@
 """Fix per-bank vector indexes to match configured extension
 
 Revision ID: a4b5c6d7e8f9
-Revises: d6e7f8a9b0c1
+Revises: 2eee35aa3cfc
 Create Date: 2026-04-01
 
 Migration d5e6f7a8b9c0 hardcoded HNSW when creating per-bank partial vector
@@ -21,7 +21,7 @@ from alembic import context, op
 from sqlalchemy import text
 
 revision: str = "a4b5c6d7e8f9"
-down_revision: str | Sequence[str] | None = "d6e7f8a9b0c1"
+down_revision: str | Sequence[str] | None = "2eee35aa3cfc"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 

hindsight-api-slim/hindsight_api/alembic/versions/b3w4x5y6z7a8_add_structured_content_to_mental_models.py

@@ -0,0 +1,44 @@
+"""Add structured_content JSONB column to mental_models
+
+Revision ID: b3w4x5y6z7a8
+Revises: a2v3w4x5y6z7
+Create Date: 2026-04-16
+
+Stores the structured representation of a mental model document (sections,
+blocks). The plain ``content`` column remains the rendered markdown shown to
+users. ``structured_content`` is the source of truth for delta-mode refreshes:
+each refresh applies a list of typed operations to the structured doc, then
+re-renders to markdown — so unchanged sections come through byte-identical
+without an LLM round-trip.
+
+Nullable: existing markdown-only mental models continue to work in full mode;
+the column is populated lazily the first time a model is refreshed in delta
+mode.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "b3w4x5y6z7a8"
+down_revision: str | Sequence[str] | None = "a2v3w4x5y6z7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD COLUMN IF NOT EXISTS structured_content JSONB
+    """)
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS structured_content")

hindsight-api-slim/hindsight_api/alembic/versions/c4x5y6z7a8b9_backsweep_orphan_observations_v2.py

@@ -0,0 +1,66 @@
+"""backsweep_orphan_observations_v2
+
+Re-run of Pass 2 from migration ``g7h8i9j0k1l2_backsweep_orphan_observations``
+to sweep observations that became orphaned between then and now.
+
+Why we need it again:
+  ``fact_storage.handle_document_tracking`` (the retain/upsert path) deleted
+  the existing document via the FK cascade — which removes the source
+  ``memory_units`` — but never invalidated the observations derived from
+  them.  Only the explicit ``MemoryEngine.delete_document`` API called
+  ``_delete_stale_observations_for_memories``.  Every document re-ingest
+  therefore left orphan observations whose ``source_memory_ids`` arrays
+  pointed at IDs that no longer existed in ``memory_units``.
+
+  ``handle_document_tracking`` now calls the same cleanup helper before the
+  cascade, so no new orphans will accumulate going forward.  This migration
+  cleans up the historical residue.
+
+Identical to Pass 2 of g7h8i9j0k1l2.  Pass 1 (memory_units whose bank is
+gone) is intentionally not re-run; that scenario has no fresh source.
+
+Revision ID: c4x5y6z7a8b9
+Revises: b3w4x5y6z7a8
+Create Date: 2026-04-16
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "c4x5y6z7a8b9"
+down_revision: str | Sequence[str] | None = "b3w4x5y6z7a8"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    mu = f"{schema}memory_units"
+
+    # Delete observations whose every source_memory_id refers to a now-deleted
+    # memory_unit (or the array is empty).  Observations with at least one
+    # surviving source are left alone — the consolidation engine will refresh
+    # their text on the next pass.
+    op.execute(
+        f"""
+        DELETE FROM {mu} orphan
+        WHERE orphan.fact_type = 'observation'
+          AND NOT EXISTS (
+              SELECT 1
+              FROM {mu} src
+              WHERE src.id = ANY(orphan.source_memory_ids)
+                AND src.bank_id = orphan.bank_id
+          )
+        """
+    )
+
+
+def downgrade() -> None:
+    # Deleted rows cannot be restored.
+    pass

hindsight-api-slim/hindsight_api/alembic/versions/d6e7f8a9b0c1_drop_documents_metadata_column.py

@@ -0,0 +1,39 @@
+"""Drop unused metadata column from documents table
+
+Revision ID: d6e7f8a9b0c1
+Revises: c2d3e4f5g6h7, c5d6e7f8a9b0
+Create Date: 2026-03-30
+
+The metadata column on documents was always stored as an empty dict {}.
+Actual document metadata is stored inside retain_params.metadata.
+
+This migration was originally shipped in v0.4.22, then its file was deleted
+in v0.5.0 (and its revision ID accidentally reused by 2eee35aa3cfc).
+Restoring the file so that databases stamped at this revision can upgrade
+cleanly to v0.5.x+.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "d6e7f8a9b0c1"
+down_revision: str | Sequence[str] | None = ("c2d3e4f5g6h7", "c5d6e7f8a9b0")
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}documents DROP COLUMN IF EXISTS metadata")
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}documents ADD COLUMN IF NOT EXISTS metadata jsonb DEFAULT '{{}}'")

hindsight-api-slim/hindsight_api/alembic/versions/h3i4j5k6l7m8_merge_heads_and_add_unit_entities_index.py

@@ -1,7 +1,7 @@
 """Merge 3 migration heads and add unit_entities composite index
 
 Revision ID: h3i4j5k6l7m8
-Revises: a4b5c6d7e8f9, c2d3e4f5g6h7, g2h3i4j5k6l7
+Revises: a4b5c6d7e8f9, g2h3i4j5k6l7
 Create Date: 2026-04-07
 
 Merges three unmerged migration heads into one, and adds a composite index
@@ -14,7 +14,7 @@ from collections.abc import Sequence
 from alembic import context, op
 
 revision: str = "h3i4j5k6l7m8"
-down_revision: str | Sequence[str] | None = ("a4b5c6d7e8f9", "c2d3e4f5g6h7", "g2h3i4j5k6l7")
+down_revision: str | Sequence[str] | None = ("a4b5c6d7e8f9", "g2h3i4j5k6l7")
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 

hindsight-api-slim/hindsight_api/api/http.py

@@ -294,6 +294,44 @@ class EntityListResponse(BaseModel):
     offset: int
 
 
+class EntityGraphResponse(BaseModel):
+    """Response model for entity co-occurrence graph endpoint."""
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "nodes": [
+                    {"data": {"id": "uuid-1", "label": "Alice", "mentionCount": 12, "color": "#42a5f5"}},
+                    {"data": {"id": "uuid-2", "label": "Google", "mentionCount": 8, "color": "#42a5f5"}},
+                ],
+                "edges": [
+                    {
+                        "data": {
+                            "id": "uuid-1-uuid-2",
+                            "source": "uuid-1",
+                            "target": "uuid-2",
+                            "linkType": "cooccurrence",
+                            "weight": 5,
+                            "color": "#ffd700",
+                            "lineStyle": "solid",
+                            "lastCooccurred": "2024-02-01T14:00:00Z",
+                        }
+                    }
+                ],
+                "total_entities": 2,
+                "total_edges": 1,
+                "limit": 1000,
+            }
+        }
+    )
+
+    nodes: list[dict[str, Any]]
+    edges: list[dict[str, Any]]
+    total_entities: int
+    total_edges: int
+    limit: int
+
+
 class EntityDetailResponse(BaseModel):
     """Response model for entity detail endpoint."""
 
@@ -1416,6 +1454,7 @@ class BankStatsResponse(BaseModel):
                 "failed_operations": 0,
                 "last_consolidated_at": "2024-01-15T10:30:00Z",
                 "pending_consolidation": 0,
+                "failed_consolidation": 0,
                 "total_observations": 45,
             }
         }
@@ -1431,12 +1470,41 @@ class BankStatsResponse(BaseModel):
     links_breakdown: dict[str, dict[str, int]]
     pending_operations: int
     failed_operations: int
+    operations_by_status: dict[str, int] = Field(
+        default_factory=dict,
+        description="Async operations grouped by status (pending, in_progress, completed, failed, cancelled).",
+    )
     # Consolidation stats
     last_consolidated_at: str | None = Field(default=None, description="When consolidation last ran (ISO format)")
     pending_consolidation: int = Field(default=0, description="Number of memories not yet processed into observations")
+    failed_consolidation: int = Field(
+        default=0,
+        description="Number of source memories (world/experience) whose consolidation permanently failed and can be retried via the consolidation recovery endpoint.",
+    )
     total_observations: int = Field(default=0, description="Total number of observations")
 
 
+class MemoryTimeseriesBucket(BaseModel):
+    """One bucket in the memory ingestion time-series."""
+
+    time: str = Field(description="Bucket start timestamp in ISO-8601 (UTC).")
+    world: int = Field(default=0, description="World-fact memories ingested in this bucket.")
+    experience: int = Field(default=0, description="Experience memories ingested in this bucket.")
+    observation: int = Field(default=0, description="Observations recorded in this bucket.")
+
+
+class MemoriesTimeseriesResponse(BaseModel):
+    """Time-series of memory ingestion bucketed by time and fact type."""
+
+    bank_id: str
+    period: str = Field(description="One of: 1h, 12h, 1d, 7d, 30d, 90d.")
+    trunc: str = Field(description="Bucket granularity: minute, hour, day.")
+    buckets: list[MemoryTimeseriesBucket] = Field(
+        default_factory=list,
+        description="Per-bucket counts, always returned fully padded for the requested period.",
+    )
+
+
 # Mental Model models
 
 
@@ -1493,6 +1561,16 @@ class UpdateDirectiveRequest(BaseModel):
 class MentalModelTrigger(BaseModel):
     """Trigger settings for a mental model."""
 
+    mode: Literal["full", "delta"] = Field(
+        default="full",
+        description=(
+            "Refresh mode. 'full' (default) regenerates the mental model content from scratch on each refresh. "
+            "'delta' performs surgical edits against the existing content: unchanged sections are preserved "
+            "byte-for-byte, stale content is removed, new content is added. If the mental model has no existing "
+            "content, or if the source_query has changed since the last refresh, delta mode falls back to a full "
+            "regeneration automatically."
+        ),
+    )
     refresh_after_consolidation: bool = Field(
         default=False,
         description="If true, refresh this mental model after observations consolidation (real-time mode)",
@@ -1526,6 +1604,27 @@ class MentalModelTrigger(BaseModel):
             "Supports nested and/or/not expressions for complex tag-based scoping."
         ),
     )
+    include_chunks: bool | None = Field(
+        default=None,
+        description=(
+            "Override whether the internal recall used during refresh returns raw chunk text. "
+            "None means use the bank/global config default (recall_include_chunks)."
+        ),
+    )
+    recall_max_tokens: int | None = Field(
+        default=None,
+        description=(
+            "Override the token budget for facts returned by the internal recall during refresh. "
+            "None means use the bank/global config default (recall_max_tokens)."
+        ),
+    )
+    recall_chunks_max_tokens: int | None = Field(
+        default=None,
+        description=(
+            "Override the token budget for raw chunks returned by the internal recall during refresh. "
+            "None means use the bank/global config default (recall_chunks_max_tokens)."
+        ),
+    )
 
     @field_validator("fact_types")
     @classmethod
@@ -1555,6 +1654,14 @@ class MentalModelResponse(BaseModel):
         default=None,
         description="Full reflect API response payload including based_on facts and observations",
     )
+    is_stale: bool | None = Field(
+        default=None,
+        description=(
+            "True when new memories matching this mental model's tag/fact_type scope have been "
+            "ingested since last_refreshed_at, or consolidation has pending items. Only populated "
+            "when detail=full."
+        ),
+    )
 
 
 class MentalModelListResponse(BaseModel):
@@ -1673,6 +1780,61 @@ class BankTemplateConfig(BaseModel):
     entities_allow_free_form: bool | None = Field(
         default=None, description="Allow entities outside the label vocabulary"
     )
+    retain_default_strategy: str | None = Field(
+        default=None, description="Name of the default retain strategy (key into retain_strategies map)"
+    )
+    retain_strategies: dict | None = Field(
+        default=None, description="Map of retain strategy name to per-strategy config dict"
+    )
+    retain_chunk_batch_size: int | None = Field(
+        default=None, description="Max chunks per streaming batch (0 disables batching)"
+    )
+    mcp_enabled_tools: list[str] | None = Field(
+        default=None, description="MCP tool allowlist for this bank (None = all tools)"
+    )
+    consolidation_llm_batch_size: int | None = Field(
+        default=None, description="LLM batch size for observation consolidation"
+    )
+    consolidation_source_facts_max_tokens: int | None = Field(
+        default=None, description="Max tokens of source facts per consolidation batch"
+    )
+    consolidation_source_facts_max_tokens_per_observation: int | None = Field(
+        default=None, description="Max tokens of source facts per observation"
+    )
+    max_observations_per_scope: int | None = Field(
+        default=None, description="Max observations to retain per consolidation scope"
+    )
+    reflect_source_facts_max_tokens: int | None = Field(
+        default=None, description="Max tokens of source facts per reflect call"
+    )
+    llm_gemini_safety_settings: list | None = Field(
+        default=None, description="Per-bank Gemini/VertexAI safety filter settings"
+    )
+    recall_budget_function: str | None = Field(
+        default=None, description="Recall budget mapping function: 'fixed' or 'adaptive'"
+    )
+    recall_budget_fixed_low: int | None = Field(
+        default=None, description="Fixed thinking_budget for budget=low (function='fixed')"
+    )
+    recall_budget_fixed_mid: int | None = Field(
+        default=None, description="Fixed thinking_budget for budget=mid (function='fixed')"
+    )
+    recall_budget_fixed_high: int | None = Field(
+        default=None, description="Fixed thinking_budget for budget=high (function='fixed')"
+    )
+    recall_budget_adaptive_low: float | None = Field(
+        default=None, description="Ratio of max_tokens for budget=low (function='adaptive')"
+    )
+    recall_budget_adaptive_mid: float | None = Field(
+        default=None, description="Ratio of max_tokens for budget=mid (function='adaptive')"
+    )
+    recall_budget_adaptive_high: float | None = Field(
+        default=None, description="Ratio of max_tokens for budget=high (function='adaptive')"
+    )
+    recall_budget_min: int | None = Field(default=None, description="Floor for the adaptive function (after clamping)")
+    recall_budget_max: int | None = Field(
+        default=None, description="Ceiling for the adaptive function (after clamping)"
+    )
 
     def get_config_updates(self) -> dict[str, Any]:
         """Return only the fields that were explicitly set (non-None)."""
@@ -2084,6 +2246,10 @@ class OperationStatusResponse(BaseModel):
     child_operations: list[ChildOperationStatus] | None = Field(
         default=None, description="Child operations for batch operations (if applicable)"
     )
+    task_payload: dict[str, Any] | None = Field(
+        default=None,
+        description="Raw task payload (params the operation was submitted with). Only populated when include_payload=true.",
+    )
 
 
 class AsyncOperationSubmitResponse(BaseModel):
@@ -2767,6 +2933,7 @@ def _register_routes(app: FastAPI):
         bank_id: str,
         type: str | None = None,
         q: str | None = None,
+        consolidation_state: str | None = None,
         limit: int = 100,
         offset: int = 0,
         request_context: RequestContext = Depends(get_request_context),
@@ -2781,6 +2948,8 @@ def _register_routes(app: FastAPI):
             bank_id: Memory Bank ID (from path)
             type: Filter by fact type (world, experience, opinion)
             q: Search query for full-text search (searches text and context)
+            consolidation_state: Filter by consolidation state for source memories
+                (world/experience). One of 'failed', 'pending', or 'done'.
             limit: Maximum number of results (default: 100)
             offset: Offset for pagination (default: 0)
         """
@@ -2789,11 +2958,14 @@ def _register_routes(app: FastAPI):
                 bank_id=bank_id,
                 fact_type=type,
                 search_query=q,
+                consolidation_state=consolidation_state,
                 limit=limit,
                 offset=offset,
                 request_context=request_context,
             )
             return data
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
         except OperationValidationError as e:
             raise HTTPException(status_code=e.status_code, detail=e.reason)
         except (AuthenticationError, HTTPException):
@@ -3248,8 +3420,10 @@ def _register_routes(app: FastAPI):
                 links_breakdown=links_breakdown,
                 pending_operations=ops.get("pending", 0),
                 failed_operations=ops.get("failed", 0),
+                operations_by_status=ops,
                 last_consolidated_at=stats["last_consolidated_at"],
                 pending_consolidation=stats["pending_consolidation"],
+                failed_consolidation=stats.get("failed_consolidation", 0),
                 total_observations=stats["total_observations"],
             )
         except OperationValidationError as e:
@@ -3263,6 +3437,35 @@ def _register_routes(app: FastAPI):
             logger.error(f"Error in /v1/default/banks/{bank_id}/stats: {error_detail}")
             raise HTTPException(status_code=500, detail=str(e))
 
+    @app.get(
+        "/v1/default/banks/{bank_id}/stats/memories-timeseries",
+        response_model=MemoriesTimeseriesResponse,
+        summary="Memory ingestion time-series",
+        description="Memories ingested over a period, bucketed by time and broken down by fact type.",
+        operation_id="get_memories_timeseries",
+        tags=["Banks"],
+    )
+    async def api_memories_timeseries(
+        bank_id: str,
+        period: str = "7d",
+        request_context: RequestContext = Depends(get_request_context),
+    ):
+        try:
+            data = await app.state.memory.get_memories_timeseries(
+                bank_id, period=period, request_context=request_context
+            )
+            return MemoriesTimeseriesResponse(**data)
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in /v1/default/banks/{bank_id}/stats/memories-timeseries: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
     @app.get(
         "/v1/default/banks/{bank_id}/entities",
         response_model=EntityListResponse,
@@ -3299,6 +3502,36 @@ def _register_routes(app: FastAPI):
             logger.error(f"Error in /v1/default/banks/{bank_id}/entities: {error_detail}")
             raise HTTPException(status_code=500, detail=str(e))
 
+    @app.get(
+        "/v1/default/banks/{bank_id}/entities/graph",
+        response_model=EntityGraphResponse,
+        summary="Get entity co-occurrence graph",
+        description="Return a graph of entities (nodes) and their co-occurrences (edges) for visualization.",
+        operation_id="get_entity_graph",
+        tags=["Entities"],
+    )
+    async def api_entity_graph(
+        bank_id: str,
+        limit: int = Query(default=1000, description="Maximum number of co-occurrence edges to return"),
+        min_count: int = Query(default=1, description="Minimum cooccurrence_count to include an edge"),
+        request_context: RequestContext = Depends(get_request_context),
+    ):
+        """Return entity co-occurrence graph for a bank."""
+        try:
+            return await app.state.memory.get_entity_graph(
+                bank_id, limit=limit, min_count=min_count, request_context=request_context
+            )
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in /v1/default/banks/{bank_id}/entities/graph: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
     @app.get(
         "/v1/default/banks/{bank_id}/entities/{entity_id}",
         response_model=EntityDetailResponse,
@@ -4168,7 +4401,13 @@ def _register_routes(app: FastAPI):
         tags=["Operations"],
     )
     async def api_get_operation_status(
-        bank_id: str, operation_id: str, request_context: RequestContext = Depends(get_request_context)
+        bank_id: str,
+        operation_id: str,
+        include_payload: bool = Query(
+            default=False,
+            description="Include the raw task payload (submission params) in the response. May be large.",
+        ),
+        request_context: RequestContext = Depends(get_request_context),
     ):
         """Get the status of an async operation."""
         try:
@@ -4178,7 +4417,9 @@ def _register_routes(app: FastAPI):
             except ValueError:
                 raise HTTPException(status_code=400, detail=f"Invalid operation_id format: {operation_id}")
 
-            result = await app.state.memory.get_operation_status(bank_id, operation_id, request_context=request_context)
+            result = await app.state.memory.get_operation_status(
+                bank_id, operation_id, request_context=request_context, include_payload=include_payload
+            )
             return OperationStatusResponse(**result)
         except OperationValidationError as e:
             raise HTTPException(status_code=e.status_code, detail=e.reason)

hindsight-api-slim/hindsight_api/config.py

@@ -238,6 +238,7 @@ ENV_RERANKER_LOCAL_BATCH_SIZE = "HINDSIGHT_API_RERANKER_LOCAL_BATCH_SIZE"
 ENV_RERANKER_TEI_URL = "HINDSIGHT_API_RERANKER_TEI_URL"
 ENV_RERANKER_TEI_BATCH_SIZE = "HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE"
 ENV_RERANKER_TEI_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT"
+ENV_RERANKER_TEI_HTTP_TIMEOUT = "HINDSIGHT_API_RERANKER_TEI_HTTP_TIMEOUT"
 ENV_RERANKER_MAX_CANDIDATES = "HINDSIGHT_API_RERANKER_MAX_CANDIDATES"
 ENV_RERANKER_FLASHRANK_MODEL = "HINDSIGHT_API_RERANKER_FLASHRANK_MODEL"
 ENV_RERANKER_FLASHRANK_CACHE_DIR = "HINDSIGHT_API_RERANKER_FLASHRANK_CACHE_DIR"
@@ -265,6 +266,7 @@ ENV_PORT = "HINDSIGHT_API_PORT"
 ENV_BASE_PATH = "HINDSIGHT_API_BASE_PATH"
 ENV_LOG_LEVEL = "HINDSIGHT_API_LOG_LEVEL"
 ENV_LOG_FORMAT = "HINDSIGHT_API_LOG_FORMAT"
+ENV_LOG_JSON_FIELDS = "HINDSIGHT_API_LOG_JSON_FIELDS"
 ENV_WORKERS = "HINDSIGHT_API_WORKERS"
 ENV_MCP_ENABLED = "HINDSIGHT_API_MCP_ENABLED"
 ENV_MCP_ENABLED_TOOLS = "HINDSIGHT_API_MCP_ENABLED_TOOLS"
@@ -333,12 +335,14 @@ ENV_FILE_DELETE_AFTER_RETAIN = "HINDSIGHT_API_FILE_DELETE_AFTER_RETAIN"
 # Observations settings (consolidated knowledge from facts)
 ENV_ENABLE_OBSERVATIONS = "HINDSIGHT_API_ENABLE_OBSERVATIONS"
 ENV_CONSOLIDATION_BATCH_SIZE = "HINDSIGHT_API_CONSOLIDATION_BATCH_SIZE"
+ENV_CONSOLIDATION_MAX_MEMORIES_PER_ROUND = "HINDSIGHT_API_CONSOLIDATION_MAX_MEMORIES_PER_ROUND"
 ENV_CONSOLIDATION_LLM_BATCH_SIZE = "HINDSIGHT_API_CONSOLIDATION_LLM_BATCH_SIZE"
 ENV_CONSOLIDATION_MAX_TOKENS = "HINDSIGHT_API_CONSOLIDATION_MAX_TOKENS"
 ENV_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS = "HINDSIGHT_API_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS"
 ENV_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION = (
     "HINDSIGHT_API_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION"
 )
+ENV_CONSOLIDATION_MAX_ATTEMPTS = "HINDSIGHT_API_CONSOLIDATION_MAX_ATTEMPTS"
 ENV_OBSERVATIONS_MISSION = "HINDSIGHT_API_OBSERVATIONS_MISSION"
 ENV_MAX_OBSERVATIONS_PER_SCOPE = "HINDSIGHT_API_MAX_OBSERVATIONS_PER_SCOPE"
 ENV_ENABLE_OBSERVATION_HISTORY = "HINDSIGHT_API_ENABLE_OBSERVATION_HISTORY"
@@ -387,6 +391,20 @@ ENV_REFLECT_MAX_CONTEXT_TOKENS = "HINDSIGHT_API_REFLECT_MAX_CONTEXT_TOKENS"
 ENV_REFLECT_WALL_TIMEOUT = "HINDSIGHT_API_REFLECT_WALL_TIMEOUT"
 ENV_REFLECT_MISSION = "HINDSIGHT_API_REFLECT_MISSION"
 ENV_REFLECT_SOURCE_FACTS_MAX_TOKENS = "HINDSIGHT_API_REFLECT_SOURCE_FACTS_MAX_TOKENS"
+ENV_RECALL_INCLUDE_CHUNKS = "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS"
+ENV_RECALL_MAX_TOKENS = "HINDSIGHT_API_RECALL_MAX_TOKENS"
+ENV_RECALL_CHUNKS_MAX_TOKENS = "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS"
+
+# Recall budget mapping (budget enum -> thinking_budget integer)
+ENV_RECALL_BUDGET_FUNCTION = "HINDSIGHT_API_RECALL_BUDGET_FUNCTION"
+ENV_RECALL_BUDGET_FIXED_LOW = "HINDSIGHT_API_RECALL_BUDGET_FIXED_LOW"
+ENV_RECALL_BUDGET_FIXED_MID = "HINDSIGHT_API_RECALL_BUDGET_FIXED_MID"
+ENV_RECALL_BUDGET_FIXED_HIGH = "HINDSIGHT_API_RECALL_BUDGET_FIXED_HIGH"
+ENV_RECALL_BUDGET_ADAPTIVE_LOW = "HINDSIGHT_API_RECALL_BUDGET_ADAPTIVE_LOW"
+ENV_RECALL_BUDGET_ADAPTIVE_MID = "HINDSIGHT_API_RECALL_BUDGET_ADAPTIVE_MID"
+ENV_RECALL_BUDGET_ADAPTIVE_HIGH = "HINDSIGHT_API_RECALL_BUDGET_ADAPTIVE_HIGH"
+ENV_RECALL_BUDGET_MIN = "HINDSIGHT_API_RECALL_BUDGET_MIN"
+ENV_RECALL_BUDGET_MAX = "HINDSIGHT_API_RECALL_BUDGET_MAX"
 
 # Audit log settings
 ENV_AUDIT_LOG_ENABLED = "HINDSIGHT_API_AUDIT_LOG_ENABLED"
@@ -432,7 +450,7 @@ DEFAULT_LLAMACPP_NO_GRAMMAR = False  # True = disable JSON grammar enforcement (
 DEFAULT_LLAMACPP_EXTRA_ARGS = None  # Space-separated extra CLI args for llama.cpp server
 
 DEFAULT_LLM_MAX_CONCURRENT = 32
-DEFAULT_LLM_MAX_RETRIES = 10  # Max retry attempts for LLM API calls
+DEFAULT_LLM_MAX_RETRIES = 3  # Max retry attempts for LLM API calls
 DEFAULT_LLM_INITIAL_BACKOFF = 1.0  # Initial backoff in seconds for retry exponential backoff
 DEFAULT_LLM_MAX_BACKOFF = 60.0  # Max backoff cap in seconds for retry exponential backoff
 DEFAULT_LLM_TIMEOUT = 120.0  # seconds
@@ -466,6 +484,7 @@ DEFAULT_RERANKER_LOCAL_BUCKET_BATCHING = False  # Length-sorted bucket batching:
 DEFAULT_RERANKER_LOCAL_BATCH_SIZE = 32  # Batch size for local reranker predict() calls
 DEFAULT_RERANKER_TEI_BATCH_SIZE = 128
 DEFAULT_RERANKER_TEI_MAX_CONCURRENT = 8
+DEFAULT_RERANKER_TEI_HTTP_TIMEOUT = 30.0  # HTTP timeout for TEI reranker requests (seconds)
 DEFAULT_RERANKER_MAX_CANDIDATES = 300
 DEFAULT_RERANKER_FLASHRANK_MODEL = "ms-marco-MiniLM-L-12-v2"  # Best balance of speed and quality
 DEFAULT_RERANKER_FLASHRANK_CACHE_DIR = None  # Use default cache directory
@@ -551,7 +570,11 @@ DEFAULT_FILE_DELETE_AFTER_RETAIN = True  # Delete file bytes after retain (saves
 DEFAULT_ENABLE_OBSERVATIONS = True  # Observations enabled by default
 DEFAULT_ENABLE_OBSERVATION_HISTORY = True  # Observation history tracking enabled by default
 DEFAULT_ENABLE_MENTAL_MODEL_HISTORY = True  # Mental model history tracking enabled by default
+DEFAULT_CONSOLIDATION_MAX_ATTEMPTS = 3  # Outer retry attempts for consolidation LLM batch calls
 DEFAULT_CONSOLIDATION_BATCH_SIZE = 50  # Memories to load per batch (internal memory optimization)
+DEFAULT_CONSOLIDATION_MAX_MEMORIES_PER_ROUND = (
+    100  # Max memories per consolidation round (0 = unlimited). Limits how long one bank holds a worker slot.
+)
 DEFAULT_CONSOLIDATION_LLM_BATCH_SIZE = 8  # Facts per LLM call (1 = no batching; >1 = batch mode)
 DEFAULT_CONSOLIDATION_MAX_TOKENS = 512  # Max tokens for recall when finding related observations
 DEFAULT_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS = (
@@ -587,6 +610,25 @@ DEFAULT_REFLECT_MAX_ITERATIONS = 10  # Max tool call iterations before forcing r
 DEFAULT_REFLECT_MAX_CONTEXT_TOKENS = 100_000  # Max accumulated context tokens before forcing final prompt
 DEFAULT_REFLECT_WALL_TIMEOUT = 300  # Wall-clock timeout in seconds for the entire reflect operation (5 minutes)
 DEFAULT_REFLECT_SOURCE_FACTS_MAX_TOKENS = -1  # Token budget for source facts in search_observations (-1 = disabled)
+DEFAULT_RECALL_INCLUDE_CHUNKS = True  # Whether internal recall (e.g. mental model refresh) returns raw chunks
+DEFAULT_RECALL_MAX_TOKENS = 2048  # Token budget for facts returned by internal recall
+DEFAULT_RECALL_CHUNKS_MAX_TOKENS = 1000  # Token budget for raw chunks returned by internal recall
+
+# Recall budget mapping
+# "fixed": thinking_budget = recall_budget_fixed_<level> (preserves legacy behavior)
+# "adaptive": thinking_budget = round(max_tokens * recall_budget_adaptive_<level>),
+#             clamped to [recall_budget_min, recall_budget_max]
+RECALL_BUDGET_FUNCTIONS = ("fixed", "adaptive")
+DEFAULT_RECALL_BUDGET_FUNCTION = "fixed"
+DEFAULT_RECALL_BUDGET_FIXED_LOW = 100
+DEFAULT_RECALL_BUDGET_FIXED_MID = 300
+DEFAULT_RECALL_BUDGET_FIXED_HIGH = 1000
+# Adaptive defaults chosen to roughly match fixed defaults at max_tokens=4096
+DEFAULT_RECALL_BUDGET_ADAPTIVE_LOW = 0.025
+DEFAULT_RECALL_BUDGET_ADAPTIVE_MID = 0.075
+DEFAULT_RECALL_BUDGET_ADAPTIVE_HIGH = 0.25
+DEFAULT_RECALL_BUDGET_MIN = 20  # Floor for the adaptive function
+DEFAULT_RECALL_BUDGET_MAX = 2000  # Ceiling for the adaptive function
 
 # Disposition defaults (None = not set, fall back to bank DB value or 3)
 DEFAULT_DISPOSITION_SKEPTICISM = None
@@ -649,6 +691,10 @@ class JsonFormatter(logging.Formatter):
         logging.CRITICAL: "CRITICAL",
     }
 
+    def __init__(self, allowed_fields: frozenset[str] | None = None):
+        super().__init__()
+        self._allowed_fields = allowed_fields
+
     def format(self, record: logging.LogRecord) -> str:
         log_entry = {
             "severity": self.SEVERITY_MAP.get(record.levelno, "DEFAULT"),
@@ -657,10 +703,20 @@ class JsonFormatter(logging.Formatter):
             "logger": record.name,
         }
 
+        # Lazy import to avoid circular dependency (engine imports from config).
+        from hindsight_api.engine.memory_engine import _current_schema
+
+        tenant = _current_schema.get()
+        if tenant:
+            log_entry["tenant"] = tenant
+
         # Add exception info if present
         if record.exc_info:
             log_entry["exception"] = self.formatException(record.exc_info)
 
+        if self._allowed_fields is not None:
+            log_entry = {k: v for k, v in log_entry.items() if k in self._allowed_fields}
+
         return json.dumps(log_entry)
 
 
@@ -681,6 +737,18 @@ def _validate_extraction_mode(mode: str) -> str:
     return mode_lower
 
 
+def _validate_recall_budget_function(function: str) -> str:
+    """Validate and normalize recall budget function."""
+    function_lower = function.lower()
+    if function_lower not in RECALL_BUDGET_FUNCTIONS:
+        logger.warning(
+            f"Invalid recall budget function '{function}', must be one of {RECALL_BUDGET_FUNCTIONS}. "
+            f"Defaulting to '{DEFAULT_RECALL_BUDGET_FUNCTION}'."
+        )
+        return DEFAULT_RECALL_BUDGET_FUNCTION
+    return function_lower
+
+
 def _get_default_model_for_provider(provider: str) -> str:
     """Get the default model for a given provider."""
     return PROVIDER_DEFAULT_MODELS.get(provider.lower(), DEFAULT_LLM_MODEL)
@@ -820,6 +888,7 @@ class HindsightConfig:
     reranker_tei_url: str | None
     reranker_tei_batch_size: int
     reranker_tei_max_concurrent: int
+    reranker_tei_http_timeout: float
     reranker_max_candidates: int
     reranker_cohere_api_key: str | None
     reranker_cohere_model: str
@@ -849,6 +918,7 @@ class HindsightConfig:
     base_path: str
     log_level: str
     log_format: str
+    log_json_fields: list[str] | None  # None = all fields; explicit list = allowlist
     mcp_enabled: bool
     mcp_enabled_tools: list[str] | None  # None = all tools; explicit list = allowlist
     mcp_stateless: bool  # True = stateless HTTP (POST-only); False = stateful (supports GET/SSE)
@@ -907,10 +977,12 @@ class HindsightConfig:
     enable_observation_history: bool
     enable_mental_model_history: bool
     consolidation_batch_size: int
+    consolidation_max_memories_per_round: int
     consolidation_llm_batch_size: int
     consolidation_max_tokens: int
     consolidation_source_facts_max_tokens: int
     consolidation_source_facts_max_tokens_per_observation: int
+    consolidation_max_attempts: int
     observations_mission: str | None
     max_observations_per_scope: int
 
@@ -925,6 +997,25 @@ class HindsightConfig:
     reflect_mission: str | None
     reflect_source_facts_max_tokens: int
 
+    # Recall settings (used by internal recall, e.g. during mental model refresh)
+    recall_include_chunks: bool
+    recall_max_tokens: int
+    recall_chunks_max_tokens: int
+
+    # Recall budget mapping: how the Budget enum (LOW/MID/HIGH) maps to thinking_budget integer.
+    # function="fixed": use the recall_budget_fixed_* values directly (legacy behavior).
+    # function="adaptive": compute round(max_tokens * recall_budget_adaptive_*),
+    #                      clamped to [recall_budget_min, recall_budget_max].
+    recall_budget_function: str
+    recall_budget_fixed_low: int
+    recall_budget_fixed_mid: int
+    recall_budget_fixed_high: int
+    recall_budget_adaptive_low: float
+    recall_budget_adaptive_mid: float
+    recall_budget_adaptive_high: float
+    recall_budget_min: int
+    recall_budget_max: int
+
     # Disposition settings (hierarchical - can be overridden per bank; None = fall back to DB)
     disposition_skepticism: int | None
     disposition_literalism: int | None
@@ -1031,6 +1122,7 @@ class HindsightConfig:
         # Consolidation settings
         "enable_observations",
         "consolidation_llm_batch_size",
+        "consolidation_max_memories_per_round",
         "consolidation_source_facts_max_tokens",
         "consolidation_source_facts_max_tokens_per_observation",
         "observations_mission",
@@ -1038,6 +1130,20 @@ class HindsightConfig:
         # Reflect settings
         "reflect_mission",
         "reflect_source_facts_max_tokens",
+        # Recall settings (used by internal recall, e.g. mental model refresh)
+        "recall_include_chunks",
+        "recall_max_tokens",
+        "recall_chunks_max_tokens",
+        # Recall budget mapping (Budget enum -> thinking_budget integer)
+        "recall_budget_function",
+        "recall_budget_fixed_low",
+        "recall_budget_fixed_mid",
+        "recall_budget_fixed_high",
+        "recall_budget_adaptive_low",
+        "recall_budget_adaptive_mid",
+        "recall_budget_adaptive_high",
+        "recall_budget_min",
+        "recall_budget_max",
         # Disposition settings
         "disposition_skepticism",
         "disposition_literalism",
@@ -1338,6 +1444,9 @@ class HindsightConfig:
             reranker_tei_max_concurrent=int(
                 os.getenv(ENV_RERANKER_TEI_MAX_CONCURRENT, str(DEFAULT_RERANKER_TEI_MAX_CONCURRENT))
             ),
+            reranker_tei_http_timeout=float(
+                os.getenv(ENV_RERANKER_TEI_HTTP_TIMEOUT, str(DEFAULT_RERANKER_TEI_HTTP_TIMEOUT))
+            ),
             reranker_max_candidates=int(os.getenv(ENV_RERANKER_MAX_CANDIDATES, str(DEFAULT_RERANKER_MAX_CANDIDATES))),
             # Cohere reranker (with backward-compatible fallback to shared API key)
             reranker_cohere_api_key=os.getenv(ENV_RERANKER_COHERE_API_KEY) or os.getenv(ENV_COHERE_API_KEY),
@@ -1382,6 +1491,7 @@ class HindsightConfig:
             base_path=os.getenv(ENV_BASE_PATH, DEFAULT_BASE_PATH),
             log_level=os.getenv(ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL),
             log_format=os.getenv(ENV_LOG_FORMAT, DEFAULT_LOG_FORMAT).lower(),
+            log_json_fields=_parse_str_list(os.getenv(ENV_LOG_JSON_FIELDS, "")) or None,
             mcp_enabled=os.getenv(ENV_MCP_ENABLED, str(DEFAULT_MCP_ENABLED)).lower() == "true",
             mcp_enabled_tools=[t.strip() for t in os.getenv(ENV_MCP_ENABLED_TOOLS).split(",") if t.strip()]
             if os.getenv(ENV_MCP_ENABLED_TOOLS)
@@ -1474,6 +1584,12 @@ class HindsightConfig:
             consolidation_batch_size=int(
                 os.getenv(ENV_CONSOLIDATION_BATCH_SIZE, str(DEFAULT_CONSOLIDATION_BATCH_SIZE))
             ),
+            consolidation_max_memories_per_round=int(
+                os.getenv(
+                    ENV_CONSOLIDATION_MAX_MEMORIES_PER_ROUND,
+                    str(DEFAULT_CONSOLIDATION_MAX_MEMORIES_PER_ROUND),
+                )
+            ),
             consolidation_llm_batch_size=int(
                 os.getenv(ENV_CONSOLIDATION_LLM_BATCH_SIZE, str(DEFAULT_CONSOLIDATION_LLM_BATCH_SIZE))
             ),
@@ -1489,6 +1605,9 @@ class HindsightConfig:
                     str(DEFAULT_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION),
                 )
             ),
+            consolidation_max_attempts=int(
+                os.getenv(ENV_CONSOLIDATION_MAX_ATTEMPTS, str(DEFAULT_CONSOLIDATION_MAX_ATTEMPTS))
+            ),
             observations_mission=os.getenv(ENV_OBSERVATIONS_MISSION) or DEFAULT_OBSERVATIONS_MISSION,
             max_observations_per_scope=int(
                 os.getenv(ENV_MAX_OBSERVATIONS_PER_SCOPE, str(DEFAULT_MAX_OBSERVATIONS_PER_SCOPE))
@@ -1523,6 +1642,31 @@ class HindsightConfig:
             reflect_source_facts_max_tokens=int(
                 os.getenv(ENV_REFLECT_SOURCE_FACTS_MAX_TOKENS, str(DEFAULT_REFLECT_SOURCE_FACTS_MAX_TOKENS))
             ),
+            recall_include_chunks=os.getenv(ENV_RECALL_INCLUDE_CHUNKS, str(DEFAULT_RECALL_INCLUDE_CHUNKS)).lower()
+            in ("true", "1", "yes"),
+            recall_max_tokens=int(os.getenv(ENV_RECALL_MAX_TOKENS, str(DEFAULT_RECALL_MAX_TOKENS))),
+            recall_chunks_max_tokens=int(
+                os.getenv(ENV_RECALL_CHUNKS_MAX_TOKENS, str(DEFAULT_RECALL_CHUNKS_MAX_TOKENS))
+            ),
+            recall_budget_function=_validate_recall_budget_function(
+                os.getenv(ENV_RECALL_BUDGET_FUNCTION, DEFAULT_RECALL_BUDGET_FUNCTION)
+            ),
+            recall_budget_fixed_low=int(os.getenv(ENV_RECALL_BUDGET_FIXED_LOW, str(DEFAULT_RECALL_BUDGET_FIXED_LOW))),
+            recall_budget_fixed_mid=int(os.getenv(ENV_RECALL_BUDGET_FIXED_MID, str(DEFAULT_RECALL_BUDGET_FIXED_MID))),
+            recall_budget_fixed_high=int(
+                os.getenv(ENV_RECALL_BUDGET_FIXED_HIGH, str(DEFAULT_RECALL_BUDGET_FIXED_HIGH))
+            ),
+            recall_budget_adaptive_low=float(
+                os.getenv(ENV_RECALL_BUDGET_ADAPTIVE_LOW, str(DEFAULT_RECALL_BUDGET_ADAPTIVE_LOW))
+            ),
+            recall_budget_adaptive_mid=float(
+                os.getenv(ENV_RECALL_BUDGET_ADAPTIVE_MID, str(DEFAULT_RECALL_BUDGET_ADAPTIVE_MID))
+            ),
+            recall_budget_adaptive_high=float(
+                os.getenv(ENV_RECALL_BUDGET_ADAPTIVE_HIGH, str(DEFAULT_RECALL_BUDGET_ADAPTIVE_HIGH))
+            ),
+            recall_budget_min=int(os.getenv(ENV_RECALL_BUDGET_MIN, str(DEFAULT_RECALL_BUDGET_MIN))),
+            recall_budget_max=int(os.getenv(ENV_RECALL_BUDGET_MAX, str(DEFAULT_RECALL_BUDGET_MAX))),
             # Disposition settings (None = fall back to DB value)
             disposition_skepticism=int(os.getenv(ENV_DISPOSITION_SKEPTICISM))
             if os.getenv(ENV_DISPOSITION_SKEPTICISM)
@@ -1613,7 +1757,8 @@ class HindsightConfig:
         handler.setLevel(self.get_python_log_level())
 
         if self.log_format == "json":
-            handler.setFormatter(JsonFormatter())
+            allowed = frozenset(self.log_json_fields) if self.log_json_fields else None
+            handler.setFormatter(JsonFormatter(allowed_fields=allowed))
         else:
             handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s"))
 

hindsight-api-slim/hindsight_api/config_resolver.py

@@ -15,7 +15,12 @@ from typing import Any
 
 import asyncpg
 
-from hindsight_api.config import HindsightConfig, _get_raw_config, normalize_config_dict
+from hindsight_api.config import (
+    RECALL_BUDGET_FUNCTIONS,
+    HindsightConfig,
+    _get_raw_config,
+    normalize_config_dict,
+)
 from hindsight_api.engine.memory_engine import fq_table
 from hindsight_api.extensions.tenant import TenantExtension
 from hindsight_api.models import RequestContext
@@ -256,6 +261,9 @@ class ConfigResolver:
                     "Strategy names must not be empty strings. Remove entries with empty names before saving."
                 )
 
+        # Validate recall budget fields
+        _validate_recall_budget_updates(normalized_updates)
+
         # Merge with existing config (JSONB || operator)
         async with self.pool.acquire() as conn:
             await conn.execute(
@@ -292,6 +300,53 @@ class ConfigResolver:
         logger.info(f"Reset bank config for {bank_id} to defaults")
 
 
+_RECALL_BUDGET_FIXED_KEYS = (
+    "recall_budget_fixed_low",
+    "recall_budget_fixed_mid",
+    "recall_budget_fixed_high",
+)
+_RECALL_BUDGET_ADAPTIVE_KEYS = (
+    "recall_budget_adaptive_low",
+    "recall_budget_adaptive_mid",
+    "recall_budget_adaptive_high",
+)
+
+
+def _validate_recall_budget_updates(updates: dict[str, Any]) -> None:
+    """Validate recall budget config updates. Raises ValueError on invalid input."""
+    if "recall_budget_function" in updates:
+        function = updates["recall_budget_function"]
+        if not isinstance(function, str) or function.lower() not in RECALL_BUDGET_FUNCTIONS:
+            raise ValueError(
+                f"recall_budget_function must be one of {sorted(RECALL_BUDGET_FUNCTIONS)}, got {function!r}"
+            )
+
+    for key in _RECALL_BUDGET_FIXED_KEYS:
+        if key in updates:
+            value = updates[key]
+            if not isinstance(value, int) or isinstance(value, bool) or value < 1:
+                raise ValueError(f"{key} must be a positive integer, got {value!r}")
+
+    for key in _RECALL_BUDGET_ADAPTIVE_KEYS:
+        if key in updates:
+            value = updates[key]
+            if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
+                raise ValueError(f"{key} must be a positive number, got {value!r}")
+
+    for key in ("recall_budget_min", "recall_budget_max"):
+        if key in updates:
+            value = updates[key]
+            if not isinstance(value, int) or isinstance(value, bool) or value < 1:
+                raise ValueError(f"{key} must be a positive integer, got {value!r}")
+
+    if "recall_budget_min" in updates and "recall_budget_max" in updates:
+        if updates["recall_budget_min"] > updates["recall_budget_max"]:
+            raise ValueError(
+                f"recall_budget_min ({updates['recall_budget_min']}) must be <= "
+                f"recall_budget_max ({updates['recall_budget_max']})"
+            )
+
+
 def apply_strategy(config: HindsightConfig, strategy_name: str) -> HindsightConfig:
     """
     Apply a named retain strategy's overrides on top of a resolved config.

hindsight-api-slim/hindsight_api/engine/consolidation/consolidator.py

@@ -42,6 +42,34 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+async def _filter_live_source_memories(
+    conn: "Connection",
+    bank_id: str,
+    source_memory_ids: list[uuid.UUID],
+) -> list[uuid.UUID]:
+    """Return only the source memory ids that still exist in the bank.
+
+    Uses FOR SHARE to block concurrent deletes from removing a row between the
+    check and the subsequent insert/update. Combined with the delete path running
+    its stale-observation sweep *after* deleting the source row, this closes the
+    race window where consolidation would otherwise produce an orphan observation.
+    """
+    if not source_memory_ids:
+        return []
+    rows = await conn.fetch(
+        f"""
+        SELECT id
+        FROM {fq_table("memory_units")}
+        WHERE id = ANY($1::uuid[]) AND bank_id = $2
+        FOR SHARE
+        """,
+        source_memory_ids,
+        bank_id,
+    )
+    live = {row["id"] for row in rows}
+    return [mid for mid in source_memory_ids if mid in live]
+
+
 class _CreateAction(BaseModel):
     text: str
     source_fact_ids: list[str]  # memory UUIDs from the NEW FACTS list
@@ -219,6 +247,7 @@ async def run_consolidation_job(
 
     perf = ConsolidationPerfLog(bank_id)
     max_memories_per_batch = config.consolidation_batch_size
+    max_memories_per_round = config.consolidation_max_memories_per_round
     llm_batch_size = max(1, config.consolidation_llm_batch_size)
 
     # Check if consolidation is enabled
@@ -281,8 +310,17 @@ async def run_consolidation_job(
     # Track all unique tags from consolidated memories for mental model refresh filtering
     consolidated_tags: set[str] = set()
 
+    round_limit_enabled = max_memories_per_round > 0
+    round_remaining = max_memories_per_round if round_limit_enabled else float("inf")
+    hit_round_limit = False
+
     llm_batch_num = 0
     while True:
+        # Cap fetch size by remaining round budget
+        fetch_limit = (
+            min(max_memories_per_batch, int(round_remaining)) if round_limit_enabled else max_memories_per_batch
+        )
+
         # Fetch next batch of unconsolidated memories
         async with pool.acquire() as conn:
             t0 = time.time()
@@ -299,7 +337,7 @@ async def run_consolidation_job(
                 LIMIT $2
                 """,
                 bank_id,
-                max_memories_per_batch,
+                fetch_limit,
             )
             perf.record_timing("fetch_memories", time.time() - t0)
 
@@ -524,6 +562,25 @@ async def run_consolidation_job(
                 f" | avg={llm_batch_time / len(llm_batch):.3f}s/memory"
             )
 
+        # Update round budget after processing this DB fetch batch
+        if round_limit_enabled:
+            round_remaining -= len(memories)
+            if round_remaining <= 0:
+                hit_round_limit = True
+                break
+
+    # Re-submit consolidation if we hit the round limit and there's likely more work
+    if hit_round_limit:
+        remaining = total_count - stats["memories_processed"]
+        logger.info(
+            f"[CONSOLIDATION] bank={bank_id} hit round limit of {max_memories_per_round} memories,"
+            f" ~{remaining} remaining. Re-queuing consolidation."
+        )
+        try:
+            await memory_engine.submit_async_consolidation(bank_id=bank_id, request_context=request_context)
+        except Exception as e:
+            logger.warning(f"[CONSOLIDATION] bank={bank_id} failed to re-queue consolidation: {e}")
+
     # Build summary
     perf.log(
         f"[3] Results: {stats['memories_processed']} memories -> "
@@ -552,16 +609,21 @@ async def run_consolidation_job(
     if timing_parts:
         perf.log(f"[4] Timing breakdown: {', '.join(timing_parts)}")
 
-    # Trigger mental model refreshes for models with refresh_after_consolidation=true
-    # SECURITY: Only refresh mental models with matching tags (or all if no tags were consolidated)
-    mental_models_refreshed = await _trigger_mental_model_refreshes(
-        memory_engine=memory_engine,
-        bank_id=bank_id,
-        request_context=request_context,
-        consolidated_tags=list(consolidated_tags) if consolidated_tags else None,
-        perf=perf,
-    )
-    stats["mental_models_refreshed"] = mental_models_refreshed
+    # Trigger mental model refreshes only on the final round (when all memories are processed).
+    # If we hit the round limit and re-queued, skip MM refresh — the next round will handle it.
+    if hit_round_limit:
+        stats["mental_models_refreshed"] = 0
+        logger.info(f"[CONSOLIDATION] bank={bank_id} skipping mental model refresh (round limit hit, re-queued)")
+    else:
+        # SECURITY: Only refresh mental models with matching tags (or all if no tags were consolidated)
+        mental_models_refreshed = await _trigger_mental_model_refreshes(
+            memory_engine=memory_engine,
+            bank_id=bank_id,
+            request_context=request_context,
+            consolidated_tags=list(consolidated_tags) if consolidated_tags else None,
+            perf=perf,
+        )
+        stats["mental_models_refreshed"] = mental_models_refreshed
 
     perf.flush()
 
@@ -593,17 +655,15 @@ async def _trigger_mental_model_refreshes(
     """
     pool = memory_engine._pool
 
-    # Find mental models with refresh_after_consolidation=true
-    # SECURITY: Control which mental models get refreshed based on tags
+    # Find mental models with refresh_after_consolidation=true that are actually stale.
+    # The tag filter on the SELECT enforces the security boundary (never look outside the
+    # relevant tag scope); compute_mental_model_is_stale then verifies that new memories
+    # in the MM's scope really were ingested since its last refresh.
     async with pool.acquire() as conn:
         if consolidated_tags:
-            # Tagged memories were consolidated - refresh:
-            # 1. Mental models with overlapping tags (security boundary)
-            # 2. Untagged mental models (they're "global" and available to all contexts)
-            # DO NOT refresh mental models with different tags
-            rows = await conn.fetch(
+            candidates = await conn.fetch(
                 f"""
-                SELECT id, name, tags
+                SELECT id, name, tags, last_refreshed_at, trigger
                 FROM {fq_table("mental_models")}
                 WHERE bank_id = $1
                   AND (trigger->>'refresh_after_consolidation')::boolean = true
@@ -616,11 +676,9 @@ async def _trigger_mental_model_refreshes(
                 consolidated_tags,
             )
         else:
-            # Untagged memories were consolidated - only refresh untagged mental models
-            # SECURITY: Tagged mental models are NOT refreshed when untagged memories are consolidated
-            rows = await conn.fetch(
+            candidates = await conn.fetch(
                 f"""
-                SELECT id, name, tags
+                SELECT id, name, tags, last_refreshed_at, trigger
                 FROM {fq_table("mental_models")}
                 WHERE bank_id = $1
                   AND (trigger->>'refresh_after_consolidation')::boolean = true
@@ -629,6 +687,11 @@ async def _trigger_mental_model_refreshes(
                 bank_id,
             )
 
+        rows = []
+        for candidate in candidates:
+            if await memory_engine.compute_mental_model_is_stale(conn, bank_id, candidate):
+                rows.append(candidate)
+
     if not rows:
         return 0
 
@@ -889,6 +952,15 @@ async def _execute_update_action(
         logger.debug(f"Update skipped: observation {observation_id} not found in recall results")
         return
 
+    live_source_memory_ids = await _filter_live_source_memories(conn, bank_id, source_memory_ids)
+    if not live_source_memory_ids:
+        logger.debug(
+            f"Update skipped: all {len(source_memory_ids)} source memories for observation "
+            f"{observation_id} were deleted concurrently"
+        )
+        return
+    source_memory_ids = live_source_memory_ids
+
     from ...config import get_config
 
     history_entry = {
@@ -1131,11 +1203,13 @@ async def _consolidate_batch_with_llm(
     memories: list[dict[str, Any]],
     union_observations: "list[MemoryFact]",
     union_source_facts: "dict[str, MemoryFact]",
-    config: Any = None,
+    config: Any,
     remaining_observation_slots: int | None = None,
     max_observations_per_scope: int = -1,
 ) -> _BatchLLMResult:
     """Single LLM call for a batch of facts against a pooled set of observations."""
+    if config is None:
+        raise ValueError("config is required for _consolidate_batch_with_llm")
     if union_observations:
         obs_list = _build_observations_for_llm(union_observations, union_source_facts)
         observations_text = json.dumps(obs_list, indent=2)
@@ -1172,8 +1246,7 @@ async def _consolidate_batch_with_llm(
                 f"(out of {max_observations_per_scope}). Prefer UPDATE over CREATE when possible."
             )
 
-    observations_mission = config.observations_mission if config is not None else None
-    prompt_template = build_batch_consolidation_prompt(observations_mission, observation_capacity_note)
+    prompt_template = build_batch_consolidation_prompt(config.observations_mission, observation_capacity_note)
     prompt = prompt_template.format(
         facts_text=facts_lines,
         observations_text=observations_text,
@@ -1182,15 +1255,29 @@ async def _consolidate_batch_with_llm(
     # Use a constrained response model when observation limit is active
     response_model = _build_response_model(max_creates=remaining_observation_slots)
 
-    max_attempts = 3
+    max_attempts = config.consolidation_max_attempts
+    inner_max_retries = config.consolidation_llm_max_retries
     last_exc: Exception | None = None
+    # Pre-compute a stable identifier set for the batch so failure logs name the
+    # exact memories whose consolidation is failing — without this, an opaque
+    # "LLM batch call failed" line gives operators no way to find the offending
+    # input until adaptive bisection narrows the batch down to a single memory.
+    memory_ids = [str(m.get("id")) for m in memories]
+    if len(memory_ids) <= 5:
+        ids_label = ", ".join(memory_ids)
+    else:
+        ids_label = f"{', '.join(memory_ids[:3])}, ... +{len(memory_ids) - 3} more"
+    batch_label = f"{len(memory_ids)} memories [{ids_label}]"
     for attempt in range(1, max_attempts + 1):
         try:
-            response: _ConsolidationBatchResponse = await llm_config.call(
-                messages=[{"role": "user", "content": prompt}],
-                response_format=response_model,
-                scope="consolidation",
-            )
+            call_kwargs: dict[str, Any] = {
+                "messages": [{"role": "user", "content": prompt}],
+                "response_format": response_model,
+                "scope": "consolidation",
+            }
+            if inner_max_retries is not None:
+                call_kwargs["max_retries"] = inner_max_retries
+            response: _ConsolidationBatchResponse = await llm_config.call(**call_kwargs)
             # Defensive truncation: some LLM providers may not enforce JSON schema max_length
             creates = response.creates
             if remaining_observation_slots is not None and remaining_observation_slots >= 0:
@@ -1209,10 +1296,13 @@ async def _consolidate_batch_with_llm(
             )
         except Exception as exc:
             last_exc = exc
-            logger.warning(f"[CONSOLIDATION] LLM batch call failed (attempt {attempt}/{max_attempts}): {exc}")
+            logger.warning(
+                f"[CONSOLIDATION] LLM batch call failed (attempt {attempt}/{max_attempts}) for {batch_label}: {exc}"
+            )
 
     logger.error(
-        f"[CONSOLIDATION] LLM batch call failed after {max_attempts} attempts, skipping batch. Last error: {last_exc}"
+        f"[CONSOLIDATION] LLM batch call failed after {max_attempts} attempts for {batch_label}, "
+        f"skipping batch. Last error: {last_exc}"
     )
     return _BatchLLMResult(obs_count=len(union_observations), prompt_chars=len(prompt), failed=True)
 
@@ -1231,6 +1321,12 @@ async def _create_observation_directly(
     perf: ConsolidationPerfLog | None = None,
 ) -> dict[str, Any]:
     """Create an observation from one or more source memories with pre-processed text."""
+    live_source_memory_ids = await _filter_live_source_memories(conn, bank_id, source_memory_ids)
+    if not live_source_memory_ids:
+        logger.debug(f"Create skipped: all {len(source_memory_ids)} source memories were deleted concurrently")
+        return {"action": "skipped", "reason": "sources_deleted"}
+    source_memory_ids = live_source_memory_ids
+
     # Generate embedding for the observation (convert to string for pgvector)
     t0 = time.time()
     embeddings = await embedding_utils.generate_embeddings_batch(memory_engine.embeddings, [observation_text])

hindsight-api-slim/hindsight_api/engine/cross_encoder.py

@@ -33,6 +33,7 @@ from ..config import (
     DEFAULT_RERANKER_SILICONFLOW_BASE_URL,
     DEFAULT_RERANKER_SILICONFLOW_MODEL,
     DEFAULT_RERANKER_TEI_BATCH_SIZE,
+    DEFAULT_RERANKER_TEI_HTTP_TIMEOUT,
     DEFAULT_RERANKER_TEI_MAX_CONCURRENT,
     DEFAULT_RERANKER_ZEROENTROPY_MODEL,
     ENV_RERANKER_COHERE_API_KEY,
@@ -48,6 +49,7 @@ from ..config import (
     ENV_RERANKER_PROVIDER,
     ENV_RERANKER_SILICONFLOW_API_KEY,
     ENV_RERANKER_TEI_BATCH_SIZE,
+    ENV_RERANKER_TEI_HTTP_TIMEOUT,
     ENV_RERANKER_TEI_MAX_CONCURRENT,
     ENV_RERANKER_TEI_URL,
     ENV_RERANKER_ZEROENTROPY_API_KEY,
@@ -1282,6 +1284,7 @@ class JinaMLXCrossEncoder(CrossEncoderModel):
     def _load_model(self) -> None:
         """Download (if needed) and load the MLX reranker. Runs in a thread."""
         import os
+        import threading
 
         from huggingface_hub import snapshot_download
 
@@ -1297,6 +1300,10 @@ class JinaMLXCrossEncoder(CrossEncoderModel):
             model_path=model_path,
             projector_path=os.path.join(model_path, "projector.safetensors"),
         )
+        # MLX Metal GPU ops are not thread-safe — concurrent calls to
+        # Device::end_encoding() crash with SIGSEGV (NULL deref).
+        # Serialize all reranker inference through this lock.
+        self._mlx_lock = threading.Lock()
         logger.info("Reranker: jina-mlx provider initialized")
 
     def _predict_sync(self, pairs: list[tuple[str, str]]) -> list[float]:
@@ -1310,13 +1317,14 @@ class JinaMLXCrossEncoder(CrossEncoderModel):
 
         all_scores = [0.0] * len(pairs)
 
-        for query, indexed_docs in query_groups.items():
-            docs = [doc for _, doc in indexed_docs]
-            indices = [idx for idx, _ in indexed_docs]
-            results = self._reranker.rerank(query, docs)
-            for result in results:
-                original_idx = result["index"]
-                all_scores[indices[original_idx]] = result["relevance_score"]
+        with self._mlx_lock:
+            for query, indexed_docs in query_groups.items():
+                docs = [doc for _, doc in indexed_docs]
+                indices = [idx for idx, _ in indexed_docs]
+                results = self._reranker.rerank(query, docs)
+                for result in results:
+                    original_idx = result["index"]
+                    all_scores[indices[original_idx]] = result["relevance_score"]
 
         return all_scores
 
@@ -1506,6 +1514,7 @@ def create_cross_encoder_from_env() -> CrossEncoderModel:
             raise ValueError(f"{ENV_RERANKER_TEI_URL} is required when {ENV_RERANKER_PROVIDER} is 'tei'")
         return RemoteTEICrossEncoder(
             base_url=url,
+            timeout=config.reranker_tei_http_timeout,
             batch_size=config.reranker_tei_batch_size,
             max_concurrent=config.reranker_tei_max_concurrent,
         )

hindsight-api-slim/hindsight_api/engine/providers/gemini_llm.py

@@ -175,7 +175,7 @@ class GeminiLLM(LLMInterface):
         Args:
             messages: List of message dicts with 'role' and 'content'.
             response_format: Optional Pydantic model for structured output.
-            max_completion_tokens: Maximum tokens in response (not supported by Gemini).
+            max_completion_tokens: Maximum tokens in response (mapped to Gemini's max_output_tokens).
             temperature: Sampling temperature (0.0-2.0).
             scope: Scope identifier for tracking.
             max_retries: Maximum retry attempts.
@@ -227,6 +227,11 @@ class GeminiLLM(LLMInterface):
             config_kwargs["response_schema"] = response_format
         if temperature is not None:
             config_kwargs["temperature"] = temperature
+        # Gemini's equivalent of OpenAI-style max_completion_tokens is max_output_tokens.
+        # Without it the model can produce arbitrarily long responses, ignoring the
+        # caller's intended cap (e.g. mental_models max_tokens during refresh).
+        if max_completion_tokens is not None:
+            config_kwargs["max_output_tokens"] = max_completion_tokens
 
         # Apply safety settings: context var (per-request bank override) takes precedence over instance default
         effective_safety_settings = _safety_settings_ctx.get()
@@ -401,7 +406,7 @@ class GeminiLLM(LLMInterface):
         Args:
             messages: List of message dicts. Can include tool results with role='tool'.
             tools: List of tool definitions in OpenAI format.
-            max_completion_tokens: Maximum tokens (not supported by Gemini).
+            max_completion_tokens: Maximum tokens (mapped to Gemini's max_output_tokens).
             temperature: Sampling temperature.
             scope: Scope identifier for tracking.
             max_retries: Maximum retry attempts.
@@ -493,6 +498,10 @@ class GeminiLLM(LLMInterface):
             config_kwargs["system_instruction"] = system_instruction
         if temperature is not None:
             config_kwargs["temperature"] = temperature
+        # See note in `call`: Gemini's max_output_tokens is the equivalent of
+        # OpenAI-style max_completion_tokens.
+        if max_completion_tokens is not None:
+            config_kwargs["max_output_tokens"] = max_completion_tokens
 
         # Map OpenAI-style tool_choice to Gemini FunctionCallingConfig
         if tool_choice == "required":

hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py

@@ -60,6 +60,32 @@ def _strip_code_fences(content: str) -> str:
         return content
 
 
+def _summarize_status_error(e: APIStatusError, body_max: int = 400) -> str:
+    """Render an APIStatusError with status code + truncated response body.
+
+    Without this, retry loops only log "API error after N attempts" with the
+    bare exception message — losing the provider's actual error payload, which
+    is the only thing that explains *why* a request failed (rate limit reason,
+    invalid tool schema, model overloaded, etc.).
+    """
+    body: Any = getattr(e, "body", None)
+    if body is None:
+        try:
+            body = e.response.text
+        except Exception:
+            body = None
+    if isinstance(body, (dict, list)):
+        try:
+            body_str = json.dumps(body, default=str)
+        except Exception:
+            body_str = str(body)
+    else:
+        body_str = str(body or "").strip()
+    if len(body_str) > body_max:
+        body_str = body_str[:body_max] + "...TRUNCATED"
+    return f"HTTP {e.status_code}: {body_str or '<no body>'}"
+
+
 class OpenAICompatibleLLM(LLMInterface):
     """
     LLM provider for OpenAI-compatible APIs.
@@ -550,12 +576,19 @@ class OpenAICompatibleLLM(LLMInterface):
 
                 last_exception = e
                 if attempt < max_retries:
+                    logger.warning(
+                        f"APIStatusError ({self.provider}/{self.model}, scope={scope}, "
+                        f"attempt {attempt + 1}/{max_retries + 1}): {_summarize_status_error(e)}"
+                    )
                     backoff = min(initial_backoff * (2**attempt), max_backoff)
                     jitter = backoff * 0.2 * (2 * (time.time() % 1) - 1)
                     sleep_time = backoff + jitter
                     await asyncio.sleep(sleep_time)
                 else:
-                    logger.error(f"API error after {max_retries + 1} attempts: {str(e)}")
+                    logger.error(
+                        f"API error after {max_retries + 1} attempts ({self.provider}/{self.model}, "
+                        f"scope={scope}): {_summarize_status_error(e)}"
+                    )
                     raise
 
             except Exception:
@@ -706,18 +739,41 @@ class OpenAICompatibleLLM(LLMInterface):
 
             except APIConnectionError as e:
                 last_exception = e
+                status_code = getattr(e, "status_code", None) or getattr(
+                    getattr(e, "response", None), "status_code", None
+                )
                 if attempt < max_retries:
+                    logger.warning(
+                        f"APIConnectionError in tool call ({self.provider}/{self.model}, scope={scope}, "
+                        f"attempt {attempt + 1}/{max_retries + 1}, HTTP {status_code}): {str(e)[:200]}"
+                    )
                     await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff))
                     continue
+                logger.error(
+                    f"Connection error in tool call after {max_retries + 1} attempts "
+                    f"({self.provider}/{self.model}, scope={scope}): {str(e)}"
+                )
                 raise
 
             except APIStatusError as e:
                 if e.status_code in (401, 403):
+                    logger.error(
+                        f"Auth error in tool call (HTTP {e.status_code}, {self.provider}/{self.model}), "
+                        f"not retrying: {_summarize_status_error(e)}"
+                    )
                     raise
                 last_exception = e
                 if attempt < max_retries:
+                    logger.warning(
+                        f"APIStatusError in tool call ({self.provider}/{self.model}, scope={scope}, "
+                        f"attempt {attempt + 1}/{max_retries + 1}): {_summarize_status_error(e)}"
+                    )
                     await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff))
                     continue
+                logger.error(
+                    f"API error in tool call after {max_retries + 1} attempts "
+                    f"({self.provider}/{self.model}, scope={scope}): {_summarize_status_error(e)}"
+                )
                 raise
 
             except Exception:
@@ -765,6 +821,7 @@ class OpenAICompatibleLLM(LLMInterface):
             "model": self.model,
             "messages": messages,
             "stream": False,
+            "think": False,  # Disable thinking for reasoning models (qwen3.5, etc.)
         }
 
         # Add schema as format parameter for structured output

hindsight-api-slim/hindsight_api/engine/reflect/agent.py

@@ -652,6 +652,46 @@ async def run_reflect_agent(
             if result.content:
                 answer = _clean_answer_text(result.content.strip())
 
+                # The call_with_tools call above is intentionally uncapped so the
+                # LLM has headroom to emit tool-call JSON plus any intermediate
+                # reasoning. But when the LLM short-circuits and returns text
+                # directly, that text becomes the user-visible final answer and
+                # must respect max_tokens like the forced-final paths do. If it
+                # overshoots, run one extra capped call to rewrite it within
+                # the cap.
+                if max_tokens is not None and len(_TIKTOKEN_ENCODING.encode(answer)) > max_tokens:
+                    rewrite_start = time.time()
+                    rewritten, rewrite_usage = await llm_config.call(
+                        messages=[
+                            {
+                                "role": "system",
+                                "content": (
+                                    "Rewrite the user's text so it fits within the requested token "
+                                    "budget. Preserve the key facts and structure; drop lower-priority "
+                                    "detail. Respond with the rewritten text only, no preamble."
+                                ),
+                            },
+                            {
+                                "role": "user",
+                                "content": f"Target budget: {max_tokens} tokens.\n\nText to rewrite:\n{answer}",
+                            },
+                        ],
+                        scope="reflect",
+                        max_completion_tokens=max_tokens,
+                        return_usage=True,
+                    )
+                    total_input_tokens += rewrite_usage.input_tokens
+                    total_output_tokens += rewrite_usage.output_tokens
+                    llm_trace.append(
+                        {
+                            "scope": "final_rewrite",
+                            "duration_ms": int((time.time() - rewrite_start) * 1000),
+                            "input_tokens": rewrite_usage.input_tokens,
+                            "output_tokens": rewrite_usage.output_tokens,
+                        }
+                    )
+                    answer = _clean_answer_text(rewritten.strip())
+
                 # Generate structured output if schema provided
                 structured_output = None
                 if response_schema and answer:

hindsight-api-slim/hindsight_api/engine/reflect/delta_ops.py

@@ -0,0 +1,307 @@
+"""Delta operations for structured mental models.
+
+The LLM's job during a delta refresh is to emit a list of these operations,
+each targeting an existing section (by id) or referencing a position relative
+to one.  ``apply_operations`` validates and applies each op in turn against a
+copy of the document; invalid ops (unknown ``section_id``, out-of-range
+``block_index``, malformed payloads) are dropped with a debug-friendly reason.
+
+Sections and blocks not mentioned by any op are physically copied through
+unchanged — there is no LLM-mediated re-emission of unchanged text, so prose
+drift is structurally impossible.
+
+Why operations and not "output the new structured doc":
+- "Output the new doc" still asks the LLM to *generate* every section's
+  blocks, including ones it didn't intend to modify, which gives it the same
+  opportunity to drift.
+- Operations make the no-change case mechanical: zero ops → identical doc.
+- Operations are auditable: each refresh produces a log of exactly what
+  changed, useful for debugging the LLM's behaviour and explaining diffs.
+
+Failure modes are by design conservative: an operation list that fails to
+parse against the Pydantic schema, or an LLM that returns invalid ops, results
+in zero changes — the document stays as-is. The structure can only get better
+or stay the same per refresh, never get worse.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Annotated, Any, Literal, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .structured_doc import (
+    Block,
+    Section,
+    StructuredDocument,
+    make_unique_id,
+    slugify_heading,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# Op payloads ---------------------------------------------------------------
+
+
+class _OpBase(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class AppendBlockOp(_OpBase):
+    """Add a new block at the end of an existing section."""
+
+    op: Literal["append_block"] = "append_block"
+    section_id: str
+    block: Block
+
+
+class InsertBlockOp(_OpBase):
+    """Insert a new block at ``index`` in an existing section.
+
+    ``index`` may equal ``len(section.blocks)`` (append) but not be greater.
+    """
+
+    op: Literal["insert_block"] = "insert_block"
+    section_id: str
+    index: int = Field(ge=0)
+    block: Block
+
+
+class ReplaceBlockOp(_OpBase):
+    """Replace the block at ``index`` of an existing section."""
+
+    op: Literal["replace_block"] = "replace_block"
+    section_id: str
+    index: int = Field(ge=0)
+    block: Block
+
+
+class RemoveBlockOp(_OpBase):
+    """Remove the block at ``index`` of an existing section."""
+
+    op: Literal["remove_block"] = "remove_block"
+    section_id: str
+    index: int = Field(ge=0)
+
+
+class AddSectionOp(_OpBase):
+    """Add a brand-new section.
+
+    ``after_section_id`` is optional; when omitted the new section is appended
+    at the end. ``new_id`` is optional; when omitted we slugify the heading
+    and disambiguate against existing IDs.
+    """
+
+    op: Literal["add_section"] = "add_section"
+    heading: str
+    level: int = Field(default=2, ge=1, le=6)
+    blocks: list[Block] = Field(default_factory=list)
+    after_section_id: str | None = None
+    new_id: str | None = None
+
+
+class RemoveSectionOp(_OpBase):
+    """Remove an entire section by id."""
+
+    op: Literal["remove_section"] = "remove_section"
+    section_id: str
+
+
+class ReplaceSectionBlocksOp(_OpBase):
+    """Replace all blocks of a section in one go.
+
+    Used when most of a section's contents are stale and rebuilding it as a
+    unit is clearer than emitting many block-level ops. The section's heading
+    and id are preserved.
+    """
+
+    op: Literal["replace_section_blocks"] = "replace_section_blocks"
+    section_id: str
+    blocks: list[Block] = Field(default_factory=list)
+
+
+class RenameSectionOp(_OpBase):
+    """Rename a section's heading. The id is unchanged so future ops still resolve."""
+
+    op: Literal["rename_section"] = "rename_section"
+    section_id: str
+    new_heading: str
+
+
+Operation = Annotated[
+    Union[
+        AppendBlockOp,
+        InsertBlockOp,
+        ReplaceBlockOp,
+        RemoveBlockOp,
+        AddSectionOp,
+        RemoveSectionOp,
+        ReplaceSectionBlocksOp,
+        RenameSectionOp,
+    ],
+    Field(discriminator="op"),
+]
+
+
+class DeltaOperationList(BaseModel):
+    """Container for the operations produced by an LLM delta call."""
+
+    model_config = ConfigDict(extra="forbid")
+    operations: list[Operation] = Field(default_factory=list)
+
+
+# Application ---------------------------------------------------------------
+
+
+class AppliedDelta(BaseModel):
+    """Outcome of applying a list of operations to a document."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    document: StructuredDocument
+    applied: list[dict[str, Any]] = Field(default_factory=list)
+    skipped: list[dict[str, Any]] = Field(default_factory=list)
+
+    @property
+    def changed(self) -> bool:
+        return len(self.applied) > 0
+
+
+def _op_summary(op: Operation) -> dict[str, Any]:
+    """Compact dict suitable for the audit trail."""
+    data = op.model_dump()
+    return {k: v for k, v in data.items() if k != "block" and k != "blocks"} | {
+        "op": data["op"],
+    }
+
+
+def apply_operations(
+    doc: StructuredDocument,
+    operations: list[Operation],
+) -> AppliedDelta:
+    """Apply a list of operations to a document, returning a new document.
+
+    The original document is never mutated. Invalid operations (unknown
+    section, out-of-range index, name collision when adding a section) are
+    skipped and recorded in ``skipped`` with a ``reason`` string.
+    """
+    new_doc = doc.model_copy(deep=True)
+    applied: list[dict[str, Any]] = []
+    skipped: list[dict[str, Any]] = []
+
+    def skip(op: Operation, reason: str) -> None:
+        entry = _op_summary(op)
+        entry["reason"] = reason
+        skipped.append(entry)
+        logger.debug(f"[STRUCTURED_DELTA] skipping op {entry}")
+
+    for op in operations:
+        if isinstance(op, AppendBlockOp):
+            section = new_doc.section_by_id(op.section_id)
+            if section is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            section.blocks.append(op.block)
+            applied.append(_op_summary(op))
+            continue
+
+        if isinstance(op, InsertBlockOp):
+            section = new_doc.section_by_id(op.section_id)
+            if section is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            if op.index > len(section.blocks):
+                skip(
+                    op,
+                    f"index out of range: {op.index} > {len(section.blocks)}",
+                )
+                continue
+            section.blocks.insert(op.index, op.block)
+            applied.append(_op_summary(op))
+            continue
+
+        if isinstance(op, ReplaceBlockOp):
+            section = new_doc.section_by_id(op.section_id)
+            if section is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            if op.index >= len(section.blocks):
+                skip(
+                    op,
+                    f"index out of range: {op.index} >= {len(section.blocks)}",
+                )
+                continue
+            section.blocks[op.index] = op.block
+            applied.append(_op_summary(op))
+            continue
+
+        if isinstance(op, RemoveBlockOp):
+            section = new_doc.section_by_id(op.section_id)
+            if section is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            if op.index >= len(section.blocks):
+                skip(
+                    op,
+                    f"index out of range: {op.index} >= {len(section.blocks)}",
+                )
+                continue
+            section.blocks.pop(op.index)
+            applied.append(_op_summary(op))
+            continue
+
+        if isinstance(op, AddSectionOp):
+            existing_ids = {s.id for s in new_doc.sections}
+            base_id = op.new_id or slugify_heading(op.heading)
+            section_id = make_unique_id(base_id, existing_ids)
+            new_section = Section(
+                id=section_id,
+                heading=op.heading,
+                level=op.level,
+                blocks=list(op.blocks),
+            )
+            if op.after_section_id is None:
+                new_doc.sections.append(new_section)
+            else:
+                idx = new_doc.section_index(op.after_section_id)
+                if idx is None:
+                    skip(op, f"unknown after_section_id: {op.after_section_id}")
+                    continue
+                new_doc.sections.insert(idx + 1, new_section)
+            entry = _op_summary(op)
+            entry["assigned_id"] = section_id
+            applied.append(entry)
+            continue
+
+        if isinstance(op, RemoveSectionOp):
+            idx = new_doc.section_index(op.section_id)
+            if idx is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            new_doc.sections.pop(idx)
+            applied.append(_op_summary(op))
+            continue
+
+        if isinstance(op, ReplaceSectionBlocksOp):
+            section = new_doc.section_by_id(op.section_id)
+            if section is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            section.blocks = list(op.blocks)
+            applied.append(_op_summary(op))
+            continue
+
+        if isinstance(op, RenameSectionOp):
+            section = new_doc.section_by_id(op.section_id)
+            if section is None:
+                skip(op, f"unknown section_id: {op.section_id}")
+                continue
+            section.heading = op.new_heading
+            applied.append(_op_summary(op))
+            continue
+
+        skip(op, f"unhandled op type: {type(op).__name__}")  # pragma: no cover
+
+    return AppliedDelta(document=new_doc, applied=applied, skipped=skipped)

hindsight-api-slim/hindsight_api/engine/reflect/prompts.py

@@ -508,3 +508,180 @@ CRITICAL: Output ONLY the final synthesized answer. Do NOT include:
 Just provide the direct answer with proper markdown formatting.
 
 CRITICAL: This is a NON-CONVERSATIONAL system. NEVER ask follow-up questions, offer to search again, suggest alternatives, or end with anything like "Would you like me to..." or "Let me know if...". The user cannot reply. Your answer must be complete and self-contained."""
+
+
+STRUCTURED_DELTA_SYSTEM_PROMPT = """You are computing a *minimal patch* to a structured document.
+
+You will be given:
+1. CURRENT DOCUMENT (JSON) — the existing structured mental model. Each section
+   has a stable ``id``, a ``heading``, a ``level`` (1..6), and an ordered list
+   of ``blocks``. Blocks are typed: ``paragraph``, ``bullet_list``,
+   ``ordered_list``, or ``code``.
+2. CANDIDATE SUMMARY (markdown) — a freshly generated synthesis of the latest
+   memories, useful only as a hint about *what new information exists*. You
+   MUST NOT copy its formatting or wording wholesale; it is not the target.
+3. SUPPORTING FACTS — the observations and facts the candidate is grounded in.
+   Treat these as the only source of new information.
+
+Your task: output a JSON object ``{"operations": [...]}``. Applied to CURRENT
+DOCUMENT, the operations must produce the smallest possible change that
+reflects the new facts.
+
+ABSOLUTE RULES
+- If CURRENT DOCUMENT already covers all the supporting facts, output
+  exactly ``{"operations": []}``. An empty operation list IS the correct
+  answer when nothing new has come in. This is the most common case.
+- Operations target sections by ``section_id`` (use the ``id`` field of the
+  section in CURRENT DOCUMENT, NOT the heading). Block operations target
+  blocks by ``index`` (0-based, against the section's current block list).
+- Add new content with ``append_block``, ``insert_block``, or ``add_section``.
+  Prefer extending an existing section over creating a new one.
+- Modify existing content with ``replace_block`` or ``replace_section_blocks``
+  ONLY when the supporting facts contradict the current text. Do NOT rewrite
+  for style, brevity, or "improvement".
+- Remove stale content with ``remove_block`` or ``remove_section`` ONLY when
+  the supporting facts directly contradict it.
+- NEVER emit operations whose only effect is to reword unchanged content.
+- NEVER emit operations to "normalize" formatting (numbered → bulleted, casing
+  changes, paragraph → list, etc).
+- Every operation MUST be justifiable by a specific fact in SUPPORTING FACTS.
+
+ALLOWED OPERATIONS (each line shows the JSON shape)
+- ``{"op": "append_block", "section_id": "...", "block": {...}}``
+- ``{"op": "insert_block", "section_id": "...", "index": N, "block": {...}}``
+- ``{"op": "replace_block", "section_id": "...", "index": N, "block": {...}}``
+- ``{"op": "remove_block", "section_id": "...", "index": N}``
+- ``{"op": "add_section", "heading": "...", "level": 2, "blocks": [...], "after_section_id": "..."}``
+- ``{"op": "remove_section", "section_id": "..."}``
+- ``{"op": "replace_section_blocks", "section_id": "...", "blocks": [...]}``
+- ``{"op": "rename_section", "section_id": "...", "new_heading": "..."}``
+
+Block shapes
+- ``{"type": "paragraph", "text": "..."}``
+- ``{"type": "bullet_list", "items": ["...", "..."]}``
+- ``{"type": "ordered_list", "items": ["...", "..."]}``
+- ``{"type": "code", "language": "json", "text": "..."}``
+
+OUTPUT FORMAT
+Return ONLY a single JSON object on its own, with no prose before or after,
+no markdown code fences, no commentary. The object must have exactly one
+top-level key, ``operations``, whose value is an array of operation objects
+(empty array when nothing changes).
+
+Examples
+- No changes needed → ``{"operations": []}``
+- Add one bullet to an existing "Members" section →
+  ``{"operations": [{"op": "append_block", "section_id": "members",
+  "block": {"type": "bullet_list", "items": ["Carol — junior engineer"]}}]}``"""
+
+
+def build_structured_delta_prompt(
+    *,
+    current_document_json: str,
+    candidate_markdown: str,
+    supporting_facts: list[dict[str, Any]],
+    source_query: str,
+    max_output_tokens: int | None = None,
+) -> str:
+    """Build the user prompt for a structured-delta mental model refresh.
+
+    The LLM's job is to emit operations against ``current_document_json``;
+    the surrounding ``candidate_markdown`` and ``supporting_facts`` are
+    references for *what new information exists*, not templates to mimic.
+
+    ``max_output_tokens`` is surfaced in the prompt so the model can keep its
+    op list within the provider's response cap. The actual cap is enforced by
+    the caller; this is just an advisory anchor — without it the model often
+    returns op lists whose JSON gets truncated mid-string.
+    """
+    fact_lines: list[str] = []
+    for f in supporting_facts:
+        fid = f.get("id", "")
+        text = (f.get("text") or "").strip().replace("\n", " ")
+        ftype = f.get("type", "")
+        fact_lines.append(f"- [{ftype}:{fid}] {text}")
+    facts_block = "\n".join(fact_lines) if fact_lines else "(no supporting facts retrieved)"
+
+    budget_hint = ""
+    if max_output_tokens is not None:
+        budget_hint = (
+            f"\n\n## Output budget\n"
+            f"Your JSON response must fit within ~{max_output_tokens} tokens. If you "
+            "would need more than this to express every change, prefer the highest-"
+            "leverage edits first (a few ``replace_section_blocks`` ops over many "
+            "block-level ops) so the response always parses as valid JSON."
+        )
+
+    return (
+        f"## Topic\n{source_query}\n\n"
+        f"## CURRENT DOCUMENT (apply ops to this; reference section ids as listed)\n"
+        f"```json\n{current_document_json}\n```\n\n"
+        f"## CANDIDATE SUMMARY (hint only — do NOT copy wording wholesale)\n"
+        f"```markdown\n{candidate_markdown}\n```\n\n"
+        f"## SUPPORTING FACTS (the only source of new information)\n{facts_block}"
+        f"{budget_hint}\n\n"
+        "## Task\n"
+        "Output a JSON object matching the operations schema. Use an empty list "
+        "if no new fact requires a change. Otherwise, emit the smallest set of "
+        "operations that reflects the new facts in CURRENT DOCUMENT, preserving "
+        "all unchanged sections and blocks by simply not mentioning them."
+    )
+
+
+DELTA_SYSTEM_PROMPT = """You are performing a surgical delta update to an existing mental model document.
+
+You will be given:
+1. CURRENT DOCUMENT: the existing mental model content (markdown).
+2. CANDIDATE UPDATE: a freshly generated synthesis based on the latest retrieved memories.
+3. SUPPORTING FACTS: the observations and facts that support the CANDIDATE UPDATE.
+
+Your task: produce an updated version of the CURRENT DOCUMENT that reflects the new reality, with the MINIMUM possible changes.
+
+ABSOLUTE RULES:
+- Preserve unchanged content BYTE-FOR-BYTE. If a sentence, heading, bullet, code block, or section is still accurate according to the CANDIDATE UPDATE and SUPPORTING FACTS, copy it verbatim — same wording, same punctuation, same whitespace, same markdown structure.
+- Do NOT reformat, rephrase, or re-style content that is still accurate. No "light edits for clarity", no reordering for flow, no synonym swaps.
+- Remove content that is contradicted by the CANDIDATE UPDATE or SUPPORTING FACTS (stale content).
+- Add new content ONLY when the SUPPORTING FACTS contain information not already in the CURRENT DOCUMENT.
+- When adding new content, prefer appending to an existing relevant section. Creating a new section is acceptable when the new information does not fit any existing section.
+- When creating a new section, match the heading style, tone, and formatting conventions used in the CURRENT DOCUMENT.
+- Every assertion in your output MUST be grounded in either (a) the CURRENT DOCUMENT (preserved) or (b) the SUPPORTING FACTS. Never introduce outside knowledge.
+- If nothing in the SUPPORTING FACTS contradicts or extends the CURRENT DOCUMENT, return the CURRENT DOCUMENT UNCHANGED, character for character.
+
+OUTPUT FORMAT:
+- Output ONLY the updated markdown document. No preamble, no explanation, no diff markers, no commentary.
+- Do not wrap the output in code fences unless the CURRENT DOCUMENT itself was entirely a code fence."""
+
+
+def build_delta_prompt(
+    *,
+    current_content: str,
+    candidate_content: str,
+    supporting_facts: list[dict[str, Any]],
+    source_query: str,
+) -> str:
+    """Build the user prompt for a delta-mode mental model refresh.
+
+    Args:
+        current_content: The existing mental model content (to preserve as much as possible).
+        candidate_content: Fresh synthesis from the reflect agent reflecting new reality.
+        supporting_facts: Flat list of fact dicts (id, text, type) supporting the candidate.
+        source_query: The mental model's source query, for topical framing.
+    """
+    fact_lines: list[str] = []
+    for f in supporting_facts:
+        fid = f.get("id", "")
+        text = (f.get("text") or "").strip().replace("\n", " ")
+        ftype = f.get("type", "")
+        fact_lines.append(f"- [{ftype}:{fid}] {text}")
+    facts_block = "\n".join(fact_lines) if fact_lines else "(no supporting facts retrieved)"
+
+    return (
+        f"## Topic\n{source_query}\n\n"
+        f"## CURRENT DOCUMENT\n```markdown\n{current_content}\n```\n\n"
+        f"## CANDIDATE UPDATE\n```markdown\n{candidate_content}\n```\n\n"
+        f"## SUPPORTING FACTS\n{facts_block}\n\n"
+        "## Task\n"
+        "Produce the updated mental model document by applying the minimum necessary changes "
+        "to CURRENT DOCUMENT so that it reflects CANDIDATE UPDATE and SUPPORTING FACTS. "
+        "Preserve unchanged content byte-for-byte. Output only the final markdown."
+    )

hindsight-api-slim/hindsight_api/engine/reflect/structured_doc.py

@@ -0,0 +1,301 @@
+"""Structured representation of a mental model document.
+
+Why this exists
+---------------
+Storing mental models as raw markdown forces every refresh to round-trip prose
+through an LLM, which then drifts on stylistic details (numbered vs bulleted
+lists, casing, separator lines, paraphrasing) even when instructed to preserve
+content byte-for-byte. The intrinsic mechanism of an LLM is to *generate* the
+next token from a gestalt of the input — not to copy tokens verbatim — so any
+"preserve unchanged content" instruction is fundamentally a soft constraint.
+
+The fix is to give the LLM no opportunity to drift on unchanged content. We
+keep an authoritative structured representation of the document; the markdown
+shown to users is a deterministic render of that structure. Delta refreshes
+emit *operations* against the structure (see ``delta_ops.py``); sections and
+blocks not mentioned by any operation are physically untouched.
+
+Schema (v1)
+-----------
+A document is an ordered list of ``Section``s.  Each section has:
+- ``id``    : stable slug derived from ``heading`` (used as the operation
+              target across refreshes; surviving renames is a separate
+              concern handled by an explicit ``rename`` op).
+- ``heading``: the markdown heading text (without the ``#`` prefix).
+- ``level`` : 1 (``#``) … 6 (``######``).  Default 2.
+- ``blocks``: ordered list of typed blocks — paragraph, bullet_list,
+              ordered_list, code.
+
+The schema is intentionally narrow: it covers what real mental-model documents
+actually contain (the kind a coding agent writes for itself or a user writes as
+a "skill" doc).  Tables, images, and raw HTML are out of scope until needed.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Annotated, Literal, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+# Blocks ---------------------------------------------------------------------
+
+
+class ParagraphBlock(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    type: Literal["paragraph"] = "paragraph"
+    text: str
+
+
+class BulletListBlock(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    type: Literal["bullet_list"] = "bullet_list"
+    items: list[str] = Field(default_factory=list)
+
+
+class OrderedListBlock(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    type: Literal["ordered_list"] = "ordered_list"
+    items: list[str] = Field(default_factory=list)
+
+
+class CodeBlock(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    type: Literal["code"] = "code"
+    language: str = ""
+    text: str
+
+
+Block = Annotated[
+    Union[ParagraphBlock, BulletListBlock, OrderedListBlock, CodeBlock],
+    Field(discriminator="type"),
+]
+
+
+# Section / Document ---------------------------------------------------------
+
+
+class Section(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    id: str
+    heading: str
+    level: int = Field(default=2, ge=1, le=6)
+    blocks: list[Block] = Field(default_factory=list)
+
+
+class StructuredDocument(BaseModel):
+    """Top-level structured representation of a mental model."""
+
+    model_config = ConfigDict(extra="forbid")
+    version: Literal[1] = 1
+    sections: list[Section] = Field(default_factory=list)
+
+    def section_by_id(self, section_id: str) -> Section | None:
+        for s in self.sections:
+            if s.id == section_id:
+                return s
+        return None
+
+    def section_index(self, section_id: str) -> int | None:
+        for i, s in enumerate(self.sections):
+            if s.id == section_id:
+                return i
+        return None
+
+
+# Slug helpers ---------------------------------------------------------------
+
+_SLUG_RX = re.compile(r"[^a-z0-9]+")
+
+
+def slugify_heading(heading: str) -> str:
+    """Stable, deterministic slug from a heading.
+
+    "Stop Conditions" -> "stop-conditions"
+    "Inputs and Context" -> "inputs-and-context"
+    """
+    slug = _SLUG_RX.sub("-", heading.strip().lower()).strip("-")
+    return slug or "section"
+
+
+def make_unique_id(base: str, existing: set[str]) -> str:
+    """Disambiguate by appending -2, -3, … if the slug is already in use."""
+    if base not in existing:
+        return base
+    i = 2
+    while f"{base}-{i}" in existing:
+        i += 1
+    return f"{base}-{i}"
+
+
+# Renderer -------------------------------------------------------------------
+
+
+def render_block(block: Block) -> str:
+    """Render a single block to markdown. No trailing newline."""
+    if isinstance(block, ParagraphBlock):
+        return block.text.rstrip()
+    if isinstance(block, BulletListBlock):
+        return "\n".join(f"- {item.rstrip()}" for item in block.items)
+    if isinstance(block, OrderedListBlock):
+        return "\n".join(f"{i + 1}. {item.rstrip()}" for i, item in enumerate(block.items))
+    if isinstance(block, CodeBlock):
+        fence_lang = block.language or ""
+        return f"```{fence_lang}\n{block.text}\n```"
+    raise TypeError(f"Unknown block type: {type(block)!r}")
+
+
+def render_section(section: Section) -> str:
+    """Render a section: heading + blank line + blocks separated by blank lines."""
+    parts = ["#" * section.level + " " + section.heading.strip()]
+    for block in section.blocks:
+        parts.append("")  # blank line before each block
+        parts.append(render_block(block))
+    return "\n".join(parts)
+
+
+def render_document(doc: StructuredDocument) -> str:
+    """Render the whole document. Sections separated by a single blank line.
+
+    The output is byte-stable: same structured input always produces the same
+    markdown, modulo the inherent ordering of sections/blocks/items.
+    """
+    if not doc.sections:
+        return ""
+    return "\n\n".join(render_section(s) for s in doc.sections) + "\n"
+
+
+# Parser ---------------------------------------------------------------------
+#
+# The parser is intentionally lenient: it accepts the markdown produced by
+# our own renderer (round-trip-safe) and the markdown an LLM tends to produce
+# for mental-model documents.  It is *not* a general CommonMark parser — it
+# does not need to be.  When it cannot classify a block it falls back to a
+# paragraph so that no content is silently dropped.
+
+_HEADING_RX = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
+_BULLET_RX = re.compile(r"^\s*[-*+]\s+(.*)$")
+_ORDERED_RX = re.compile(r"^\s*\d+[.)]\s+(.*)$")
+_FENCE_RX = re.compile(r"^```([A-Za-z0-9_+-]*)\s*$")
+
+
+def _strip_separators(lines: list[str]) -> list[str]:
+    """Drop horizontal-rule lines (`---`, `***`) used as section separators.
+
+    Our renderer never emits these, but LLM output frequently includes them
+    between sections; treating them as blank lines avoids parsing them as
+    paragraphs.
+    """
+    return ["" if re.fullmatch(r"\s*([-*_])\1{2,}\s*", line) else line for line in lines]
+
+
+def _split_blocks(lines: list[str]) -> list[list[str]]:
+    """Group consecutive non-blank lines into block chunks."""
+    chunks: list[list[str]] = []
+    current: list[str] = []
+    in_fence = False
+    for line in lines:
+        if _FENCE_RX.match(line):
+            current.append(line)
+            in_fence = not in_fence
+            continue
+        if in_fence:
+            current.append(line)
+            continue
+        if line.strip() == "":
+            if current:
+                chunks.append(current)
+                current = []
+        else:
+            current.append(line)
+    if current:
+        chunks.append(current)
+    return chunks
+
+
+def _parse_block(chunk: list[str]) -> Block:
+    """Parse a single non-empty chunk into a block."""
+    if chunk and _FENCE_RX.match(chunk[0]):
+        m = _FENCE_RX.match(chunk[0])
+        lang = m.group(1) if m else ""
+        body_lines = chunk[1:]
+        if body_lines and _FENCE_RX.match(body_lines[-1]):
+            body_lines = body_lines[:-1]
+        return CodeBlock(language=lang, text="\n".join(body_lines))
+
+    if all(_BULLET_RX.match(line) for line in chunk):
+        items = []
+        for line in chunk:
+            m = _BULLET_RX.match(line)
+            assert m is not None
+            items.append(m.group(1).strip())
+        return BulletListBlock(items=items)
+
+    if all(_ORDERED_RX.match(line) for line in chunk):
+        items = []
+        for line in chunk:
+            m = _ORDERED_RX.match(line)
+            assert m is not None
+            items.append(m.group(1).strip())
+        return OrderedListBlock(items=items)
+
+    return ParagraphBlock(text=" ".join(line.strip() for line in chunk).strip())
+
+
+def parse_markdown(markdown: str) -> StructuredDocument:
+    """Best-effort parse of a markdown document into the structured schema.
+
+    Sections are introduced by ATX headings (``#``..``######``).  Anything
+    before the first heading is wrapped into an implicit "Overview" section
+    so we never silently drop user content.  Section IDs are unique slugs of
+    their headings.
+    """
+    raw_lines = (markdown or "").splitlines()
+    lines = _strip_separators(raw_lines)
+
+    sections: list[Section] = []
+    used_ids: set[str] = set()
+    pending: list[str] = []
+    current: Section | None = None
+
+    def flush_pending_into(section: Section) -> None:
+        if not pending:
+            return
+        for chunk in _split_blocks(pending):
+            section.blocks.append(_parse_block(chunk))
+        pending.clear()
+
+    for line in lines:
+        m = _HEADING_RX.match(line)
+        if m:
+            if current is not None:
+                flush_pending_into(current)
+                sections.append(current)
+            elif pending:
+                # Content before the first heading: wrap in implicit section.
+                base = "overview"
+                section_id = make_unique_id(base, used_ids)
+                used_ids.add(section_id)
+                implicit = Section(id=section_id, heading="Overview", level=2)
+                flush_pending_into(implicit)
+                sections.append(implicit)
+            level = len(m.group(1))
+            heading = m.group(2).strip()
+            section_id = make_unique_id(slugify_heading(heading), used_ids)
+            used_ids.add(section_id)
+            current = Section(id=section_id, heading=heading, level=level)
+        else:
+            pending.append(line)
+
+    if current is not None:
+        flush_pending_into(current)
+        sections.append(current)
+    elif pending:
+        base = "overview"
+        section_id = make_unique_id(base, used_ids)
+        used_ids.add(section_id)
+        implicit = Section(id=section_id, heading="Overview", level=2)
+        flush_pending_into(implicit)
+        sections.append(implicit)
+
+    return StructuredDocument(sections=sections)

hindsight-api-slim/hindsight_api/engine/reflect/tools.py

@@ -23,6 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 async def tool_search_mental_models(
+    memory_engine: "MemoryEngine",
     conn: "Connection",
     bank_id: str,
     query: str,
@@ -32,7 +33,6 @@ async def tool_search_mental_models(
     tags_match: str = "any",
     tag_groups: "list | None" = None,
     exclude_ids: list[str] | None = None,
-    pending_consolidation: int = 0,
 ) -> dict[str, Any]:
     """
     Search user-curated mental models by semantic similarity.
@@ -82,7 +82,7 @@ async def tool_search_mental_models(
         f"""
         SELECT
             id, name, content,
-            tags, created_at, last_refreshed_at,
+            tags, created_at, last_refreshed_at, trigger,
             1 - (embedding <=> $2::vector) as relevance
         FROM {fq_table("mental_models")}
         WHERE bank_id = $1 AND embedding IS NOT NULL {filters}
@@ -99,10 +99,9 @@ async def tool_search_mental_models(
         if last_refreshed_at and last_refreshed_at.tzinfo is None:
             last_refreshed_at = last_refreshed_at.replace(tzinfo=timezone.utc)
 
-        # A mental model is stale when there are memories that haven't been consolidated yet —
-        # the same signal used for observations staleness.
-        is_stale = pending_consolidation > 0
-        staleness_reason = f"{pending_consolidation} memories pending consolidation" if is_stale else None
+        # Per-MM staleness: new in-scope memories since last refresh (includes pending).
+        is_stale = await memory_engine.compute_mental_model_is_stale(conn, bank_id, row)
+        staleness_reason = "new in-scope memories ingested since last refresh" if is_stale else None
 
         mental_models.append(
             {
@@ -214,6 +213,7 @@ async def tool_recall(
     connection_budget: int = 1,
     max_chunk_tokens: int = 1000,
     fact_types: list[str] | None = None,
+    include_chunks: bool = True,
 ) -> dict[str, Any]:
     """
     Search memories using TEMPR retrieval.
@@ -230,15 +230,15 @@ async def tool_recall(
         tags: Filter by tags (includes untagged memories)
         tags_match: How to match tags - "any" (OR), "all" (AND), or "exact"
         connection_budget: Max DB connections for this recall (default 1 for internal ops)
-        max_chunk_tokens: Maximum tokens for raw source chunk text (default 1000, always included)
+        max_chunk_tokens: Maximum tokens for raw source chunk text (default 1000)
         fact_types: Optional filter for fact types to retrieve. Defaults to ["experience", "world"].
+        include_chunks: Whether to fetch raw chunk text alongside facts (default True).
 
     Returns:
-        Dict with list of matching memories including raw chunk text
+        Dict with list of matching memories including raw chunk text (when include_chunks)
     """
     # Only world/experience are valid for raw recall (observation is handled by search_observations)
     recall_fact_type = [ft for ft in (fact_types or ["experience", "world"]) if ft in ("world", "experience")]
-    include_chunks = True
     internal_ctx = replace(request_context, internal=True)
     result = await memory_engine.recall_async(
         bank_id=bank_id,

hindsight-api-slim/hindsight_api/engine/retain/embedding_utils.py

@@ -47,6 +47,16 @@ async def generate_embeddings_batch(embeddings_backend, texts: list[str]) -> lis
             embeddings_backend.encode,
             texts,
         )
-        return embeddings
     except Exception as e:
         raise Exception(f"Failed to generate batch embeddings: {str(e)}")
+
+    # Guarantee 1:1 alignment with input texts. A silent length mismatch here
+    # propagates downstream as zip() drops items, eventually surfacing as an
+    # IndexError in retain mapping (see issue #1037).
+    if len(embeddings) != len(texts):
+        raise RuntimeError(
+            f"Embeddings backend returned {len(embeddings)} vectors for {len(texts)} input texts; "
+            "expected exact 1:1 alignment"
+        )
+
+    return embeddings

hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py

@@ -224,6 +224,85 @@ async def ensure_bank_exists(conn, bank_id: str) -> None:
         await create_bank_vector_indexes(conn, bank_id, str(internal_id))
 
 
+async def delete_stale_observations_for_memories(
+    conn,
+    bank_id: str,
+    fact_ids: "list[str | uuid.UUID]",
+) -> int:
+    """Delete observations whose source memories are about to be removed.
+
+    Mirrors the cleanup performed by ``MemoryEngine.delete_document`` so that
+    every code path that removes ``memory_units`` also removes the
+    observations derived from them. Without this, ingesting a fresh version
+    of a document via the retain pipeline (which does a full-replace
+    ``DELETE FROM documents`` cascade) used to leave orphan observations
+    pointing at memory IDs that no longer existed.
+
+    For each observation referencing any of ``fact_ids``:
+    1. Delete the observation row (its text is stale once even one source
+       memory disappears).
+    2. Reset ``consolidated_at = NULL`` on the surviving source memories so
+       they get re-consolidated under fresh observations on the next run.
+
+    Must be called within an active transaction, before the source memories
+    are deleted.
+
+    Returns the number of observations deleted.
+    """
+    if not fact_ids:
+        return 0
+
+    fact_uuids = [uuid.UUID(str(fid)) if not isinstance(fid, uuid.UUID) else fid for fid in fact_ids]
+
+    affected_obs = await conn.fetch(
+        f"""
+        SELECT id, source_memory_ids
+        FROM {fq_table("memory_units")}
+        WHERE bank_id = $1
+          AND fact_type = 'observation'
+          AND source_memory_ids && $2::uuid[]
+        """,
+        bank_id,
+        fact_uuids,
+    )
+
+    if not affected_obs:
+        return 0
+
+    deleted_set = {str(uid) for uid in fact_uuids}
+    obs_ids = [obs["id"] for obs in affected_obs]
+    seen_remaining: set[str] = set()
+    remaining_source_ids: list[uuid.UUID] = []
+    for obs in affected_obs:
+        for src_id in obs["source_memory_ids"] or []:
+            src_str = str(src_id)
+            if src_str not in deleted_set and src_str not in seen_remaining:
+                remaining_source_ids.append(src_id)
+                seen_remaining.add(src_str)
+
+    await conn.execute(
+        f"DELETE FROM {fq_table('memory_units')} WHERE id = ANY($1::uuid[])",
+        obs_ids,
+    )
+
+    if remaining_source_ids:
+        await conn.execute(
+            f"""
+            UPDATE {fq_table("memory_units")}
+            SET consolidated_at = NULL
+            WHERE id = ANY($1::uuid[])
+              AND fact_type IN ('experience', 'world')
+            """,
+            remaining_source_ids,
+        )
+
+    logger.info(
+        f"[OBSERVATIONS] Deleted {len(obs_ids)} observations, reset {len(remaining_source_ids)} "
+        f"source memories for re-consolidation in bank {bank_id}"
+    )
+    return len(obs_ids)
+
+
 async def handle_document_tracking(
     conn,
     bank_id: str,
@@ -254,9 +333,29 @@ async def handle_document_tracking(
     combined_content = _sanitize_text(combined_content) or ""
     content_hash = hashlib.sha256(combined_content.encode()).hexdigest()
 
-    # Delete old document first (cascades to units and links)
-    # Only delete on the first batch to avoid deleting data we just inserted
+    # Delete old document first (cascades to units and links).
+    # Only delete on the first batch to avoid deleting data we just inserted.
+    # Before the cascade, fan out to delete observations derived from the
+    # outgoing memory_units — otherwise the FK ON DELETE CASCADE removes the
+    # source memory_units but leaves observation rows pointing at IDs that
+    # no longer exist (consolidated_at on co-source memories also stays
+    # frozen). Same cleanup the explicit ``delete_document`` API performs.
     if is_first_batch:
+        existing_unit_rows = await conn.fetch(
+            f"""
+            SELECT id FROM {fq_table("memory_units")}
+            WHERE document_id = $1 AND fact_type IN ('experience', 'world')
+            """,
+            document_id,
+        )
+        existing_unit_ids = [row["id"] for row in existing_unit_rows]
+        if existing_unit_ids:
+            invalidated = await delete_stale_observations_for_memories(conn, bank_id, existing_unit_ids)
+            if invalidated:
+                logger.info(
+                    f"[RETAIN] Document {document_id} re-ingested: invalidated "
+                    f"{invalidated} observation(s) derived from {len(existing_unit_ids)} outgoing memory_units"
+                )
         await conn.fetchval(
             f"DELETE FROM {fq_table('documents')} WHERE id = $1 AND bank_id = $2 RETURNING id",
             document_id,

hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py

@@ -72,7 +72,6 @@ from . import (
 from .types import (
     ChunkMetadata,
     EntityResolutionResult,
-    ExtractedFact,
     Phase1Result,
     Phase3Context,
     ProcessedFact,
@@ -302,8 +301,11 @@ async def _insert_facts_and_links(
         causal_link_count = await link_creation.create_causal_links_batch(conn, bank_id, unit_ids, processed_facts)
         log_buffer.append(f"  Causal links: {causal_link_count} links in {time.time() - step_start:.3f}s")
 
-    # Map results back to original content items
-    result_unit_ids = _map_results_to_contents(contents, extracted_facts, unit_ids if unit_ids else [])
+    # Map results back to original content items. Use processed_facts (not
+    # extracted_facts) because unit_ids has 1:1 alignment with processed_facts —
+    # any upstream drop between extraction and processing would otherwise cause
+    # an IndexError (see issue #1037).
+    result_unit_ids = _map_results_to_contents(contents, processed_facts, unit_ids if unit_ids else [])
 
     if outbox_callback:
         await outbox_callback(conn)
@@ -487,8 +489,8 @@ async def retain_batch(
             return result_unit_ids, total_usage
 
     # Resolve effective document_id early so both delta and streaming paths
-    # can find existing chunks from a prior attempt. On retry, the generated
-    # document_id is recovered from operation result_metadata.
+    # can find existing chunks from a prior attempt. On retry, a generated
+    # document_id is recovered from operation result_metadata.document_ids[0].
     effective_doc_id = document_id
     if not effective_doc_id:
         doc_ids = {item.get("document_id") for item in contents_dicts if item.get("document_id")}
@@ -507,26 +509,41 @@ async def retain_batch(
                         if isinstance(row["result_metadata"], dict)
                         else json.loads(row["result_metadata"])
                     )
-                    effective_doc_id = meta.get("generated_document_id")
+                    recovered = meta.get("document_ids") or []
+                    if recovered:
+                        effective_doc_id = recovered[0]
         except Exception:
             pass
     if not effective_doc_id:
         effective_doc_id = str(uuid.uuid4())
-        # Persist so retries reuse the same document_id
-        if operation_id:
-            try:
-                async with acquire_with_retry(pool) as conn:
-                    await conn.execute(
-                        f"""
-                        UPDATE {fq_table("async_operations")}
-                        SET result_metadata = result_metadata || $1::jsonb, updated_at = now()
-                        WHERE operation_id = $2
-                        """,
-                        json.dumps({"generated_document_id": effective_doc_id}),
-                        uuid.UUID(operation_id),
-                    )
-            except Exception:
-                logger.warning("Failed to persist generated document_id", exc_info=True)
+
+    # Record effective_doc_id on the operation (idempotent set-append). Captures
+    # both user-provided and generated ids so the operation shows every document
+    # it touched, and lets retries reuse the same generated id.
+    if operation_id:
+        try:
+            async with acquire_with_retry(pool) as conn:
+                await conn.execute(
+                    f"""
+                    UPDATE {fq_table("async_operations")}
+                    SET result_metadata = jsonb_set(
+                        COALESCE(result_metadata, '{{}}'::jsonb),
+                        '{{document_ids}}',
+                        CASE
+                            WHEN COALESCE(result_metadata->'document_ids', '[]'::jsonb) @> $1::jsonb
+                                THEN result_metadata->'document_ids'
+                            ELSE COALESCE(result_metadata->'document_ids', '[]'::jsonb) || $1::jsonb
+                        END,
+                        true
+                    ),
+                    updated_at = now()
+                    WHERE operation_id = $2
+                    """,
+                    json.dumps([effective_doc_id]),
+                    uuid.UUID(operation_id),
+                )
+        except Exception:
+            logger.warning("Failed to persist document_id", exc_info=True)
 
     # --- Append mode: prepend existing document content to new content ---
     # When update_mode="append", fetch the existing document text and prepend it
@@ -1550,12 +1567,19 @@ def _build_delta_contents(
 
 def _map_results_to_contents(
     contents: list[RetainContent],
-    extracted_facts: list[ExtractedFact],
+    processed_facts: list[ProcessedFact],
     unit_ids: list[str],
 ) -> list[list[str]]:
-    """Map created unit IDs back to original content items."""
+    """Map created unit IDs back to original content items.
+
+    `processed_facts` and `unit_ids` must have the same length: each unit_id
+    corresponds to the processed_fact at the same index.
+    """
+    if len(processed_facts) != len(unit_ids):
+        raise ValueError(f"processed_facts ({len(processed_facts)}) and unit_ids ({len(unit_ids)}) length mismatch")
+
     facts_by_content: dict[int, list[int]] = {i: [] for i in range(len(contents))}
-    for i, fact in enumerate(extracted_facts):
+    for i, fact in enumerate(processed_facts):
         # Normalize content_index: some LLM providers return 1-indexed values.
         # Clamp to valid range to prevent KeyError.
         idx = fact.content_index
@@ -1564,12 +1588,8 @@ def _map_results_to_contents(
         facts_by_content[idx].append(i)
 
     result_unit_ids = []
-    unit_idx = 0
     for content_index in range(len(contents)):
-        content_unit_ids = []
-        for _ in facts_by_content[content_index]:
-            content_unit_ids.append(unit_ids[unit_idx])
-            unit_idx += 1
+        content_unit_ids = [unit_ids[i] for i in facts_by_content[content_index]]
         result_unit_ids.append(content_unit_ids)
 
     return result_unit_ids

hindsight-api-slim/hindsight_api/engine/task_backend.py

@@ -193,17 +193,21 @@ class BrokerTaskBackend(TaskBackend):
         table = fq_table("async_operations", schema)
 
         if operation_id:
-            # Update existing operation with task payload
+            # Callers now include task_payload in the same INSERT that creates the
+            # async_operations row (see MemoryEngine._submit_async_operation). The
+            # WHERE clause guards against overwriting that payload — the UPDATE is a
+            # no-op when the row is already claimable, and only fills in a NULL payload
+            # for any legacy caller that still creates the row first.
             await pool.execute(
                 f"""
                 UPDATE {table}
                 SET task_payload = $1::jsonb, updated_at = now()
-                WHERE operation_id = $2
+                WHERE operation_id = $2 AND task_payload IS NULL
                 """,
                 payload_json,
                 operation_id,
             )
-            logger.debug(f"Updated task payload for operation {operation_id}")
+            logger.debug(f"submit_task UPDATE for operation {operation_id} (no-op if payload already set)")
         else:
             # Insert new operation (for tasks without pre-created records)
             # e.g., access_count_update tasks

hindsight-api-slim/hindsight_api/extensions/__init__.py

@@ -55,6 +55,7 @@ from hindsight_api.extensions.tenant import (
     TenantExtension,
 )
 from hindsight_api.models import RequestContext
+from hindsight_api.worker.exceptions import DeferOperation
 
 __all__ = [
     # Base
@@ -68,6 +69,7 @@ __all__ = [
     # MCP Extension
     "MCPExtension",
     # Operation Validator - Core
+    "DeferOperation",
     "OperationValidationError",
     "OperationValidatorExtension",
     "RecallContext",

hindsight-api-slim/hindsight_api/extensions/operation_validator.py

@@ -376,6 +376,16 @@ class OperationValidatorExtension(Extension, ABC):
         2. [operation executes]
         3. on_*_complete (post-operation)
 
+    Outcomes for `validate_*` hooks:
+        - accept: return `ValidationResult.accept()` (or `accept_with(...)`)
+        - reject: return `ValidationResult.reject(reason, status_code)`
+          (raises `OperationValidationError` upstream)
+        - defer: raise `DeferOperation(exec_date, reason)` from
+          `hindsight_api.worker.exceptions` to requeue the task for a
+          future time without bumping `retry_count`. Worker-only — do
+          not raise from `validate_recall` / `validate_reflect` in
+          synchronous HTTP request paths, where it surfaces as a 500.
+
     Supported operations:
         - retain, recall, reflect (core memory operations)
         - consolidate (mental models consolidation)

hindsight-api-slim/hindsight_api/models.py

@@ -62,7 +62,6 @@ class Document(Base):
     bank_id: Mapped[str] = mapped_column(Text, primary_key=True)
     original_text: Mapped[str | None] = mapped_column(Text)
     content_hash: Mapped[str | None] = mapped_column(Text)
-    doc_metadata: Mapped[dict] = mapped_column("metadata", JSONB, server_default=sql_text("'{}'::jsonb"))
     created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now())
     updated_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now())
 

hindsight-api-slim/hindsight_api/worker/exceptions.py

@@ -7,3 +7,24 @@ class RetryTaskAt(Exception):
     def __init__(self, retry_at: datetime, message: str = ""):
         self.retry_at = retry_at
         super().__init__(message)
+
+
+class DeferOperation(Exception):
+    """Raise from an extension hook (or task handler) to requeue the
+    operation for execution at a later time, without counting as a retry.
+
+    Unlike `RetryTaskAt`, this is not a failure: `retry_count` is not
+    incremented and `error_message` is not written. Use this for
+    backpressure / "not yet, try later" decisions made before or during
+    task execution (e.g. quota windows, warming dependencies, upstream
+    rate limits).
+
+    Worker-only: raising this from a hook called in HTTP request context
+    (e.g. `validate_recall` for a synchronous recall) will surface as an
+    unhandled 500 — there is no queue to defer to.
+    """
+
+    def __init__(self, exec_date: datetime, reason: str = ""):
+        self.exec_date = exec_date
+        self.reason = reason
+        super().__init__(reason)

hindsight-api-slim/hindsight_api/worker/poller.py

@@ -15,7 +15,7 @@ from collections.abc import Awaitable, Callable
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any
 
-from .exceptions import RetryTaskAt
+from .exceptions import DeferOperation, RetryTaskAt
 from .stage import StageHolder, bind_holder
 
 if TYPE_CHECKING:
@@ -130,6 +130,9 @@ class WorkerPoller:
         self._active_tasks: dict[str, ActiveTaskInfo] = {}
         # Track in-flight tasks by operation type
         self._in_flight_by_type: dict[str, int] = {}
+        # Rotation offset for per-tenant fair claiming. Advances past the last
+        # schema we serviced so a busy tenant can't monopolize the poll order.
+        self._next_schema_idx: int = 0
 
     async def _get_schemas(self) -> list[str | None]:
         """Get list of schemas to poll. Returns [None] for default schema (no prefix)."""
@@ -196,6 +199,15 @@ class WorkerPoller:
 
         Uses FOR UPDATE SKIP LOCKED to ensure no conflicts with other workers.
 
+        Schema iteration is round-robin to prevent one busy tenant from
+        starving others. Each poll starts at ``self._next_schema_idx`` and
+        wraps around the full list. First pass caps at 1 claim per schema
+        so every tenant with pending work gets a fair chance; a second
+        pass backfills remaining slots from any schema when there's spare
+        capacity. After the call, the offset advances past the last
+        schema we serviced (or by 1 if nothing was claimed) so the next
+        poll starts at a different position.
+
         Returns:
             List of ClaimedTask objects containing operation_id, task_dict, and schema
         """
@@ -206,15 +218,29 @@ class WorkerPoller:
             return []
 
         schemas = await self._get_schemas()
+        if not schemas:
+            return []
+
+        # Rotate the schema order so no tenant is always first.
+        start = self._next_schema_idx % len(schemas)
+        rotated = list(enumerate(schemas))
+        rotated = rotated[start:] + rotated[:start]
+
         all_tasks: list[ClaimedTask] = []
         remaining_non_consolidation = non_consolidation_available
         remaining_consolidation = consolidation_available
+        last_serviced_idx: int | None = None
 
-        for schema in schemas:
+        # Pass 1: fairness pass — at most 1 claim per pool per schema,
+        # so every tenant with pending work is considered before we
+        # return to a tenant we already claimed from.
+        for orig_idx, schema in rotated:
             if remaining_non_consolidation <= 0 and remaining_consolidation <= 0:
                 break
 
-            tasks = await self._claim_batch_for_schema(schema, remaining_non_consolidation, remaining_consolidation)
+            nc_limit = min(1, remaining_non_consolidation)
+            c_limit = min(1, remaining_consolidation)
+            tasks = await self._claim_batch_for_schema(schema, nc_limit, c_limit)
 
             for task in tasks:
                 op_type = task.task_dict.get("operation_type", "unknown")
@@ -223,8 +249,41 @@ class WorkerPoller:
                 else:
                     remaining_non_consolidation -= 1
 
+            if tasks:
+                last_serviced_idx = orig_idx
+
             all_tasks.extend(tasks)
 
+        # Pass 2: capacity pass — fill any remaining slots from whichever
+        # schemas still have work. Preserves rotation order so a tenant
+        # earlier in the rotation doesn't monopolize again when only one
+        # tenant has more work.
+        if remaining_non_consolidation > 0 or remaining_consolidation > 0:
+            for orig_idx, schema in rotated:
+                if remaining_non_consolidation <= 0 and remaining_consolidation <= 0:
+                    break
+
+                tasks = await self._claim_batch_for_schema(schema, remaining_non_consolidation, remaining_consolidation)
+
+                for task in tasks:
+                    op_type = task.task_dict.get("operation_type", "unknown")
+                    if op_type == "consolidation":
+                        remaining_consolidation -= 1
+                    else:
+                        remaining_non_consolidation -= 1
+
+                if tasks:
+                    last_serviced_idx = orig_idx
+
+                all_tasks.extend(tasks)
+
+        # Advance offset past the last schema we serviced, or by 1 if
+        # nothing was claimed (so we don't keep re-hitting an empty head).
+        if last_serviced_idx is not None:
+            self._next_schema_idx = (last_serviced_idx + 1) % len(schemas)
+        else:
+            self._next_schema_idx = (start + 1) % len(schemas)
+
         return all_tasks
 
     async def _claim_batch_for_schema(
@@ -461,6 +520,25 @@ class WorkerPoller:
         )
         logger.warning(f"Task {operation_id} scheduled for retry at {retry_at}: {error_message}")
 
+    async def _defer_operation(self, operation_id: str, exec_date: "Any", reason: str, schema: str | None):
+        """Reset task to pending for re-pickup at exec_date without counting as a retry.
+
+        Unlike `_schedule_retry`, this does not bump `retry_count` and does not
+        populate `error_message` — defer is intentional backpressure, not a failure.
+        """
+        table = fq_table("async_operations", schema)
+        await self._pool.execute(
+            f"""
+            UPDATE {table}
+            SET status = 'pending', next_retry_at = $2, worker_id = NULL, claimed_at = NULL,
+                updated_at = now()
+            WHERE operation_id = $1
+            """,
+            operation_id,
+            exec_date,
+        )
+        logger.info(f"Task {operation_id} deferred until {exec_date}: {reason}")
+
     async def execute_task(self, task: ClaimedTask):
         """Execute a single task as a background job (fire-and-forget)."""
         task_type = task.task_dict.get("type", "unknown")
@@ -532,6 +610,8 @@ class WorkerPoller:
                 task.task_dict["_schema"] = task.schema
             await self._executor(task.task_dict)
             logger.debug(f"Task {task.operation_id} execution finished")
+        except DeferOperation as e:
+            await self._defer_operation(task.operation_id, e.exec_date, e.reason, task.schema)
         except RetryTaskAt as e:
             await self._schedule_retry(task.operation_id, e.retry_at, str(e), task.schema)
         except Exception as e:
@@ -812,13 +892,42 @@ class WorkerPoller:
             schemas = await self._get_schemas()
             global_pending = 0
             all_worker_counts: dict[str, int] = {}
+            # operation_type -> aggregated bucket counts across schemas
+            pending_breakdown: dict[str, dict[str, int]] = {}
 
             async with self._pool.acquire() as conn:
                 for schema in schemas:
                     table = fq_table("async_operations", schema)
 
-                    row = await conn.fetchrow(f"SELECT COUNT(*) as count FROM {table} WHERE status = 'pending'")
-                    global_pending += row["count"] if row else 0
+                    # Bucket pending rows by the same predicates the claim query
+                    # filters on, so an operator can see why pending > 0 but
+                    # nothing is being claimed (orphaned batch_retain parents,
+                    # retry backoff, etc.).
+                    breakdown_rows = await conn.fetch(
+                        f"""
+                        SELECT
+                            operation_type,
+                            COUNT(*) AS total,
+                            COUNT(*) FILTER (WHERE task_payload IS NULL) AS payload_null,
+                            COUNT(*) FILTER (
+                                WHERE next_retry_at IS NOT NULL AND next_retry_at > now()
+                            ) AS retry_blocked,
+                            COUNT(*) FILTER (WHERE worker_id IS NOT NULL) AS assigned
+                        FROM {table}
+                        WHERE status = 'pending'
+                        GROUP BY operation_type
+                        """
+                    )
+                    for br in breakdown_rows:
+                        op_type = br["operation_type"] or "unknown"
+                        bucket = pending_breakdown.setdefault(
+                            op_type, {"total": 0, "payload_null": 0, "retry_blocked": 0, "assigned": 0}
+                        )
+                        bucket["total"] += br["total"]
+                        bucket["payload_null"] += br["payload_null"]
+                        bucket["retry_blocked"] += br["retry_blocked"]
+                        bucket["assigned"] += br["assigned"]
+                        global_pending += br["total"]
 
                     worker_rows = await conn.fetch(
                         f"""
@@ -856,6 +965,13 @@ class WorkerPoller:
                 f"my_active: {processing_str}"
             )
 
+            # Pending breakdown - explains why pending rows aren't being claimed
+            # (orphaned batch_retain parents have payload_null > 0, retry storms
+            # show up as retry_blocked, etc.). Skip when nothing is pending so
+            # the line doesn't add noise on idle deployments.
+            if global_pending > 0:
+                self._log_pending_breakdown(pending_breakdown)
+
             # Per-task lines, sorted oldest-first so stuck tasks bubble to the top.
             self._log_per_task_lines(active_tasks, now=time.monotonic())
 
@@ -895,7 +1011,14 @@ class WorkerPoller:
             min_size = pool.get_min_size() if hasattr(pool, "get_min_size") else None
             max_size = pool.get_max_size() if hasattr(pool, "get_max_size") else None
             queue = getattr(pool, "_queue", None)
-            waiters = queue.qsize() if queue is not None and hasattr(queue, "qsize") else None
+            # asyncpg's _queue is a LifoQueue pre-filled to max_size with
+            # PoolConnectionHolder objects. qsize() therefore counts *available
+            # holders*, not callers waiting on the pool — the previous "waiters"
+            # label here was the opposite of what it suggested. The actual count
+            # of awaiters is len(_queue._getters), nonzero only when qsize()==0.
+            free_holders = queue.qsize() if queue is not None and hasattr(queue, "qsize") else None
+            getters = getattr(queue, "_getters", None) if queue is not None else None
+            pending_acquires = len(getters) if getters is not None else None
 
             parts = [f"size={size}"]
             if min_size is not None and max_size is not None:
@@ -903,13 +1026,44 @@ class WorkerPoller:
             if free is not None:
                 parts.append(f"idle={free}")
                 parts.append(f"in_use={size - free}")
-            if waiters is not None:
-                parts.append(f"waiters={waiters}")
+            if free_holders is not None:
+                parts.append(f"free_holders={free_holders}")
+            if pending_acquires is not None:
+                parts.append(f"pending_acquires={pending_acquires}")
             return " ".join(parts)
         except Exception as e:
             logger.debug(f"Pool stats unavailable: {e}")
             return "unavailable"
 
+    def _log_pending_breakdown(self, breakdown: dict[str, dict[str, int]]) -> None:
+        """Emit one [PENDING_BREAKDOWN] line bucketing pending rows by claimability.
+
+        Each bucket mirrors a predicate in the claim query:
+          * payload_null   - row has no task_payload (e.g. batch_retain parent
+                             whose reconciliation never fired); claim query
+                             skips it forever
+          * retry_blocked  - next_retry_at is still in the future
+          * assigned       - worker_id already set; another worker owns it
+
+        ``claimable`` is the residual that *should* be picked up on the next
+        poll. If ``claimable > 0`` while workers report free slots, the bug is
+        somewhere else (lock contention, tenant discovery, etc.) - this line
+        narrows the search.
+        """
+        if not breakdown:
+            return
+
+        parts = []
+        for op_type in sorted(breakdown):
+            b = breakdown[op_type]
+            claimable = b["total"] - b["payload_null"] - b["retry_blocked"] - b["assigned"]
+            parts.append(
+                f"{op_type}: total={b['total']} claimable={claimable} "
+                f"payload_null={b['payload_null']} retry_blocked={b['retry_blocked']} "
+                f"assigned={b['assigned']}"
+            )
+        logger.info(f"[PENDING_BREAKDOWN] {' | '.join(parts)}")
+
     def _log_per_task_lines(self, active_tasks: dict[str, ActiveTaskInfo], now: float) -> None:
         """Emit one [WORKER_TASK] line per in-flight task and dump stuck stacks.
 

hindsight-api-slim/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-api-slim"
-version = "0.5.1"
+version = "0.5.3"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"

hindsight-api-slim/tests/test_async_batch_retain.py

@@ -433,3 +433,181 @@ async def test_config_retain_batch_tokens_respected(memory, request_context):
     # Even small batches use parent-child pattern now (simpler code path)
     assert "child_operations" in status
     assert status["result_metadata"]["num_sub_batches"] == 1
+
+
+async def _child_metadata(memory, bank_id: str, parent_operation_id: str, request_context):
+    """Fetch the first child operation's result_metadata for a parent batch_retain."""
+    parent = await memory.get_operation_status(
+        bank_id=bank_id,
+        operation_id=parent_operation_id,
+        request_context=request_context,
+    )
+    assert parent["status"] == "completed", parent
+    assert parent["child_operations"], "expected at least one child operation"
+    child_id = parent["child_operations"][0]["operation_id"]
+    child = await memory.get_operation_status(
+        bank_id=bank_id,
+        operation_id=child_id,
+        request_context=request_context,
+    )
+    return child["result_metadata"]
+
+
+@pytest.mark.asyncio
+async def test_retain_records_user_provided_document_ids(memory, request_context):
+    """User-supplied document_ids land in child op result_metadata.document_ids."""
+    bank_id = "test_doc_ids_user_supplied"
+    d1 = str(uuid.uuid4())
+    d2 = str(uuid.uuid4())
+    contents = [
+        {"content": "User-supplied doc one content.", "document_id": d1},
+        {"content": "User-supplied doc two content.", "document_id": d2},
+    ]
+
+    result = await memory.submit_async_retain(
+        bank_id=bank_id,
+        contents=contents,
+        request_context=request_context,
+    )
+    await asyncio.sleep(0.2)
+
+    meta = await _child_metadata(memory, bank_id, result["operation_id"], request_context)
+    assert "document_ids" in meta, meta
+    assert set(meta["document_ids"]) == {d1, d2}
+
+
+@pytest.mark.asyncio
+async def test_retain_records_generated_document_id(memory, request_context):
+    """With no document_ids supplied, retain records the single generated id."""
+    bank_id = "test_doc_ids_generated"
+    contents = [
+        {"content": "Generated doc item one."},
+        {"content": "Generated doc item two."},
+    ]
+
+    result = await memory.submit_async_retain(
+        bank_id=bank_id,
+        contents=contents,
+        request_context=request_context,
+    )
+    await asyncio.sleep(0.2)
+
+    meta = await _child_metadata(memory, bank_id, result["operation_id"], request_context)
+    assert "document_ids" in meta, meta
+    assert isinstance(meta["document_ids"], list)
+    assert len(meta["document_ids"]) == 1
+    # Must be a valid UUID string (generated by the orchestrator)
+    uuid.UUID(meta["document_ids"][0])
+
+
+@pytest.mark.asyncio
+async def test_retain_records_shared_document_id_once(memory, request_context):
+    """Items sharing one document_id record it exactly once (idempotent set-append)."""
+    bank_id = "test_doc_ids_shared"
+    shared = str(uuid.uuid4())
+    # Duplicate per-item doc_ids are rejected up front, so shared-doc mode
+    # is exercised by a single item carrying the id.
+    contents = [{"content": "Shared doc, chunk A.", "document_id": shared}]
+
+    result = await memory.submit_async_retain(
+        bank_id=bank_id,
+        contents=contents,
+        request_context=request_context,
+    )
+    await asyncio.sleep(0.2)
+
+    meta = await _child_metadata(memory, bank_id, result["operation_id"], request_context)
+    assert meta.get("document_ids") == [shared]
+
+
+@pytest.mark.asyncio
+async def test_get_operation_status_include_payload(memory, request_context):
+    """include_payload=True returns the original submission payload; default omits it."""
+    bank_id = "test_include_payload"
+    contents = [{"content": "Payload roundtrip test item."}]
+
+    result = await memory.submit_async_retain(
+        bank_id=bank_id,
+        contents=contents,
+        request_context=request_context,
+    )
+    await asyncio.sleep(0.2)
+
+    parent = await memory.get_operation_status(
+        bank_id=bank_id,
+        operation_id=result["operation_id"],
+        request_context=request_context,
+    )
+    child_id = parent["child_operations"][0]["operation_id"]
+
+    # Default: no payload
+    without = await memory.get_operation_status(
+        bank_id=bank_id,
+        operation_id=child_id,
+        request_context=request_context,
+    )
+    assert without.get("task_payload") is None
+
+    # With flag: payload populated
+    with_payload = await memory.get_operation_status(
+        bank_id=bank_id,
+        operation_id=child_id,
+        request_context=request_context,
+        include_payload=True,
+    )
+    payload = with_payload.get("task_payload")
+    assert payload is not None, with_payload
+    assert payload.get("bank_id") == bank_id
+    assert payload.get("contents")
+    assert payload["contents"][0]["content"] == "Payload roundtrip test item."
+
+
+@pytest.mark.asyncio
+async def test_submit_async_operation_leaves_claimable_row_when_submit_task_fails(memory):
+    """Regression for the crash-window orphan bug fixed in #1091.
+
+    Previously, _submit_async_operation INSERTed the async_operations row without
+    task_payload, then called submit_task as a separate step to fill it in. If
+    submit_task failed (crash, timeout, dropped connection) after the INSERT
+    committed, the row was left with task_payload IS NULL and became permanently
+    stuck because the worker claim query filters on task_payload IS NOT NULL.
+
+    With the atomic INSERT, even if submit_task raises afterwards the row is born
+    claimable. This test simulates the crash by forcing submit_task to raise.
+    """
+    bank_id = f"test_orphan_prevention_{uuid.uuid4().hex[:8]}"
+    pool = await memory._get_pool()
+    await _ensure_bank(pool, bank_id)
+
+    async def failing_submit_task(_task_dict):
+        raise RuntimeError("Simulated crash between INSERT and submit_task")
+
+    memory._task_backend.submit_task = failing_submit_task  # type: ignore[method-assign]
+
+    with pytest.raises(RuntimeError, match="Simulated crash"):
+        await memory._submit_async_operation(
+            bank_id=bank_id,
+            operation_type="retain",
+            task_type="batch_retain",
+            task_payload={"contents": [{"content": "hello", "document_id": "d1"}]},
+        )
+
+    rows = await pool.fetch(
+        """
+        SELECT status, task_payload
+        FROM async_operations
+        WHERE bank_id = $1 AND operation_type = 'retain'
+        """,
+        bank_id,
+    )
+    assert len(rows) == 1, f"Expected exactly one retain row for bank_id={bank_id}, got {len(rows)}"
+    row = rows[0]
+    assert row["status"] == "pending"
+    assert row["task_payload"] is not None, (
+        "task_payload must be set atomically by the INSERT — a NULL here means "
+        "the worker claim query (task_payload IS NOT NULL) will never pick this row up"
+    )
+    payload = json.loads(row["task_payload"])
+    assert payload["type"] == "batch_retain"
+    assert payload["bank_id"] == bank_id
+    assert payload["contents"] == [{"content": "hello", "document_id": "d1"}]

hindsight-api-slim/tests/test_bank_stats.py

@@ -0,0 +1,242 @@
+"""
+Tests for the bank stats endpoint and the memories-timeseries endpoint.
+
+Covers the new fields exposed by GET /v1/default/banks/{bank_id}/stats
+(operations_by_status) and the new endpoint
+GET /v1/default/banks/{bank_id}/stats/memories-timeseries.
+"""
+import uuid
+from datetime import datetime
+
+import httpx
+import pytest
+import pytest_asyncio
+
+from hindsight_api.api import create_app
+
+
+@pytest_asyncio.fixture
+async def api_client(memory):
+    app = create_app(memory, initialize_memory=False)
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
+        yield client
+
+
+@pytest.fixture
+def test_bank_id():
+    return f"stats_test_{datetime.now().timestamp()}"
+
+
+async def _insert_memory(memory, bank_id: str, text: str, *, failed: bool = False) -> str:
+    """Insert a single experience memory, optionally marked as consolidation-failed."""
+    mem_id = uuid.uuid4()
+    async with memory._pool.acquire() as conn:
+        await conn.execute(
+            """
+            INSERT INTO memory_units (id, bank_id, text, fact_type, created_at, consolidation_failed_at)
+            VALUES ($1, $2, $3, 'experience', now(), CASE WHEN $4 THEN now() ELSE NULL END)
+            """,
+            mem_id,
+            bank_id,
+            text,
+            failed,
+        )
+    return str(mem_id)
+
+
+@pytest.mark.asyncio
+async def test_bank_stats_exposes_operations_by_status(api_client, test_bank_id):
+    """/stats should return operations_by_status with all finished operations."""
+    try:
+        # Kick off a retain so at least one completed operation exists.
+        response = await api_client.post(
+            f"/v1/default/banks/{test_bank_id}/memories",
+            json={"items": [{"content": "Alice is a software engineer.", "context": "team"}]},
+        )
+        assert response.status_code == 200
+
+        response = await api_client.get(f"/v1/default/banks/{test_bank_id}/stats")
+        assert response.status_code == 200
+        stats = response.json()
+
+        assert "operations_by_status" in stats
+        assert isinstance(stats["operations_by_status"], dict)
+        # A synchronous retain finishes as "completed".
+        assert stats["operations_by_status"].get("completed", 0) >= 1
+        # pending/failed counters should still be present as scalar mirrors.
+        assert stats["pending_operations"] == stats["operations_by_status"].get("pending", 0)
+        assert stats["failed_operations"] == stats["operations_by_status"].get("failed", 0)
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "period,expected_count,expected_trunc",
+    [
+        ("1h", 60, "minute"),
+        ("12h", 12, "hour"),
+        ("1d", 24, "hour"),
+        ("7d", 7, "day"),
+        ("30d", 30, "day"),
+        ("90d", 90, "day"),
+    ],
+)
+async def test_memories_timeseries_periods(
+    api_client, test_bank_id, period, expected_count, expected_trunc
+):
+    """Every period must return the full expected bucket count and trunc."""
+    try:
+        response = await api_client.post(
+            f"/v1/default/banks/{test_bank_id}/memories",
+            json={"items": [{"content": "Bob works on infrastructure.", "context": "team"}]},
+        )
+        assert response.status_code == 200
+
+        response = await api_client.get(
+            f"/v1/default/banks/{test_bank_id}/stats/memories-timeseries",
+            params={"period": period},
+        )
+        assert response.status_code == 200
+        body = response.json()
+
+        assert body["bank_id"] == test_bank_id
+        assert body["period"] == period
+        assert body["trunc"] == expected_trunc
+        assert len(body["buckets"]) == expected_count
+
+        for bucket in body["buckets"]:
+            assert "time" in bucket
+            assert bucket["world"] >= 0
+            assert bucket["experience"] >= 0
+            assert bucket["observation"] >= 0
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+async def test_memories_timeseries_invalid_period_falls_back(api_client, test_bank_id):
+    """An unknown period must fall back to the 7d default."""
+    try:
+        response = await api_client.get(
+            f"/v1/default/banks/{test_bank_id}/stats/memories-timeseries",
+            params={"period": "nonsense"},
+        )
+        assert response.status_code == 200
+        body = response.json()
+        assert body["period"] == "7d"
+        assert body["trunc"] == "day"
+        assert len(body["buckets"]) == 7
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+async def test_memories_timeseries_empty_bank_returns_zero_filled_buckets(
+    api_client, test_bank_id
+):
+    """A bank with no memories must still return the full zero-filled bucket set."""
+    try:
+        # Ensure the bank exists.
+        response = await api_client.get(f"/v1/default/banks/{test_bank_id}/profile")
+        assert response.status_code == 200
+
+        response = await api_client.get(
+            f"/v1/default/banks/{test_bank_id}/stats/memories-timeseries",
+            params={"period": "7d"},
+        )
+        assert response.status_code == 200
+        body = response.json()
+        assert len(body["buckets"]) == 7
+        for bucket in body["buckets"]:
+            assert bucket["world"] == 0
+            assert bucket["experience"] == 0
+            assert bucket["observation"] == 0
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+async def test_memories_timeseries_reflects_retained_memories(api_client, test_bank_id):
+    """Freshly-retained memories must show up in today's bucket counts."""
+    try:
+        response = await api_client.post(
+            f"/v1/default/banks/{test_bank_id}/memories",
+            json={
+                "items": [
+                    {"content": "Alice is a software engineer.", "context": "team"},
+                    {"content": "Bob works on infrastructure.", "context": "team"},
+                ]
+            },
+        )
+        assert response.status_code == 200
+
+        response = await api_client.get(
+            f"/v1/default/banks/{test_bank_id}/stats/memories-timeseries",
+            params={"period": "7d"},
+        )
+        assert response.status_code == 200
+        body = response.json()
+        totals = sum(b["world"] + b["experience"] + b["observation"] for b in body["buckets"])
+        assert totals >= 2, "expected at least two memories across all buckets"
+
+        # Those memories should land in the most-recent bucket.
+        latest = body["buckets"][-1]
+        assert latest["world"] + latest["experience"] + latest["observation"] >= 2
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+async def test_bank_stats_reports_failed_consolidation(api_client, memory, test_bank_id):
+    """/stats must surface the count of memories with consolidation_failed_at set."""
+    try:
+        await _insert_memory(memory, test_bank_id, "Alice failed 1.", failed=True)
+        await _insert_memory(memory, test_bank_id, "Alice failed 2.", failed=True)
+        await _insert_memory(memory, test_bank_id, "Alice pending.", failed=False)
+
+        response = await api_client.get(f"/v1/default/banks/{test_bank_id}/stats")
+        assert response.status_code == 200
+        stats = response.json()
+
+        assert stats["failed_consolidation"] == 2
+        # The two failed memories also count as "not-yet-consolidated".
+        assert stats["pending_consolidation"] >= 3
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+async def test_list_memories_filter_by_consolidation_state_failed(api_client, memory, test_bank_id):
+    """?consolidation_state=failed returns only memories with consolidation_failed_at set."""
+    try:
+        failed_id = await _insert_memory(memory, test_bank_id, "Broken item.", failed=True)
+        await _insert_memory(memory, test_bank_id, "Healthy item.", failed=False)
+
+        response = await api_client.get(
+            f"/v1/default/banks/{test_bank_id}/memories/list",
+            params={"consolidation_state": "failed"},
+        )
+        assert response.status_code == 200
+        body = response.json()
+
+        ids = [item["id"] for item in body["items"]]
+        assert failed_id in ids
+        assert body["total"] == 1
+        assert body["items"][0]["consolidation_failed_at"] is not None
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")
+
+
+@pytest.mark.asyncio
+async def test_list_memories_filter_by_consolidation_state_rejects_unknown(api_client, test_bank_id):
+    """An invalid consolidation_state value must return a 400 (not 500)."""
+    try:
+        response = await api_client.get(
+            f"/v1/default/banks/{test_bank_id}/memories/list",
+            params={"consolidation_state": "bogus"},
+        )
+        assert response.status_code == 400
+    finally:
+        await api_client.delete(f"/v1/default/banks/{test_bank_id}")

hindsight-api-slim/tests/test_bank_template_configurable_fields.py

@@ -0,0 +1,115 @@
+"""Verify that BankTemplateConfig exposes every hierarchical field that
+_CONFIGURABLE_FIELDS already accepts at the engine layer.
+
+This test guards the fix for the gap described in the upstream PR title
+"fix(bank-template): align BankTemplateConfig with _CONFIGURABLE_FIELDS".
+Each new field is POSTed through /v1/default/banks/{id}/import and then
+read back via the bank-config endpoint; assertion is that the applied
+value round-trips through the engine.
+
+Runs via: uv run pytest tests/test_bank_template_configurable_fields.py -v
+
+The api_client fixture (shared with tests/test_bank_templates.py) wraps
+create_app(memory, initialize_memory=False) in an httpx.ASGITransport
+with base_url http://test — in-process, no network, no tenant extension.
+Copy the fixture inline here so the test file does not depend on a
+conftest we do not ship in the patch.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+import httpx
+import pytest
+import pytest_asyncio
+
+from hindsight_api.api import create_app
+from hindsight_api.api.http import BankTemplateConfig
+
+# Each tuple is (field_name, applied_value). Values chosen to differ
+# visibly from defaults so round-trip bugs surface.
+NEW_FIELDS: list[tuple[str, object]] = [
+    ("retain_default_strategy", "strategy-a"),
+    ("retain_strategies", {"strategy-a": {"mode": "concise", "max_tokens": 512}}),
+    ("retain_chunk_batch_size", 7),
+    ("mcp_enabled_tools", ["list_banks", "get_bank_profile"]),
+    ("consolidation_llm_batch_size", 11),
+    ("consolidation_source_facts_max_tokens", 2048),
+    ("consolidation_source_facts_max_tokens_per_observation", 256),
+    ("max_observations_per_scope", 13),
+    ("reflect_source_facts_max_tokens", 4096),
+    ("llm_gemini_safety_settings", [{"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"}]),
+    ("recall_budget_function", "adaptive"),
+    ("recall_budget_fixed_low", 50),
+    ("recall_budget_fixed_mid", 250),
+    ("recall_budget_fixed_high", 800),
+    ("recall_budget_adaptive_low", 0.05),
+    ("recall_budget_adaptive_mid", 0.1),
+    ("recall_budget_adaptive_high", 0.4),
+    ("recall_budget_min", 30),
+    ("recall_budget_max", 1500),
+]
+
+
+@pytest_asyncio.fixture
+async def api_client(memory):
+    """Matches the fixture in tests/test_bank_templates.py — in-process
+    ASGI test client, no tenant extension, no auth."""
+    app = create_app(memory, initialize_memory=False)
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
+        yield client
+
+
+@pytest.fixture
+def bank_id():
+    return f"tmpl_config_{datetime.now().timestamp()}"
+
+
+def test_bank_template_config_declares_every_configurable_field():
+    """Pydantic-level guard: every field in NEW_FIELDS must be a declared
+    attribute of BankTemplateConfig so get_config_updates() picks it up."""
+    declared = set(BankTemplateConfig.model_fields.keys())
+    missing = [name for name, _ in NEW_FIELDS if name not in declared]
+    assert not missing, f"BankTemplateConfig missing fields: {missing}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("field_name,applied_value", NEW_FIELDS, ids=[n for n, _ in NEW_FIELDS])
+async def test_new_field_round_trips_through_import(
+    api_client: httpx.AsyncClient,
+    bank_id: str,
+    field_name: str,
+    applied_value: object,
+):
+    """POST a minimal manifest with one new field set, then read bank
+    config back and assert the value made it through.
+
+    Bank config response shape per upstream's test_import_applies_config:
+    top-level keys are resolved hierarchical config; per-bank overrides
+    live under config["overrides"][<field>]. Assert on the override slot.
+    """
+    unique_bank_id = f"{bank_id}_{field_name}"
+    manifest = {
+        "version": "1",
+        "bank": {field_name: applied_value},
+    }
+
+    resp = await api_client.post(
+        f"/v1/default/banks/{unique_bank_id}/import",
+        json=manifest,
+    )
+    assert resp.status_code == 200, resp.text
+
+    # Read bank config back — field must reflect the applied value
+    # under the "overrides" slot, matching upstream's own test shape.
+    read = await api_client.get(f"/v1/default/banks/{unique_bank_id}/config")
+    assert read.status_code == 200, read.text
+    config = read.json()
+    overrides = config.get("overrides", {})
+    assert overrides.get(field_name) == applied_value, (
+        f"round-trip mismatch for {field_name}: "
+        f"sent {applied_value!r}, got {overrides.get(field_name)!r} "
+        f"(full overrides: {overrides!r})"
+    )

hindsight-api-slim/tests/test_consolidation.py

@@ -1515,6 +1515,7 @@ class TestHierarchicalRetrieval:
         async with memory._pool.acquire() as conn:
             query_embedding = memory.embeddings.encode(["What does John like?"])[0]
             mental_model_result = await tool_search_mental_models(
+                memory_engine=memory,
                 conn=conn,
                 bank_id=bank_id,
                 query="What does John like?",
@@ -1576,6 +1577,7 @@ class TestHierarchicalRetrieval:
         async with memory._pool.acquire() as conn:
             query_embedding = memory.embeddings.encode(["Where does Sarah work?"])[0]
             mental_model_result = await tool_search_mental_models(
+                memory_engine=memory,
                 conn=conn,
                 bank_id=bank_id,
                 query="Where does Sarah work?",

hindsight-api-slim/tests/test_consolidation_retry_budget.py

@@ -0,0 +1,100 @@
+"""Tests for consolidation retry budget configurability (issue #1042)."""
+
+import pytest
+
+from unittest.mock import AsyncMock, MagicMock
+
+from hindsight_api.engine.consolidation.consolidator import _consolidate_batch_with_llm
+
+
+@pytest.fixture
+def mock_llm_config():
+    llm = AsyncMock()
+    response = MagicMock()
+    response.creates = []
+    response.updates = []
+    response.deletes = []
+    llm.call.return_value = response
+    return llm
+
+
+@pytest.fixture
+def mock_config():
+    config = MagicMock()
+    config.observations_mission = None
+    config.consolidation_max_attempts = 3
+    config.consolidation_llm_max_retries = None
+    return config
+
+
+class TestConsolidationRetryBudget:
+    @pytest.mark.asyncio
+    async def test_config_is_required(self, mock_llm_config):
+        """Passing config=None raises — it's a programmer error, not a runtime fallback."""
+        with pytest.raises(ValueError, match="config is required"):
+            await _consolidate_batch_with_llm(
+                llm_config=mock_llm_config,
+                memories=[{"id": "m1", "text": "test"}],
+                union_observations=[],
+                union_source_facts={},
+                config=None,
+            )
+
+    @pytest.mark.asyncio
+    async def test_configurable_max_attempts(self, mock_llm_config, mock_config):
+        """consolidation_max_attempts controls the outer retry loop."""
+        mock_config.consolidation_max_attempts = 5
+        mock_llm_config.call.side_effect = RuntimeError("fail")
+        result = await _consolidate_batch_with_llm(
+            llm_config=mock_llm_config,
+            memories=[{"id": "m1", "text": "test"}],
+            union_observations=[],
+            union_source_facts={},
+            config=mock_config,
+        )
+        assert result.failed
+        assert mock_llm_config.call.call_count == 5
+
+    @pytest.mark.asyncio
+    async def test_max_retries_threaded_to_call(self, mock_llm_config, mock_config):
+        """consolidation_llm_max_retries is passed to llm_config.call()."""
+        mock_config.consolidation_llm_max_retries = 3
+        await _consolidate_batch_with_llm(
+            llm_config=mock_llm_config,
+            memories=[{"id": "m1", "text": "test"}],
+            union_observations=[],
+            union_source_facts={},
+            config=mock_config,
+        )
+        assert mock_llm_config.call.call_args.kwargs.get("max_retries") == 3
+
+    @pytest.mark.asyncio
+    async def test_max_retries_not_passed_when_none(self, mock_llm_config, mock_config):
+        """When consolidation_llm_max_retries is None, max_retries is not passed."""
+        mock_config.consolidation_llm_max_retries = None
+        await _consolidate_batch_with_llm(
+            llm_config=mock_llm_config,
+            memories=[{"id": "m1", "text": "test"}],
+            union_observations=[],
+            union_source_facts={},
+            config=mock_config,
+        )
+        assert "max_retries" not in mock_llm_config.call.call_args.kwargs
+
+    @pytest.mark.asyncio
+    async def test_reduced_budget_limits_total_calls(self, mock_llm_config, mock_config):
+        """Setting both to low values caps total failure attempts."""
+        mock_config.consolidation_max_attempts = 2
+        mock_config.consolidation_llm_max_retries = 2
+        mock_llm_config.call.side_effect = RuntimeError("upstream 503")
+        result = await _consolidate_batch_with_llm(
+            llm_config=mock_llm_config,
+            memories=[{"id": "m1", "text": "test"}],
+            union_observations=[],
+            union_source_facts={},
+            config=mock_config,
+        )
+        assert result.failed
+        assert mock_llm_config.call.call_count == 2
+        for call_args in mock_llm_config.call.call_args_list:
+            assert call_args.kwargs.get("max_retries") == 2

hindsight-api-slim/tests/test_consolidation_round_limit.py

@@ -0,0 +1,146 @@
+"""Integration tests for consolidation_max_memories_per_round config."""
+
+import uuid
+from unittest.mock import patch
+
+import pytest
+
+from hindsight_api.config import _get_raw_config
+from hindsight_api.engine.consolidation.consolidator import run_consolidation_job
+from hindsight_api.engine.memory_engine import MemoryEngine
+
+
+@pytest.fixture(autouse=True)
+def enable_observations():
+    config = _get_raw_config()
+    original = config.enable_observations
+    config.enable_observations = True
+    yield
+    config.enable_observations = original
+
+
+def _make_config(**overrides):
+    raw = _get_raw_config()
+    return type(raw)(
+        **{
+            **{f: getattr(raw, f) for f in raw.__dataclass_fields__},
+            **overrides,
+        }
+    )
+
+
+@pytest.mark.asyncio
+async def test_round_limit_caps_processed_memories(memory: MemoryEngine, request_context):
+    """When max_memories_per_round is set, consolidation processes at most that many memories
+    and re-submits itself for the remaining backlog."""
+    bank_id = f"test-round-limit-{uuid.uuid4().hex[:8]}"
+    await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
+
+    # Disable consolidation during retain so we build up a backlog
+    fake_config_no_obs = _make_config(enable_observations=False)
+
+    with patch.object(memory._config_resolver, "resolve_full_config", return_value=fake_config_no_obs):
+        for i in range(6):
+            await memory.retain_async(
+                bank_id=bank_id,
+                content=f"Fact number {i}: The user enjoys activity {i} on weekends.",
+                request_context=request_context,
+            )
+
+    # Verify we have unconsolidated memories
+    async with memory._pool.acquire() as conn:
+        unconsolidated = await conn.fetchval(
+            """
+            SELECT COUNT(*) FROM memory_units
+            WHERE bank_id = $1 AND consolidated_at IS NULL
+              AND consolidation_failed_at IS NULL AND fact_type IN ('experience', 'world')
+            """,
+            bank_id,
+        )
+    assert unconsolidated >= 6, f"Expected at least 6 unconsolidated memories, got {unconsolidated}"
+
+    # Run consolidation with a round limit of 3
+    round_limit = 3
+    fake_config = _make_config(consolidation_max_memories_per_round=round_limit)
+
+    with (
+        patch.object(memory._config_resolver, "resolve_full_config", return_value=fake_config),
+        patch.object(memory, "submit_async_consolidation") as mock_requeue,
+    ):
+        result = await run_consolidation_job(
+            memory_engine=memory,
+            bank_id=bank_id,
+            request_context=request_context,
+        )
+
+    assert result["status"] == "completed"
+    assert result["memories_processed"] <= round_limit
+
+    # Must have re-queued consolidation for remaining work
+    mock_requeue.assert_called_once_with(bank_id=bank_id, request_context=request_context)
+
+    # Mental model refresh should be skipped on intermediate round
+    assert result.get("mental_models_refreshed", 0) == 0
+
+    # Verify some memories are still unconsolidated
+    async with memory._pool.acquire() as conn:
+        still_unconsolidated = await conn.fetchval(
+            """
+            SELECT COUNT(*) FROM memory_units
+            WHERE bank_id = $1 AND consolidated_at IS NULL
+              AND consolidation_failed_at IS NULL AND fact_type IN ('experience', 'world')
+            """,
+            bank_id,
+        )
+    assert still_unconsolidated > 0, "Some memories should still be unconsolidated after hitting round limit"
+
+    await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_unlimited_round_processes_all(memory: MemoryEngine, request_context):
+    """When max_memories_per_round is 0 (unlimited), all memories are processed without re-queue."""
+    bank_id = f"test-unlimited-{uuid.uuid4().hex[:8]}"
+    await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
+
+    # Disable consolidation during retain
+    fake_config_no_obs = _make_config(enable_observations=False)
+
+    with patch.object(memory._config_resolver, "resolve_full_config", return_value=fake_config_no_obs):
+        for i in range(4):
+            await memory.retain_async(
+                bank_id=bank_id,
+                content=f"Fact {i}: The user visited city {i} last year.",
+                request_context=request_context,
+            )
+
+    # Run consolidation with unlimited round (0)
+    fake_config = _make_config(consolidation_max_memories_per_round=0)
+
+    with (
+        patch.object(memory._config_resolver, "resolve_full_config", return_value=fake_config),
+        patch.object(memory, "submit_async_consolidation") as mock_requeue,
+    ):
+        result = await run_consolidation_job(
+            memory_engine=memory,
+            bank_id=bank_id,
+            request_context=request_context,
+        )
+
+    assert result["status"] == "completed"
+    # Should NOT re-queue
+    mock_requeue.assert_not_called()
+
+    # All memories should be consolidated
+    async with memory._pool.acquire() as conn:
+        still_unconsolidated = await conn.fetchval(
+            """
+            SELECT COUNT(*) FROM memory_units
+            WHERE bank_id = $1 AND consolidated_at IS NULL
+              AND consolidation_failed_at IS NULL AND fact_type IN ('experience', 'world')
+            """,
+            bank_id,
+        )
+    assert still_unconsolidated == 0
+
+    await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_file_retain.py

@@ -543,6 +543,196 @@ async def test_async_file_retain_serializes_datetime_timestamp(memory_no_llm_ver
     assert row["timestamp"] == "2024-01-15T10:30:00+00:00"
 
 
+@pytest.mark.asyncio
+async def test_file_retain_maps_timestamp_to_event_date(memory_no_llm_verify, sample_txt_content):
+    """Regression (PR #1092): file retain must translate 'timestamp' -> 'event_date'.
+
+    The retain orchestrator only reads 'event_date' from each content dict.
+    _handle_file_convert_retain previously forwarded 'timestamp' unchanged, so every
+    file-retained memory silently defaulted to utcnow() and the 'unset' sentinel
+    was a no-op. This test intercepts the inner batch_retain task the handler
+    submits and asserts the key mapping is correct for all three inputs:
+    explicit ISO timestamp, 'unset' sentinel, and omitted (None).
+    """
+    from hindsight_api.engine.parsers.base import FileParser
+    from hindsight_api.models import RequestContext
+
+    memory = memory_no_llm_verify
+
+    class NoopParser(FileParser):
+        async def convert(self, file_data: bytes, filename: str) -> str:
+            return file_data.decode("utf-8")
+
+        def supports(self, filename: str, content_type: str | None = None) -> bool:
+            return filename.endswith(".txt")
+
+        def name(self) -> str:
+            return "event_date_regression_parser"
+
+    memory._parser_registry.register(NoopParser())
+
+    class MockFile:
+        def __init__(self, content, filename, content_type):
+            self.content = content
+            self.filename = filename
+            self.content_type = content_type
+
+        async def read(self):
+            return self.content
+
+    # Capture the inner batch_retain submission from _handle_file_convert_retain so we
+    # can inspect its content dict without running the (LLM-dependent) retain pipeline.
+    original_submit = memory._task_backend.submit_task
+    captured: list[dict] = []
+
+    async def capturing_submit(task_dict):
+        if task_dict.get("type") == "batch_retain":
+            captured.append(task_dict)
+            return
+        await original_submit(task_dict)
+
+    memory._task_backend.submit_task = capturing_submit
+    try:
+        context = RequestContext(internal=True)
+
+        async def run_case(label: str, timestamp_value) -> dict:
+            bank_id = f"test_file_event_date_{label}_{datetime.now(timezone.utc).timestamp()}"
+            await memory.get_bank_profile(bank_id, request_context=context)
+
+            captured.clear()
+            await memory.submit_async_file_retain(
+                bank_id=bank_id,
+                file_items=[
+                    {
+                        "file": MockFile(sample_txt_content, f"{label}.txt", "text/plain"),
+                        "document_id": f"doc_{label}",
+                        "context": "regression test",
+                        "metadata": {},
+                        "tags": [],
+                        "timestamp": timestamp_value,
+                        "parser": ["event_date_regression_parser"],
+                    }
+                ],
+                document_tags=None,
+                request_context=context,
+            )
+            assert len(captured) == 1, f"{label}: expected exactly one batch_retain submission"
+            contents = captured[0]["contents"]
+            assert len(contents) == 1
+            return contents[0]
+
+        # Explicit ISO timestamp -> event_date must equal that string.
+        content = await run_case("explicit", "2024-01-15T10:30:00+00:00")
+        assert "timestamp" not in content, "raw 'timestamp' must not leak into retain content"
+        assert content["event_date"] == "2024-01-15T10:30:00+00:00"
+
+        # 'unset' sentinel -> event_date must be explicit None (orchestrator stores NULL).
+        content = await run_case("unset", "unset")
+        assert "timestamp" not in content
+        assert "event_date" in content, "'unset' must produce an explicit event_date=None"
+        assert content["event_date"] is None
+
+        # Omitted timestamp -> event_date key must be absent (orchestrator defaults to utcnow).
+        content = await run_case("missing", None)
+        assert "timestamp" not in content
+        assert "event_date" not in content
+    finally:
+        memory._task_backend.submit_task = original_submit
+
+
+@pytest.mark.asyncio
+async def test_file_retain_forwards_all_content_fields(memory_no_llm_verify, sample_txt_content):
+    """Regression: _handle_file_convert_retain must forward every FileRetainMetadata
+    field to the inner batch_retain task without renaming or dropping it.
+
+    Covers document_id, context, metadata, tags (per-content), plus strategy
+    and document_tags (per-request). The timestamp -> event_date mapping has
+    its own test above. Existing file retain tests only assert HTTP 200 or
+    inspect the outer file_convert_retain task_payload; none verify what
+    arrives at the retain pipeline. If any of these fields were silently
+    dropped or mis-keyed -- the same failure mode as #1092 for timestamp --
+    those tests would still pass.
+    """
+    from hindsight_api.engine.parsers.base import FileParser
+    from hindsight_api.models import RequestContext
+
+    memory = memory_no_llm_verify
+
+    class NoopParser(FileParser):
+        async def convert(self, file_data: bytes, filename: str) -> str:
+            return file_data.decode("utf-8")
+
+        def supports(self, filename: str, content_type: str | None = None) -> bool:
+            return filename.endswith(".txt")
+
+        def name(self) -> str:
+            return "all_fields_regression_parser"
+
+    memory._parser_registry.register(NoopParser())
+
+    class MockFile:
+        def __init__(self, content, filename, content_type):
+            self.content = content
+            self.filename = filename
+            self.content_type = content_type
+
+        async def read(self):
+            return self.content
+
+    original_submit = memory._task_backend.submit_task
+    captured: list[dict] = []
+
+    async def capturing_submit(task_dict):
+        if task_dict.get("type") == "batch_retain":
+            captured.append(task_dict)
+            return
+        await original_submit(task_dict)
+
+    memory._task_backend.submit_task = capturing_submit
+    try:
+        request_context = RequestContext(internal=True)
+        bank_id = f"test_file_all_fields_{datetime.now(timezone.utc).timestamp()}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        await memory.submit_async_file_retain(
+            bank_id=bank_id,
+            file_items=[
+                {
+                    "file": MockFile(sample_txt_content, "doc.txt", "text/plain"),
+                    "document_id": "my_doc_id",
+                    "context": "meeting notes from Alice",
+                    "metadata": {"author": "Alice", "year": "2024"},
+                    "tags": ["report", "q1"],
+                    "timestamp": None,
+                    "parser": ["all_fields_regression_parser"],
+                    "strategy": "my_strategy",
+                }
+            ],
+            document_tags=["batch_tag"],
+            request_context=request_context,
+        )
+
+        assert len(captured) == 1, "expected exactly one batch_retain submission"
+        payload = captured[0]
+        assert payload["type"] == "batch_retain"
+
+        # Per-request fields (live on the outer task payload, not per-content).
+        assert payload.get("strategy") == "my_strategy", "strategy must be forwarded at request level"
+        assert payload.get("document_tags") == ["batch_tag"], "document_tags must be forwarded at request level"
+
+        # Per-content fields.
+        assert len(payload["contents"]) == 1
+        content = payload["contents"][0]
+        assert content["document_id"] == "my_doc_id"
+        assert content["context"] == "meeting notes from Alice"
+        assert content["metadata"] == {"author": "Alice", "year": "2024"}
+        assert content["tags"] == ["report", "q1"]
+        # content is the converted markdown (raw bytes decoded by NoopParser).
+        assert content["content"] == sample_txt_content.decode("utf-8")
+    finally:
+        memory._task_backend.submit_task = original_submit
+
+
 @pytest.mark.asyncio
 async def test_file_conversion_failure_sets_status_to_failed(memory_no_llm_verify, sample_txt_content):
     """Test that when file conversion fails, the operation status is set to 'failed' not 'completed'."""

hindsight-api-slim/tests/test_hierarchical_config.py

@@ -98,7 +98,7 @@ async def test_hierarchical_fields_categorization():
     assert "retain_chunk_batch_size" in configurable
 
     # Verify count is correct
-    assert len(configurable) == 22
+    assert len(configurable) == 35
 
     # Verify credential fields (NEVER exposed)
     assert "llm_api_key" in credentials
@@ -458,7 +458,7 @@ async def test_config_get_bank_config_no_static_or_credential_fields_leak(memory
             assert field in config, f"Expected configurable field '{field}' missing from config"
 
         # Should have a small number of configurable fields (not hundreds)
-        assert len(config) < 25, f"Too many fields returned: {len(config)}"
+        assert len(config) < 50, f"Too many fields returned: {len(config)}"
 
     finally:
         await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_http_api_integration.py

@@ -286,6 +286,34 @@ async def test_full_api_workflow(api_client, test_bank_id):
         )
         assert response.status_code == 410  # Deprecated endpoint
 
+    # Entity co-occurrence graph — shape is stable even when there are no
+    # co-occurrences; every edge must reference two nodes that are also present.
+    response = await api_client.get(f"/v1/default/banks/{test_bank_id}/entities/graph")
+    assert response.status_code == 200
+    entity_graph = response.json()
+    assert set(entity_graph.keys()) >= {"nodes", "edges", "total_entities", "total_edges", "limit"}
+    assert entity_graph["limit"] == 1000
+    assert len(entity_graph["nodes"]) == entity_graph["total_entities"]
+    assert len(entity_graph["edges"]) == entity_graph["total_edges"]
+    node_ids = {n["data"]["id"] for n in entity_graph["nodes"]}
+    for edge in entity_graph["edges"]:
+        assert edge["data"]["source"] in node_ids
+        assert edge["data"]["target"] in node_ids
+        assert edge["data"]["linkType"] == "cooccurrence"
+        assert edge["data"]["weight"] >= 1
+
+    # min_count filter — raising the threshold can only shrink the edge set.
+    response = await api_client.get(
+        f"/v1/default/banks/{test_bank_id}/entities/graph?min_count=1000000"
+    )
+    assert response.status_code == 200
+    filtered_graph = response.json()
+    assert filtered_graph["total_edges"] == 0
+
+    # "graph" must route to the graph endpoint, not be parsed as an entity_id.
+    # Regression guard in case someone reorders the FastAPI route registration.
+    assert entity_graph["total_entities"] >= 0
+
     # ================================================================
     # 9. List All Banks (should include our test bank)
     # ================================================================

hindsight-api-slim/tests/test_load_large_batch.py

@@ -25,6 +25,8 @@ from hindsight_api.engine.llm_wrapper import TokenUsage
 
 logger = logging.getLogger(__name__)
 
+pytestmark = pytest.mark.xdist_group("load_batch_tests")
+
 
 def generate_content(char_count: int) -> str:
     """Generate realistic content of approximately char_count characters."""
@@ -117,9 +119,18 @@ class TestLargeBatchRetain:
         except Exception:
             pass
 
+    @pytest.fixture
+    def disable_observations(self):
+        from hindsight_api.config import _get_raw_config
+        config = _get_raw_config()
+        original = config.enable_observations
+        config.enable_observations = False
+        yield
+        config.enable_observations = original
+
     @pytest.mark.asyncio
     @pytest.mark.timeout(300)  # 5 minute timeout
-    async def test_large_batch_500k_chars_20_items(self, memory_with_mock_llm, request_context):
+    async def test_large_batch_500k_chars_20_items(self, memory_with_mock_llm, request_context, disable_observations):
         """
         Test retaining a batch of 20 content items totaling ~500k chars.
 
@@ -283,7 +294,7 @@ class TestLargeBatchRetain:
 
     @pytest.mark.asyncio
     @pytest.mark.timeout(60)
-    async def test_db_connection_pool_under_load(self, memory_with_mock_llm, request_context):
+    async def test_db_connection_pool_under_load(self, memory_with_mock_llm, request_context, disable_observations):
         """
         Test that DB connection pool handles concurrent operations.
 

hindsight-api-slim/tests/test_mental_model_delta.py

@@ -0,0 +1,892 @@
+"""Tests for delta-mode mental model refresh.
+
+Delta mode performs a surgical update on the existing mental model content:
+- Unchanged sections are preserved byte-for-byte.
+- Stale content is removed.
+- New content from observations/facts is added, preferably by extending existing sections.
+
+Fallback rules:
+- If the mental model has no existing content, delta falls back to a full regeneration.
+- If the source_query has changed since the last refresh, delta falls back to a full regeneration.
+
+This file contains two kinds of tests:
+
+1. TestDeltaRefreshPlumbing: fast, deterministic tests that monkey-patch reflect_async
+   and the LLM call to verify branching logic (fallback conditions, provenance tracking).
+
+2. TestDeltaRefreshGeminiEval: real-LLM behavioral evals against Gemini. These are
+   gated on HINDSIGHT_RUN_GEMINI_EVALS=1 (plus a Gemini API key) because they cost
+   money/time and require network access. They verify the actual quality of delta
+   updates — format preservation, surgical edits, observation-grounding.
+"""
+
+import os
+import uuid
+from typing import Any
+
+import pytest
+
+from hindsight_api import MemoryEngine, RequestContext
+from hindsight_api.engine.llm_wrapper import LLMConfig
+from hindsight_api.engine.response_models import ReflectResult
+
+
+def _canned_reflect_result(text: str, facts: list[dict] | None = None) -> ReflectResult:
+    """Build a minimal ReflectResult for monkey-patching reflect_async."""
+    return ReflectResult.model_validate(
+        {
+            "text": text,
+            "based_on": {
+                "observation": facts or [],
+                "world": [],
+                "experience": [],
+                "mental-models": [],
+                "directives": [],
+            },
+        }
+    )
+
+
+@pytest.fixture
+def patch_reflect(monkeypatch):
+    """Helper that patches memory.reflect_async to return a canned result and records the call.
+
+    Usage:
+        calls = patch_reflect(memory, text="hello", facts=[...])
+        await memory.refresh_mental_model(...)
+        assert len(calls) == 1
+    """
+
+    def _install(memory: MemoryEngine, *, text: str, facts: list[dict] | None = None):
+        calls: list[dict] = []
+
+        async def fake_reflect_async(**kwargs):
+            calls.append(kwargs)
+            return _canned_reflect_result(text, facts)
+
+        monkeypatch.setattr(memory, "reflect_async", fake_reflect_async)
+        return calls
+
+    return _install
+
+
+@pytest.fixture
+def patch_llm_call(monkeypatch):
+    """Patch the reflect LLM config's ``.call()`` used for the structured delta call.
+
+    The structured-delta path passes ``response_format=DeltaOperationList``, so the
+    LLM returns a Pydantic instance.  Each invocation of ``patch_llm_call`` installs
+    a single canned response, in any of these shapes:
+
+    - ``DeltaOperationList`` instance → returned as-is
+    - ``[]`` (empty list) → no operations (this is the no-change case)
+    - ``[{"op": "...", ...}, ...]`` → wrapped into ``{"operations": [...]}``
+    - ``{"operations": [...]}`` → validated directly
+    """
+    from hindsight_api.engine.reflect.delta_ops import DeltaOperationList
+
+    def _to_op_list(resp: Any) -> DeltaOperationList:
+        if isinstance(resp, DeltaOperationList):
+            return resp
+        if isinstance(resp, dict):
+            if "operations" in resp:
+                return DeltaOperationList.model_validate(resp)
+            # Treat a bare op dict as a one-op list for ergonomics.
+            return DeltaOperationList.model_validate({"operations": [resp]})
+        if isinstance(resp, list):
+            return DeltaOperationList.model_validate({"operations": resp})
+        if isinstance(resp, str):
+            # Tests that expect *no* call ever still install a sentinel; treat as no-op.
+            return DeltaOperationList()
+        raise TypeError(f"unsupported canned LLM response: {type(resp)!r}")
+
+    def _install(memory: MemoryEngine, *, returns):
+        calls: list[dict] = []
+        canned = _to_op_list(returns)
+
+        async def fake_call(*, messages, **kwargs):
+            calls.append({"messages": messages, **kwargs})
+            return canned
+
+        monkeypatch.setattr(memory._reflect_llm_config, "call", fake_call)
+        return calls
+
+    return _install
+
+
+class TestDeltaRefreshPlumbing:
+    """Deterministic tests that verify the branching/plumbing of delta-mode refresh."""
+
+    async def test_full_mode_does_not_call_delta_merge(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        patch_llm_call,
+    ):
+        """When trigger.mode='full', no second LLM call for delta merge occurs."""
+        bank_id = f"test-delta-full-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content="# Team\n\nOriginal content.",
+            trigger={"mode": "full"},
+            request_context=request_context,
+        )
+
+        patch_reflect(memory, text="# Team\n\nRegenerated from scratch.")
+        llm_calls = patch_llm_call(memory, returns="should-not-be-called")
+
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        assert refreshed is not None
+        assert refreshed["content"] == "# Team\n\nRegenerated from scratch."
+        assert len(llm_calls) == 0, "Delta merge LLM call must not happen in full mode"
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_delta_mode_empty_content_falls_back_to_full(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        patch_llm_call,
+    ):
+        """When the mental model has no existing content there is nothing to anchor
+        a surgical edit on, so delta falls back to full regeneration. The user's
+        candidate from reflect_async is used verbatim.
+        """
+        bank_id = f"test-delta-empty-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content="",  # no existing content
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+
+        patch_reflect(memory, text="# Team\n\nFull fresh synthesis.")
+        llm_calls = patch_llm_call(memory, returns=[])
+
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        assert refreshed["content"] == "# Team\n\nFull fresh synthesis."
+        assert len(llm_calls) == 0  # delta path skipped entirely
+        rr = refreshed.get("reflect_response") or {}
+        assert rr.get("delta_applied") is not True
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_delta_mode_source_query_change_falls_back_to_full(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        patch_llm_call,
+    ):
+        """If source_query changes after a refresh, the next delta run must do a full rewrite."""
+        bank_id = f"test-delta-query-change-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content="# Team\n\nBaseline.",
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+
+        # First refresh: establishes last_refreshed_source_query.
+        patch_reflect(memory, text="# Team\n\nFirst pass.")
+        patch_llm_call(memory, returns="unused-first")
+        await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        # Now change the source_query — a genuine topic shift.
+        await memory.update_mental_model(
+            bank_id=bank_id,
+            mental_model_id=mm["id"],
+            source_query="Tell me about customers instead",
+            request_context=request_context,
+        )
+
+        # Second refresh under the new query must do a FULL rewrite, not a delta merge.
+        patch_reflect(memory, text="# Customers\n\nBrand new topic.")
+        llm_calls = patch_llm_call(memory, returns="should-not-be-called")
+
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        assert refreshed["content"] == "# Customers\n\nBrand new topic."
+        assert len(llm_calls) == 0, "Source-query change must bypass the delta merge"
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_delta_mode_applies_ops_when_query_stable(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        patch_llm_call,
+    ):
+        """When content exists and source_query is stable, the delta LLM produces ops
+        that are applied against the parsed structured doc. The unchanged section
+        renders byte-identical, the new fact lands in a new block.
+        """
+        bank_id = f"test-delta-apply-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        existing = (
+            "# Team\n"
+            "\n"
+            "Alice is the lead.\n"
+            "\n"
+            "## Members\n"
+            "\n"
+            "- Alice — lead\n"
+        )
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content=existing,
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+
+        # First refresh: empty op list → structured doc unchanged → markdown is the
+        # render of the parsed existing content. This also seeds the tracking column.
+        patch_reflect(memory, text="ignored — full mode candidate")
+        patch_llm_call(memory, returns=[])  # zero ops
+        await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        # Second refresh: a new fact arrives; LLM returns one append_block op.
+        candidate = "# Team\n\nAlice is the lead. Bob joined as junior engineer."
+        patch_reflect(
+            memory,
+            text=candidate,
+            facts=[
+                {
+                    "id": "obs-bob",
+                    "text": "Bob joined the team as junior engineer",
+                    "type": "observation",
+                    "context": None,
+                }
+            ],
+        )
+        ops = [
+            {
+                "op": "append_block",
+                "section_id": "members",
+                "block": {
+                    "type": "bullet_list",
+                    "items": ["Bob — junior engineer"],
+                },
+            }
+        ]
+        llm_calls = patch_llm_call(memory, returns=ops)
+
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        assert len(llm_calls) == 1, "Structured-delta LLM call must fire exactly once"
+        system_msg = llm_calls[0]["messages"][0]["content"]
+        user_msg = llm_calls[0]["messages"][1]["content"]
+        # Prompt must include the structured doc + supporting facts + the system prompt.
+        assert "minimal patch" in system_msg.lower()
+        assert "operations" in system_msg.lower()
+        assert "obs-bob" in user_msg
+        assert "Bob joined" in user_msg
+        # The structured JSON of the current doc must include the section id "members".
+        assert '"id": "members"' in user_msg
+
+        # New content includes the new bullet.
+        assert "Bob — junior engineer" in refreshed["content"]
+        # Unchanged section ("Alice is the lead.") still present.
+        assert "Alice is the lead." in refreshed["content"]
+        rr = refreshed.get("reflect_response") or {}
+        assert rr.get("delta_applied") is True
+        applied = rr.get("delta_operations_applied") or []
+        assert len(applied) == 1
+        assert applied[0]["op"] == "append_block"
+        assert applied[0]["section_id"] == "members"
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_delta_zero_ops_keeps_existing_content_byte_identical(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        patch_llm_call,
+    ):
+        """Zero operations from the LLM must mean zero changes in the rendered output.
+
+        This is the structural guarantee: any sections/blocks not mentioned by an
+        op come through byte-identical. A no-op refresh therefore re-renders the
+        same structured doc — which (after the first refresh has parsed and
+        re-rendered it) is byte-stable.
+        """
+        bank_id = f"test-delta-noop-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        existing = (
+            "# Team\n"
+            "\n"
+            "Alice is the lead.\n"
+            "\n"
+            "## Members\n"
+            "\n"
+            "- Alice\n"
+        )
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content=existing,
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+        # First refresh: parses + renders existing into structured form. The output
+        # may not match `existing` byte-for-byte (whitespace normalised by renderer).
+        patch_reflect(memory, text="ignored — full mode candidate")
+        patch_llm_call(memory, returns=[])
+        first = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+        normalised = first["content"]
+
+        # Second refresh: zero ops again → same bytes as first refresh.
+        patch_reflect(memory, text="something completely different from existing")
+        patch_llm_call(memory, returns=[])
+        second = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+        assert second["content"] == normalised
+        rr = second.get("reflect_response") or {}
+        assert rr.get("delta_applied") is True  # delta path ran; produced no changes
+        assert rr.get("delta_operations_applied") == []
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_delta_llm_failure_falls_back_to_candidate(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        monkeypatch,
+    ):
+        """When the structured-delta LLM call raises, refresh falls back to the
+        candidate markdown so the user still sees a fresh synthesis instead of
+        an opaque failure.
+        """
+        bank_id = f"test-delta-llm-fail-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content="# Team\n\nExisting.\n",
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+        # Seed tracking column with a successful zero-op refresh.
+        patch_reflect(memory, text="ignored")
+
+        async def ok_call(*, messages, **kwargs):
+            from hindsight_api.engine.reflect.delta_ops import DeltaOperationList
+
+            return DeltaOperationList()
+
+        monkeypatch.setattr(memory._reflect_llm_config, "call", ok_call)
+        await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        # Now the second refresh: LLM raises. Refresh must not crash; it should
+        # store the candidate markdown.
+        candidate = "# Team\n\nFallback candidate from reflect_async.\n"
+        patch_reflect(memory, text=candidate)
+
+        async def boom(*, messages, **kwargs):
+            raise RuntimeError("simulated provider 500")
+
+        monkeypatch.setattr(memory._reflect_llm_config, "call", boom)
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        assert "Fallback candidate" in refreshed["content"]
+        rr = refreshed.get("reflect_response") or {}
+        assert rr.get("delta_applied") is False
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_empty_reflect_answer_preserves_existing_content(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        patch_llm_call,
+        monkeypatch,
+    ):
+        """Regression: when the reflect agent returns an empty answer (small models
+        sometimes hit this after exhausting tool-call retries), the refresh must
+        NOT overwrite the existing content with an empty string.
+
+        Previously this destroyed the working document on every transient upstream
+        failure, and the next refresh saw current_content == "" and skipped the
+        delta path entirely — a snowball that emptied valuable mental models.
+
+        The scenario covered here is the realistic failure path: the structured
+        delta call also fails (because the empty supporting facts produce empty
+        / invalid JSON) so the fallback path kicks in. Without the guard, the
+        fallback would write "" to the DB; with it, the existing content stays.
+        """
+        bank_id = f"test-empty-reflect-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        existing = (
+            "# Team\n"
+            "\n"
+            "Alice is the lead.\n"
+            "\n"
+            "## Members\n"
+            "\n"
+            "- Alice\n"
+        )
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content=existing,
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+
+        # Reflect returns "" — this is the upstream failure mode.
+        patch_reflect(memory, text="")
+
+        # Delta call also fails (mirrors the real groq behaviour where empty
+        # supporting facts often produce empty / invalid JSON). Refresh then
+        # falls back to the empty candidate, which the guard rejects.
+        async def boom(*, messages, **kwargs):
+            raise RuntimeError("simulated empty/invalid JSON from provider")
+
+        monkeypatch.setattr(memory._reflect_llm_config, "call", boom)
+
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        # Existing content preserved exactly.
+        assert refreshed["content"] == existing, (
+            "Empty reflect answer overwrote existing content — guard regressed"
+        )
+        rr = refreshed.get("reflect_response") or {}
+        assert rr.get("refresh_skipped") == "empty_candidate"
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+# ---------------------------------------------------------------------------
+# Real-Gemini evaluation tests
+# ---------------------------------------------------------------------------
+
+_GEMINI_API_KEY = (
+    os.getenv("HINDSIGHT_GEMINI_API_KEY")
+    or os.getenv("GEMINI_API_KEY")
+    or os.getenv("GOOGLE_API_KEY")
+)
+_OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+_RUN_LLM_EVAL = os.getenv("HINDSIGHT_RUN_GEMINI_EVALS") == "1" and (
+    bool(_GEMINI_API_KEY) or bool(_OPENAI_API_KEY)
+)
+
+
+pytestmark_gemini = pytest.mark.skipif(
+    not _RUN_LLM_EVAL,
+    reason=(
+        "Real-LLM delta evals are gated. Set HINDSIGHT_RUN_GEMINI_EVALS=1 and provide "
+        "GEMINI_API_KEY (preferred) or OPENAI_API_KEY to run."
+    ),
+)
+
+
+@pytest.fixture
+async def gemini_memory(memory_no_llm_verify: MemoryEngine):
+    """MemoryEngine wired to a real LLM for reflect + structured delta.
+
+    Prefers Gemini (the original target) but falls back to OpenAI when the
+    Gemini key is unavailable — the structured-delta architecture works
+    against either, and waiting on a single provider's key would block
+    iteration. The chosen model is logged so test failures are unambiguous
+    about which provider produced them.
+    """
+    if _GEMINI_API_KEY:
+        provider = "gemini"
+        model = os.getenv("HINDSIGHT_GEMINI_EVAL_MODEL", "gemini-2.0-flash")
+        cfg = LLMConfig(provider=provider, api_key=_GEMINI_API_KEY, base_url="", model=model)
+    else:
+        provider = "openai"
+        model = os.getenv("HINDSIGHT_OPENAI_EVAL_MODEL", "gpt-4o-mini")
+        cfg = LLMConfig(provider=provider, api_key=_OPENAI_API_KEY or "", base_url="", model=model)
+    print(f"\n[delta-eval] using provider={provider} model={model}")
+    memory_no_llm_verify._reflect_llm_config = cfg
+    memory_no_llm_verify._llm_config = cfg
+    memory_no_llm_verify._retain_llm_config = cfg
+    memory_no_llm_verify._consolidation_llm_config = cfg
+    yield memory_no_llm_verify
+
+
+_NEWS_FEED_SKILL_MARKDOWN = """## Purpose
+
+Generate a concise, top-N personalized AI/ML news brief in response to user-triggered requests such as "ai news", "top 5 this week", or "what matters for builders today".
+
+## Scope
+
+- **In scope**: collecting, filtering, and summarizing AI/ML articles from user-preferred RSS feeds, applying user preferences stored in the AI News Feed Preferences mental model, and delivering the brief to the user.
+- **Out of scope**: non-AI news, detailed article content, legal or privacy reviews beyond user preferences, and posting the brief to external platforms without explicit user approval.
+
+## Rules
+
+- **Always**:
+  1. Use the AI News Feed Preferences mental model to retrieve user preferences; do not embed preferences in the skill file.
+  2. Do not post the brief to any platform unless the user explicitly approves.
+  3. Do not persist preferences locally; rely solely on the mental model.
+  4. Refresh the feed after consolidation if the trigger-refresh-after-consolidation flag is true.
+- **Prefer**:
+  1. Provide a concise summary (about 2-3 sentences per article) for the top-N articles.
+  2. Default to the top-5 articles unless the user specifies otherwise.
+  3. Order articles chronologically or by relevance as per user preference.
+  4. Highlight any user-specified topics or tags if present.
+
+## Procedure
+
+1. **Trigger detection** — identify a request containing keywords like "ai news", "top N", or "what matters".
+2. **Preference retrieval** — call memory recall for the AI News Feed Preferences mental model to obtain RSS feed URLs and any filtering criteria.
+3. **Feed consolidation** — fetch all feeds, de-duplicate entries, and apply any user-specified filters.
+4. **Article selection** — choose the top-N articles based on date or user preference; if trigger-refresh-after-consolidation is true, re-fetch feeds before selection.
+5. **Summarization** — generate a brief summary for each article, keeping it short and to the point.
+6. **Approval check** — if the brief is to be posted externally, verify explicit user approval; otherwise, deliver it directly to the user.
+7. **Memory retention** — store any new learnings or preferences observed during the task using memory retain.
+
+## Inputs and Context
+
+- **Source feeds**: user-specified RSS URLs stored in the mental model (e.g., https://aiagentmemory.org/index.xml).
+- **Time window**: the latest update from each feed; typically the last 7 days for weekly briefs.
+- **User preferences**: stored in the AI News Feed Preferences mental model; may include topics, tags, or language.
+
+## Output Shape
+
+- **Structure**: list of articles with title, publication date, source, and a 2-sentence summary.
+- **Format**: plain text or markdown (as requested by the user).
+- **Length**: concise — approximately 2-3 sentences per article; total brief about 200-300 words for top-5.
+- **Voice/Tone**: neutral, informative, and concise; use bullet points for clarity.
+
+## Stop Conditions
+
+- If the mental model cannot be retrieved, refuse or request clarification.
+- If the user has not provided any RSS feed URLs, ask for a preferred source.
+- If the brief is requested for posting and explicit approval is missing, refuse.
+- If the user explicitly requests to remove a skill or stop the briefing, comply immediately.
+
+## Open Questions
+
+- Desired brief length or word count?
+- Preferred summary style (bullet vs paragraph).
+- Whether the user wants to include non-AI but AI-related topics.
+- Frequency or schedule for automated briefs (if any).
+- Specific user-defined tags or topics to highlight.
+"""
+
+
+@pytestmark_gemini
+class TestDeltaRefreshGeminiEval:
+    """Real-LLM evals for the structured-delta refresh path.
+
+    The structural guarantee these tests verify: sections and blocks not
+    targeted by an LLM-emitted operation are byte-identical between the
+    pre-refresh and post-refresh markdown render. This is what the
+    structured-ops architecture buys us — the LLM cannot drift on text it
+    never re-emits.
+
+    Real Gemini is used (not a mock) because the failure mode we're guarding
+    against is precisely "the LLM doesn't reliably do what the prompt says,
+    even at temperature 0". Mocked output would prove the wiring works but
+    not that the contract holds against an actual model.
+    """
+
+    async def _seed(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        bank_id: str,
+        existing_markdown: str,
+        memories: list[str],
+    ) -> dict[str, Any]:
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Skill Doc",
+            source_query="Document the news-feed skill: purpose, rules, procedure, stop conditions.",
+            content=existing_markdown,
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+        await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[{"content": m} for m in memories],
+            request_context=request_context,
+        )
+        await memory.wait_for_background_tasks()
+        # First refresh: parses existing into structured form. With well-aligned
+        # memories the LLM should emit zero ops, so the structured doc is just
+        # the parsed existing content. The rendered markdown is canonicalised.
+        first = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+        return {"mm": mm, "first": first}
+
+    async def test_no_change_when_observations_agree_with_existing(
+        self, gemini_memory: MemoryEngine, request_context: RequestContext
+    ):
+        """When observations only restate the existing doc, a second delta
+        refresh produces output byte-identical to the first refresh's output.
+
+        The first refresh canonicalises whitespace via the parser+renderer; we
+        compare the *second* refresh against the *first* (not against the raw
+        seed markdown), which is the actual repeat-refresh behaviour users
+        will see in production.
+        """
+        bank_id = f"eval-delta-noop-{uuid.uuid4().hex[:8]}"
+        seeded = await self._seed(
+            gemini_memory,
+            request_context,
+            bank_id,
+            existing_markdown=_NEWS_FEED_SKILL_MARKDOWN,
+            memories=[
+                "The news-feed skill produces a concise top-N AI/ML news brief.",
+                "Default brief size is top 5 unless the user specifies otherwise.",
+                "Source feed: https://aiagentmemory.org/index.xml.",
+                "The skill must not post externally without explicit approval.",
+            ],
+        )
+        first_content = seeded["first"]["content"]
+
+        second = await gemini_memory.refresh_mental_model(
+            bank_id=bank_id,
+            mental_model_id=seeded["mm"]["id"],
+            request_context=request_context,
+        )
+        second_content = second["content"]
+
+        # Byte-identical render across refreshes when no new fact has arrived.
+        assert second_content == first_content, (
+            "Repeat delta refresh changed bytes when no new facts arrived.\n"
+            f"--- diff sample (first 300 chars different) ---\n"
+            f"first:  {first_content[:300]!r}\n"
+            f"second: {second_content[:300]!r}"
+        )
+        rr = second.get("reflect_response") or {}
+        # The LLM may emit zero ops (best case) or non-effective ops (still no
+        # change to render); both are acceptable so long as the bytes match.
+        assert rr.get("delta_applied") is True
+
+        await gemini_memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_new_observation_is_merged_surgically(
+        self, gemini_memory: MemoryEngine, request_context: RequestContext
+    ):
+        """A new fact arrives; only the section relevant to it should change.
+
+        Asserts the architectural guarantee at the section level: every
+        section that the LLM did NOT name in an operation must render exactly
+        the same bytes after the refresh as before. The new fact itself must
+        appear somewhere in the output.
+        """
+        from hindsight_api.engine.reflect.structured_doc import (
+            StructuredDocument,
+            render_section,
+        )
+
+        bank_id = f"eval-delta-add-{uuid.uuid4().hex[:8]}"
+        seeded = await self._seed(
+            gemini_memory,
+            request_context,
+            bank_id,
+            existing_markdown=_NEWS_FEED_SKILL_MARKDOWN,
+            memories=[
+                "The news-feed skill produces a concise top-N AI/ML news brief.",
+                "Default brief size is top 5.",
+                "Source feed: https://aiagentmemory.org/index.xml.",
+            ],
+        )
+        first_content = seeded["first"]["content"]
+        first_struct = StructuredDocument.model_validate(
+            seeded["first"]["reflect_response"]["delta_operations_applied"]
+            and seeded["first"].get("structured_content")
+            or {"version": 1, "sections": []}
+        )
+        # The first refresh's structured snapshot is what the second refresh
+        # will operate on. Re-fetch via get_mental_model would also work.
+        # For preservation comparison we re-parse first_content.
+        from hindsight_api.engine.reflect.structured_doc import parse_markdown
+
+        before = parse_markdown(first_content)
+
+        # Introduce a brand-new fact that fits into "Inputs and Context" or
+        # similar — but the model may pick any reasonable section.
+        await gemini_memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {
+                    "content": (
+                        "The default time window for the news brief is the last 7 days, "
+                        "matching the weekly cadence preferred by the user."
+                    )
+                },
+            ],
+            request_context=request_context,
+        )
+        await gemini_memory.wait_for_background_tasks()
+
+        refreshed = await gemini_memory.refresh_mental_model(
+            bank_id=bank_id,
+            mental_model_id=seeded["mm"]["id"],
+            request_context=request_context,
+        )
+        content = refreshed["content"]
+        rr = refreshed.get("reflect_response") or {}
+        applied_ops = rr.get("delta_operations_applied") or []
+        touched_section_ids = {op.get("section_id") for op in applied_ops if op.get("section_id")}
+
+        # The fact must show up.
+        assert "7 days" in content or "seven days" in content.lower(), (
+            f"New fact about 7-day window missing from delta output: {content!r}"
+        )
+
+        # Every untouched section must render byte-identical to its pre-refresh form.
+        after = parse_markdown(content)
+        before_by_id = {s.id: s for s in before.sections}
+        for section in after.sections:
+            if section.id in touched_section_ids:
+                continue
+            orig = before_by_id.get(section.id)
+            if orig is None:
+                continue  # newly added section, no preservation contract
+            assert render_section(orig) == render_section(section), (
+                f"Untouched section {section.id!r} drifted between refreshes — the "
+                f"structured-ops architecture's preservation guarantee was violated.\n"
+                f"BEFORE:\n{render_section(orig)!r}\n"
+                f"AFTER:\n{render_section(section)!r}"
+            )
+
+        assert rr.get("delta_applied") is True
+
+        await gemini_memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_no_change_repeated_three_times_stays_byte_stable(
+        self, gemini_memory: MemoryEngine, request_context: RequestContext
+    ):
+        """Three consecutive no-change refreshes must produce three identical
+        markdown outputs. This is the regression test for the original
+        complaint where prose-merge delta drifted content across versions even
+        when no observation changed.
+        """
+        bank_id = f"eval-delta-stable-{uuid.uuid4().hex[:8]}"
+        seeded = await self._seed(
+            gemini_memory,
+            request_context,
+            bank_id,
+            existing_markdown=_NEWS_FEED_SKILL_MARKDOWN,
+            memories=[
+                "The news-feed skill produces a top-N AI brief on demand.",
+                "It must not post without explicit user approval.",
+            ],
+        )
+        c1 = seeded["first"]["content"]
+        r2 = await gemini_memory.refresh_mental_model(
+            bank_id=bank_id,
+            mental_model_id=seeded["mm"]["id"],
+            request_context=request_context,
+        )
+        r3 = await gemini_memory.refresh_mental_model(
+            bank_id=bank_id,
+            mental_model_id=seeded["mm"]["id"],
+            request_context=request_context,
+        )
+        assert r2["content"] == c1, "second refresh drifted vs first"
+        assert r3["content"] == c1, "third refresh drifted vs first"
+
+        await gemini_memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_source_query_change_forces_full_rewrite(
+        self, gemini_memory: MemoryEngine, request_context: RequestContext
+    ):
+        """Changing source_query must bypass delta and produce a full regeneration."""
+        bank_id = f"eval-delta-query-change-{uuid.uuid4().hex[:8]}"
+        await gemini_memory.get_bank_profile(bank_id, request_context=request_context)
+
+        mm = await gemini_memory.create_mental_model(
+            bank_id=bank_id,
+            name="Subject",
+            source_query="Summarize the team and how it operates.",
+            content="# Team Overview\n\nAlice leads the team.\n",
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+
+        await gemini_memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {"content": "Alice leads the team."},
+                {"content": "The product is a memory system for AI agents."},
+                {"content": "Customers include small SaaS startups and enterprise pilots."},
+            ],
+            request_context=request_context,
+        )
+        await gemini_memory.wait_for_background_tasks()
+
+        # First refresh seeds tracking column under the team query.
+        await gemini_memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        # Change the topic entirely.
+        await gemini_memory.update_mental_model(
+            bank_id=bank_id,
+            mental_model_id=mm["id"],
+            source_query="Summarize our customers and what we sell them.",
+            request_context=request_context,
+        )
+
+        refreshed = await gemini_memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+        content = refreshed["content"].lower()
+        # Content should now be about customers/product, not (only) about Alice leading the team.
+        assert "customer" in content or "product" in content, (
+            f"Full rewrite should cover the new topic, got: {refreshed['content']!r}"
+        )
+        # delta_applied should be absent/False because we took the full path.
+        assert (refreshed.get("reflect_response") or {}).get("delta_applied") is not True
+
+        await gemini_memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_mental_models.py

@@ -8,7 +8,8 @@ import uuid
 
 import pytest
 
-from hindsight_api.engine.memory_engine import MemoryEngine
+from hindsight_api.engine.memory_engine import MemoryEngine, fq_table
+from hindsight_api.engine.retain import embedding_utils
 
 
 @pytest.fixture
@@ -688,6 +689,51 @@ class TestMentalModelHistory:
         assert len(history) == 1
         assert history[0]["previous_content"] == "Original content"
         assert "changed_at" in history[0]
+        assert "previous_reflect_response" in history[0]
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_history_snapshots_previous_reflect_response(
+        self, memory: MemoryEngine, request_context
+    ):
+        """Each history entry snapshots the reflect_response that produced previous_content."""
+        bank_id = f"test-mm-history-reflect-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Test Model",
+            source_query="What is the test?",
+            content="v1",
+            request_context=request_context,
+        )
+
+        rr_v1 = {"text": "v1", "based_on": {"observation": [{"id": "o1", "text": "obs1"}]}, "mental_models": []}
+        await memory.update_mental_model(
+            bank_id=bank_id,
+            mental_model_id=mm["id"],
+            content="v2",
+            reflect_response=rr_v1,
+            request_context=request_context,
+        )
+
+        rr_v2 = {"text": "v2", "based_on": {"observation": [{"id": "o2", "text": "obs2"}]}, "mental_models": []}
+        await memory.update_mental_model(
+            bank_id=bank_id,
+            mental_model_id=mm["id"],
+            content="v3",
+            reflect_response=rr_v2,
+            request_context=request_context,
+        )
+
+        history = await memory.get_mental_model_history(bank_id, mm["id"], request_context=request_context)
+        assert len(history) == 2
+        # Most recent first: replacing v2 snapshotted rr_v1 (the reflect that produced v2).
+        assert history[0]["previous_content"] == "v2"
+        assert history[0]["previous_reflect_response"] == rr_v1
+        # The first update replaced v1, which had no reflect_response stored yet.
+        assert history[1]["previous_content"] == "v1"
+        assert history[1]["previous_reflect_response"] is None
 
         await memory.delete_bank(bank_id, request_context=request_context)
 
@@ -763,6 +809,222 @@ class TestMentalModelHistory:
         await memory.delete_bank(bank_id, request_context=request_context)
 
 
+class TestMentalModelStaleness:
+    """Tests for compute_mental_model_is_stale scope semantics.
+
+    Memories are inserted directly into ``memory_units`` so the scenarios don't
+    depend on the LLM fact-extraction pipeline.
+    """
+
+    @staticmethod
+    async def _insert_memory(
+        memory: MemoryEngine,
+        bank_id: str,
+        *,
+        tags: list[str] | None = None,
+        fact_type: str = "experience",
+    ) -> str:
+        from datetime import datetime, timezone
+
+        pool = await memory._get_pool()
+        mem_id = str(uuid.uuid4())
+        now = datetime.now(timezone.utc)
+        async with pool.acquire() as conn:
+            await conn.execute(
+                f"""
+                INSERT INTO {fq_table("memory_units")}
+                    (id, bank_id, text, event_date, fact_type, tags, created_at)
+                VALUES ($1, $2, $3, $4, $5, $6::varchar[], $4)
+                """,
+                mem_id,
+                bank_id,
+                "test memory",
+                now,
+                fact_type,
+                tags if tags is not None else [],
+            )
+        return mem_id
+
+    async def test_fresh_mental_model_is_not_stale(self, memory: MemoryEngine, request_context):
+        bank_id = f"test-mm-stale-fresh-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id, name="MM", source_query="q", content="c", request_context=request_context
+        )
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is False
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_untagged_mm_stale_on_any_new_memory(
+        self, memory: MemoryEngine, request_context
+    ):
+        bank_id = f"test-mm-stale-untagged-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id, name="MM", source_query="q", content="c", request_context=request_context
+        )
+        await self._insert_memory(memory, bank_id, tags=["something"])
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is True
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_tagged_mm_ignores_out_of_scope_memory(
+        self, memory: MemoryEngine, request_context
+    ):
+        bank_id = f"test-mm-stale-oos-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="MM",
+            source_query="q",
+            content="c",
+            tags=["user_a"],
+            request_context=request_context,
+        )
+        # Memory tagged with unrelated tag → not in scope, MM should not be stale
+        await self._insert_memory(memory, bank_id, tags=["user_b"])
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is False
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_tagged_mm_stale_on_overlapping_memory(
+        self, memory: MemoryEngine, request_context
+    ):
+        bank_id = f"test-mm-stale-overlap-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="MM",
+            source_query="q",
+            content="c",
+            tags=["user_a"],
+            request_context=request_context,
+        )
+        await self._insert_memory(memory, bank_id, tags=["user_a", "extra"])
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is True
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_tags_match_all_strict_requires_all_tags(
+        self, memory: MemoryEngine, request_context
+    ):
+        """tags_match='all_strict' → memory must contain ALL MM tags (and be tagged)."""
+        bank_id = f"test-mm-stale-all-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="MM",
+            source_query="q",
+            content="c",
+            tags=["user_a", "proj_x"],
+            trigger={"refresh_after_consolidation": False, "tags_match": "all_strict"},
+            request_context=request_context,
+        )
+        # Memory only has one of the tags → does NOT match all_strict
+        await self._insert_memory(memory, bank_id, tags=["user_a"])
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is False, "all_strict must require ALL MM tags"
+
+        # Now add a memory with both tags → matches
+        await self._insert_memory(memory, bank_id, tags=["user_a", "proj_x"])
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is True
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_tags_match_any_strict_excludes_untagged(
+        self, memory: MemoryEngine, request_context
+    ):
+        """tags_match='any_strict' → untagged memory does NOT keep MM in scope."""
+        bank_id = f"test-mm-stale-anystrict-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="MM",
+            source_query="q",
+            content="c",
+            tags=["user_a"],
+            trigger={"refresh_after_consolidation": False, "tags_match": "any_strict"},
+            request_context=request_context,
+        )
+        await self._insert_memory(memory, bank_id, tags=None)
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is False
+
+        await self._insert_memory(memory, bank_id, tags=["user_a"])
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is True
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_fact_type_filter_narrows_scope(
+        self, memory: MemoryEngine, request_context
+    ):
+        bank_id = f"test-mm-stale-fact-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="MM",
+            source_query="q",
+            content="c",
+            trigger={"refresh_after_consolidation": False, "fact_types": ["world"]},
+            request_context=request_context,
+        )
+        # Out-of-scope fact_type → not stale
+        await self._insert_memory(memory, bank_id, fact_type="experience")
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is False
+
+        # Matching fact_type → stale
+        await self._insert_memory(memory, bank_id, fact_type="world")
+        got = await memory.get_mental_model(bank_id, mm["id"], request_context=request_context)
+        assert got["is_stale"] is True
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    async def test_tool_search_mental_models_returns_is_stale_per_mm(
+        self, memory: MemoryEngine, request_context
+    ):
+        """Regression: tool_search_mental_models must compute is_stale per-MM via scope,
+        not via a bank-wide pending_consolidation short-circuit."""
+        from hindsight_api.engine.reflect.tools import tool_search_mental_models
+
+        bank_id = f"test-mm-stale-tool-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+        fresh = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="fresh MM",
+            source_query="q",
+            content="fresh",
+            tags=["user_b"],
+            request_context=request_context,
+        )
+        stale = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="stale MM",
+            source_query="q",
+            content="stale",
+            tags=["user_a"],
+            request_context=request_context,
+        )
+        # Memory only in user_a's scope → only `stale` MM should be flagged.
+        await self._insert_memory(memory, bank_id, tags=["user_a"])
+
+        pool = await memory._get_pool()
+        async with pool.acquire() as conn:
+            embedding = (
+                await embedding_utils.generate_embeddings_batch(memory.embeddings, ["q"])
+            )[0]
+            result = await tool_search_mental_models(
+                memory, conn, bank_id, "q", embedding, max_results=10
+            )
+        by_id = {m["id"]: m for m in result["mental_models"]}
+        assert by_id[fresh["id"]]["is_stale"] is False
+        assert by_id[stale["id"]]["is_stale"] is True
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
 class TestMentalModelRefreshTagSecurity:
     """Test that mental model refresh respects tag-based security boundaries."""
 
@@ -1253,6 +1515,145 @@ class TestMentalModelTriggerTagsConfig:
         await memory.delete_bank(bank_id, request_context=request_context)
 
 
+class TestMentalModelRefreshMaxTokens:
+    """Verify that refresh_mental_model honors the per-model max_tokens column.
+
+    These tests mock the engine's collaborators so we can assert the exact kwargs
+    passed to reflect_async without spinning up a DB or LLM. The bug being guarded
+    against: the per-model ``max_tokens`` column was ignored during refresh, so
+    reflect_async fell back to its default (4096) and the generated content could
+    exceed the user-configured limit when there were many facts to synthesize.
+    """
+
+    async def test_refresh_passes_stored_max_tokens_to_reflect(self, request_context):
+        from unittest.mock import AsyncMock
+
+        from hindsight_api.engine.memory_engine import MemoryEngine
+        from hindsight_api.engine.response_models import ReflectResult
+
+        custom_max_tokens = 777
+        mental_model = {
+            "id": "mm-1",
+            "bank_id": "bank-1",
+            "name": "Capped Model",
+            "source_query": "Summarize the facts",
+            "content": "initial",
+            "tags": None,
+            "max_tokens": custom_max_tokens,
+            "trigger": {"refresh_after_consolidation": False},
+        }
+
+        engine = MemoryEngine.__new__(MemoryEngine)
+        engine._authenticate_tenant = AsyncMock(return_value=None)  # type: ignore[method-assign]
+        engine.get_mental_model = AsyncMock(return_value=mental_model)  # type: ignore[method-assign]
+        engine.reflect_async = AsyncMock(  # type: ignore[method-assign]
+            return_value=ReflectResult(text="stub synthesis", based_on={})
+        )
+        engine.update_mental_model = AsyncMock(return_value=mental_model)  # type: ignore[method-assign]
+
+        await engine.refresh_mental_model(
+            bank_id="bank-1",
+            mental_model_id="mm-1",
+            request_context=request_context,
+        )
+
+        assert engine.reflect_async.await_count == 1
+        kwargs = engine.reflect_async.await_args.kwargs
+        assert kwargs.get("max_tokens") == custom_max_tokens, (
+            f"refresh_mental_model should forward the stored max_tokens ({custom_max_tokens}) "
+            f"to reflect_async, but got max_tokens={kwargs.get('max_tokens')!r}"
+        )
+
+    async def test_refresh_content_respects_max_tokens(self, memory: MemoryEngine, request_context):
+        """End-to-end: refreshed content must stay within the model's max_tokens cap.
+
+        We seed the bank with enough varied facts that an unconstrained synthesis
+        would happily produce a long answer, then refresh a mental model with a
+        small max_tokens and assert the resulting content is actually within the
+        cap (with a small tolerance for cross-tokenizer drift, since the LLM may
+        not use cl100k_base).
+        """
+        from hindsight_api.engine.memory_engine import count_tokens
+
+        bank_id = f"test-refresh-cap-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        # Seed enough content that an uncapped reflect would produce a long answer.
+        await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {"content": (
+                    "Alice is the staff frontend engineer. She owns the design system, "
+                    "leads accessibility reviews, mentors three junior engineers, and runs "
+                    "the weekly UI guild meeting every Thursday at 2pm Pacific."
+                )},
+                {"content": (
+                    "Bob is the backend tech lead. He owns the payments service, the "
+                    "billing reconciliation pipeline, and the on-call rotation for the "
+                    "platform team. He is the primary reviewer for any database migration."
+                )},
+                {"content": (
+                    "Carol manages the data platform. Her team operates the warehouse, "
+                    "the streaming ingestion layer, and the metrics pipeline that feeds "
+                    "the executive dashboards refreshed every fifteen minutes."
+                )},
+                {"content": (
+                    "The team holds a company-wide demo every other Friday. Engineering "
+                    "presents shipped work, design walks through prototypes, and product "
+                    "shares roadmap updates for the upcoming quarter."
+                )},
+                {"content": (
+                    "Dan is the security lead. He runs the quarterly threat-modeling "
+                    "exercises, owns the incident response runbook, and coordinates the "
+                    "annual external penetration test with the vendor."
+                )},
+                {"content": (
+                    "Erin runs developer experience. She maintains the local-dev tooling, "
+                    "the CI pipelines, the release automation, and the internal "
+                    "documentation portal that everyone uses to onboard new hires."
+                )},
+            ],
+            request_context=request_context,
+        )
+        await memory.wait_for_background_tasks()
+
+        cap = 200
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Summary (capped)",
+            source_query="Give me a complete overview of every team member, what they own, and the recurring meetings.",
+            content="initial",
+            max_tokens=cap,
+            request_context=request_context,
+        )
+
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id,
+            mental_model_id=mm["id"],
+            request_context=request_context,
+        )
+
+        assert refreshed is not None
+        content = refreshed["content"]
+        assert content, "refresh produced empty content"
+
+        # The provider enforces the cap exactly in its own tokenizer, but our
+        # local count uses tiktoken (cl100k_base) which can disagree with
+        # provider tokenizers (Gemini's SentencePiece in particular tends to run
+        # ~30% higher for English prose). We use a generous tolerance — the test
+        # is guarding against the regression where the cap was ignored entirely
+        # and content grew toward reflect_async's default of 4096 tokens.
+        observed_tokens = count_tokens(content)
+        tolerance = 1.5
+        assert observed_tokens <= cap * tolerance, (
+            f"refreshed content exceeds max_tokens cap: "
+            f"observed≈{observed_tokens} tokens, cap={cap} (tolerance x{tolerance}). "
+            f"content={content!r}"
+        )
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
 class TestMentalModelTriggerSchema:
     """Unit tests for MentalModelTrigger schema validation (no DB needed)."""
 

hindsight-api-slim/tests/test_observation_invalidation.py

@@ -8,6 +8,7 @@ These tests verify that:
    resets the target memory itself for re-consolidation
 4. delete_bank(fact_type=...) also cleans up affected observations
 """
+
 import uuid
 from unittest.mock import AsyncMock, patch
 
@@ -20,6 +21,7 @@ from hindsight_api.engine.memory_engine import MemoryEngine
 # Helpers
 # ---------------------------------------------------------------------------
 
+
 async def _insert_memory(conn, bank_id: str, text: str, fact_type: str = "experience") -> uuid.UUID:
     """Insert a memory unit directly, bypassing LLM retain pipeline."""
     mem_id = uuid.uuid4()
@@ -36,9 +38,7 @@ async def _insert_memory(conn, bank_id: str, text: str, fact_type: str = "experi
     return mem_id
 
 
-async def _insert_observation(
-    conn, bank_id: str, text: str, source_memory_ids: list[uuid.UUID]
-) -> uuid.UUID:
+async def _insert_observation(conn, bank_id: str, text: str, source_memory_ids: list[uuid.UUID]) -> uuid.UUID:
     """Insert an observation unit directly."""
     obs_id = uuid.uuid4()
     await conn.execute(
@@ -79,8 +79,8 @@ async def _ensure_bank(memory: MemoryEngine, bank_id: str, request_context: Requ
 # Tests: delete_memory_unit
 # ---------------------------------------------------------------------------
 
-class TestDeleteMemoryUnitObservationCleanup:
 
+class TestDeleteMemoryUnitObservationCleanup:
     @pytest.mark.asyncio
     async def test_deleting_source_memory_removes_observation(
         self, memory: MemoryEngine, request_context: RequestContext
@@ -207,12 +207,10 @@ class TestDeleteMemoryUnitObservationCleanup:
 # Tests: delete_document
 # ---------------------------------------------------------------------------
 
-class TestDeleteDocumentObservationCleanup:
 
+class TestDeleteDocumentObservationCleanup:
     @pytest.mark.asyncio
-    async def test_deleting_document_removes_observations(
-        self, memory: MemoryEngine, request_context: RequestContext
-    ):
+    async def test_deleting_document_removes_observations(self, memory: MemoryEngine, request_context: RequestContext):
         """Deleting a document removes observations derived from its memory units."""
         bank_id = f"test-invalidate-doc-{uuid.uuid4().hex[:8]}"
         await _ensure_bank(memory, bank_id, request_context)
@@ -248,9 +246,7 @@ class TestDeleteDocumentObservationCleanup:
             m3 = await _insert_memory(conn, bank_id, "Alice is an avid outdoor person.")
 
             # Observation referencing both doc memories and the standalone memory
-            obs_id = await _insert_observation(
-                conn, bank_id, "Alice enjoys outdoor activities.", [m1, m2, m3]
-            )
+            obs_id = await _insert_observation(conn, bank_id, "Alice enjoys outdoor activities.", [m1, m2, m3])
 
         # Patch out consolidation so it doesn't re-set consolidated_at before we can check it
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
@@ -267,12 +263,117 @@ class TestDeleteDocumentObservationCleanup:
         await memory.delete_bank(bank_id, request_context=request_context)
 
 
+# ---------------------------------------------------------------------------
+# Tests: document upsert via retain pipeline (regression for orphan observations)
+# ---------------------------------------------------------------------------
+
+class TestDocumentUpsertObservationCleanup:
+    """Regression: re-ingesting a document via the retain pipeline must clean up
+    observations derived from the outgoing memory_units, the same way the
+    explicit ``MemoryEngine.delete_document`` API does.
+
+    Before the fix, ``fact_storage.handle_document_tracking`` deleted the
+    document via FK cascade — removing the source memory_units silently — but
+    never invalidated the dependent observations. They became orphans whose
+    ``source_memory_ids`` arrays pointed at IDs that no longer existed in
+    ``memory_units``.
+    """
+
+    @pytest.mark.asyncio
+    async def test_upsert_document_removes_observations_from_outgoing_memories(
+        self, memory: MemoryEngine, request_context: RequestContext
+    ):
+        from hindsight_api.engine.retain.fact_storage import handle_document_tracking
+
+        bank_id = f"test-upsert-obs-cleanup-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(memory, bank_id, request_context)
+
+        pool = await memory._get_pool()
+        doc_id = str(uuid.uuid4())
+
+        # Pre-populate: one document, two source memories under it, one
+        # standalone memory not in the document, and an observation that joins
+        # all three. After the upsert, the two doc memories should be gone
+        # (cascade) AND the observation should be invalidated (the bug we're
+        # fixing). The standalone memory should be reset for re-consolidation.
+        async with pool.acquire() as conn:
+            await conn.execute(
+                """
+                INSERT INTO documents (id, bank_id, original_text, content_hash, created_at, updated_at)
+                VALUES ($1, $2, 'old version', 'hash-old', NOW(), NOW())
+                """,
+                doc_id,
+                bank_id,
+            )
+            doc_mem_a = uuid.uuid4()
+            doc_mem_b = uuid.uuid4()
+            for mem_id, text in [(doc_mem_a, "Old fact A."), (doc_mem_b, "Old fact B.")]:
+                await conn.execute(
+                    """
+                    INSERT INTO memory_units (id, bank_id, text, fact_type, event_date, document_id,
+                                              created_at, updated_at, consolidated_at)
+                    VALUES ($1, $2, $3, 'experience', NOW(), $4, NOW(), NOW(), NOW())
+                    """,
+                    mem_id,
+                    bank_id,
+                    text,
+                    doc_id,
+                )
+            standalone_mem = await _insert_memory(conn, bank_id, "Standalone fact C.")
+            obs_id = await _insert_observation(
+                conn,
+                bank_id,
+                "Aggregated observation joining doc + standalone facts.",
+                [doc_mem_a, doc_mem_b, standalone_mem],
+            )
+
+        # Trigger the upsert path directly. ``handle_document_tracking`` is
+        # what the retain orchestrator calls on every document re-ingest.
+        async with pool.acquire() as conn:
+            async with conn.transaction():
+                await handle_document_tracking(
+                    conn,
+                    bank_id=bank_id,
+                    document_id=doc_id,
+                    combined_content="new version replacing old facts",
+                    is_first_batch=True,
+                    retain_params=None,
+                    document_tags=None,
+                )
+
+        async with pool.acquire() as conn:
+            obs_ids = await _get_observation_ids(conn, bank_id)
+            assert str(obs_id) not in obs_ids, (
+                "Observation derived from the outgoing memory_units should have been "
+                "deleted during the upsert (regression: orphan observations were "
+                "previously left behind because handle_document_tracking didn't call "
+                "delete_stale_observations_for_memories)"
+            )
+
+            # The standalone memory survives (different document_id) and should
+            # be reset for re-consolidation since one of its observations was
+            # invalidated by the upsert.
+            consolidated_at = await _get_consolidated_at(conn, standalone_mem)
+            assert consolidated_at is None, (
+                "Surviving co-source memory should be reset for re-consolidation"
+            )
+
+            # The two doc-scoped memories are gone via FK cascade.
+            doc_mem_count = await conn.fetchval(
+                "SELECT COUNT(*) FROM memory_units WHERE id = ANY($1::uuid[])",
+                [doc_mem_a, doc_mem_b],
+            )
+            assert doc_mem_count == 0
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
 # ---------------------------------------------------------------------------
 # Tests: delete_bank with fact_type filter
 # ---------------------------------------------------------------------------
 
-class TestDeleteBankByTypeObservationCleanup:
 
+class TestDeleteBankByTypeObservationCleanup:
     @pytest.mark.asyncio
     async def test_clearing_experience_memories_removes_affected_observations(
         self, memory: MemoryEngine, request_context: RequestContext
@@ -285,9 +386,7 @@ class TestDeleteBankByTypeObservationCleanup:
         async with pool.acquire() as conn:
             exp1 = await _insert_memory(conn, bank_id, "Alice went hiking last week.", "experience")
             world1 = await _insert_memory(conn, bank_id, "Alice is a hiker.", "world")
-            obs_id = await _insert_observation(
-                conn, bank_id, "Alice is a regular hiker.", [exp1, world1]
-            )
+            obs_id = await _insert_observation(conn, bank_id, "Alice is a regular hiker.", [exp1, world1])
 
         # Patch out consolidation so it doesn't re-set consolidated_at before we can check it
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
@@ -330,8 +429,8 @@ class TestDeleteBankByTypeObservationCleanup:
 # Tests: clear_observations_for_memory
 # ---------------------------------------------------------------------------
 
-class TestClearObservationsForMemory:
 
+class TestClearObservationsForMemory:
     @pytest.mark.asyncio
     async def test_clears_observations_and_resets_all_source_memories(
         self, memory: MemoryEngine, request_context: RequestContext
@@ -348,9 +447,7 @@ class TestClearObservationsForMemory:
 
         # Patch out consolidation so it doesn't re-set consolidated_at before we can check it
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            result = await memory.clear_observations_for_memory(
-                bank_id, str(m1), request_context=request_context
-            )
+            result = await memory.clear_observations_for_memory(bank_id, str(m1), request_context=request_context)
 
         assert result["deleted_count"] == 1
 
@@ -365,9 +462,7 @@ class TestClearObservationsForMemory:
         await memory.delete_bank(bank_id, request_context=request_context)
 
     @pytest.mark.asyncio
-    async def test_no_observations_returns_zero(
-        self, memory: MemoryEngine, request_context: RequestContext
-    ):
+    async def test_no_observations_returns_zero(self, memory: MemoryEngine, request_context: RequestContext):
         """Returns 0 when the memory has no associated observations."""
         bank_id = f"test-clear-obs-noop-{uuid.uuid4().hex[:8]}"
         await _ensure_bank(memory, bank_id, request_context)
@@ -376,9 +471,7 @@ class TestClearObservationsForMemory:
         async with pool.acquire() as conn:
             m1 = await _insert_memory(conn, bank_id, "Alice loves hiking.")
 
-        result = await memory.clear_observations_for_memory(
-            bank_id, str(m1), request_context=request_context
-        )
+        result = await memory.clear_observations_for_memory(bank_id, str(m1), request_context=request_context)
 
         assert result["deleted_count"] == 0
 
@@ -405,9 +498,7 @@ class TestClearObservationsForMemory:
             obs1_id = await _insert_observation(conn, bank_id, "Alice is an avid hiker.", [m1, m2])
             obs2_id = await _insert_observation(conn, bank_id, "Alice is a mountaineer.", [m3])
 
-        result = await memory.clear_observations_for_memory(
-            bank_id, str(m1), request_context=request_context
-        )
+        result = await memory.clear_observations_for_memory(bank_id, str(m1), request_context=request_context)
 
         assert result["deleted_count"] == 1
 
@@ -439,9 +530,7 @@ class TestClearObservationsForMemory:
 
         # Patch out consolidation so it doesn't re-set consolidated_at before we can check it
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            result = await memory.clear_observations_for_memory(
-                bank_id, str(m1), request_context=request_context
-            )
+            result = await memory.clear_observations_for_memory(bank_id, str(m1), request_context=request_context)
 
         assert result["deleted_count"] == 2
 
@@ -493,11 +582,8 @@ async def _insert_document_with_memories(
 
 
 class TestUpdateDocumentTagsObservationCleanup:
-
     @pytest.mark.asyncio
-    async def test_update_tags_returns_updated_document(
-        self, memory: MemoryEngine, request_context: RequestContext
-    ):
+    async def test_update_tags_returns_updated_document(self, memory: MemoryEngine, request_context: RequestContext):
         """update_document returns the updated document with new tags."""
         bank_id = f"test-tag-update-basic-{uuid.uuid4().hex[:8]}"
         await _ensure_bank(memory, bank_id, request_context)
@@ -507,9 +593,7 @@ class TestUpdateDocumentTagsObservationCleanup:
             doc_id = f"doc-{uuid.uuid4().hex[:8]}"
             await _insert_document_with_memories(conn, bank_id, doc_id, [("Alice loves hiking.", "experience")])
 
-        result = await memory.update_document(
-            doc_id, bank_id, tags=["new-tag"], request_context=request_context
-        )
+        result = await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
 
         assert result is True
 
@@ -523,18 +607,14 @@ class TestUpdateDocumentTagsObservationCleanup:
         bank_id = f"test-tag-update-missing-{uuid.uuid4().hex[:8]}"
         await _ensure_bank(memory, bank_id, request_context)
 
-        result = await memory.update_document(
-            "nonexistent-doc", bank_id, tags=["tag"], request_context=request_context
-        )
+        result = await memory.update_document("nonexistent-doc", bank_id, tags=["tag"], request_context=request_context)
 
         assert result is False
 
         await memory.delete_bank(bank_id, request_context=request_context)
 
     @pytest.mark.asyncio
-    async def test_update_tags_propagates_to_memory_units(
-        self, memory: MemoryEngine, request_context: RequestContext
-    ):
+    async def test_update_tags_propagates_to_memory_units(self, memory: MemoryEngine, request_context: RequestContext):
         """Changing document tags also updates all associated memory unit tags."""
         bank_id = f"test-tag-update-propagate-{uuid.uuid4().hex[:8]}"
         await _ensure_bank(memory, bank_id, request_context)
@@ -547,23 +627,17 @@ class TestUpdateDocumentTagsObservationCleanup:
             )
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
 
         async with pool.acquire() as conn:
             for mem_id in mem_ids:
-                tags = await conn.fetchval(
-                    "SELECT tags FROM memory_units WHERE id = $1", mem_id
-                )
+                tags = await conn.fetchval("SELECT tags FROM memory_units WHERE id = $1", mem_id)
                 assert list(tags) == ["new-tag"], f"Memory unit {mem_id} should have updated tags"
 
         await memory.delete_bank(bank_id, request_context=request_context)
 
     @pytest.mark.asyncio
-    async def test_update_tags_invalidates_observations(
-        self, memory: MemoryEngine, request_context: RequestContext
-    ):
+    async def test_update_tags_invalidates_observations(self, memory: MemoryEngine, request_context: RequestContext):
         """Observations referencing the document's memory units are deleted on tag change."""
         bank_id = f"test-tag-update-obs-{uuid.uuid4().hex[:8]}"
         await _ensure_bank(memory, bank_id, request_context)
@@ -577,9 +651,7 @@ class TestUpdateDocumentTagsObservationCleanup:
             obs_id = await _insert_observation(conn, bank_id, "Alice is a hiker.", mem_ids)
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
 
         async with pool.acquire() as conn:
             obs_ids = await _get_observation_ids(conn, bank_id)
@@ -607,9 +679,7 @@ class TestUpdateDocumentTagsObservationCleanup:
             assert await _get_consolidated_at(conn, mem_ids[0]) is not None
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
 
         async with pool.acquire() as conn:
             consolidated_at = await _get_consolidated_at(conn, mem_ids[0])
@@ -634,9 +704,7 @@ class TestUpdateDocumentTagsObservationCleanup:
             await _insert_observation(conn, bank_id, "Alice is a hiker.", mem_ids)
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()) as mock_consolidate:
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
             mock_consolidate.assert_awaited_once()
 
         await memory.delete_bank(bank_id, request_context=request_context)
@@ -652,15 +720,11 @@ class TestUpdateDocumentTagsObservationCleanup:
         pool = await memory._get_pool()
         async with pool.acquire() as conn:
             doc_id = f"doc-{uuid.uuid4().hex[:8]}"
-            await _insert_document_with_memories(
-                conn, bank_id, doc_id, [("Alice loves hiking.", "experience")]
-            )
+            await _insert_document_with_memories(conn, bank_id, doc_id, [("Alice loves hiking.", "experience")])
             # No observations inserted
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()) as mock_consolidate:
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
             mock_consolidate.assert_not_awaited()
 
         await memory.delete_bank(bank_id, request_context=request_context)
@@ -689,9 +753,7 @@ class TestUpdateDocumentTagsObservationCleanup:
             assert await _get_consolidated_at(conn, other_mem) is not None
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
 
         async with pool.acquire() as conn:
             obs_ids = await _get_observation_ids(conn, bank_id)
@@ -719,17 +781,128 @@ class TestUpdateDocumentTagsObservationCleanup:
             )
             # Unrelated memory not in the document
             unrelated = await _insert_memory(conn, bank_id, "Bob likes cycling.")
-            unrelated_obs_id = await _insert_observation(
-                conn, bank_id, "Bob is a cyclist.", [unrelated]
-            )
+            unrelated_obs_id = await _insert_observation(conn, bank_id, "Bob is a cyclist.", [unrelated])
 
         with patch.object(memory, "submit_async_consolidation", new=AsyncMock()):
-            await memory.update_document(
-                doc_id, bank_id, tags=["new-tag"], request_context=request_context
-            )
+            await memory.update_document(doc_id, bank_id, tags=["new-tag"], request_context=request_context)
 
         async with pool.acquire() as conn:
             obs_ids = await _get_observation_ids(conn, bank_id)
             assert str(unrelated_obs_id) in obs_ids, "Unrelated observation should remain untouched"
 
         await memory.delete_bank(bank_id, request_context=request_context)
+
+
+# ---------------------------------------------------------------------------
+# Tests: consolidation-vs-delete race — filtering stale source_memory_ids
+# ---------------------------------------------------------------------------
+
+
+class TestConsolidationSourceMemoryFiltering:
+    """
+    When a source memory is deleted concurrently with consolidation, the
+    observation must not be written referencing the dead uuid. We exercise
+    the guard by calling the consolidator helpers directly with a deleted
+    source id in the input list.
+    """
+
+    @pytest.mark.asyncio
+    async def test_create_observation_filters_deleted_source_memories(
+        self, memory: MemoryEngine, request_context: RequestContext
+    ):
+        from hindsight_api.engine.consolidation.consolidator import _create_observation_directly
+
+        bank_id = f"test-race-create-filter-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(memory, bank_id, request_context)
+
+        pool = await memory._get_pool()
+        async with pool.acquire() as conn:
+            live = await _insert_memory(conn, bank_id, "Alice loves hiking.")
+            dead = uuid.uuid4()  # never existed — stands in for a concurrently deleted source
+
+            result = await _create_observation_directly(
+                conn=conn,
+                memory_engine=memory,
+                bank_id=bank_id,
+                source_memory_ids=[live, dead],
+                observation_text="Alice enjoys hiking regularly.",
+            )
+
+            assert result["action"] == "created"
+            stored = await conn.fetchval(
+                "SELECT source_memory_ids FROM memory_units WHERE id = $1",
+                uuid.UUID(result["observation_id"]),
+            )
+            stored_set = {str(s) for s in stored}
+            assert str(live) in stored_set
+            assert str(dead) not in stored_set, "Deleted source must not appear in stored observation"
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    @pytest.mark.asyncio
+    async def test_create_observation_skipped_when_all_sources_deleted(
+        self, memory: MemoryEngine, request_context: RequestContext
+    ):
+        from hindsight_api.engine.consolidation.consolidator import _create_observation_directly
+
+        bank_id = f"test-race-create-skip-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(memory, bank_id, request_context)
+
+        pool = await memory._get_pool()
+        async with pool.acquire() as conn:
+            result = await _create_observation_directly(
+                conn=conn,
+                memory_engine=memory,
+                bank_id=bank_id,
+                source_memory_ids=[uuid.uuid4(), uuid.uuid4()],
+                observation_text="All sources gone.",
+            )
+
+            assert result["action"] == "skipped"
+            assert result["reason"] == "sources_deleted"
+
+            obs_ids = await _get_observation_ids(conn, bank_id)
+            assert obs_ids == [], "No observation row should exist"
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+    @pytest.mark.asyncio
+    async def test_update_observation_skipped_when_all_new_sources_deleted(
+        self, memory: MemoryEngine, request_context: RequestContext
+    ):
+        from hindsight_api.engine.consolidation.consolidator import _execute_update_action
+        from hindsight_api.engine.response_models import MemoryFact
+
+        bank_id = f"test-race-update-skip-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(memory, bank_id, request_context)
+
+        pool = await memory._get_pool()
+        async with pool.acquire() as conn:
+            original_source = await _insert_memory(conn, bank_id, "Alice hikes.")
+            obs_id = await _insert_observation(conn, bank_id, "Alice is a hiker.", [original_source])
+            original_text = "Alice is a hiker."
+
+            observation_model = MemoryFact(
+                id=str(obs_id),
+                text=original_text,
+                fact_type="observation",
+                source_fact_ids=[str(original_source)],
+                tags=[],
+            )
+
+            await _execute_update_action(
+                conn=conn,
+                memory_engine=memory,
+                bank_id=bank_id,
+                source_memory_ids=[uuid.uuid4(), uuid.uuid4()],  # all dead
+                observation_id=str(obs_id),
+                new_text="This update must not land.",
+                observations=[observation_model],
+            )
+
+            row = await conn.fetchrow("SELECT text, source_memory_ids FROM memory_units WHERE id = $1", obs_id)
+            assert row["text"] == original_text, "Observation text must not change"
+            stored_sources = {str(s) for s in row["source_memory_ids"]}
+            assert stored_sources == {str(original_source)}, "Dead sources must not be appended"
+
+        await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_recall_budget_config.py

@@ -0,0 +1,254 @@
+"""
+Tests for the configurable recall-budget mapping (Budget enum -> thinking_budget int).
+
+Two functions are supported:
+- "fixed": returns the recall_budget_fixed_<level> integer directly (legacy default).
+- "adaptive": returns round(max_tokens * recall_budget_adaptive_<level>),
+              clamped to [recall_budget_min, recall_budget_max].
+
+Both the function selector and the per-level numbers are hierarchical config
+fields (global env -> tenant -> bank), so they can be overridden per bank.
+"""
+
+import dataclasses
+
+import pytest
+
+from hindsight_api.config import (
+    DEFAULT_RECALL_BUDGET_ADAPTIVE_HIGH,
+    DEFAULT_RECALL_BUDGET_ADAPTIVE_LOW,
+    DEFAULT_RECALL_BUDGET_ADAPTIVE_MID,
+    DEFAULT_RECALL_BUDGET_FIXED_HIGH,
+    DEFAULT_RECALL_BUDGET_FIXED_LOW,
+    DEFAULT_RECALL_BUDGET_FIXED_MID,
+    DEFAULT_RECALL_BUDGET_MAX,
+    DEFAULT_RECALL_BUDGET_MIN,
+    DEFAULT_RECALL_BUDGET_FUNCTION,
+    ENV_RECALL_BUDGET_ADAPTIVE_LOW,
+    ENV_RECALL_BUDGET_ADAPTIVE_MID,
+    ENV_RECALL_BUDGET_FIXED_HIGH,
+    ENV_RECALL_BUDGET_FIXED_LOW,
+    ENV_RECALL_BUDGET_FIXED_MID,
+    ENV_RECALL_BUDGET_MAX,
+    ENV_RECALL_BUDGET_MIN,
+    ENV_RECALL_BUDGET_FUNCTION,
+    RECALL_BUDGET_FUNCTIONS,
+    HindsightConfig,
+)
+from hindsight_api.config_resolver import _validate_recall_budget_updates
+from hindsight_api.engine.memory_engine import Budget, _resolve_thinking_budget
+
+
+_BUDGET_FIELD_NAMES = (
+    "recall_budget_function",
+    "recall_budget_fixed_low",
+    "recall_budget_fixed_mid",
+    "recall_budget_fixed_high",
+    "recall_budget_adaptive_low",
+    "recall_budget_adaptive_mid",
+    "recall_budget_adaptive_high",
+    "recall_budget_min",
+    "recall_budget_max",
+)
+
+
+class TestBudgetConfigFields:
+    def test_fields_exist_on_dataclass(self):
+        names = {f.name for f in dataclasses.fields(HindsightConfig)}
+        for field_name in _BUDGET_FIELD_NAMES:
+            assert field_name in names, f"Missing dataclass field: {field_name}"
+
+    def test_fields_are_configurable(self):
+        configurable = HindsightConfig.get_configurable_fields()
+        for field_name in _BUDGET_FIELD_NAMES:
+            assert field_name in configurable, f"Field not in _CONFIGURABLE_FIELDS: {field_name}"
+
+    def test_default_function_is_fixed_for_backwards_compat(self):
+        # The whole point of function="fixed" being default is to preserve legacy behavior.
+        assert DEFAULT_RECALL_BUDGET_FUNCTION == "fixed"
+        assert "fixed" in RECALL_BUDGET_FUNCTIONS
+        assert "adaptive" in RECALL_BUDGET_FUNCTIONS
+
+    def test_default_fixed_values_match_legacy_hardcoded_mapping(self):
+        # These are the values that used to live in the hardcoded budget_mapping dict.
+        assert DEFAULT_RECALL_BUDGET_FIXED_LOW == 100
+        assert DEFAULT_RECALL_BUDGET_FIXED_MID == 300
+        assert DEFAULT_RECALL_BUDGET_FIXED_HIGH == 1000
+
+    def test_default_adaptive_clamps_are_sane(self):
+        assert DEFAULT_RECALL_BUDGET_MIN >= 1
+        assert DEFAULT_RECALL_BUDGET_MAX > DEFAULT_RECALL_BUDGET_MIN
+
+    def test_env_var_constants(self):
+        assert ENV_RECALL_BUDGET_FUNCTION == "HINDSIGHT_API_RECALL_BUDGET_FUNCTION"
+        assert ENV_RECALL_BUDGET_FIXED_LOW == "HINDSIGHT_API_RECALL_BUDGET_FIXED_LOW"
+        assert ENV_RECALL_BUDGET_FIXED_MID == "HINDSIGHT_API_RECALL_BUDGET_FIXED_MID"
+        assert ENV_RECALL_BUDGET_FIXED_HIGH == "HINDSIGHT_API_RECALL_BUDGET_FIXED_HIGH"
+        assert ENV_RECALL_BUDGET_ADAPTIVE_LOW == "HINDSIGHT_API_RECALL_BUDGET_ADAPTIVE_LOW"
+        assert ENV_RECALL_BUDGET_ADAPTIVE_MID == "HINDSIGHT_API_RECALL_BUDGET_ADAPTIVE_MID"
+        assert ENV_RECALL_BUDGET_MIN == "HINDSIGHT_API_RECALL_BUDGET_MIN"
+        assert ENV_RECALL_BUDGET_MAX == "HINDSIGHT_API_RECALL_BUDGET_MAX"
+
+    def test_from_env_reads_overrides(self, monkeypatch):
+        monkeypatch.setenv(ENV_RECALL_BUDGET_FUNCTION, "adaptive")
+        monkeypatch.setenv(ENV_RECALL_BUDGET_FIXED_MID, "777")
+        monkeypatch.setenv(ENV_RECALL_BUDGET_ADAPTIVE_MID, "0.5")
+        monkeypatch.setenv(ENV_RECALL_BUDGET_MIN, "5")
+        monkeypatch.setenv(ENV_RECALL_BUDGET_MAX, "9999")
+
+        config = HindsightConfig.from_env()
+        assert config.recall_budget_function == "adaptive"
+        assert config.recall_budget_fixed_mid == 777
+        assert config.recall_budget_adaptive_mid == 0.5
+        assert config.recall_budget_min == 5
+        assert config.recall_budget_max == 9999
+
+    def test_from_env_invalid_function_falls_back_to_default(self, monkeypatch):
+        # Defensive parsing: an invalid env value logs a warning and falls back.
+        monkeypatch.setenv(ENV_RECALL_BUDGET_FUNCTION, "garbage")
+        config = HindsightConfig.from_env()
+        assert config.recall_budget_function == DEFAULT_RECALL_BUDGET_FUNCTION
+
+
+class TestResolveThinkingBudgetFixedFunction:
+    @pytest.fixture
+    def fixed_config(self):
+        return {
+            "recall_budget_function": "fixed",
+            "recall_budget_fixed_low": 100,
+            "recall_budget_fixed_mid": 300,
+            "recall_budget_fixed_high": 1000,
+            "recall_budget_adaptive_low": 0.025,
+            "recall_budget_adaptive_mid": 0.075,
+            "recall_budget_adaptive_high": 0.25,
+            "recall_budget_min": 20,
+            "recall_budget_max": 2000,
+        }
+
+    def test_low_mid_high_match_fixed_values(self, fixed_config):
+        assert _resolve_thinking_budget(fixed_config, Budget.LOW, 4096) == 100
+        assert _resolve_thinking_budget(fixed_config, Budget.MID, 4096) == 300
+        assert _resolve_thinking_budget(fixed_config, Budget.HIGH, 4096) == 1000
+
+    def test_none_budget_defaults_to_mid(self, fixed_config):
+        assert _resolve_thinking_budget(fixed_config, None, 4096) == 300
+
+    def test_max_tokens_does_not_affect_fixed_function(self, fixed_config):
+        # Whole point of "fixed": result is independent of max_tokens.
+        assert _resolve_thinking_budget(fixed_config, Budget.MID, 1) == 300
+        assert _resolve_thinking_budget(fixed_config, Budget.MID, 1_000_000) == 300
+
+    def test_per_bank_overrides_take_effect(self, fixed_config):
+        fixed_config["recall_budget_fixed_mid"] = 42
+        assert _resolve_thinking_budget(fixed_config, Budget.MID, 4096) == 42
+
+
+class TestResolveThinkingBudgetAdaptiveFunction:
+    @pytest.fixture
+    def adaptive_config(self):
+        return {
+            "recall_budget_function": "adaptive",
+            "recall_budget_fixed_low": 100,
+            "recall_budget_fixed_mid": 300,
+            "recall_budget_fixed_high": 1000,
+            "recall_budget_adaptive_low": 0.025,
+            "recall_budget_adaptive_mid": 0.075,
+            "recall_budget_adaptive_high": 0.25,
+            "recall_budget_min": 20,
+            "recall_budget_max": 2000,
+        }
+
+    def test_scales_with_max_tokens(self, adaptive_config):
+        # 4096 * 0.075 = 307.2 -> 307
+        assert _resolve_thinking_budget(adaptive_config, Budget.MID, 4096) == 307
+        # 8192 * 0.075 = 614.4 -> 614
+        assert _resolve_thinking_budget(adaptive_config, Budget.MID, 8192) == 614
+
+    def test_clamps_to_floor_when_max_tokens_tiny(self, adaptive_config):
+        # 100 * 0.025 = 2.5 -> 2 -> clamped to floor 20
+        assert _resolve_thinking_budget(adaptive_config, Budget.LOW, 100) == 20
+
+    def test_clamps_to_ceiling_when_max_tokens_huge(self, adaptive_config):
+        # 100_000 * 0.25 = 25_000 -> clamped to ceiling 2000
+        assert _resolve_thinking_budget(adaptive_config, Budget.HIGH, 100_000) == 2000
+
+    def test_none_budget_defaults_to_mid(self, adaptive_config):
+        assert _resolve_thinking_budget(adaptive_config, None, 4096) == 307
+
+    def test_custom_clamps_per_bank(self, adaptive_config):
+        adaptive_config["recall_budget_min"] = 500
+        adaptive_config["recall_budget_max"] = 600
+        # 4096 * 0.075 = 307 -> below floor 500
+        assert _resolve_thinking_budget(adaptive_config, Budget.MID, 4096) == 500
+        # 4096 * 0.25 = 1024 -> above ceiling 600
+        assert _resolve_thinking_budget(adaptive_config, Budget.HIGH, 4096) == 600
+
+
+class TestResolveThinkingBudgetFallbacks:
+    def test_empty_config_uses_legacy_defaults(self):
+        # Resilience: missing keys should not crash; fallback to legacy mapping.
+        assert _resolve_thinking_budget({}, Budget.LOW, 4096) == 100
+        assert _resolve_thinking_budget({}, Budget.MID, 4096) == 300
+        assert _resolve_thinking_budget({}, Budget.HIGH, 4096) == 1000
+
+    def test_unknown_function_falls_back_to_fixed(self):
+        # Defensive: if some bad config slipped past validation, behave like "fixed".
+        assert _resolve_thinking_budget({"recall_budget_function": "garbage"}, Budget.MID, 4096) == 300
+
+
+class TestValidateRecallBudgetUpdates:
+    def test_no_op_passes(self):
+        _validate_recall_budget_updates({})
+        _validate_recall_budget_updates({"unrelated_field": 123})
+
+    def test_valid_function_values(self):
+        _validate_recall_budget_updates({"recall_budget_function": "fixed"})
+        _validate_recall_budget_updates({"recall_budget_function": "adaptive"})
+
+    def test_invalid_function_raises(self):
+        with pytest.raises(ValueError, match="recall_budget_function"):
+            _validate_recall_budget_updates({"recall_budget_function": "wrong"})
+        with pytest.raises(ValueError, match="recall_budget_function"):
+            _validate_recall_budget_updates({"recall_budget_function": 123})
+
+    def test_fixed_must_be_positive_integer(self):
+        for key in ("recall_budget_fixed_low", "recall_budget_fixed_mid", "recall_budget_fixed_high"):
+            _validate_recall_budget_updates({key: 1})
+            _validate_recall_budget_updates({key: 100_000})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: 0})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: -5})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: 1.5})  # float not allowed for fixed
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: True})  # bool sneaks past int check
+
+    def test_adaptive_must_be_positive_number(self):
+        for key in ("recall_budget_adaptive_low", "recall_budget_adaptive_mid", "recall_budget_adaptive_high"):
+            _validate_recall_budget_updates({key: 0.001})
+            _validate_recall_budget_updates({key: 1.0})
+            _validate_recall_budget_updates({key: 5})  # int is acceptable as a number
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: 0})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: -0.1})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: True})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: "0.5"})
+
+    def test_min_must_be_le_max_when_both_set(self):
+        _validate_recall_budget_updates({"recall_budget_min": 10, "recall_budget_max": 1000})
+        _validate_recall_budget_updates({"recall_budget_min": 100, "recall_budget_max": 100})
+        with pytest.raises(ValueError, match="recall_budget_min"):
+            _validate_recall_budget_updates({"recall_budget_min": 5000, "recall_budget_max": 100})
+
+    def test_min_max_must_be_positive_integers(self):
+        for key in ("recall_budget_min", "recall_budget_max"):
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: 0})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: -1})
+            with pytest.raises(ValueError, match=key):
+                _validate_recall_budget_updates({key: 1.5})

hindsight-api-slim/tests/test_recall_config.py

@@ -0,0 +1,231 @@
+"""
+Tests for the internal recall configuration knobs used during mental model
+refresh: recall_include_chunks, recall_max_tokens, recall_chunks_max_tokens.
+
+These are exposed both as hierarchical config fields (env → tenant → bank)
+and as overrides on a mental model's `trigger` JSONB field.
+"""
+
+import dataclasses
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from hindsight_api.engine.reflect.tools import tool_recall
+from hindsight_api.engine.response_models import RecallResult as RecallResultModel
+from hindsight_api.models import RequestContext
+
+
+def _make_mock_engine():
+    engine = MagicMock()
+    engine.recall_async = AsyncMock(return_value=RecallResultModel(results=[], entities={}, chunks={}))
+    return engine
+
+
+@pytest.fixture
+def mock_request_context():
+    # internal=True bypasses the tenant extension, letting these unit tests
+    # exercise engine methods without standing up auth.
+    return RequestContext(internal=True)
+
+
+class TestToolRecallIncludeChunks:
+    """tool_recall must honor the include_chunks parameter (was hardcoded True)."""
+
+    @pytest.mark.asyncio
+    async def test_default_includes_chunks(self, mock_request_context):
+        engine = _make_mock_engine()
+
+        await tool_recall(engine, "bank-1", "q", mock_request_context)
+
+        kwargs = engine.recall_async.call_args.kwargs
+        assert kwargs["include_chunks"] is True
+
+    @pytest.mark.asyncio
+    async def test_include_chunks_false_propagates(self, mock_request_context):
+        engine = _make_mock_engine()
+
+        await tool_recall(engine, "bank-1", "q", mock_request_context, include_chunks=False)
+
+        kwargs = engine.recall_async.call_args.kwargs
+        assert kwargs["include_chunks"] is False
+
+    @pytest.mark.asyncio
+    async def test_max_chunk_tokens_propagates(self, mock_request_context):
+        engine = _make_mock_engine()
+
+        await tool_recall(
+            engine, "bank-1", "q", mock_request_context, max_chunk_tokens=2500, max_tokens=512
+        )
+
+        kwargs = engine.recall_async.call_args.kwargs
+        assert kwargs["max_chunk_tokens"] == 2500
+        assert kwargs["max_tokens"] == 512
+
+
+class TestRecallConfigFields:
+    """Hierarchical config fields for internal recall."""
+
+    def test_fields_exist_on_dataclass(self):
+        from hindsight_api.config import HindsightConfig
+
+        names = {f.name for f in dataclasses.fields(HindsightConfig)}
+        assert "recall_include_chunks" in names
+        assert "recall_max_tokens" in names
+        assert "recall_chunks_max_tokens" in names
+
+    def test_fields_are_configurable(self):
+        from hindsight_api.config import HindsightConfig
+
+        configurable = HindsightConfig.get_configurable_fields()
+        assert "recall_include_chunks" in configurable
+        assert "recall_max_tokens" in configurable
+        assert "recall_chunks_max_tokens" in configurable
+
+    def test_default_values(self):
+        from hindsight_api.config import (
+            DEFAULT_RECALL_CHUNKS_MAX_TOKENS,
+            DEFAULT_RECALL_INCLUDE_CHUNKS,
+            DEFAULT_RECALL_MAX_TOKENS,
+        )
+
+        assert DEFAULT_RECALL_INCLUDE_CHUNKS is True
+        assert DEFAULT_RECALL_MAX_TOKENS == 2048
+        assert DEFAULT_RECALL_CHUNKS_MAX_TOKENS == 1000
+
+    def test_env_var_constants(self):
+        from hindsight_api.config import (
+            ENV_RECALL_CHUNKS_MAX_TOKENS,
+            ENV_RECALL_INCLUDE_CHUNKS,
+            ENV_RECALL_MAX_TOKENS,
+        )
+
+        assert ENV_RECALL_INCLUDE_CHUNKS == "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS"
+        assert ENV_RECALL_MAX_TOKENS == "HINDSIGHT_API_RECALL_MAX_TOKENS"
+        assert ENV_RECALL_CHUNKS_MAX_TOKENS == "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS"
+
+    @patch.dict(
+        "os.environ",
+        {
+            "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS": "false",
+            "HINDSIGHT_API_RECALL_MAX_TOKENS": "777",
+            "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS": "333",
+        },
+    )
+    def test_from_env_reads_overrides(self):
+        from hindsight_api.config import HindsightConfig
+
+        config = HindsightConfig.from_env()
+        assert config.recall_include_chunks is False
+        assert config.recall_max_tokens == 777
+        assert config.recall_chunks_max_tokens == 333
+
+
+class TestMentalModelTriggerRecallFields:
+    """MentalModelTrigger Pydantic model accepts the new override fields."""
+
+    def test_trigger_accepts_new_fields(self):
+        from hindsight_api.api.http import MentalModelTrigger
+
+        trigger = MentalModelTrigger(
+            include_chunks=False,
+            recall_max_tokens=512,
+            recall_chunks_max_tokens=0,
+        )
+        assert trigger.include_chunks is False
+        assert trigger.recall_max_tokens == 512
+        assert trigger.recall_chunks_max_tokens == 0
+
+    def test_trigger_defaults_are_none(self):
+        from hindsight_api.api.http import MentalModelTrigger
+
+        trigger = MentalModelTrigger()
+        assert trigger.include_chunks is None
+        assert trigger.recall_max_tokens is None
+        assert trigger.recall_chunks_max_tokens is None
+
+
+class TestRefreshTriggerWiring:
+    """Verify mental-model refresh forwards trigger overrides into reflect_async kwargs."""
+
+    @pytest.mark.asyncio
+    async def test_trigger_overrides_passed_to_reflect_async(self, mock_request_context):
+        from hindsight_api.engine.memory_engine import MemoryEngine
+        from hindsight_api.engine.response_models import ReflectResult
+
+        engine = MemoryEngine.__new__(MemoryEngine)
+
+        async def fake_get_mental_model(bank_id, mental_model_id, request_context):
+            return {
+                "id": mental_model_id,
+                "source_query": "What do we know?",
+                "tags": [],
+                "trigger": {
+                    "include_chunks": False,
+                    "recall_max_tokens": 512,
+                    "recall_chunks_max_tokens": 0,
+                    "fact_types": ["world"],
+                },
+            }
+
+        captured = {}
+
+        async def fake_reflect_async(**kwargs):
+            captured.update(kwargs)
+            return ReflectResult(text="ok", based_on={})
+
+        async def fake_update_mental_model(*args, **kwargs):
+            return None
+
+        engine.get_mental_model = fake_get_mental_model
+        engine.reflect_async = fake_reflect_async
+        engine.update_mental_model = fake_update_mental_model
+        engine._operation_validator = None
+        engine._tenant_extension = None
+
+        await engine.refresh_mental_model(
+            bank_id="bank-1",
+            mental_model_id="mm-1",
+            request_context=mock_request_context,
+        )
+
+        assert captured["recall_include_chunks"] is False
+        assert captured["recall_max_tokens_override"] == 512
+        assert captured["recall_chunks_max_tokens_override"] == 0
+        assert captured["fact_types"] == ["world"]
+
+    @pytest.mark.asyncio
+    async def test_missing_trigger_fields_pass_none(self, mock_request_context):
+        from hindsight_api.engine.memory_engine import MemoryEngine
+        from hindsight_api.engine.response_models import ReflectResult
+
+        engine = MemoryEngine.__new__(MemoryEngine)
+
+        async def fake_get_mental_model(bank_id, mental_model_id, request_context):
+            return {"id": mental_model_id, "source_query": "q", "tags": [], "trigger": {}}
+
+        captured = {}
+
+        async def fake_reflect_async(**kwargs):
+            captured.update(kwargs)
+            return ReflectResult(text="ok", based_on={})
+
+        async def fake_update_mental_model(*args, **kwargs):
+            return None
+
+        engine.get_mental_model = fake_get_mental_model
+        engine.reflect_async = fake_reflect_async
+        engine.update_mental_model = fake_update_mental_model
+        engine._operation_validator = None
+        engine._tenant_extension = None
+
+        await engine.refresh_mental_model(
+            bank_id="bank-1",
+            mental_model_id="mm-1",
+            request_context=mock_request_context,
+        )
+
+        # When trigger fields are absent, None is forwarded so reflect_async falls back to bank/global config.
+        assert captured["recall_include_chunks"] is None
+        assert captured["recall_max_tokens_override"] is None
+        assert captured["recall_chunks_max_tokens_override"] is None

hindsight-api-slim/tests/test_reflect_agent.py

@@ -396,6 +396,88 @@ class TestReflectAgentMocked:
         # Verify recall was actually called (normalization worked)
         mock_functions["recall_fn"].assert_called_once()
 
+    @pytest.mark.asyncio
+    async def test_short_circuit_answer_is_capped_by_max_tokens(self, mock_llm, mock_functions):
+        """When the LLM short-circuits (returns text without calling a tool) and the text
+        exceeds max_tokens, the agent must rewrite it through a capped call so the final
+        user-visible answer respects the configured limit.
+        """
+        # Build a long response that's well over the cap in cl100k_base tokens.
+        long_answer = " ".join(
+            [
+                "This is a detailed paragraph about the team, their roles, and their recurring meetings."
+            ]
+            * 80
+        )
+        # The short-circuit path: tool_calls empty, content populated.
+        mock_llm.call_with_tools.return_value = LLMToolCallResult(
+            tool_calls=[],
+            content=long_answer,
+            finish_reason="stop",
+            input_tokens=10,
+            output_tokens=500,
+        )
+        mock_llm.call = AsyncMock(
+            return_value=(
+                "Short rewritten answer.",
+                TokenUsage(input_tokens=50, output_tokens=10, total_tokens=60),
+            )
+        )
+
+        cap = 50
+        result = await run_reflect_agent(
+            llm_config=mock_llm,
+            bank_id="test-bank",
+            query="test query",
+            bank_profile={"name": "Test", "mission": "Testing"},
+            max_tokens=cap,
+            **mock_functions,
+        )
+
+        # The rewrite call must have been made, and it must carry the cap.
+        assert mock_llm.call.await_count == 1, (
+            f"expected exactly one capped rewrite call, got {mock_llm.call.await_count}"
+        )
+        rewrite_kwargs = mock_llm.call.await_args.kwargs
+        assert rewrite_kwargs.get("max_completion_tokens") == cap, (
+            f"rewrite call should use max_completion_tokens={cap}, "
+            f"got {rewrite_kwargs.get('max_completion_tokens')}"
+        )
+
+        # The final answer is the rewritten text, not the oversized original.
+        assert result.text == "Short rewritten answer."
+
+        # The trace records the rewrite step so we can see it was invoked.
+        assert any(entry.scope == "final_rewrite" for entry in result.llm_trace), (
+            f"llm_trace should include a final_rewrite entry, got {result.llm_trace}"
+        )
+
+    @pytest.mark.asyncio
+    async def test_short_circuit_answer_under_cap_is_not_rewritten(self, mock_llm, mock_functions):
+        """If the short-circuit answer already fits within max_tokens, no extra rewrite
+        call should happen — we don't want to pay for a second LLM call in the common case.
+        """
+        short_answer = "Small answer that already fits."
+        mock_llm.call_with_tools.return_value = LLMToolCallResult(
+            tool_calls=[],
+            content=short_answer,
+            finish_reason="stop",
+            input_tokens=10,
+            output_tokens=8,
+        )
+
+        result = await run_reflect_agent(
+            llm_config=mock_llm,
+            bank_id="test-bank",
+            query="test query",
+            bank_profile={"name": "Test", "mission": "Testing"},
+            max_tokens=200,
+            **mock_functions,
+        )
+
+        assert result.text == short_answer
+        mock_llm.call.assert_not_called()
+
     @pytest.mark.asyncio
     async def test_max_iterations_reached(self, mock_llm, mock_functions):
         """Test that agent stops after max iterations even with errors."""

hindsight-api-slim/tests/test_retain.py

@@ -378,6 +378,7 @@ async def test_mentioned_at_vs_occurred(memory, request_context):
             content="Alice graduated from MIT in March 2020.",
             context="education history",
             event_date=conversation_date,  # When this conversation happened
+            fact_type_override="world",
             request_context=request_context,
         )
 
@@ -1149,6 +1150,7 @@ async def test_chunk_fact_mapping(memory, request_context):
             content=content,
             context="technical documentation",
             document_id=document_id,
+            fact_type_override="world",
             request_context=request_context,
         )
 

hindsight-api-slim/tests/test_retain_orchestrator_mapping.py

@@ -0,0 +1,117 @@
+"""Unit tests for retain orchestrator mapping and embeddings length guarantee.
+
+Regression coverage for issue #1037: a silent length mismatch between the
+extracted facts and the generated embeddings caused
+`_map_results_to_contents` to raise IndexError during batch_retain.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime
+from unittest.mock import MagicMock
+
+import pytest
+
+from hindsight_api.engine.retain import embedding_utils
+from hindsight_api.engine.retain.orchestrator import _map_results_to_contents
+from hindsight_api.engine.retain.types import ProcessedFact, RetainContent
+
+
+def _make_processed_fact(content_index: int, text: str = "fact") -> ProcessedFact:
+    return ProcessedFact(
+        fact_text=text,
+        fact_type="world",
+        embedding=[0.0, 0.0, 0.0],
+        occurred_start=None,
+        occurred_end=None,
+        mentioned_at=datetime(2026, 1, 1),
+        context="",
+        metadata={},
+        content_index=content_index,
+    )
+
+
+def _make_content(text: str = "x") -> RetainContent:
+    return RetainContent(content=text)
+
+
+class TestMapResultsToContents:
+    def test_groups_unit_ids_by_content_index(self):
+        contents = [_make_content("a"), _make_content("b"), _make_content("c")]
+        processed = [
+            _make_processed_fact(0, "a1"),
+            _make_processed_fact(0, "a2"),
+            _make_processed_fact(2, "c1"),
+        ]
+        unit_ids = ["u-a1", "u-a2", "u-c1"]
+
+        result = _map_results_to_contents(contents, processed, unit_ids)
+
+        assert result == [["u-a1", "u-a2"], [], ["u-c1"]]
+
+    def test_handles_out_of_range_content_index(self):
+        contents = [_make_content("a"), _make_content("b")]
+        processed = [
+            _make_processed_fact(-1, "f1"),
+            _make_processed_fact(99, "f2"),
+        ]
+        unit_ids = ["u1", "u2"]
+
+        result = _map_results_to_contents(contents, processed, unit_ids)
+
+        assert result == [["u1"], ["u2"]]
+
+    def test_empty_inputs(self):
+        assert _map_results_to_contents([], [], []) == []
+
+    def test_length_mismatch_raises(self):
+        # Regression for #1037: previously the function silently overran unit_ids.
+        contents = [_make_content("a")]
+        processed = [_make_processed_fact(0), _make_processed_fact(0)]
+        unit_ids = ["u1"]  # one fewer than processed_facts
+
+        with pytest.raises(ValueError, match="length mismatch"):
+            _map_results_to_contents(contents, processed, unit_ids)
+
+    def test_unit_ids_assigned_by_processed_fact_position(self):
+        # Even if processed_facts are interleaved across contents, each unit_id
+        # must follow its corresponding processed_fact (positional alignment).
+        contents = [_make_content("a"), _make_content("b")]
+        processed = [
+            _make_processed_fact(1, "b1"),
+            _make_processed_fact(0, "a1"),
+            _make_processed_fact(1, "b2"),
+        ]
+        unit_ids = ["u-b1", "u-a1", "u-b2"]
+
+        result = _map_results_to_contents(contents, processed, unit_ids)
+
+        assert result == [["u-a1"], ["u-b1", "u-b2"]]
+
+
+class TestEmbeddingsBatchLengthGuarantee:
+    def test_raises_when_backend_returns_fewer_embeddings(self):
+        # Regression for #1037: backends that silently truncate must not pass
+        # through — `zip(extracted_facts, embeddings)` would otherwise drop
+        # facts and break unit_id alignment downstream.
+        backend = MagicMock()
+        backend.encode.return_value = [[0.1, 0.2]]  # only 1 vector for 3 inputs
+
+        with pytest.raises(RuntimeError, match="returned 1 vectors for 3 input texts"):
+            asyncio.run(embedding_utils.generate_embeddings_batch(backend, ["a", "b", "c"]))
+
+    def test_raises_when_backend_returns_more_embeddings(self):
+        backend = MagicMock()
+        backend.encode.return_value = [[0.1], [0.2], [0.3]]
+
+        with pytest.raises(RuntimeError, match="returned 3 vectors for 2 input texts"):
+            asyncio.run(embedding_utils.generate_embeddings_batch(backend, ["a", "b"]))
+
+    def test_passes_through_aligned_embeddings(self):
+        backend = MagicMock()
+        backend.encode.return_value = [[0.1], [0.2]]
+
+        result = asyncio.run(embedding_utils.generate_embeddings_batch(backend, ["a", "b"]))
+
+        assert result == [[0.1], [0.2]]

hindsight-api-slim/tests/test_structured_doc.py

@@ -0,0 +1,438 @@
+"""Unit tests for the structured document schema, renderer, parser, and
+delta-operation applicator.
+
+These tests are pure-Python (no DB, no LLM) and run fast. They guard the
+mechanical guarantees that the structured-delta architecture relies on:
+
+- Deterministic rendering (same input → same bytes).
+- Round-trip parse → render is stable for canonical markdown.
+- Section IDs are stable slugs and survive disambiguation.
+- Operations target sections/blocks by id/index and never silently corrupt
+  the document; invalid ops are dropped, not applied half-way.
+- Sections and blocks not mentioned by any op come through byte-identical.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from hindsight_api.engine.reflect.delta_ops import (
+    AddSectionOp,
+    AppendBlockOp,
+    DeltaOperationList,
+    InsertBlockOp,
+    RemoveBlockOp,
+    RemoveSectionOp,
+    RenameSectionOp,
+    ReplaceBlockOp,
+    ReplaceSectionBlocksOp,
+    apply_operations,
+)
+from hindsight_api.engine.reflect.structured_doc import (
+    BulletListBlock,
+    CodeBlock,
+    OrderedListBlock,
+    ParagraphBlock,
+    Section,
+    StructuredDocument,
+    make_unique_id,
+    parse_markdown,
+    render_block,
+    render_document,
+    render_section,
+    slugify_heading,
+)
+
+# Helpers --------------------------------------------------------------------
+
+
+def _team_overview_doc() -> StructuredDocument:
+    return StructuredDocument(
+        sections=[
+            Section(
+                id="team-overview",
+                heading="Team Overview",
+                level=1,
+                blocks=[ParagraphBlock(text="Quick summary of the engineering team.")],
+            ),
+            Section(
+                id="members",
+                heading="Members",
+                level=2,
+                blocks=[
+                    BulletListBlock(
+                        items=[
+                            "**Alice** — team lead, owns planning.",
+                            "**Bob** — senior engineer, mentors juniors.",
+                        ]
+                    )
+                ],
+            ),
+            Section(
+                id="cadence",
+                heading="Cadence",
+                level=2,
+                blocks=[ParagraphBlock(text="Standups happen daily at 9am.")],
+            ),
+        ]
+    )
+
+
+# Slug ----------------------------------------------------------------------
+
+
+class TestSlugify:
+    def test_basic(self):
+        assert slugify_heading("Purpose") == "purpose"
+
+    def test_multi_word(self):
+        assert slugify_heading("Stop Conditions") == "stop-conditions"
+
+    def test_punctuation_collapses(self):
+        assert slugify_heading("Inputs / Context !") == "inputs-context"
+
+    def test_unicode_falls_back(self):
+        # Non-ASCII chars are stripped; if nothing remains, slug becomes "section".
+        assert slugify_heading("???") == "section"
+
+    def test_make_unique_id_no_collision(self):
+        assert make_unique_id("rules", set()) == "rules"
+
+    def test_make_unique_id_collision(self):
+        assert make_unique_id("rules", {"rules"}) == "rules-2"
+        assert make_unique_id("rules", {"rules", "rules-2"}) == "rules-3"
+
+
+# Renderer ------------------------------------------------------------------
+
+
+class TestRenderer:
+    def test_paragraph(self):
+        assert render_block(ParagraphBlock(text="hello world")) == "hello world"
+
+    def test_bullet_list(self):
+        block = BulletListBlock(items=["one", "two"])
+        assert render_block(block) == "- one\n- two"
+
+    def test_ordered_list_uses_sequential_numbering(self):
+        block = OrderedListBlock(items=["one", "two", "three"])
+        assert render_block(block) == "1. one\n2. two\n3. three"
+
+    def test_code_block_with_language(self):
+        block = CodeBlock(language="json", text='{"a": 1}')
+        assert render_block(block) == '```json\n{"a": 1}\n```'
+
+    def test_code_block_no_language(self):
+        block = CodeBlock(text="raw text")
+        assert render_block(block) == "```\nraw text\n```"
+
+    def test_section_heading_level(self):
+        section = Section(
+            id="purpose", heading="Purpose", level=3, blocks=[ParagraphBlock(text="hi")]
+        )
+        assert render_section(section).startswith("### Purpose\n\nhi")
+
+    def test_document_round_trip_is_stable(self):
+        doc = _team_overview_doc()
+        rendered = render_document(doc)
+        # Re-rendering must produce the same bytes.
+        assert render_document(doc) == rendered
+        # Headings, members, cadence all present.
+        assert "# Team Overview" in rendered
+        assert "## Members" in rendered
+        assert "## Cadence" in rendered
+        assert "- **Alice**" in rendered
+        assert "Standups happen daily at 9am" in rendered
+        # Sections separated by exactly one blank line, document ends with newline.
+        assert rendered.endswith("\n")
+        assert "\n\n\n" not in rendered
+
+    def test_empty_document_renders_empty(self):
+        assert render_document(StructuredDocument()) == ""
+
+
+# Parser --------------------------------------------------------------------
+
+
+class TestParser:
+    def test_simple_document(self):
+        markdown = (
+            "# Team Overview\n"
+            "\n"
+            "Quick summary.\n"
+            "\n"
+            "## Members\n"
+            "\n"
+            "- Alice\n"
+            "- Bob\n"
+            "\n"
+            "## Cadence\n"
+            "\n"
+            "Standups daily.\n"
+        )
+        doc = parse_markdown(markdown)
+        assert [s.id for s in doc.sections] == ["team-overview", "members", "cadence"]
+        assert [s.level for s in doc.sections] == [1, 2, 2]
+        assert isinstance(doc.sections[0].blocks[0], ParagraphBlock)
+        assert isinstance(doc.sections[1].blocks[0], BulletListBlock)
+        assert doc.sections[1].blocks[0].items == ["Alice", "Bob"]
+
+    def test_horizontal_rule_treated_as_blank(self):
+        markdown = "## Rules\n\n- one\n\n---\n\n## Stop\n\nstop here.\n"
+        doc = parse_markdown(markdown)
+        assert [s.id for s in doc.sections] == ["rules", "stop"]
+        # Horizontal rule must NOT become a paragraph.
+        assert all(
+            not (isinstance(b, ParagraphBlock) and "---" in b.text)
+            for s in doc.sections
+            for b in s.blocks
+        )
+
+    def test_ordered_list(self):
+        markdown = "## Steps\n\n1. one\n2. two\n3. three\n"
+        doc = parse_markdown(markdown)
+        block = doc.sections[0].blocks[0]
+        assert isinstance(block, OrderedListBlock)
+        assert block.items == ["one", "two", "three"]
+
+    def test_code_block(self):
+        markdown = '## Example\n\n```json\n{"a": 1}\n```\n'
+        doc = parse_markdown(markdown)
+        block = doc.sections[0].blocks[0]
+        assert isinstance(block, CodeBlock)
+        assert block.language == "json"
+        assert block.text == '{"a": 1}'
+
+    def test_implicit_overview_when_content_before_first_heading(self):
+        markdown = "preamble paragraph.\n\n## Members\n\n- Alice\n"
+        doc = parse_markdown(markdown)
+        assert doc.sections[0].id == "overview"
+        assert isinstance(doc.sections[0].blocks[0], ParagraphBlock)
+        assert doc.sections[1].id == "members"
+
+    def test_duplicate_headings_get_unique_ids(self):
+        markdown = "## Notes\n\nfirst.\n\n## Notes\n\nsecond.\n"
+        doc = parse_markdown(markdown)
+        assert [s.id for s in doc.sections] == ["notes", "notes-2"]
+
+    def test_round_trip_via_render(self):
+        original = _team_overview_doc()
+        markdown = render_document(original)
+        roundtripped = parse_markdown(markdown)
+        # Re-render must match the original render exactly.
+        assert render_document(roundtripped) == markdown
+
+
+# Operation applicator ------------------------------------------------------
+
+
+class TestApplyOperations:
+    def test_zero_ops_returns_identical_document(self):
+        doc = _team_overview_doc()
+        result = apply_operations(doc, [])
+        assert result.document.model_dump() == doc.model_dump()
+        assert render_document(result.document) == render_document(doc)
+        assert result.applied == []
+        assert result.changed is False
+
+    def test_unknown_section_op_is_skipped(self):
+        doc = _team_overview_doc()
+        op = AppendBlockOp(section_id="does-not-exist", block=ParagraphBlock(text="x"))
+        result = apply_operations(doc, [op])
+        assert result.applied == []
+        assert len(result.skipped) == 1
+        assert "unknown section_id" in result.skipped[0]["reason"]
+        # Document unchanged.
+        assert render_document(result.document) == render_document(doc)
+
+    def test_append_block_to_existing_section(self):
+        doc = _team_overview_doc()
+        op = AppendBlockOp(
+            section_id="members",
+            block=BulletListBlock(items=["**Carol** — junior engineer."]),
+        )
+        result = apply_operations(doc, [op])
+        members = result.document.section_by_id("members")
+        assert members is not None
+        assert len(members.blocks) == 2  # original list + new bullet block
+        # Other sections byte-identical
+        original = doc.model_dump()
+        new = result.document.model_dump()
+        assert new["sections"][0] == original["sections"][0]  # team-overview
+        assert new["sections"][2] == original["sections"][2]  # cadence
+
+    def test_insert_block_at_index(self):
+        doc = _team_overview_doc()
+        op = InsertBlockOp(
+            section_id="members",
+            index=0,
+            block=ParagraphBlock(text="Roster as of 2026:"),
+        )
+        result = apply_operations(doc, [op])
+        members = result.document.section_by_id("members")
+        assert isinstance(members.blocks[0], ParagraphBlock)
+        assert members.blocks[0].text.startswith("Roster")
+
+    def test_insert_block_out_of_range_skipped(self):
+        doc = _team_overview_doc()
+        op = InsertBlockOp(
+            section_id="members", index=99, block=ParagraphBlock(text="x")
+        )
+        result = apply_operations(doc, [op])
+        assert result.applied == []
+        assert "index out of range" in result.skipped[0]["reason"]
+
+    def test_replace_block(self):
+        doc = _team_overview_doc()
+        op = ReplaceBlockOp(
+            section_id="cadence",
+            index=0,
+            block=ParagraphBlock(text="Standups happen daily at 10am."),
+        )
+        result = apply_operations(doc, [op])
+        cadence = result.document.section_by_id("cadence")
+        assert isinstance(cadence.blocks[0], ParagraphBlock)
+        assert cadence.blocks[0].text.endswith("10am.")
+
+    def test_remove_block(self):
+        doc = _team_overview_doc()
+        op = RemoveBlockOp(section_id="members", index=0)
+        result = apply_operations(doc, [op])
+        members = result.document.section_by_id("members")
+        assert members.blocks == []
+
+    def test_add_section_at_end(self):
+        doc = _team_overview_doc()
+        op = AddSectionOp(
+            heading="Open Questions",
+            blocks=[ParagraphBlock(text="None right now.")],
+        )
+        result = apply_operations(doc, [op])
+        assert result.document.sections[-1].id == "open-questions"
+        assert result.document.sections[-1].heading == "Open Questions"
+
+    def test_add_section_after_existing(self):
+        doc = _team_overview_doc()
+        op = AddSectionOp(
+            heading="Charter",
+            after_section_id="team-overview",
+            blocks=[ParagraphBlock(text="Mission statement.")],
+        )
+        result = apply_operations(doc, [op])
+        ids = [s.id for s in result.document.sections]
+        assert ids == ["team-overview", "charter", "members", "cadence"]
+
+    def test_add_section_after_unknown_skipped(self):
+        doc = _team_overview_doc()
+        op = AddSectionOp(
+            heading="Charter",
+            after_section_id="nope",
+            blocks=[],
+        )
+        result = apply_operations(doc, [op])
+        assert result.applied == []
+        assert "unknown after_section_id" in result.skipped[0]["reason"]
+
+    def test_add_section_with_id_collision_disambiguates(self):
+        doc = _team_overview_doc()
+        op = AddSectionOp(heading="Members", blocks=[])
+        result = apply_operations(doc, [op])
+        # Two sections with heading "Members": the new one gets "members-2".
+        ids = [s.id for s in result.document.sections]
+        assert "members" in ids
+        assert "members-2" in ids
+
+    def test_remove_section(self):
+        doc = _team_overview_doc()
+        op = RemoveSectionOp(section_id="cadence")
+        result = apply_operations(doc, [op])
+        assert [s.id for s in result.document.sections] == ["team-overview", "members"]
+
+    def test_replace_section_blocks_preserves_id_and_heading(self):
+        doc = _team_overview_doc()
+        op = ReplaceSectionBlocksOp(
+            section_id="members",
+            blocks=[ParagraphBlock(text="See the org chart.")],
+        )
+        result = apply_operations(doc, [op])
+        members = result.document.section_by_id("members")
+        assert members.heading == "Members"
+        assert members.id == "members"
+        assert len(members.blocks) == 1
+        assert isinstance(members.blocks[0], ParagraphBlock)
+
+    def test_rename_section_keeps_id(self):
+        doc = _team_overview_doc()
+        op = RenameSectionOp(section_id="cadence", new_heading="Operating Cadence")
+        result = apply_operations(doc, [op])
+        section = result.document.section_by_id("cadence")
+        assert section.heading == "Operating Cadence"
+        # ID stable so future ops still resolve.
+        assert section.id == "cadence"
+
+    def test_unmodified_sections_byte_identical_in_render(self):
+        """The structural guarantee: sections not touched by any op render
+        identically character-for-character.
+        """
+        doc = _team_overview_doc()
+        op = AppendBlockOp(
+            section_id="members",
+            block=ParagraphBlock(text="New: Carol joined as junior engineer."),
+        )
+        result = apply_operations(doc, [op])
+        before_overview = render_section(doc.section_by_id("team-overview"))
+        after_overview = render_section(
+            result.document.section_by_id("team-overview")
+        )
+        before_cadence = render_section(doc.section_by_id("cadence"))
+        after_cadence = render_section(
+            result.document.section_by_id("cadence")
+        )
+        assert before_overview == after_overview
+        assert before_cadence == after_cadence
+
+
+class TestDeltaOperationListSchema:
+    """Sanity-check that the discriminated-union schema serialises as the LLM
+    will see it: each op has a literal ``op`` string that picks the variant.
+    """
+
+    def test_round_trip_via_json(self):
+        ops = DeltaOperationList(
+            operations=[
+                AppendBlockOp(
+                    section_id="members",
+                    block=ParagraphBlock(text="hi"),
+                ),
+                AddSectionOp(
+                    heading="Open Questions",
+                    after_section_id="cadence",
+                    blocks=[ParagraphBlock(text="None.")],
+                ),
+                RemoveSectionOp(section_id="charter"),
+            ]
+        )
+        payload = ops.model_dump_json()
+        roundtripped = DeltaOperationList.model_validate_json(payload)
+        assert len(roundtripped.operations) == 3
+
+    def test_invalid_op_field_rejected(self):
+        with pytest.raises(Exception):  # pydantic ValidationError
+            DeltaOperationList.model_validate(
+                {"operations": [{"op": "not_a_real_op", "section_id": "x"}]}
+            )
+
+    def test_extra_field_rejected(self):
+        with pytest.raises(Exception):
+            DeltaOperationList.model_validate(
+                {
+                    "operations": [
+                        {
+                            "op": "append_block",
+                            "section_id": "members",
+                            "block": {"type": "paragraph", "text": "hi"},
+                            "extra_field": "no",
+                        }
+                    ]
+                }
+            )

hindsight-api-slim/tests/test_tei_cross_encoder.py

@@ -549,6 +549,29 @@ class TestRemoteTEICrossEncoderConfig:
 
         clear_config_cache()  # Clear cache after test
 
+    def test_create_from_env_with_custom_timeout(self):
+        """Test that HINDSIGHT_API_RERANKER_TEI_HTTP_TIMEOUT is respected."""
+        import os
+
+        from hindsight_api.config import clear_config_cache
+        from hindsight_api.engine.cross_encoder import create_cross_encoder_from_env
+
+        with patch.dict(
+            os.environ,
+            {
+                "HINDSIGHT_API_RERANKER_PROVIDER": "tei",
+                "HINDSIGHT_API_RERANKER_TEI_URL": "http://test:9000",
+                "HINDSIGHT_API_RERANKER_TEI_HTTP_TIMEOUT": "120.0",
+            },
+        ):
+            clear_config_cache()
+            encoder = create_cross_encoder_from_env()
+
+            assert isinstance(encoder, RemoteTEICrossEncoder)
+            assert encoder.timeout == 120.0
+
+        clear_config_cache()
+
 
 # ============================================================================
 # TEI Reranker Performance Benchmark Tests

hindsight-api-slim/tests/test_verify_connection_soft_fail.py

@@ -0,0 +1,157 @@
+"""
+Tests that LLM connection verification failures don't crash server startup.
+
+When the LLM provider is unavailable (e.g. 429 quota exhaustion), the server
+should log a warning and continue booting rather than crash-looping.
+See: https://github.com/vectorize-io/hindsight/issues/1147
+"""
+
+import logging
+from unittest.mock import AsyncMock, patch
+
+import pytest
+import pytest_asyncio
+
+from hindsight_api import MemoryEngine
+from hindsight_api.engine.task_backend import SyncTaskBackend
+
+
+@pytest_asyncio.fixture(scope="function")
+async def engine_with_failing_llm(pg0_db_url, embeddings, cross_encoder, query_analyzer):
+    """Create a MemoryEngine whose LLM verify_connection raises."""
+    mem = MemoryEngine(
+        db_url=pg0_db_url,
+        memory_llm_provider="mock",
+        memory_llm_api_key="",
+        memory_llm_model="mock",
+        embeddings=embeddings,
+        cross_encoder=cross_encoder,
+        query_analyzer=query_analyzer,
+        pool_min_size=1,
+        pool_max_size=5,
+        run_migrations=False,
+        task_backend=SyncTaskBackend(),
+        skip_llm_verification=False,  # Enable verification — we want to test the soft-fail path
+    )
+    yield mem
+    try:
+        if mem._pool and not mem._pool._closing:
+            await mem.close()
+    except Exception:
+        pass
+
+
+@pytest.mark.asyncio
+async def test_initialize_succeeds_when_verify_connection_fails(
+    engine_with_failing_llm,
+    caplog,
+):
+    """Server should start even if LLM verify_connection raises (e.g. 429)."""
+    engine = engine_with_failing_llm
+
+    # Patch the mock provider's verify_connection to simulate a 429 error
+    with patch.object(
+        engine._llm_config._provider_impl,
+        "verify_connection",
+        new_callable=AsyncMock,
+        side_effect=RuntimeError("429 RESOURCE_EXHAUSTED: Quota exceeded"),
+    ):
+        with caplog.at_level(logging.WARNING):
+            # Should NOT raise — the server boots despite the LLM being unavailable
+            await engine.initialize()
+
+    # Verify the warning was logged
+    assert any("LLM connection verification failed" in record.message for record in caplog.records)
+    assert any("429 RESOURCE_EXHAUSTED" in record.message for record in caplog.records)
+
+
+@pytest.mark.asyncio
+async def test_initialize_logs_warning_per_failing_config(
+    pg0_db_url,
+    embeddings,
+    cross_encoder,
+    query_analyzer,
+    caplog,
+):
+    """Each distinct LLM config that fails verification gets its own warning."""
+    engine = MemoryEngine(
+        db_url=pg0_db_url,
+        memory_llm_provider="mock",
+        memory_llm_api_key="",
+        memory_llm_model="default-model",
+        retain_llm_provider="mock",
+        retain_llm_model="retain-model",  # Different model → separate verification
+        embeddings=embeddings,
+        cross_encoder=cross_encoder,
+        query_analyzer=query_analyzer,
+        pool_min_size=1,
+        pool_max_size=5,
+        run_migrations=False,
+        task_backend=SyncTaskBackend(),
+        skip_llm_verification=False,
+    )
+
+    # Both default and retain verify_connection will raise
+    with (
+        patch.object(
+            engine._llm_config._provider_impl,
+            "verify_connection",
+            new_callable=AsyncMock,
+            side_effect=RuntimeError("429 quota exceeded for default"),
+        ),
+        patch.object(
+            engine._retain_llm_config._provider_impl,
+            "verify_connection",
+            new_callable=AsyncMock,
+            side_effect=RuntimeError("429 quota exceeded for retain"),
+        ),
+    ):
+        with caplog.at_level(logging.WARNING):
+            await engine.initialize()
+
+    warning_messages = [r.message for r in caplog.records if "LLM connection verification failed" in r.message]
+    assert len(warning_messages) == 2
+    assert any("'default'" in msg for msg in warning_messages)
+    assert any("'retain'" in msg for msg in warning_messages)
+
+    try:
+        if engine._pool and not engine._pool._closing:
+            await engine.close()
+    except Exception:
+        pass
+
+
+@pytest.mark.asyncio
+async def test_initialize_succeeds_when_verify_connection_succeeds(
+    pg0_db_url,
+    embeddings,
+    cross_encoder,
+    query_analyzer,
+    caplog,
+):
+    """Verify the happy path still works — no warnings when verification passes."""
+    engine = MemoryEngine(
+        db_url=pg0_db_url,
+        memory_llm_provider="mock",
+        memory_llm_api_key="",
+        memory_llm_model="mock",
+        embeddings=embeddings,
+        cross_encoder=cross_encoder,
+        query_analyzer=query_analyzer,
+        pool_min_size=1,
+        pool_max_size=5,
+        run_migrations=False,
+        task_backend=SyncTaskBackend(),
+        skip_llm_verification=False,
+    )
+
+    with caplog.at_level(logging.WARNING):
+        await engine.initialize()
+
+    assert not any("LLM connection verification failed" in r.message for r in caplog.records)
+
+    try:
+        if engine._pool and not engine._pool._closing:
+            await engine.close()
+    except Exception:
+        pass

hindsight-api-slim/tests/test_worker.py

@@ -137,6 +137,55 @@ class TestBrokerTaskBackend:
         payload = json.loads(row["task_payload"])
         assert payload["node_ids"] == ["node1", "node2"]
 
+    @pytest.mark.asyncio
+    async def test_submit_task_preserves_existing_payload(self, pool, clean_operations):
+        """Callers now INSERT task_payload atomically, then call submit_task as a
+        no-op for the BrokerTaskBackend path. submit_task must not overwrite a
+        payload that is already set, otherwise a stale updated_at bump on a
+        possibly-already-processing row reintroduces noise the fix aimed to remove.
+        """
+        operation_id = uuid.uuid4()
+        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(pool, bank_id)
+
+        original_payload = {"type": "test_task", "bank_id": bank_id, "version": "inserted"}
+        await pool.execute(
+            """
+            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
+            VALUES ($1, $2, 'test_operation', 'pending', $3::jsonb)
+            """,
+            operation_id,
+            bank_id,
+            json.dumps(original_payload),
+        )
+
+        row_before = await pool.fetchrow(
+            "SELECT updated_at FROM async_operations WHERE operation_id = $1",
+            operation_id,
+        )
+
+        backend = BrokerTaskBackend(pool_getter=lambda: pool)
+        await backend.initialize()
+
+        await backend.submit_task(
+            {
+                "operation_id": str(operation_id),
+                "type": "test_task",
+                "bank_id": bank_id,
+                "version": "resubmitted",
+            }
+        )
+
+        row_after = await pool.fetchrow(
+            "SELECT task_payload, updated_at FROM async_operations WHERE operation_id = $1",
+            operation_id,
+        )
+        payload = json.loads(row_after["task_payload"])
+        assert payload["version"] == "inserted", "submit_task must not overwrite an existing payload"
+        assert row_after["updated_at"] == row_before["updated_at"], (
+            "submit_task must not bump updated_at when payload was already set"
+        )
+
 
 class TestWorkerPoller:
     """Tests for WorkerPoller task claiming and execution."""
@@ -466,6 +515,199 @@ class TestWorkerPoller:
         )
         assert "Simulated conversion error" in row["error_message"]
 
+    @pytest.mark.asyncio
+    async def test_executor_defer_requeues_without_bumping_retry_count(self, pool, clean_operations):
+        """DeferOperation requeues the task without counting as a retry.
+
+        Unlike RetryTaskAt (failure-driven), DeferOperation is intentional
+        backpressure: the row goes back to 'pending' with next_retry_at set,
+        but retry_count is unchanged and error_message stays NULL.
+        """
+        from datetime import datetime, timedelta, timezone
+
+        from hindsight_api.worker import WorkerPoller
+        from hindsight_api.worker.exceptions import DeferOperation
+        from hindsight_api.worker.poller import ClaimedTask
+
+        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+        op_id = uuid.uuid4()
+        payload = json.dumps({"type": "retain", "operation_id": str(op_id), "bank_id": bank_id})
+        await _ensure_bank(pool, bank_id)
+        await pool.execute(
+            """
+            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id, claimed_at, retry_count)
+            VALUES ($1, $2, 'retain', 'processing', $3::jsonb, 'test-worker-1', now(), 0)
+            """,
+            op_id,
+            bank_id,
+            payload,
+        )
+
+        defer_until = datetime.now(timezone.utc) + timedelta(minutes=5)
+
+        async def deferring_executor(task_dict):
+            raise DeferOperation(exec_date=defer_until, reason="upstream quota window not yet open")
+
+        poller = WorkerPoller(
+            pool=pool,
+            worker_id="test-worker-1",
+            executor=deferring_executor,
+        )
+
+        task_dict = json.loads(payload)
+        claimed_task = ClaimedTask(operation_id=str(op_id), task_dict=task_dict, schema=None)
+        await poller.execute_task(claimed_task)
+
+        completed = await poller.wait_for_active_tasks(timeout=5.0)
+        assert completed, "Task did not complete within timeout"
+
+        row = await pool.fetchrow(
+            "SELECT status, worker_id, claimed_at, retry_count, error_message, next_retry_at "
+            "FROM async_operations WHERE operation_id = $1",
+            op_id,
+        )
+        assert row["status"] == "pending"
+        assert row["worker_id"] is None
+        assert row["claimed_at"] is None
+        assert row["retry_count"] == 0, "defer must NOT increment retry_count"
+        assert row["error_message"] is None, "defer must NOT write error_message"
+        assert row["next_retry_at"] is not None
+        # exec_date should round-trip; allow 1s slack for db precision
+        assert abs((row["next_retry_at"] - defer_until).total_seconds()) < 1
+
+    @pytest.mark.asyncio
+    async def test_deferred_task_not_picked_up_until_exec_date(self, pool, clean_operations):
+        """A deferred task is invisible to claim_batch until next_retry_at <= NOW()."""
+        from datetime import datetime, timedelta, timezone
+
+        from hindsight_api.worker import WorkerPoller
+
+        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+        op_id = uuid.uuid4()
+        payload = json.dumps({"type": "retain", "operation_id": str(op_id), "bank_id": bank_id})
+        await _ensure_bank(pool, bank_id)
+        future = datetime.now(timezone.utc) + timedelta(minutes=5)
+        await pool.execute(
+            """
+            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, next_retry_at)
+            VALUES ($1, $2, 'retain', 'pending', $3::jsonb, $4)
+            """,
+            op_id,
+            bank_id,
+            payload,
+            future,
+        )
+
+        poller = WorkerPoller(
+            pool=pool,
+            worker_id="test-worker-1",
+            executor=lambda x: None,
+        )
+
+        claimed = await poller.claim_batch()
+        assert all(c.operation_id != str(op_id) for c in claimed), "deferred task must not be claimed before exec_date"
+
+        # Move next_retry_at into the past — task becomes claimable.
+        await pool.execute(
+            "UPDATE async_operations SET next_retry_at = now() - interval '1 minute' WHERE operation_id = $1",
+            op_id,
+        )
+        claimed = await poller.claim_batch()
+        assert any(c.operation_id == str(op_id) for c in claimed), "task must be claimed once next_retry_at has passed"
+
+    @pytest.mark.asyncio
+    async def test_defer_operation_exported_from_extensions(self):
+        """DeferOperation must be importable from hindsight_api.extensions for extension authors."""
+        from hindsight_api.extensions import DeferOperation as DeferFromExtensions
+        from hindsight_api.worker.exceptions import DeferOperation as DeferFromWorker
+
+        assert DeferFromExtensions is DeferFromWorker
+
+    @pytest.mark.asyncio
+    async def test_extension_validate_retain_defer_propagates_to_poller(self, pool, clean_operations):
+        """An OperationValidatorExtension that raises DeferOperation in validate_retain
+        causes the worker to requeue the task at the requested exec_date.
+
+        Mimics the MemoryEngine flow: the executor calls validate_retain before doing
+        any real work, the exception bubbles up to the poller, which defers the row.
+        """
+        from datetime import datetime, timedelta, timezone
+
+        from hindsight_api.extensions import (
+            DeferOperation,
+            OperationValidatorExtension,
+            RecallContext,
+            ReflectContext,
+            RetainContext,
+            ValidationResult,
+        )
+        from hindsight_api.worker import WorkerPoller
+        from hindsight_api.worker.poller import ClaimedTask
+
+        defer_until = datetime.now(timezone.utc) + timedelta(minutes=10)
+
+        class DeferringValidator(OperationValidatorExtension):
+            def __init__(self):
+                super().__init__({})
+
+            async def validate_retain(self, ctx: RetainContext) -> ValidationResult:
+                raise DeferOperation(exec_date=defer_until, reason="quota window closed")
+
+            async def validate_recall(self, ctx: RecallContext) -> ValidationResult:
+                return ValidationResult.accept()
+
+            async def validate_reflect(self, ctx: ReflectContext) -> ValidationResult:
+                return ValidationResult.accept()
+
+        validator = DeferringValidator()
+
+        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+        op_id = uuid.uuid4()
+        payload = json.dumps({"type": "retain", "operation_id": str(op_id), "bank_id": bank_id})
+        await _ensure_bank(pool, bank_id)
+        await pool.execute(
+            """
+            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id, claimed_at, retry_count)
+            VALUES ($1, $2, 'retain', 'processing', $3::jsonb, 'test-worker-1', now(), 0)
+            """,
+            op_id,
+            bank_id,
+            payload,
+        )
+
+        async def executor_calling_validator(task_dict):
+            ctx = RetainContext(
+                bank_id=task_dict["bank_id"],
+                contents=[],
+                request_context=None,  # type: ignore[arg-type]
+            )
+            await validator.validate_retain(ctx)
+
+        poller = WorkerPoller(
+            pool=pool,
+            worker_id="test-worker-1",
+            executor=executor_calling_validator,
+        )
+
+        task_dict = json.loads(payload)
+        claimed_task = ClaimedTask(operation_id=str(op_id), task_dict=task_dict, schema=None)
+        await poller.execute_task(claimed_task)
+
+        completed = await poller.wait_for_active_tasks(timeout=5.0)
+        assert completed, "Task did not complete within timeout"
+
+        row = await pool.fetchrow(
+            "SELECT status, worker_id, claimed_at, retry_count, error_message, next_retry_at "
+            "FROM async_operations WHERE operation_id = $1",
+            op_id,
+        )
+        assert row["status"] == "pending"
+        assert row["worker_id"] is None
+        assert row["claimed_at"] is None
+        assert row["retry_count"] == 0
+        assert row["error_message"] is None
+        assert abs((row["next_retry_at"] - defer_until).total_seconds()) < 1
+
     @pytest.mark.asyncio
     async def test_claim_batch_skips_consolidation_when_same_bank_processing(self, pool, clean_operations):
         """Test that pending consolidation is skipped if same bank has one processing."""
@@ -1505,8 +1747,7 @@ async def test_consolidation_slots_reserved_when_retain_saturates(pool, clean_op
         consolidation_started = [op for op, t in started.items() if t == "consolidation"]
 
         assert len(retain_started) == 3, (
-            f"Retain should be capped at max_slots - consolidation_max_slots = 3, "
-            f"got {len(retain_started)}"
+            f"Retain should be capped at max_slots - consolidation_max_slots = 3, got {len(retain_started)}"
         )
         assert len(consolidation_started) == 1, (
             f"Consolidation should claim its reserved slot even while retain saturates, "
@@ -1529,6 +1770,96 @@ async def test_consolidation_slots_reserved_when_retain_saturates(pool, clean_op
             pass
 
 
+@pytest.mark.asyncio
+async def test_pending_breakdown_explains_unclaimable_rows(pool, clean_operations, caplog):
+    """Pending rows that the claim query filters out must be visible in logs.
+
+    Background: production incident where a 'pending' retain sat in the queue for
+    hours while workers had free slots. With only the global pending count in
+    [WORKER_STATS] there's no way to tell whether the rows are claimable-but-not-
+    being-claimed (real bug) vs filtered out by the claim WHERE clause (data
+    state). This test verifies [PENDING_BREAKDOWN] surfaces each filter bucket
+    so operators can diagnose without DB access.
+    """
+    import logging
+
+    from hindsight_api.worker.poller import WorkerPoller
+
+    poller = WorkerPoller(
+        pool=pool,
+        worker_id="test-worker-pending-breakdown",
+        executor=lambda _t: asyncio.sleep(0),
+        poll_interval_ms=50,
+        max_slots=5,
+    )
+
+    bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+    await _ensure_bank(pool, bank_id)
+
+    # Mix of pending rows that the claim query treats differently:
+    #   * payload_null  - batch_retain parent (orphan candidate)
+    #   * retry_blocked - failed once, scheduled an hour out
+    #   * assigned      - worker_id stamped (e.g. left over from a prior crash
+    #                     that re-queued without clearing worker_id)
+    #   * claimable     - normal retain ready to go
+    #   * consolidation - normal consolidation, also claimable
+    rows = [
+        ("batch_retain", None, None, None),  # payload_null
+        ("retain", json.dumps({"type": "test"}), "future", None),  # retry_blocked
+        ("retain", json.dumps({"type": "test"}), None, "ghost-worker"),  # assigned
+        ("retain", json.dumps({"type": "test"}), None, None),  # claimable
+        ("consolidation", json.dumps({"type": "test"}), None, None),  # claimable
+    ]
+    for op_type, payload, retry_marker, worker_id in rows:
+        op_id = uuid.uuid4()
+        await pool.execute(
+            """
+            INSERT INTO async_operations
+                (operation_id, bank_id, operation_type, status, task_payload,
+                 next_retry_at, worker_id)
+            VALUES ($1, $2, $3, 'pending', $4::jsonb,
+                    CASE WHEN $5::text = 'future' THEN now() + interval '1 hour' ELSE NULL END,
+                    $6)
+            """,
+            op_id,
+            bank_id,
+            op_type,
+            payload,
+            retry_marker,
+            worker_id,
+        )
+
+    # Trigger one stats emit. _last_progress_log starts at 0, so the first call
+    # always logs.
+    with caplog.at_level(logging.INFO, logger="hindsight_api.worker.poller"):
+        await poller._log_progress_if_due()
+
+    breakdown_lines = [r.message for r in caplog.records if r.message.startswith("[PENDING_BREAKDOWN]")]
+    assert len(breakdown_lines) == 1, f"Expected exactly one breakdown line, got: {breakdown_lines}"
+
+    # The breakdown is global (not bank-scoped), so other rows in the table may
+    # contribute. Parse the per-op_type buckets from the line and assert that
+    # our additions appear (>= 1 for each bucket we populated).
+    line = breakdown_lines[0]
+    buckets: dict[str, dict[str, int]] = {}
+    for section in line.removeprefix("[PENDING_BREAKDOWN]").split("|"):
+        section = section.strip()
+        if ":" not in section:
+            continue
+        op_type, fields = section.split(":", 1)
+        kv = {}
+        for token in fields.strip().split():
+            k, _, v = token.partition("=")
+            kv[k] = int(v)
+        buckets[op_type.strip()] = kv
+
+    assert buckets["batch_retain"]["payload_null"] >= 1
+    assert buckets["retain"]["retry_blocked"] >= 1
+    assert buckets["retain"]["assigned"] >= 1
+    assert buckets["retain"]["claimable"] >= 1
+    assert buckets["consolidation"]["claimable"] >= 1
+
+
 class TestMarkFailedParentPropagation:
     """Tests for _mark_failed parent propagation in WorkerPoller.
 
@@ -1747,3 +2078,413 @@ class TestMarkFailedParentPropagation:
             f"Parent batch_retain should be 'failed' after child fails via unhandled exception, "
             f"got '{parent_row['status']}'"
         )
+
+
+class TestClaimBatchRotation:
+    """Tests for round-robin schema rotation in claim_batch.
+
+    These use a mocked _claim_batch_for_schema so the tests are hermetic
+    and exercise rotation logic without needing multiple real tenant schemas.
+    """
+
+    def _make_poller_with_fake_work(self, pool, pending_per_schema, max_slots=1):
+        """Build a poller whose schemas and per-schema claims are scripted.
+
+        ``pending_per_schema`` maps schema name -> current pending count.
+        The fake claim handler decrements the count and returns a ClaimedTask
+        if the schema still has work, else returns an empty list.
+        """
+        from hindsight_api.extensions.tenant import Tenant, TenantExtension
+        from hindsight_api.worker import WorkerPoller
+        from hindsight_api.worker.poller import ClaimedTask
+
+        schemas = list(pending_per_schema.keys())
+
+        class StaticTenantExtension(TenantExtension):
+            def __init__(self):
+                super().__init__(config={})
+
+            async def authenticate(self, context):
+                raise NotImplementedError
+
+            async def list_tenants(self) -> list[Tenant]:
+                return [Tenant(schema=s) for s in schemas]
+
+        poller = WorkerPoller(
+            pool=pool,
+            worker_id="test-rotation",
+            executor=lambda x: None,
+            tenant_extension=StaticTenantExtension(),
+            max_slots=max_slots,
+            # No consolidation reservation — all slots available for non-consolidation
+            # test tasks. Keeps the fair-rotation behavior easy to assert.
+            consolidation_max_slots=0,
+        )
+
+        serviced: list[str] = []
+
+        async def fake_claim(schema, non_consolidation_limit, consolidation_limit):
+            # Tests only exercise non-consolidation ("test") tasks, so we only
+            # consult the non-consolidation limit.
+            remaining = pending_per_schema.get(schema, 0)
+            if remaining <= 0 or non_consolidation_limit <= 0:
+                return []
+            take = min(remaining, non_consolidation_limit)
+            pending_per_schema[schema] = remaining - take
+            out = []
+            for _ in range(take):
+                serviced.append(schema)
+                out.append(
+                    ClaimedTask(
+                        operation_id=str(uuid.uuid4()),
+                        task_dict={"operation_type": "test", "bank_id": schema or "default"},
+                        schema=schema,
+                    )
+                )
+            return out
+
+        poller._claim_batch_for_schema = fake_claim  # type: ignore[method-assign]
+        return poller, serviced
+
+    @pytest.mark.asyncio
+    async def test_rotation_advances_past_serviced_schema(self, pool):
+        """After claiming from schema at offset N, next poll starts at N+1.
+
+        This is the 'crucial detail' that separates working rotation from
+        broken rotation: advancing +1 from the previous offset would cause
+        the first schema with work to always win.
+        """
+        # Only schema "b" has work; "a" and "c" are idle.
+        pending = {"a": 0, "b": 5, "c": 0}
+        poller, serviced = self._make_poller_with_fake_work(pool, pending, max_slots=1)
+
+        await poller.claim_batch()
+        # Found work at index 1 ("b"), so next offset should be 2 ("c").
+        assert poller._next_schema_idx == 2
+        assert serviced == ["b"]
+
+    @pytest.mark.asyncio
+    async def test_rotation_advances_by_one_when_no_work(self, pool):
+        """Empty sweep advances offset by 1 so we don't keep re-hitting the same head."""
+        pending = {"a": 0, "b": 0, "c": 0}
+        poller, serviced = self._make_poller_with_fake_work(pool, pending, max_slots=1)
+
+        poller._next_schema_idx = 0
+        await poller.claim_batch()
+        assert poller._next_schema_idx == 1
+        assert serviced == []
+
+        await poller.claim_batch()
+        assert poller._next_schema_idx == 2
+        assert serviced == []
+
+    @pytest.mark.asyncio
+    async def test_small_tenant_not_starved_by_busy_tenant(self, pool):
+        """Small tenant with 1 pending task gets serviced within bounded polls
+        even when another tenant has a huge backlog. Prevents the regression
+        observed in prod where one tenant's 1000+ retains monopolized workers.
+        """
+        pending = {"friday-main": 1000, "tenant-b": 1}
+        poller, serviced = self._make_poller_with_fake_work(pool, pending, max_slots=1)
+
+        # MAX_SLOTS=1 means one claim per poll. Over ~2 polls the rotation
+        # must reach tenant-b, regardless of which started first.
+        for _ in range(5):
+            await poller.claim_batch()
+            if "tenant-b" in serviced:
+                break
+
+        assert "tenant-b" in serviced, f"tenant-b was starved; serviced={serviced[:20]}"
+
+    @pytest.mark.asyncio
+    async def test_max_slots_greater_than_one_spreads_across_tenants(self, pool):
+        """With MAX_SLOTS>1 the first pass caps at 1 claim per schema so
+        a single poll services multiple tenants rather than draining one.
+        """
+        pending = {"a": 10, "b": 10, "c": 10, "d": 10, "e": 10}
+        poller, serviced = self._make_poller_with_fake_work(pool, pending, max_slots=3)
+
+        await poller.claim_batch()
+        # First pass gives 1 claim each to 3 different schemas — not 3 from the same one.
+        assert len(serviced) == 3
+        assert len(set(serviced)) == 3, f"Expected 3 different tenants, got {serviced}"
+
+    @pytest.mark.asyncio
+    async def test_max_slots_greater_than_one_backfills_when_only_one_tenant_has_work(self, pool):
+        """Second pass fills remaining slots when only one tenant has work,
+        so fairness doesn't sacrifice throughput in the single-tenant case.
+        """
+        pending = {"a": 0, "b": 10, "c": 0}
+        poller, serviced = self._make_poller_with_fake_work(pool, pending, max_slots=3)
+
+        await poller.claim_batch()
+        # Pass 1: 1 from "b" (only one with work). Pass 2: 2 more from "b".
+        assert serviced == ["b", "b", "b"]
+
+
+class TestDecommissionAllWorkers:
+    """Tests for decommission-workers (all workers) functionality."""
+
+    @pytest.mark.asyncio
+    async def test_decommission_all_releases_all_processing_tasks(self, pool, clean_operations):
+        """Test that decommissioning all workers releases tasks from every worker."""
+        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(pool, bank_id)
+
+        # Create tasks for multiple workers
+        for worker in ["worker-a", "worker-b", "worker-c"]:
+            for i in range(2):
+                op_id = uuid.uuid4()
+                payload = json.dumps({"type": "test_task", "index": i, "bank_id": bank_id})
+                await pool.execute(
+                    """
+                    INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id, claimed_at)
+                    VALUES ($1, $2, 'test', 'processing', $3::jsonb, $4, now())
+                    """,
+                    op_id,
+                    bank_id,
+                    payload,
+                    worker,
+                )
+
+        # Decommission all
+        result = await pool.fetch(
+            """
+            UPDATE async_operations
+            SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
+            WHERE status = 'processing' AND bank_id = $1
+            RETURNING operation_id, worker_id, operation_type
+            """,
+            bank_id,
+        )
+
+        assert len(result) == 6
+
+        # All should be pending now
+        rows = await pool.fetch(
+            "SELECT status, worker_id, claimed_at FROM async_operations WHERE bank_id = $1",
+            bank_id,
+        )
+        for row in rows:
+            assert row["status"] == "pending"
+            assert row["worker_id"] is None
+            assert row["claimed_at"] is None
+
+    @pytest.mark.asyncio
+    async def test_decommission_all_does_not_affect_pending_or_completed(self, pool, clean_operations):
+        """Test that decommissioning all workers only touches 'processing' tasks."""
+        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(pool, bank_id)
+
+        # Create a pending task
+        pending_id = uuid.uuid4()
+        await pool.execute(
+            """
+            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
+            VALUES ($1, $2, 'test', 'pending', '{"type":"test","bank_id":"x"}'::jsonb)
+            """,
+            pending_id,
+            bank_id,
+        )
+
+        # Create a completed task
+        completed_id = uuid.uuid4()
+        await pool.execute(
+            """
+            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, completed_at)
+            VALUES ($1, $2, 'test', 'completed', '{"type":"test","bank_id":"x"}'::jsonb, now())
+            """,
+            completed_id,
+            bank_id,
+        )
+
+        # Create a processing task
+        processing_id = uuid.uuid4()
+        await pool.execute(
+            """
+            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id, claimed_at)
+            VALUES ($1, $2, 'test', 'processing', '{"type":"test","bank_id":"x"}'::jsonb, 'dead-worker', now())
+            """,
+            processing_id,
+            bank_id,
+        )
+
+        # Decommission all
+        result = await pool.fetch(
+            """
+            UPDATE async_operations
+            SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
+            WHERE status = 'processing' AND bank_id = $1
+            RETURNING operation_id
+            """,
+            bank_id,
+        )
+
+        assert len(result) == 1
+        assert result[0]["operation_id"] == processing_id
+
+        # Pending task unchanged
+        pending_row = await pool.fetchrow(
+            "SELECT status FROM async_operations WHERE operation_id = $1", pending_id
+        )
+        assert pending_row["status"] == "pending"
+
+        # Completed task unchanged
+        completed_row = await pool.fetchrow(
+            "SELECT status FROM async_operations WHERE operation_id = $1", completed_id
+        )
+        assert completed_row["status"] == "completed"
+
+    @pytest.mark.asyncio
+    async def test_decommission_all_returns_empty_when_no_processing(self, pool, clean_operations):
+        """Test decommissioning when there are no processing tasks."""
+        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(pool, bank_id)
+
+        # Only pending tasks
+        for i in range(3):
+            op_id = uuid.uuid4()
+            payload = json.dumps({"type": "test_task", "index": i, "bank_id": bank_id})
+            await pool.execute(
+                """
+                INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
+                VALUES ($1, $2, 'test', 'pending', $3::jsonb)
+                """,
+                op_id,
+                bank_id,
+                payload,
+            )
+
+        result = await pool.fetch(
+            """
+            UPDATE async_operations
+            SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
+            WHERE status = 'processing' AND bank_id = $1
+            RETURNING operation_id
+            """,
+            bank_id,
+        )
+
+        assert len(result) == 0
+
+
+class TestWorkerStatus:
+    """Tests for worker-status functionality."""
+
+    @pytest.mark.asyncio
+    async def test_worker_status_shows_processing_tasks(self, pool, clean_operations):
+        """Test that worker status returns all processing tasks with their details."""
+        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(pool, bank_id)
+
+        # Create processing tasks for two workers
+        for worker, op_type in [("worker-x", "retain"), ("worker-x", "consolidation"), ("worker-y", "retain")]:
+            op_id = uuid.uuid4()
+            payload = json.dumps({"type": "test_task", "bank_id": bank_id})
+            await pool.execute(
+                """
+                INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id, claimed_at)
+                VALUES ($1, $2, $3, 'processing', $4::jsonb, $5, now())
+                """,
+                op_id,
+                bank_id,
+                op_type,
+                payload,
+                worker,
+            )
+
+        rows = await pool.fetch(
+            """
+            SELECT worker_id, operation_id, operation_type, bank_id,
+                   claimed_at, updated_at,
+                   now() - claimed_at AS running_for,
+                   now() - updated_at AS last_update_ago
+            FROM async_operations
+            WHERE status = 'processing' AND bank_id = $1
+            ORDER BY worker_id, claimed_at
+            """,
+            bank_id,
+        )
+
+        assert len(rows) == 3
+
+        # Verify all expected columns are present
+        for row in rows:
+            assert row["worker_id"] in ("worker-x", "worker-y")
+            assert row["operation_type"] in ("retain", "consolidation")
+            assert row["bank_id"] == bank_id
+            assert row["claimed_at"] is not None
+            assert row["updated_at"] is not None
+            assert row["running_for"] is not None
+            assert row["last_update_ago"] is not None
+
+        # Verify grouping: worker-x has 2, worker-y has 1
+        worker_x_rows = [r for r in rows if r["worker_id"] == "worker-x"]
+        worker_y_rows = [r for r in rows if r["worker_id"] == "worker-y"]
+        assert len(worker_x_rows) == 2
+        assert len(worker_y_rows) == 1
+
+    @pytest.mark.asyncio
+    async def test_worker_status_excludes_non_processing(self, pool, clean_operations):
+        """Test that worker status only shows processing tasks, not pending/completed/failed."""
+        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(pool, bank_id)
+
+        # Create tasks in various statuses
+        for status in ["pending", "processing", "completed", "failed"]:
+            op_id = uuid.uuid4()
+            payload = json.dumps({"type": "test_task", "bank_id": bank_id})
+            worker = "status-worker" if status == "processing" else None
+            claimed = "now()" if status == "processing" else "NULL"
+            await pool.execute(
+                f"""
+                INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id, claimed_at)
+                VALUES ($1, $2, 'test', $3, $4::jsonb, $5, {claimed})
+                """,
+                op_id,
+                bank_id,
+                status,
+                payload,
+                worker,
+            )
+
+        rows = await pool.fetch(
+            """
+            SELECT worker_id, operation_type, bank_id
+            FROM async_operations
+            WHERE status = 'processing' AND bank_id = $1
+            """,
+            bank_id,
+        )
+
+        assert len(rows) == 1
+        assert rows[0]["worker_id"] == "status-worker"
+
+    @pytest.mark.asyncio
+    async def test_worker_status_empty_when_no_processing(self, pool, clean_operations):
+        """Test that worker status returns empty when no tasks are processing."""
+        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+        await _ensure_bank(pool, bank_id)
+
+        # Only pending tasks
+        op_id = uuid.uuid4()
+        payload = json.dumps({"type": "test_task", "bank_id": bank_id})
+        await pool.execute(
+            """
+            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
+            VALUES ($1, $2, 'test', 'pending', $3::jsonb)
+            """,
+            op_id,
+            bank_id,
+            payload,
+        )
+
+        rows = await pool.fetch(
+            """
+            SELECT worker_id FROM async_operations
+            WHERE status = 'processing' AND bank_id = $1
+            """,
+            bank_id,
+        )
+
+        assert len(rows) == 0

hindsight-api/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hindsight-api"
-version = "0.5.1"
+version = "0.5.3"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"

hindsight-cli/.openapi-coverage.toml

@@ -21,7 +21,13 @@
 # Operation-level skips
 # ---------------------------------------------------------------------------
 [skip]
-# (empty — every operation is currently wired)
+# UI-only endpoint powering the control-plane stats chart.
+# Zero-filled bucket arrays don't map to a useful CLI command.
+get_memories_timeseries = "UI-only endpoint for the control plane stats chart"
+
+# UI-only endpoint powering the control-plane entity constellation view.
+# Returns nodes/edges in cytoscape shape; not a useful CLI command.
+get_entity_graph = "UI-only endpoint for the control plane entity constellation"
 
 # ---------------------------------------------------------------------------
 # Per-operation parameter skips

hindsight-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hindsight-cli"
-version = "0.5.1"
+version = "0.5.3"
 edition = "2021"
 authors = ["Hindsight Team"]
 description = "A beautiful CLI for Hindsight - semantic memory system"

hindsight-cli/src/api.rs

@@ -513,7 +513,7 @@ impl ApiClient {
         self.runtime.block_on(async {
             let response = self
                 .client
-                .list_memories(bank_id, limit, offset, q, type_filter, None)
+                .list_memories(bank_id, None, limit, offset, q, type_filter, None)
                 .await?;
             Ok(response.into_inner())
         })
@@ -735,7 +735,7 @@ impl ApiClient {
         self.runtime.block_on(async {
             let response = self
                 .client
-                .get_operation_status(bank_id, operation_id, None)
+                .get_operation_status(bank_id, operation_id, None, None)
                 .await?;
             Ok(response.into_inner())
         })

hindsight-cli/src/commands/mental_model.rs

@@ -118,12 +118,16 @@ pub fn create(
     // behaviour is preserved otherwise.
     let trigger = if trigger_refresh_after_consolidation {
         Some(types::MentalModelTriggerInput {
+            mode: types::Mode::Full,
             refresh_after_consolidation: true,
             exclude_mental_models: false,
             exclude_mental_model_ids: None,
             fact_types: None,
             tag_groups: None,
             tags_match: None,
+            include_chunks: None,
+            recall_max_tokens: None,
+            recall_chunks_max_tokens: None,
         })
     } else {
         None
@@ -192,12 +196,16 @@ pub fn update(
     // Only build a trigger override when the user actually passed the flag;
     // sending None leaves the existing trigger config untouched on the server.
     let trigger = trigger_refresh_after_consolidation.map(|refresh| types::MentalModelTriggerInput {
+        mode: types::Mode::Full,
         refresh_after_consolidation: refresh,
         exclude_mental_models: false,
         exclude_mental_model_ids: None,
         fact_types: None,
         tag_groups: None,
         tags_match: None,
+        include_chunks: None,
+        recall_max_tokens: None,
+        recall_chunks_max_tokens: None,
     });
 
     let request = types::UpdateMentalModelRequest {

hindsight-cli/src/config.rs

@@ -7,6 +7,8 @@ use std::path::PathBuf;
 const DEFAULT_API_URL: &str = "http://localhost:8888";
 const CONFIG_FILE_NAME: &str = "config";
 const CONFIG_DIR_NAME: &str = ".hindsight";
+const PROFILE_DIR_NAME: &str = "cli-profiles";
+const PROFILE_ENV_VAR: &str = "HINDSIGHT_PROFILE";
 
 #[derive(Debug)]
 pub struct Config {
@@ -18,6 +20,7 @@ pub struct Config {
 #[derive(Debug, Clone, PartialEq)]
 pub enum ConfigSource {
     LocalFile,
+    Profile(String),
     Environment,
     Default,
 }
@@ -26,6 +29,7 @@ impl std::fmt::Display for ConfigSource {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             ConfigSource::LocalFile => write!(f, "config file"),
+            ConfigSource::Profile(name) => write!(f, "profile '{}'", name),
             ConfigSource::Environment => write!(f, "environment variable"),
             ConfigSource::Default => write!(f, "default"),
         }
@@ -33,27 +37,43 @@ impl std::fmt::Display for ConfigSource {
 }
 
 impl Config {
-    /// Load configuration with the following priority:
-    /// 1. Environment variable (HINDSIGHT_API_URL, HINDSIGHT_API_KEY) - highest priority, for overrides
-    /// 2. Local config file (~/.hindsight/config.toml)
-    /// 3. Default (http://localhost:8888)
+    /// Load configuration with no explicit profile. See [`Self::load_with_profile`].
     pub fn load() -> Result<Self> {
-        // Load API key from environment (highest priority)
+        Self::load_with_profile(None)
+    }
+
+    /// Load configuration with the following priority:
+    /// 1. Environment variable (HINDSIGHT_API_URL/HINDSIGHT_API_KEY) - highest priority
+    /// 2. Named profile (from `profile_name` arg, else `$HINDSIGHT_PROFILE`)
+    ///    at `~/.hindsight/cli-profiles/<name>.toml`
+    /// 3. Local config file (`~/.hindsight/config`)
+    /// 4. Default (http://localhost:8888)
+    pub fn load_with_profile(profile_name: Option<&str>) -> Result<Self> {
         let env_api_key = env::var("HINDSIGHT_API_KEY").ok();
 
-        // 1. Environment variable takes highest priority (for overrides)
+        // 1. Environment variable takes highest priority
         if let Ok(api_url) = env::var("HINDSIGHT_API_URL") {
             return Self::validate_and_create(api_url, env_api_key, ConfigSource::Environment);
         }
 
-        // 2. Try local config file
+        // 2. Named profile (explicit flag takes precedence over env var)
+        let resolved_profile: Option<String> = profile_name
+            .map(|s| s.to_string())
+            .or_else(|| env::var(PROFILE_ENV_VAR).ok().filter(|s| !s.is_empty()));
+
+        if let Some(name) = resolved_profile {
+            let (api_url, file_api_key) = Self::load_profile(&name)?;
+            let api_key = env_api_key.or(file_api_key);
+            return Self::validate_and_create(api_url, api_key, ConfigSource::Profile(name));
+        }
+
+        // 3. Local config file
         if let Some((api_url, file_api_key)) = Self::load_from_file()? {
-            // Environment api_key takes precedence over file api_key
             let api_key = env_api_key.or(file_api_key);
             return Self::validate_and_create(api_url, api_key, ConfigSource::LocalFile);
         }
 
-        // 3. Fall back to default
+        // 4. Fall back to default
         Self::validate_and_create(DEFAULT_API_URL.to_string(), env_api_key, ConfigSource::Default)
     }
 
@@ -151,6 +171,173 @@ impl Config {
     pub fn api_url(&self) -> &str {
         &self.api_url
     }
+
+    // ---------- profile support ----------
+
+    pub fn profile_dir() -> Option<PathBuf> {
+        Self::config_dir().map(|dir| dir.join(PROFILE_DIR_NAME))
+    }
+
+    pub fn profile_file_path(name: &str) -> Option<PathBuf> {
+        Self::profile_dir().map(|dir| dir.join(format!("{}.toml", name)))
+    }
+
+    /// Load a named profile. Returns (api_url, api_key) or an error if the profile
+    /// file is missing / malformed.
+    pub fn load_profile(name: &str) -> Result<(String, Option<String>)> {
+        validate_profile_name(name)?;
+        let dir = Self::profile_dir()
+            .ok_or_else(|| anyhow::anyhow!("Could not determine home directory"))?;
+        load_profile_from_dir(&dir, name)
+    }
+
+    /// Save a named profile to `~/.hindsight/cli-profiles/<name>.toml`.
+    /// Sets file permissions to 0600 on Unix to protect the API key.
+    pub fn save_profile(name: &str, api_url: &str, api_key: Option<&str>) -> Result<PathBuf> {
+        validate_profile_name(name)?;
+        let dir = Self::profile_dir()
+            .ok_or_else(|| anyhow::anyhow!("Could not determine home directory"))?;
+        save_profile_to_dir(&dir, name, api_url, api_key)
+    }
+
+    /// List profile names (without `.toml` extension), sorted alphabetically.
+    pub fn list_profiles() -> Result<Vec<String>> {
+        let dir = match Self::profile_dir() {
+            Some(d) => d,
+            None => return Ok(vec![]),
+        };
+        list_profiles_in_dir(&dir)
+    }
+
+    /// Delete a named profile. Returns Ok(path) on success. Errors if the profile
+    /// does not exist.
+    pub fn delete_profile(name: &str) -> Result<PathBuf> {
+        validate_profile_name(name)?;
+        let path = Self::profile_file_path(name)
+            .ok_or_else(|| anyhow::anyhow!("Could not determine home directory"))?;
+        if !path.exists() {
+            anyhow::bail!("profile '{}' not found at {}", name, path.display());
+        }
+        fs::remove_file(&path)
+            .with_context(|| format!("Failed to delete profile file: {}", path.display()))?;
+        Ok(path)
+    }
+}
+
+fn load_profile_from_dir(dir: &std::path::Path, name: &str) -> Result<(String, Option<String>)> {
+    let path = dir.join(format!("{}.toml", name));
+    if !path.exists() {
+        anyhow::bail!(
+            "profile '{}' not found at {}; create with: hindsight profile create {} --api-url <url>",
+            name,
+            path.display(),
+            name
+        );
+    }
+
+    let content = fs::read_to_string(&path)
+        .with_context(|| format!("Failed to read profile file: {}", path.display()))?;
+
+    let mut api_url: Option<String> = None;
+    let mut api_key: Option<String> = None;
+
+    for line in content.lines() {
+        if let Some(v) = parse_config_value(line, "api_url") {
+            api_url = Some(v);
+        } else if let Some(v) = parse_config_value(line, "api_key") {
+            api_key = Some(v);
+        }
+    }
+
+    let api_url = api_url.ok_or_else(|| {
+        anyhow::anyhow!(
+            "profile '{}' at {} is missing required 'api_url' field",
+            name,
+            path.display()
+        )
+    })?;
+
+    Ok((api_url, api_key))
+}
+
+fn save_profile_to_dir(
+    dir: &std::path::Path,
+    name: &str,
+    api_url: &str,
+    api_key: Option<&str>,
+) -> Result<PathBuf> {
+    if !api_url.starts_with("http://") && !api_url.starts_with("https://") {
+        anyhow::bail!(
+            "Invalid API URL: {}. Must start with http:// or https://",
+            api_url
+        );
+    }
+
+    if !dir.exists() {
+        fs::create_dir_all(dir)
+            .with_context(|| format!("Failed to create profile directory: {}", dir.display()))?;
+    }
+
+    let path = dir.join(format!("{}.toml", name));
+    let mut content = format!("api_url = \"{}\"\n", api_url);
+    if let Some(key) = api_key {
+        content.push_str(&format!("api_key = \"{}\"\n", key));
+    }
+
+    fs::write(&path, content)
+        .with_context(|| format!("Failed to write profile file: {}", path.display()))?;
+
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        let perms = fs::Permissions::from_mode(0o600);
+        fs::set_permissions(&path, perms).with_context(|| {
+            format!("Failed to set permissions on profile file: {}", path.display())
+        })?;
+    }
+
+    Ok(path)
+}
+
+fn list_profiles_in_dir(dir: &std::path::Path) -> Result<Vec<String>> {
+    if !dir.exists() {
+        return Ok(vec![]);
+    }
+    let mut names: Vec<String> = fs::read_dir(dir)
+        .with_context(|| format!("Failed to read profile directory: {}", dir.display()))?
+        .filter_map(|entry| entry.ok())
+        .filter_map(|entry| {
+            let path = entry.path();
+            if path.extension().and_then(|s| s.to_str()) != Some("toml") {
+                return None;
+            }
+            path.file_stem()
+                .and_then(|s| s.to_str())
+                .map(|s| s.to_string())
+        })
+        .collect();
+    names.sort();
+    Ok(names)
+}
+
+/// Reject empty, path-like, or hidden profile names so they can't escape the
+/// profile directory.
+fn validate_profile_name(name: &str) -> Result<()> {
+    if name.is_empty() {
+        anyhow::bail!("profile name cannot be empty");
+    }
+    if name.starts_with('.')
+        || name.contains('/')
+        || name.contains('\\')
+        || name.contains("..")
+        || name.contains(char::is_whitespace)
+    {
+        anyhow::bail!(
+            "invalid profile name '{}': must not contain path separators, whitespace, or start with '.'",
+            name
+        );
+    }
+    Ok(())
 }
 
 /// Prompt user for API URL interactively
@@ -197,6 +384,104 @@ mod tests {
         assert_eq!(format!("{}", ConfigSource::LocalFile), "config file");
         assert_eq!(format!("{}", ConfigSource::Environment), "environment variable");
         assert_eq!(format!("{}", ConfigSource::Default), "default");
+        assert_eq!(
+            format!("{}", ConfigSource::Profile("prod".to_string())),
+            "profile 'prod'"
+        );
+    }
+
+    fn tempdir() -> PathBuf {
+        let pid = std::process::id();
+        let nanos = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_nanos();
+        let counter = std::sync::atomic::AtomicU64::new(0);
+        let n = counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        let dir = std::env::temp_dir().join(format!("hindsight-cli-test-{}-{}-{}", pid, nanos, n));
+        std::fs::create_dir_all(&dir).unwrap();
+        dir
+    }
+
+    #[test]
+    fn test_validate_profile_name_ok() {
+        assert!(validate_profile_name("prod").is_ok());
+        assert!(validate_profile_name("staging-1").is_ok());
+        assert!(validate_profile_name("openclaw-plugin").is_ok());
+        assert!(validate_profile_name("a_b_c").is_ok());
+    }
+
+    #[test]
+    fn test_validate_profile_name_rejects_unsafe() {
+        assert!(validate_profile_name("").is_err());
+        assert!(validate_profile_name(".hidden").is_err());
+        assert!(validate_profile_name("a/b").is_err());
+        assert!(validate_profile_name("a\\b").is_err());
+        assert!(validate_profile_name("..").is_err());
+        assert!(validate_profile_name("foo bar").is_err());
+    }
+
+    #[test]
+    fn test_save_and_load_profile_roundtrip() {
+        let dir = tempdir();
+        let path = save_profile_to_dir(&dir, "prod", "https://api.example.com", Some("hsk_abc"))
+            .unwrap();
+        assert!(path.exists());
+
+        let (url, key) = load_profile_from_dir(&dir, "prod").unwrap();
+        assert_eq!(url, "https://api.example.com");
+        assert_eq!(key.as_deref(), Some("hsk_abc"));
+
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::PermissionsExt;
+            let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777;
+            assert_eq!(mode, 0o600);
+        }
+    }
+
+    #[test]
+    fn test_save_profile_rejects_invalid_url() {
+        let dir = tempdir();
+        let err = save_profile_to_dir(&dir, "foo", "localhost:8888", None).unwrap_err();
+        assert!(err.to_string().contains("Invalid API URL"));
+    }
+
+    #[test]
+    fn test_load_profile_missing_returns_helpful_error() {
+        let dir = tempdir();
+        let err = load_profile_from_dir(&dir, "nope").unwrap_err().to_string();
+        assert!(err.contains("profile 'nope' not found"));
+        assert!(err.contains("hindsight profile create nope"));
+    }
+
+    #[test]
+    fn test_load_profile_missing_api_url_fails() {
+        let dir = tempdir();
+        let path = dir.join("broken.toml");
+        std::fs::write(&path, "api_key = \"x\"\n").unwrap();
+        let err = load_profile_from_dir(&dir, "broken").unwrap_err().to_string();
+        assert!(err.contains("missing required 'api_url'"));
+    }
+
+    #[test]
+    fn test_list_profiles_returns_sorted_names() {
+        let dir = tempdir();
+        save_profile_to_dir(&dir, "prod", "https://prod.example.com", None).unwrap();
+        save_profile_to_dir(&dir, "dev", "https://dev.example.com", None).unwrap();
+        save_profile_to_dir(&dir, "staging", "https://staging.example.com", None).unwrap();
+        // Non-toml files should be ignored.
+        std::fs::write(dir.join("README"), "hi").unwrap();
+
+        let names = list_profiles_in_dir(&dir).unwrap();
+        assert_eq!(names, vec!["dev", "prod", "staging"]);
+    }
+
+    #[test]
+    fn test_list_profiles_missing_dir_is_empty() {
+        let dir = tempdir().join("nonexistent");
+        let names = list_profiles_in_dir(&dir).unwrap();
+        assert!(names.is_empty());
     }
 
     #[test]

hindsight-cli/src/main.rs

@@ -45,6 +45,12 @@ struct Cli {
     #[arg(short = 'v', long, global = true)]
     verbose: bool,
 
+    /// Named profile to load from ~/.hindsight/cli-profiles/<name>.toml
+    /// (env var HINDSIGHT_PROFILE is used if this flag is omitted).
+    /// Environment variables (HINDSIGHT_API_URL / HINDSIGHT_API_KEY) still override profile values.
+    #[arg(short = 'p', long, global = true, env = "HINDSIGHT_PROFILE")]
+    profile: Option<String>,
+
     #[command(subcommand)]
     command: Commands,
 }
@@ -129,7 +135,7 @@ enum Commands {
 
     /// Configure the CLI (API URL, API key, etc.)
     #[command(
-        after_help = "Configuration priority:\n  1. Environment variables (HINDSIGHT_API_URL, HINDSIGHT_API_KEY) - highest priority\n  2. Config file (~/.hindsight/config)\n  3. Default (http://localhost:8888)"
+        after_help = "Configuration priority:\n  1. Environment variables (HINDSIGHT_API_URL, HINDSIGHT_API_KEY) - highest priority\n  2. Named profile (-p / HINDSIGHT_PROFILE, see 'hindsight profile')\n  3. Config file (~/.hindsight/config)\n  4. Default (http://localhost:8888)"
     )]
     Configure {
         /// API URL to connect to (interactive prompt if not provided)
@@ -139,6 +145,40 @@ enum Commands {
         #[arg(long)]
         api_key: Option<String>,
     },
+
+    /// Manage named connection profiles (~/.hindsight/cli-profiles/<name>.toml)
+    #[command(subcommand)]
+    Profile(ProfileCommands),
+}
+
+#[derive(Subcommand)]
+enum ProfileCommands {
+    /// Create or overwrite a profile
+    Create {
+        /// Profile name (used with -p/--profile or $HINDSIGHT_PROFILE)
+        name: String,
+        /// API URL (required)
+        #[arg(long)]
+        api_url: String,
+        /// API key (optional; stored in profile file with 0600 permissions)
+        #[arg(long)]
+        api_key: Option<String>,
+    },
+    /// List all known profiles
+    List,
+    /// Show the contents of a profile
+    Show {
+        /// Profile name
+        name: String,
+    },
+    /// Delete a profile
+    Delete {
+        /// Profile name
+        name: String,
+        /// Skip confirmation prompt
+        #[arg(short = 'y', long)]
+        yes: bool,
+    },
 }
 
 #[derive(Subcommand)]
@@ -1091,7 +1131,8 @@ enum DirectiveCommands {
 }
 
 fn main() {
-    if let Err(_) = run() {
+    if let Err(e) = run() {
+        ui::print_error(&format!("{:#}", e));
         std::process::exit(1);
     }
 }
@@ -1101,19 +1142,25 @@ fn run() -> Result<()> {
 
     let output_format: OutputFormat = cli.output.into();
     let verbose = cli.verbose;
+    let profile = cli.profile.clone();
 
     // Handle configure command before loading full config (it doesn't need API client)
     if let Commands::Configure { api_url, api_key } = cli.command {
         return handle_configure(api_url, api_key, output_format);
     }
 
+    // Handle profile management commands — no API client required.
+    if let Commands::Profile(cmd) = cli.command {
+        return handle_profile(cmd, output_format);
+    }
+
     // Handle ui command - needs config but not API client
     if let Commands::Ui = cli.command {
-        return handle_ui(output_format);
+        return handle_ui(profile.as_deref(), output_format);
     }
 
     // Load configuration
-    let config = Config::from_env().unwrap_or_else(|e| {
+    let config = Config::load_with_profile(profile.as_deref()).unwrap_or_else(|e| {
         ui::print_error(&format!("Configuration error: {}", e));
         errors::print_config_help();
         std::process::exit(1);
@@ -1130,6 +1177,7 @@ fn run() -> Result<()> {
     // Execute command and handle errors
     let result: Result<()> = match cli.command {
         Commands::Configure { .. } => unreachable!(), // Handled above
+        Commands::Profile(_) => unreachable!(),       // Handled above
         Commands::Ui => unreachable!(),               // Handled above
         Commands::Explore => commands::explore::run(&client),
 
@@ -1875,11 +1923,11 @@ fn handle_configure(
     Ok(())
 }
 
-fn handle_ui(output_format: OutputFormat) -> Result<()> {
+fn handle_ui(profile: Option<&str>, output_format: OutputFormat) -> Result<()> {
     use std::process::Command;
 
     // Load configuration to get the API URL
-    let config = Config::load().unwrap_or_else(|e| {
+    let config = Config::load_with_profile(profile).unwrap_or_else(|e| {
         ui::print_error(&format!("Configuration error: {}", e));
         errors::print_config_help();
         std::process::exit(1);
@@ -1921,3 +1969,118 @@ fn handle_ui(output_format: OutputFormat) -> Result<()> {
 
     Ok(())
 }
+
+fn mask_api_key(key: &str) -> String {
+    if key.len() > 8 {
+        format!("{}...{}", &key[..4], &key[key.len() - 4..])
+    } else {
+        "****".to_string()
+    }
+}
+
+fn handle_profile(cmd: ProfileCommands, output_format: OutputFormat) -> Result<()> {
+    match cmd {
+        ProfileCommands::Create {
+            name,
+            api_url,
+            api_key,
+        } => {
+            let path = Config::save_profile(&name, &api_url, api_key.as_deref())?;
+            if output_format == OutputFormat::Pretty {
+                ui::print_success(&format!("Profile '{}' saved to {}", name, path.display()));
+                println!();
+                println!("  API URL: {}", api_url);
+                if let Some(ref key) = api_key {
+                    println!("  API Key: {}", mask_api_key(key));
+                }
+                println!();
+                println!(
+                    "Use with: hindsight -p {} <command>  (or export HINDSIGHT_PROFILE={})",
+                    name, name
+                );
+            } else {
+                let result = serde_json::json!({
+                    "name": name,
+                    "api_url": api_url,
+                    "api_key_set": api_key.is_some(),
+                    "path": path.display().to_string(),
+                });
+                output::print_output(&result, output_format)?;
+            }
+            Ok(())
+        }
+        ProfileCommands::List => {
+            let names = Config::list_profiles()?;
+            if output_format == OutputFormat::Pretty {
+                if names.is_empty() {
+                    ui::print_info("No profiles found.");
+                    println!();
+                    println!("Create one with: hindsight profile create <name> --api-url <url>");
+                } else {
+                    ui::print_info("Profiles:");
+                    for name in &names {
+                        println!("  • {}", name);
+                    }
+                }
+            } else {
+                output::print_output(&serde_json::json!({ "profiles": names }), output_format)?;
+            }
+            Ok(())
+        }
+        ProfileCommands::Show { name } => {
+            let (api_url, api_key) = Config::load_profile(&name)?;
+            let path = Config::profile_file_path(&name)
+                .map(|p| p.display().to_string())
+                .unwrap_or_default();
+            if output_format == OutputFormat::Pretty {
+                ui::print_info(&format!("Profile '{}'", name));
+                println!();
+                println!("  Path:    {}", path);
+                println!("  API URL: {}", api_url);
+                if let Some(ref key) = api_key {
+                    println!("  API Key: {}", mask_api_key(key));
+                }
+            } else {
+                let result = serde_json::json!({
+                    "name": name,
+                    "path": path,
+                    "api_url": api_url,
+                    "api_key_set": api_key.is_some(),
+                });
+                output::print_output(&result, output_format)?;
+            }
+            Ok(())
+        }
+        ProfileCommands::Delete { name, yes } => {
+            let path = Config::profile_file_path(&name)
+                .ok_or_else(|| anyhow::anyhow!("Could not determine home directory"))?;
+            if !path.exists() {
+                anyhow::bail!("profile '{}' not found at {}", name, path.display());
+            }
+            if !yes && output_format == OutputFormat::Pretty {
+                print!("Delete profile '{}' at {}? [y/N]: ", name, path.display());
+                std::io::Write::flush(&mut std::io::stdout())?;
+                let mut input = String::new();
+                std::io::stdin().read_line(&mut input)?;
+                if !matches!(input.trim().to_lowercase().as_str(), "y" | "yes") {
+                    ui::print_info("Aborted.");
+                    return Ok(());
+                }
+            }
+            let deleted = Config::delete_profile(&name)?;
+            if output_format == OutputFormat::Pretty {
+                ui::print_success(&format!("Deleted profile '{}' ({})", name, deleted.display()));
+            } else {
+                output::print_output(
+                    &serde_json::json!({
+                        "name": name,
+                        "path": deleted.display().to_string(),
+                        "deleted": true,
+                    }),
+                    output_format,
+                )?;
+            }
+            Ok(())
+        }
+    }
+}

hindsight-cli/tests/cli_profile.rs

@@ -0,0 +1,321 @@
+//! End-to-end tests for the `hindsight profile` CRUD subcommands and the
+//! global `-p/--profile` flag.
+//!
+//! These tests do not require a running Hindsight API server: they exercise
+//! the binary against a temporary HOME directory and assert on the profile
+//! files it reads/writes. The only time a command is expected to contact the
+//! API is the `-p` precedence test, which uses `hindsight version` pointed at
+//! a deliberately-unreachable URL so we can assert that the profile value
+//! (not the default `http://localhost:8888`) was picked up from the error
+//! message.
+//!
+//! These integration tests are Unix-only because they rely on overriding
+//! `$HOME` to redirect `dirs::home_dir()` at a tempdir. On Windows
+//! `dirs::home_dir()` resolves via `FOLDERID_Profile` (the Win32 shell API)
+//! and ignores the env var, so running these tests there would pollute the
+//! real user profile directory. The Windows runtime path is still exercised
+//! by the `config::tests::*` unit tests, which drive the path-based
+//! `save_profile_to_dir` / `load_profile_from_dir` helpers directly.
+#![cfg(unix)]
+
+use std::path::PathBuf;
+use std::process::{Command, Output};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+static COUNTER: AtomicU64 = AtomicU64::new(0);
+
+fn unique_tempdir(tag: &str) -> PathBuf {
+    let pid = std::process::id();
+    let nanos = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_nanos();
+    let n = COUNTER.fetch_add(1, Ordering::SeqCst);
+    let dir = std::env::temp_dir().join(format!("hindsight-profile-test-{}-{}-{}-{}", tag, pid, nanos, n));
+    std::fs::create_dir_all(&dir).unwrap();
+    dir
+}
+
+fn hindsight_binary() -> String {
+    std::env::var("CARGO_BIN_EXE_hindsight").unwrap_or_else(|_| {
+        let debug = "./target/debug/hindsight";
+        let release = "./target/release/hindsight";
+        if std::path::Path::new(debug).exists() {
+            debug.to_string()
+        } else if std::path::Path::new(release).exists() {
+            release.to_string()
+        } else {
+            "hindsight".to_string()
+        }
+    })
+}
+
+fn run_with_home(home: &std::path::Path, args: &[&str]) -> Output {
+    Command::new(hindsight_binary())
+        .env("HOME", home)
+        // Unset anything that would bypass the profile/config-file resolution
+        // we're trying to exercise here.
+        .env_remove("HINDSIGHT_API_URL")
+        .env_remove("HINDSIGHT_API_KEY")
+        .env_remove("HINDSIGHT_PROFILE")
+        .args(args)
+        .output()
+        .expect("failed to spawn hindsight binary")
+}
+
+fn assert_success(out: &Output) {
+    if !out.status.success() {
+        panic!(
+            "command failed: status={:?}\n--- stdout ---\n{}\n--- stderr ---\n{}",
+            out.status,
+            String::from_utf8_lossy(&out.stdout),
+            String::from_utf8_lossy(&out.stderr),
+        );
+    }
+}
+
+fn stdout(out: &Output) -> String {
+    String::from_utf8_lossy(&out.stdout).into_owned()
+}
+
+fn stderr(out: &Output) -> String {
+    String::from_utf8_lossy(&out.stderr).into_owned()
+}
+
+#[test]
+fn profile_create_writes_toml_and_json_output() {
+    let home = unique_tempdir("create");
+    let out = run_with_home(
+        &home,
+        &[
+            "--output",
+            "json",
+            "profile",
+            "create",
+            "prod",
+            "--api-url",
+            "https://api.example.com",
+            "--api-key",
+            "hsk_abcdef1234",
+        ],
+    );
+    assert_success(&out);
+
+    let payload: serde_json::Value =
+        serde_json::from_str(&stdout(&out)).expect("expected JSON output");
+    assert_eq!(payload["name"], "prod");
+    assert_eq!(payload["api_url"], "https://api.example.com");
+    assert_eq!(payload["api_key_set"], true);
+
+    let path = home.join(".hindsight/cli-profiles/prod.toml");
+    assert!(path.exists(), "profile file was not created at {}", path.display());
+    let body = std::fs::read_to_string(&path).unwrap();
+    assert!(body.contains("api_url = \"https://api.example.com\""));
+    assert!(body.contains("api_key = \"hsk_abcdef1234\""));
+
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777;
+        assert_eq!(mode, 0o600, "profile file should be mode 0600");
+    }
+}
+
+#[test]
+fn profile_create_rejects_invalid_api_url() {
+    let home = unique_tempdir("invalid-url");
+    let out = run_with_home(
+        &home,
+        &["profile", "create", "foo", "--api-url", "localhost:8888"],
+    );
+    assert!(!out.status.success());
+    assert!(stderr(&out).contains("Invalid API URL"));
+}
+
+#[test]
+fn profile_create_rejects_unsafe_names() {
+    let home = unique_tempdir("unsafe-name");
+    for bad in &["..", ".hidden", "a/b", "a b"] {
+        let out = run_with_home(
+            &home,
+            &["profile", "create", bad, "--api-url", "https://example.com"],
+        );
+        assert!(
+            !out.status.success(),
+            "expected failure for profile name {:?}",
+            bad
+        );
+    }
+}
+
+#[test]
+fn profile_list_returns_sorted_names() {
+    let home = unique_tempdir("list");
+    for name in &["prod", "dev", "staging"] {
+        let out = run_with_home(
+            &home,
+            &[
+                "profile",
+                "create",
+                name,
+                "--api-url",
+                &format!("https://{}.example.com", name),
+            ],
+        );
+        assert_success(&out);
+    }
+
+    let out = run_with_home(&home, &["--output", "json", "profile", "list"]);
+    assert_success(&out);
+    let payload: serde_json::Value = serde_json::from_str(&stdout(&out)).unwrap();
+    let names: Vec<&str> = payload["profiles"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|v| v.as_str().unwrap())
+        .collect();
+    assert_eq!(names, vec!["dev", "prod", "staging"]);
+}
+
+#[test]
+fn profile_list_empty_when_no_profiles() {
+    let home = unique_tempdir("list-empty");
+    let out = run_with_home(&home, &["--output", "json", "profile", "list"]);
+    assert_success(&out);
+    let payload: serde_json::Value = serde_json::from_str(&stdout(&out)).unwrap();
+    assert!(payload["profiles"].as_array().unwrap().is_empty());
+}
+
+#[test]
+fn profile_show_returns_stored_values() {
+    let home = unique_tempdir("show");
+    assert_success(&run_with_home(
+        &home,
+        &[
+            "profile",
+            "create",
+            "prod",
+            "--api-url",
+            "https://api.example.com",
+            "--api-key",
+            "hsk_xyz",
+        ],
+    ));
+
+    let out = run_with_home(&home, &["--output", "json", "profile", "show", "prod"]);
+    assert_success(&out);
+    let payload: serde_json::Value = serde_json::from_str(&stdout(&out)).unwrap();
+    assert_eq!(payload["name"], "prod");
+    assert_eq!(payload["api_url"], "https://api.example.com");
+    assert_eq!(payload["api_key_set"], true);
+}
+
+#[test]
+fn profile_show_missing_profile_errors_with_hint() {
+    let home = unique_tempdir("show-missing");
+    let out = run_with_home(&home, &["profile", "show", "nope"]);
+    assert!(!out.status.success());
+    let err = stderr(&out) + &stdout(&out);
+    assert!(err.contains("profile 'nope' not found"));
+    assert!(err.contains("hindsight profile create nope"));
+}
+
+#[test]
+fn profile_delete_removes_file() {
+    let home = unique_tempdir("delete");
+    assert_success(&run_with_home(
+        &home,
+        &[
+            "profile",
+            "create",
+            "prod",
+            "--api-url",
+            "https://api.example.com",
+        ],
+    ));
+    let path = home.join(".hindsight/cli-profiles/prod.toml");
+    assert!(path.exists());
+
+    let out = run_with_home(&home, &["profile", "delete", "prod", "-y"]);
+    assert_success(&out);
+    assert!(!path.exists(), "profile file should have been removed");
+}
+
+#[test]
+fn profile_delete_missing_errors() {
+    let home = unique_tempdir("delete-missing");
+    let out = run_with_home(&home, &["profile", "delete", "nope", "-y"]);
+    assert!(!out.status.success());
+}
+
+#[test]
+fn p_flag_overrides_config_file_api_url() {
+    // Build a HOME that contains BOTH a legacy ~/.hindsight/config pointing
+    // at URL_A and a named profile pointing at URL_B. Running `hindsight -p`
+    // should pick the profile URL, not the config-file URL — we verify by
+    // looking at the connection-error message (no API server needed).
+    let home = unique_tempdir("precedence");
+    let hindsight_dir = home.join(".hindsight");
+    std::fs::create_dir_all(&hindsight_dir).unwrap();
+    std::fs::write(
+        hindsight_dir.join("config"),
+        "api_url = \"http://127.0.0.1:9/from-config\"\n",
+    )
+    .unwrap();
+
+    assert_success(&run_with_home(
+        &home,
+        &[
+            "profile",
+            "create",
+            "prod",
+            "--api-url",
+            "http://127.0.0.1:9/from-profile",
+        ],
+    ));
+
+    // `version` will fail to connect (port 9 is "discard"), but the error
+    // message echoes the API URL actually used.
+    let out = run_with_home(&home, &["-p", "prod", "version"]);
+    assert!(!out.status.success());
+    let err = stderr(&out) + &stdout(&out);
+    assert!(
+        err.contains("from-profile"),
+        "expected profile URL in output, got:\n{}",
+        err
+    );
+    assert!(
+        !err.contains("from-config"),
+        "config-file URL should not have been used:\n{}",
+        err
+    );
+}
+
+#[test]
+fn hindsight_profile_env_var_is_honored() {
+    let home = unique_tempdir("env-var");
+    assert_success(&run_with_home(
+        &home,
+        &[
+            "profile",
+            "create",
+            "staging",
+            "--api-url",
+            "http://127.0.0.1:9/from-env",
+        ],
+    ));
+
+    // Re-run without `-p` but with HINDSIGHT_PROFILE set.
+    let out = Command::new(hindsight_binary())
+        .env("HOME", &home)
+        .env_remove("HINDSIGHT_API_URL")
+        .env_remove("HINDSIGHT_API_KEY")
+        .env("HINDSIGHT_PROFILE", "staging")
+        .args(["version"])
+        .output()
+        .unwrap();
+    assert!(!out.status.success());
+    let err = String::from_utf8_lossy(&out.stderr).into_owned()
+        + &String::from_utf8_lossy(&out.stdout);
+    assert!(err.contains("from-env"), "expected profile URL in output:\n{}", err);
+}

hindsight-clients/go/api/openapi.yaml

@@ -7,7 +7,7 @@ info:
     name: Apache 2.0
     url: https://www.apache.org/licenses/LICENSE-2.0.html
   title: Hindsight HTTP API
-  version: 0.5.1
+  version: 0.5.3
 servers:
 - url: /
 paths:
@@ -166,6 +166,14 @@ paths:
           nullable: true
           type: string
         style: form
+      - explode: true
+        in: query
+        name: consolidation_state
+        required: false
+        schema:
+          nullable: true
+          type: string
+        style: form
       - explode: true
         in: query
         name: limit
@@ -464,6 +472,53 @@ paths:
       summary: Get statistics for memory bank
       tags:
       - Banks
+  /v1/default/banks/{bank_id}/stats/memories-timeseries:
+    get:
+      description: "Memories ingested over a period, bucketed by time and broken down\
+        \ by fact type."
+      operationId: get_memories_timeseries
+      parameters:
+      - explode: false
+        in: path
+        name: bank_id
+        required: true
+        schema:
+          title: Bank Id
+          type: string
+        style: simple
+      - explode: true
+        in: query
+        name: period
+        required: false
+        schema:
+          default: 7d
+          title: Period
+          type: string
+        style: form
+      - explode: false
+        in: header
+        name: authorization
+        required: false
+        schema:
+          nullable: true
+          type: string
+        style: simple
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/MemoriesTimeseriesResponse'
+          description: Successful Response
+        "422":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      summary: Memory ingestion time-series
+      tags:
+      - Banks
   /v1/default/banks/{bank_id}/entities:
     get:
       description: "List all entities (people, organizations, etc.) known by the bank,\
@@ -524,6 +579,66 @@ paths:
       summary: List entities
       tags:
       - Entities
+  /v1/default/banks/{bank_id}/entities/graph:
+    get:
+      description: Return a graph of entities (nodes) and their co-occurrences (edges)
+        for visualization.
+      operationId: get_entity_graph
+      parameters:
+      - explode: false
+        in: path
+        name: bank_id
+        required: true
+        schema:
+          title: Bank Id
+          type: string
+        style: simple
+      - description: Maximum number of co-occurrence edges to return
+        explode: true
+        in: query
+        name: limit
+        required: false
+        schema:
+          default: 1000
+          description: Maximum number of co-occurrence edges to return
+          title: Limit
+          type: integer
+        style: form
+      - description: Minimum cooccurrence_count to include an edge
+        explode: true
+        in: query
+        name: min_count
+        required: false
+        schema:
+          default: 1
+          description: Minimum cooccurrence_count to include an edge
+          title: Min Count
+          type: integer
+        style: form
+      - explode: false
+        in: header
+        name: authorization
+        required: false
+        schema:
+          nullable: true
+          type: string
+        style: simple
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/EntityGraphResponse'
+          description: Successful Response
+        "422":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      summary: Get entity co-occurrence graph
+      tags:
+      - Entities
   /v1/default/banks/{bank_id}/entities/{entity_id}:
     get:
       description: Get detailed information about an entity including observations
@@ -1781,6 +1896,19 @@ paths:
           title: Operation Id
           type: string
         style: simple
+      - description: Include the raw task payload (submission params) in the response.
+          May be large.
+        explode: true
+        in: query
+        name: include_payload
+        required: false
+        schema:
+          default: false
+          description: Include the raw task payload (submission params) in the response.
+            May be large.
+          title: Include Payload
+          type: boolean
+        style: form
       - explode: false
         in: header
         name: authorization
@@ -3439,6 +3567,7 @@ components:
       description: Response model for bank statistics endpoint.
       example:
         bank_id: user123
+        failed_consolidation: 0
         failed_operations: 0
         last_consolidated_at: 2024-01-15T10:30:00Z
         links_breakdown:
@@ -3500,6 +3629,12 @@ components:
         failed_operations:
           title: Failed Operations
           type: integer
+        operations_by_status:
+          additionalProperties:
+            type: integer
+          description: "Async operations grouped by status (pending, in_progress,\
+            \ completed, failed, cancelled)."
+          title: Operations By Status
         last_consolidated_at:
           nullable: true
           type: string
@@ -3508,6 +3643,12 @@ components:
           description: Number of memories not yet processed into observations
           title: Pending Consolidation
           type: integer
+        failed_consolidation:
+          default: 0
+          description: Number of source memories (world/experience) whose consolidation
+            permanently failed and can be retried via the consolidation recovery endpoint.
+          title: Failed Consolidation
+          type: integer
         total_observations:
           default: 0
           description: Total number of observations
@@ -3576,6 +3717,66 @@ components:
         entities_allow_free_form:
           nullable: true
           type: boolean
+        retain_default_strategy:
+          nullable: true
+          type: string
+        retain_strategies:
+          additionalProperties: {}
+          nullable: true
+        retain_chunk_batch_size:
+          nullable: true
+          type: integer
+        mcp_enabled_tools:
+          items:
+            type: string
+          nullable: true
+          type: array
+        consolidation_llm_batch_size:
+          nullable: true
+          type: integer
+        consolidation_source_facts_max_tokens:
+          nullable: true
+          type: integer
+        consolidation_source_facts_max_tokens_per_observation:
+          nullable: true
+          type: integer
+        max_observations_per_scope:
+          nullable: true
+          type: integer
+        reflect_source_facts_max_tokens:
+          nullable: true
+          type: integer
+        llm_gemini_safety_settings:
+          items: {}
+          nullable: true
+          type: array
+        recall_budget_function:
+          nullable: true
+          type: string
+        recall_budget_fixed_low:
+          nullable: true
+          type: integer
+        recall_budget_fixed_mid:
+          nullable: true
+          type: integer
+        recall_budget_fixed_high:
+          nullable: true
+          type: integer
+        recall_budget_adaptive_low:
+          nullable: true
+          type: number
+        recall_budget_adaptive_mid:
+          nullable: true
+          type: number
+        recall_budget_adaptive_high:
+          nullable: true
+          type: number
+        recall_budget_min:
+          nullable: true
+          type: integer
+        recall_budget_max:
+          nullable: true
+          type: integer
       title: BankTemplateConfig
     BankTemplateDirective:
       description: |-
@@ -4381,6 +4582,58 @@ components:
       - mention_count
       - observations
       title: EntityDetailResponse
+    EntityGraphResponse:
+      description: Response model for entity co-occurrence graph endpoint.
+      example:
+        edges:
+        - data:
+            color: '#ffd700'
+            id: uuid-1-uuid-2
+            lastCooccurred: 2024-02-01T14:00:00Z
+            lineStyle: solid
+            linkType: cooccurrence
+            source: uuid-1
+            target: uuid-2
+            weight: 5
+        limit: 1000
+        nodes:
+        - data:
+            color: '#42a5f5'
+            id: uuid-1
+            label: Alice
+            mentionCount: 12
+        - data:
+            color: '#42a5f5'
+            id: uuid-2
+            label: Google
+            mentionCount: 8
+        total_edges: 1
+        total_entities: 2
+      properties:
+        nodes:
+          items:
+            additionalProperties: {}
+          type: array
+        edges:
+          items:
+            additionalProperties: {}
+          type: array
+        total_entities:
+          title: Total Entities
+          type: integer
+        total_edges:
+          title: Total Edges
+          type: integer
+        limit:
+          title: Limit
+          type: integer
+      required:
+      - edges
+      - limit
+      - nodes
+      - total_edges
+      - total_entities
+      title: EntityGraphResponse
     EntityIncludeOptions:
       description: Options for including entity observations in recall results.
       properties:
@@ -4736,6 +4989,44 @@ components:
       - offset
       - total
       title: ListTagsResponse
+    MemoriesTimeseriesResponse:
+      description: Time-series of memory ingestion bucketed by time and fact type.
+      example:
+        period: period
+        trunc: trunc
+        bank_id: bank_id
+        buckets:
+        - world: 0
+          observation: 1
+          time: time
+          experience: 6
+        - world: 0
+          observation: 1
+          time: time
+          experience: 6
+      properties:
+        bank_id:
+          title: Bank Id
+          type: string
+        period:
+          description: "One of: 1h, 12h, 1d, 7d, 30d, 90d."
+          title: Period
+          type: string
+        trunc:
+          description: "Bucket granularity: minute, hour, day."
+          title: Trunc
+          type: string
+        buckets:
+          description: "Per-bucket counts, always returned fully padded for the requested\
+            \ period."
+          items:
+            $ref: '#/components/schemas/MemoryTimeseriesBucket'
+          type: array
+      required:
+      - bank_id
+      - period
+      - trunc
+      title: MemoriesTimeseriesResponse
     MemoryItem:
       description: Single memory item for retain.
       example:
@@ -4793,6 +5084,36 @@ components:
       required:
       - content
       title: MemoryItem
+    MemoryTimeseriesBucket:
+      description: One bucket in the memory ingestion time-series.
+      example:
+        world: 0
+        observation: 1
+        time: time
+        experience: 6
+      properties:
+        time:
+          description: Bucket start timestamp in ISO-8601 (UTC).
+          title: Time
+          type: string
+        world:
+          default: 0
+          description: World-fact memories ingested in this bucket.
+          title: World
+          type: integer
+        experience:
+          default: 0
+          description: Experience memories ingested in this bucket.
+          title: Experience
+          type: integer
+        observation:
+          default: 0
+          description: Observations recorded in this bucket.
+          title: Observation
+          type: integer
+      required:
+      - time
+      title: MemoryTimeseriesBucket
     MentalModelListResponse:
       description: Response model for listing mental models.
       example:
@@ -4806,7 +5127,9 @@ components:
           created_at: created_at
           id: id
           trigger:
+            mode: full
             refresh_after_consolidation: false
+            recall_chunks_max_tokens: 1
             tag_groups:
             - match: any_strict
               tags:
@@ -4822,9 +5145,12 @@ components:
             exclude_mental_model_ids:
             - exclude_mental_model_ids
             - exclude_mental_model_ids
+            include_chunks: true
             tags_match: any
             exclude_mental_models: false
+            recall_max_tokens: 6
           last_refreshed_at: last_refreshed_at
+          is_stale: true
           content: content
           tags:
           - tags
@@ -4838,7 +5164,9 @@ components:
           created_at: created_at
           id: id
           trigger:
+            mode: full
             refresh_after_consolidation: false
+            recall_chunks_max_tokens: 1
             tag_groups:
             - match: any_strict
               tags:
@@ -4854,9 +5182,12 @@ components:
             exclude_mental_model_ids:
             - exclude_mental_model_ids
             - exclude_mental_model_ids
+            include_chunks: true
             tags_match: any
             exclude_mental_models: false
+            recall_max_tokens: 6
           last_refreshed_at: last_refreshed_at
+          is_stale: true
           content: content
           tags:
           - tags
@@ -4881,7 +5212,9 @@ components:
         created_at: created_at
         id: id
         trigger:
+          mode: full
           refresh_after_consolidation: false
+          recall_chunks_max_tokens: 1
           tag_groups:
           - match: any_strict
             tags:
@@ -4897,9 +5230,12 @@ components:
           exclude_mental_model_ids:
           - exclude_mental_model_ids
           - exclude_mental_model_ids
+          include_chunks: true
           tags_match: any
           exclude_mental_models: false
+          recall_max_tokens: 6
         last_refreshed_at: last_refreshed_at
+        is_stale: true
         content: content
         tags:
         - tags
@@ -4939,6 +5275,9 @@ components:
         reflect_response:
           additionalProperties: {}
           nullable: true
+        is_stale:
+          nullable: true
+          type: boolean
       required:
       - bank_id
       - id
@@ -4947,6 +5286,19 @@ components:
     MentalModelTrigger-Input:
       description: Trigger settings for a mental model.
       properties:
+        mode:
+          default: full
+          description: "Refresh mode. 'full' (default) regenerates the mental model\
+            \ content from scratch on each refresh. 'delta' performs surgical edits\
+            \ against the existing content: unchanged sections are preserved byte-for-byte,\
+            \ stale content is removed, new content is added. If the mental model\
+            \ has no existing content, or if the source_query has changed since the\
+            \ last refresh, delta mode falls back to a full regeneration automatically."
+          enum:
+          - full
+          - delta
+          title: Mode
+          type: string
         refresh_after_consolidation:
           default: false
           description: "If true, refresh this mental model after observations consolidation\
@@ -4986,11 +5338,22 @@ components:
             $ref: '#/components/schemas/MentalModelTrigger_Input_tag_groups_inner'
           nullable: true
           type: array
+        include_chunks:
+          nullable: true
+          type: boolean
+        recall_max_tokens:
+          nullable: true
+          type: integer
+        recall_chunks_max_tokens:
+          nullable: true
+          type: integer
       title: MentalModelTrigger
     MentalModelTrigger-Output:
       description: Trigger settings for a mental model.
       example:
+        mode: full
         refresh_after_consolidation: false
+        recall_chunks_max_tokens: 1
         tag_groups:
         - match: any_strict
           tags:
@@ -5006,9 +5369,24 @@ components:
         exclude_mental_model_ids:
         - exclude_mental_model_ids
         - exclude_mental_model_ids
+        include_chunks: true
         tags_match: any
         exclude_mental_models: false
+        recall_max_tokens: 6
       properties:
+        mode:
+          default: full
+          description: "Refresh mode. 'full' (default) regenerates the mental model\
+            \ content from scratch on each refresh. 'delta' performs surgical edits\
+            \ against the existing content: unchanged sections are preserved byte-for-byte,\
+            \ stale content is removed, new content is added. If the mental model\
+            \ has no existing content, or if the source_query has changed since the\
+            \ last refresh, delta mode falls back to a full regeneration automatically."
+          enum:
+          - full
+          - delta
+          title: Mode
+          type: string
         refresh_after_consolidation:
           default: false
           description: "If true, refresh this mental model after observations consolidation\
@@ -5048,6 +5426,15 @@ components:
             $ref: '#/components/schemas/MentalModelTrigger_Output_tag_groups_inner'
           nullable: true
           type: array
+        include_chunks:
+          nullable: true
+          type: boolean
+        recall_max_tokens:
+          nullable: true
+          type: integer
+        recall_chunks_max_tokens:
+          nullable: true
+          type: integer
       title: MentalModelTrigger
     OperationResponse:
       description: Response model for a single async operation.
@@ -5131,6 +5518,9 @@ components:
             $ref: '#/components/schemas/ChildOperationStatus'
           nullable: true
           type: array
+        task_payload:
+          additionalProperties: {}
+          nullable: true
       required:
       - operation_id
       - status

hindsight-clients/go/api_audit.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_bank_templates.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_banks.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -905,6 +905,140 @@ func (a *BanksAPIService) GetBankProfileExecute(r ApiGetBankProfileRequest) (*Ba
 	return localVarReturnValue, localVarHTTPResponse, nil
 }
 
+type ApiGetMemoriesTimeseriesRequest struct {
+	ctx context.Context
+	ApiService *BanksAPIService
+	bankId string
+	period *string
+	authorization *string
+}
+
+func (r ApiGetMemoriesTimeseriesRequest) Period(period string) ApiGetMemoriesTimeseriesRequest {
+	r.period = &period
+	return r
+}
+
+func (r ApiGetMemoriesTimeseriesRequest) Authorization(authorization string) ApiGetMemoriesTimeseriesRequest {
+	r.authorization = &authorization
+	return r
+}
+
+func (r ApiGetMemoriesTimeseriesRequest) Execute() (*MemoriesTimeseriesResponse, *http.Response, error) {
+	return r.ApiService.GetMemoriesTimeseriesExecute(r)
+}
+
+/*
+GetMemoriesTimeseries Memory ingestion time-series
+
+Memories ingested over a period, bucketed by time and broken down by fact type.
+
+ @param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
+ @param bankId
+ @return ApiGetMemoriesTimeseriesRequest
+*/
+func (a *BanksAPIService) GetMemoriesTimeseries(ctx context.Context, bankId string) ApiGetMemoriesTimeseriesRequest {
+	return ApiGetMemoriesTimeseriesRequest{
+		ApiService: a,
+		ctx: ctx,
+		bankId: bankId,
+	}
+}
+
+// Execute executes the request
+//  @return MemoriesTimeseriesResponse
+func (a *BanksAPIService) GetMemoriesTimeseriesExecute(r ApiGetMemoriesTimeseriesRequest) (*MemoriesTimeseriesResponse, *http.Response, error) {
+	var (
+		localVarHTTPMethod   = http.MethodGet
+		localVarPostBody     interface{}
+		formFiles            []formFile
+		localVarReturnValue  *MemoriesTimeseriesResponse
+	)
+
+	localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "BanksAPIService.GetMemoriesTimeseries")
+	if err != nil {
+		return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
+	}
+
+	localVarPath := localBasePath + "/v1/default/banks/{bank_id}/stats/memories-timeseries"
+	localVarPath = strings.Replace(localVarPath, "{"+"bank_id"+"}", url.PathEscape(parameterValueToString(r.bankId, "bankId")), -1)
+
+	localVarHeaderParams := make(map[string]string)
+	localVarQueryParams := url.Values{}
+	localVarFormParams := url.Values{}
+
+	if r.period != nil {
+		parameterAddToHeaderOrQuery(localVarQueryParams, "period", r.period, "form", "")
+	} else {
+		var defaultValue string = "7d"
+		r.period = &defaultValue
+	}
+	// to determine the Content-Type header
+	localVarHTTPContentTypes := []string{}
+
+	// set Content-Type header
+	localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
+	if localVarHTTPContentType != "" {
+		localVarHeaderParams["Content-Type"] = localVarHTTPContentType
+	}
+
+	// to determine the Accept header
+	localVarHTTPHeaderAccepts := []string{"application/json"}
+
+	// set Accept header
+	localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
+	if localVarHTTPHeaderAccept != "" {
+		localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
+	}
+	if r.authorization != nil {
+		parameterAddToHeaderOrQuery(localVarHeaderParams, "authorization", r.authorization, "simple", "")
+	}
+	req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
+	if err != nil {
+		return localVarReturnValue, nil, err
+	}
+
+	localVarHTTPResponse, err := a.client.callAPI(req)
+	if err != nil || localVarHTTPResponse == nil {
+		return localVarReturnValue, localVarHTTPResponse, err
+	}
+
+	localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
+	localVarHTTPResponse.Body.Close()
+	localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
+	if err != nil {
+		return localVarReturnValue, localVarHTTPResponse, err
+	}
+
+	if localVarHTTPResponse.StatusCode >= 300 {
+		newErr := &GenericOpenAPIError{
+			body:  localVarBody,
+			error: localVarHTTPResponse.Status,
+		}
+		if localVarHTTPResponse.StatusCode == 422 {
+			var v HTTPValidationError
+			err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+			if err != nil {
+				newErr.error = err.Error()
+				return localVarReturnValue, localVarHTTPResponse, newErr
+			}
+					newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
+					newErr.model = v
+		}
+		return localVarReturnValue, localVarHTTPResponse, newErr
+	}
+
+	err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+	if err != nil {
+		newErr := &GenericOpenAPIError{
+			body:  localVarBody,
+			error: err.Error(),
+		}
+		return localVarReturnValue, localVarHTTPResponse, newErr
+	}
+
+	return localVarReturnValue, localVarHTTPResponse, nil
+}
+
 type ApiListBanksRequest struct {
 	ctx context.Context
 	ApiService *BanksAPIService

hindsight-clients/go/api_directives.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_documents.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_entities.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -149,6 +149,154 @@ func (a *EntitiesAPIService) GetEntityExecute(r ApiGetEntityRequest) (*EntityDet
 	return localVarReturnValue, localVarHTTPResponse, nil
 }
 
+type ApiGetEntityGraphRequest struct {
+	ctx context.Context
+	ApiService *EntitiesAPIService
+	bankId string
+	limit *int32
+	minCount *int32
+	authorization *string
+}
+
+// Maximum number of co-occurrence edges to return
+func (r ApiGetEntityGraphRequest) Limit(limit int32) ApiGetEntityGraphRequest {
+	r.limit = &limit
+	return r
+}
+
+// Minimum cooccurrence_count to include an edge
+func (r ApiGetEntityGraphRequest) MinCount(minCount int32) ApiGetEntityGraphRequest {
+	r.minCount = &minCount
+	return r
+}
+
+func (r ApiGetEntityGraphRequest) Authorization(authorization string) ApiGetEntityGraphRequest {
+	r.authorization = &authorization
+	return r
+}
+
+func (r ApiGetEntityGraphRequest) Execute() (*EntityGraphResponse, *http.Response, error) {
+	return r.ApiService.GetEntityGraphExecute(r)
+}
+
+/*
+GetEntityGraph Get entity co-occurrence graph
+
+Return a graph of entities (nodes) and their co-occurrences (edges) for visualization.
+
+ @param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
+ @param bankId
+ @return ApiGetEntityGraphRequest
+*/
+func (a *EntitiesAPIService) GetEntityGraph(ctx context.Context, bankId string) ApiGetEntityGraphRequest {
+	return ApiGetEntityGraphRequest{
+		ApiService: a,
+		ctx: ctx,
+		bankId: bankId,
+	}
+}
+
+// Execute executes the request
+//  @return EntityGraphResponse
+func (a *EntitiesAPIService) GetEntityGraphExecute(r ApiGetEntityGraphRequest) (*EntityGraphResponse, *http.Response, error) {
+	var (
+		localVarHTTPMethod   = http.MethodGet
+		localVarPostBody     interface{}
+		formFiles            []formFile
+		localVarReturnValue  *EntityGraphResponse
+	)
+
+	localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "EntitiesAPIService.GetEntityGraph")
+	if err != nil {
+		return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
+	}
+
+	localVarPath := localBasePath + "/v1/default/banks/{bank_id}/entities/graph"
+	localVarPath = strings.Replace(localVarPath, "{"+"bank_id"+"}", url.PathEscape(parameterValueToString(r.bankId, "bankId")), -1)
+
+	localVarHeaderParams := make(map[string]string)
+	localVarQueryParams := url.Values{}
+	localVarFormParams := url.Values{}
+
+	if r.limit != nil {
+		parameterAddToHeaderOrQuery(localVarQueryParams, "limit", r.limit, "form", "")
+	} else {
+		var defaultValue int32 = 1000
+		r.limit = &defaultValue
+	}
+	if r.minCount != nil {
+		parameterAddToHeaderOrQuery(localVarQueryParams, "min_count", r.minCount, "form", "")
+	} else {
+		var defaultValue int32 = 1
+		r.minCount = &defaultValue
+	}
+	// to determine the Content-Type header
+	localVarHTTPContentTypes := []string{}
+
+	// set Content-Type header
+	localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
+	if localVarHTTPContentType != "" {
+		localVarHeaderParams["Content-Type"] = localVarHTTPContentType
+	}
+
+	// to determine the Accept header
+	localVarHTTPHeaderAccepts := []string{"application/json"}
+
+	// set Accept header
+	localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
+	if localVarHTTPHeaderAccept != "" {
+		localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
+	}
+	if r.authorization != nil {
+		parameterAddToHeaderOrQuery(localVarHeaderParams, "authorization", r.authorization, "simple", "")
+	}
+	req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
+	if err != nil {
+		return localVarReturnValue, nil, err
+	}
+
+	localVarHTTPResponse, err := a.client.callAPI(req)
+	if err != nil || localVarHTTPResponse == nil {
+		return localVarReturnValue, localVarHTTPResponse, err
+	}
+
+	localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
+	localVarHTTPResponse.Body.Close()
+	localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
+	if err != nil {
+		return localVarReturnValue, localVarHTTPResponse, err
+	}
+
+	if localVarHTTPResponse.StatusCode >= 300 {
+		newErr := &GenericOpenAPIError{
+			body:  localVarBody,
+			error: localVarHTTPResponse.Status,
+		}
+		if localVarHTTPResponse.StatusCode == 422 {
+			var v HTTPValidationError
+			err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+			if err != nil {
+				newErr.error = err.Error()
+				return localVarReturnValue, localVarHTTPResponse, newErr
+			}
+					newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
+					newErr.model = v
+		}
+		return localVarReturnValue, localVarHTTPResponse, newErr
+	}
+
+	err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+	if err != nil {
+		newErr := &GenericOpenAPIError{
+			body:  localVarBody,
+			error: err.Error(),
+		}
+		return localVarReturnValue, localVarHTTPResponse, newErr
+	}
+
+	return localVarReturnValue, localVarHTTPResponse, nil
+}
+
 type ApiListEntitiesRequest struct {
 	ctx context.Context
 	ApiService *EntitiesAPIService

hindsight-clients/go/api_files.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_memory.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -721,6 +721,7 @@ type ApiListMemoriesRequest struct {
 	bankId string
 	type_ *string
 	q *string
+	consolidationState *string
 	limit *int32
 	offset *int32
 	authorization *string
@@ -736,6 +737,11 @@ func (r ApiListMemoriesRequest) Q(q string) ApiListMemoriesRequest {
 	return r
 }
 
+func (r ApiListMemoriesRequest) ConsolidationState(consolidationState string) ApiListMemoriesRequest {
+	r.consolidationState = &consolidationState
+	return r
+}
+
 func (r ApiListMemoriesRequest) Limit(limit int32) ApiListMemoriesRequest {
 	r.limit = &limit
 	return r
@@ -800,6 +806,9 @@ func (a *MemoryAPIService) ListMemoriesExecute(r ApiListMemoriesRequest) (*ListM
 	if r.q != nil {
 		parameterAddToHeaderOrQuery(localVarQueryParams, "q", r.q, "form", "")
 	}
+	if r.consolidationState != nil {
+		parameterAddToHeaderOrQuery(localVarQueryParams, "consolidation_state", r.consolidationState, "form", "")
+	}
 	if r.limit != nil {
 		parameterAddToHeaderOrQuery(localVarQueryParams, "limit", r.limit, "form", "")
 	} else {

hindsight-clients/go/api_mental_models.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_monitoring.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_operations.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -154,9 +154,16 @@ type ApiGetOperationStatusRequest struct {
 	ApiService *OperationsAPIService
 	bankId string
 	operationId string
+	includePayload *bool
 	authorization *string
 }
 
+// Include the raw task payload (submission params) in the response. May be large.
+func (r ApiGetOperationStatusRequest) IncludePayload(includePayload bool) ApiGetOperationStatusRequest {
+	r.includePayload = &includePayload
+	return r
+}
+
 func (r ApiGetOperationStatusRequest) Authorization(authorization string) ApiGetOperationStatusRequest {
 	r.authorization = &authorization
 	return r
@@ -208,6 +215,12 @@ func (a *OperationsAPIService) GetOperationStatusExecute(r ApiGetOperationStatus
 	localVarQueryParams := url.Values{}
 	localVarFormParams := url.Values{}
 
+	if r.includePayload != nil {
+		parameterAddToHeaderOrQuery(localVarQueryParams, "include_payload", r.includePayload, "form", "")
+	} else {
+		var defaultValue bool = false
+		r.includePayload = &defaultValue
+	}
 	// to determine the Content-Type header
 	localVarHTTPContentTypes := []string{}
 

hindsight-clients/go/api_webhooks.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/client.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -41,7 +41,7 @@ var (
 	queryDescape    = strings.NewReplacer( "%5B", "[", "%5D", "]" )
 )
 
-// APIClient manages communication with the Hindsight HTTP API API v0.5.1
+// APIClient manages communication with the Hindsight HTTP API API v0.5.3
 // In most cases there should be only one, shared, APIClient.
 type APIClient struct {
 	cfg    *Configuration

hindsight-clients/go/configuration.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/hindsight_client.go

@@ -2,9 +2,31 @@ package hindsight
 
 import (
 	"net/http"
+	"runtime/debug"
 	"time"
 )
 
+// defaultUserAgent returns the User-Agent string sent on every request unless
+// the caller overrides cfg.UserAgent. The version is read from build info so
+// it stays in sync with the module version automatically; falls back to
+// "devel" when running from an unpinned local checkout.
+func defaultUserAgent() string {
+	version := "devel"
+	if info, ok := debug.ReadBuildInfo(); ok {
+		for _, dep := range info.Deps {
+			if dep.Path == "github.com/vectorize-io/hindsight/hindsight-clients/go" {
+				version = dep.Version
+				break
+			}
+		}
+	}
+	return "hindsight-client-go/" + version
+}
+
+// DefaultUserAgent is the User-Agent string sent on every request unless the
+// caller overrides cfg.UserAgent (e.g. for integrations identifying themselves).
+var DefaultUserAgent = defaultUserAgent()
+
 // NewAPIClientWithToken creates a new API client configured with a base URL and API token.
 // The token is sent as a Bearer token in the Authorization header for all requests.
 // Note: this uses http.DefaultClient which has no timeout. Use NewAPIClientWithTimeout
@@ -16,6 +38,7 @@ import (
 //	resp, _, err := client.MemoryAPI.RetainMemories(ctx, bankID).RetainRequest(req).Execute()
 func NewAPIClientWithToken(baseURL, token string) *APIClient {
 	cfg := NewConfiguration()
+	cfg.UserAgent = DefaultUserAgent
 	cfg.Servers = ServerConfigurations{
 		{URL: baseURL},
 	}
@@ -32,6 +55,7 @@ func NewAPIClientWithToken(baseURL, token string) *APIClient {
 //	resp, _, err := client.MemoryAPI.RetainMemories(ctx, bankID).RetainRequest(req).Execute()
 func NewAPIClientWithTimeout(baseURL, token string, timeout time.Duration) *APIClient {
 	cfg := NewConfiguration()
+	cfg.UserAgent = DefaultUserAgent
 	cfg.Servers = ServerConfigurations{
 		{URL: baseURL},
 	}

hindsight-clients/go/model_add_background_request.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_async_operation_submit_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_audit_log_entry.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_audit_log_list_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_audit_log_stats_bucket.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_audit_log_stats_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_background_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_config_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_config_update.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_list_item.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_list_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_profile_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_stats_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -31,9 +31,13 @@ type BankStatsResponse struct {
 	LinksBreakdown map[string]map[string]int32 `json:"links_breakdown"`
 	PendingOperations int32 `json:"pending_operations"`
 	FailedOperations int32 `json:"failed_operations"`
+	// Async operations grouped by status (pending, in_progress, completed, failed, cancelled).
+	OperationsByStatus map[string]int32 `json:"operations_by_status,omitempty"`
 	LastConsolidatedAt NullableString `json:"last_consolidated_at,omitempty"`
 	// Number of memories not yet processed into observations
 	PendingConsolidation *int32 `json:"pending_consolidation,omitempty"`
+	// Number of source memories (world/experience) whose consolidation permanently failed and can be retried via the consolidation recovery endpoint.
+	FailedConsolidation *int32 `json:"failed_consolidation,omitempty"`
 	// Total number of observations
 	TotalObservations *int32 `json:"total_observations,omitempty"`
 }
@@ -58,6 +62,8 @@ func NewBankStatsResponse(bankId string, totalNodes int32, totalLinks int32, tot
 	this.FailedOperations = failedOperations
 	var pendingConsolidation int32 = 0
 	this.PendingConsolidation = &pendingConsolidation
+	var failedConsolidation int32 = 0
+	this.FailedConsolidation = &failedConsolidation
 	var totalObservations int32 = 0
 	this.TotalObservations = &totalObservations
 	return &this
@@ -70,6 +76,8 @@ func NewBankStatsResponseWithDefaults() *BankStatsResponse {
 	this := BankStatsResponse{}
 	var pendingConsolidation int32 = 0
 	this.PendingConsolidation = &pendingConsolidation
+	var failedConsolidation int32 = 0
+	this.FailedConsolidation = &failedConsolidation
 	var totalObservations int32 = 0
 	this.TotalObservations = &totalObservations
 	return &this
@@ -315,6 +323,38 @@ func (o *BankStatsResponse) SetFailedOperations(v int32) {
 	o.FailedOperations = v
 }
 
+// GetOperationsByStatus returns the OperationsByStatus field value if set, zero value otherwise.
+func (o *BankStatsResponse) GetOperationsByStatus() map[string]int32 {
+	if o == nil || IsNil(o.OperationsByStatus) {
+		var ret map[string]int32
+		return ret
+	}
+	return o.OperationsByStatus
+}
+
+// GetOperationsByStatusOk returns a tuple with the OperationsByStatus field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *BankStatsResponse) GetOperationsByStatusOk() (map[string]int32, bool) {
+	if o == nil || IsNil(o.OperationsByStatus) {
+		return map[string]int32{}, false
+	}
+	return o.OperationsByStatus, true
+}
+
+// HasOperationsByStatus returns a boolean if a field has been set.
+func (o *BankStatsResponse) HasOperationsByStatus() bool {
+	if o != nil && !IsNil(o.OperationsByStatus) {
+		return true
+	}
+
+	return false
+}
+
+// SetOperationsByStatus gets a reference to the given map[string]int32 and assigns it to the OperationsByStatus field.
+func (o *BankStatsResponse) SetOperationsByStatus(v map[string]int32) {
+	o.OperationsByStatus = v
+}
+
 // GetLastConsolidatedAt returns the LastConsolidatedAt field value if set, zero value otherwise (both if not set or set to explicit null).
 func (o *BankStatsResponse) GetLastConsolidatedAt() string {
 	if o == nil || IsNil(o.LastConsolidatedAt.Get()) {
@@ -389,6 +429,38 @@ func (o *BankStatsResponse) SetPendingConsolidation(v int32) {
 	o.PendingConsolidation = &v
 }
 
+// GetFailedConsolidation returns the FailedConsolidation field value if set, zero value otherwise.
+func (o *BankStatsResponse) GetFailedConsolidation() int32 {
+	if o == nil || IsNil(o.FailedConsolidation) {
+		var ret int32
+		return ret
+	}
+	return *o.FailedConsolidation
+}
+
+// GetFailedConsolidationOk returns a tuple with the FailedConsolidation field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *BankStatsResponse) GetFailedConsolidationOk() (*int32, bool) {
+	if o == nil || IsNil(o.FailedConsolidation) {
+		return nil, false
+	}
+	return o.FailedConsolidation, true
+}
+
+// HasFailedConsolidation returns a boolean if a field has been set.
+func (o *BankStatsResponse) HasFailedConsolidation() bool {
+	if o != nil && !IsNil(o.FailedConsolidation) {
+		return true
+	}
+
+	return false
+}
+
+// SetFailedConsolidation gets a reference to the given int32 and assigns it to the FailedConsolidation field.
+func (o *BankStatsResponse) SetFailedConsolidation(v int32) {
+	o.FailedConsolidation = &v
+}
+
 // GetTotalObservations returns the TotalObservations field value if set, zero value otherwise.
 func (o *BankStatsResponse) GetTotalObservations() int32 {
 	if o == nil || IsNil(o.TotalObservations) {
@@ -441,12 +513,18 @@ func (o BankStatsResponse) ToMap() (map[string]interface{}, error) {
 	toSerialize["links_breakdown"] = o.LinksBreakdown
 	toSerialize["pending_operations"] = o.PendingOperations
 	toSerialize["failed_operations"] = o.FailedOperations
+	if !IsNil(o.OperationsByStatus) {
+		toSerialize["operations_by_status"] = o.OperationsByStatus
+	}
 	if o.LastConsolidatedAt.IsSet() {
 		toSerialize["last_consolidated_at"] = o.LastConsolidatedAt.Get()
 	}
 	if !IsNil(o.PendingConsolidation) {
 		toSerialize["pending_consolidation"] = o.PendingConsolidation
 	}
+	if !IsNil(o.FailedConsolidation) {
+		toSerialize["failed_consolidation"] = o.FailedConsolidation
+	}
 	if !IsNil(o.TotalObservations) {
 		toSerialize["total_observations"] = o.TotalObservations
 	}

hindsight-clients/go/model_bank_template_config.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -31,6 +31,25 @@ type BankTemplateConfig struct {
 	DispositionEmpathy NullableInt32 `json:"disposition_empathy,omitempty"`
 	EntityLabels []map[string]interface{} `json:"entity_labels,omitempty"`
 	EntitiesAllowFreeForm NullableBool `json:"entities_allow_free_form,omitempty"`
+	RetainDefaultStrategy NullableString `json:"retain_default_strategy,omitempty"`
+	RetainStrategies map[string]interface{} `json:"retain_strategies,omitempty"`
+	RetainChunkBatchSize NullableInt32 `json:"retain_chunk_batch_size,omitempty"`
+	McpEnabledTools []string `json:"mcp_enabled_tools,omitempty"`
+	ConsolidationLlmBatchSize NullableInt32 `json:"consolidation_llm_batch_size,omitempty"`
+	ConsolidationSourceFactsMaxTokens NullableInt32 `json:"consolidation_source_facts_max_tokens,omitempty"`
+	ConsolidationSourceFactsMaxTokensPerObservation NullableInt32 `json:"consolidation_source_facts_max_tokens_per_observation,omitempty"`
+	MaxObservationsPerScope NullableInt32 `json:"max_observations_per_scope,omitempty"`
+	ReflectSourceFactsMaxTokens NullableInt32 `json:"reflect_source_facts_max_tokens,omitempty"`
+	LlmGeminiSafetySettings []interface{} `json:"llm_gemini_safety_settings,omitempty"`
+	RecallBudgetFunction NullableString `json:"recall_budget_function,omitempty"`
+	RecallBudgetFixedLow NullableInt32 `json:"recall_budget_fixed_low,omitempty"`
+	RecallBudgetFixedMid NullableInt32 `json:"recall_budget_fixed_mid,omitempty"`
+	RecallBudgetFixedHigh NullableInt32 `json:"recall_budget_fixed_high,omitempty"`
+	RecallBudgetAdaptiveLow NullableFloat32 `json:"recall_budget_adaptive_low,omitempty"`
+	RecallBudgetAdaptiveMid NullableFloat32 `json:"recall_budget_adaptive_mid,omitempty"`
+	RecallBudgetAdaptiveHigh NullableFloat32 `json:"recall_budget_adaptive_high,omitempty"`
+	RecallBudgetMin NullableInt32 `json:"recall_budget_min,omitempty"`
+	RecallBudgetMax NullableInt32 `json:"recall_budget_max,omitempty"`
 }
 
 // NewBankTemplateConfig instantiates a new BankTemplateConfig object
@@ -545,6 +564,777 @@ func (o *BankTemplateConfig) UnsetEntitiesAllowFreeForm() {
 	o.EntitiesAllowFreeForm.Unset()
 }
 
+// GetRetainDefaultStrategy returns the RetainDefaultStrategy field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetRetainDefaultStrategy() string {
+	if o == nil || IsNil(o.RetainDefaultStrategy.Get()) {
+		var ret string
+		return ret
+	}
+	return *o.RetainDefaultStrategy.Get()
+}
+
+// GetRetainDefaultStrategyOk returns a tuple with the RetainDefaultStrategy field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetRetainDefaultStrategyOk() (*string, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.RetainDefaultStrategy.Get(), o.RetainDefaultStrategy.IsSet()
+}
+
+// HasRetainDefaultStrategy returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasRetainDefaultStrategy() bool {
+	if o != nil && o.RetainDefaultStrategy.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetRetainDefaultStrategy gets a reference to the given NullableString and assigns it to the RetainDefaultStrategy field.
+func (o *BankTemplateConfig) SetRetainDefaultStrategy(v string) {
+	o.RetainDefaultStrategy.Set(&v)
+}
+// SetRetainDefaultStrategyNil sets the value for RetainDefaultStrategy to be an explicit nil
+func (o *BankTemplateConfig) SetRetainDefaultStrategyNil() {
+	o.RetainDefaultStrategy.Set(nil)
+}
+
+// UnsetRetainDefaultStrategy ensures that no value is present for RetainDefaultStrategy, not even an explicit nil
+func (o *BankTemplateConfig) UnsetRetainDefaultStrategy() {
+	o.RetainDefaultStrategy.Unset()
+}
+
+// GetRetainStrategies returns the RetainStrategies field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetRetainStrategies() map[string]interface{} {
+	if o == nil {
+		var ret map[string]interface{}
+		return ret
+	}
+	return o.RetainStrategies
+}
+
+// GetRetainStrategiesOk returns a tuple with the RetainStrategies field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetRetainStrategiesOk() (map[string]interface{}, bool) {
+	if o == nil || IsNil(o.RetainStrategies) {
+		return map[string]interface{}{}, false
+	}
+	return o.RetainStrategies, true
+}
+
+// HasRetainStrategies returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasRetainStrategies() bool {
+	if o != nil && !IsNil(o.RetainStrategies) {
+		return true
+	}
+
+	return false
+}
+
+// SetRetainStrategies gets a reference to the given map[string]interface{} and assigns it to the RetainStrategies field.
+func (o *BankTemplateConfig) SetRetainStrategies(v map[string]interface{}) {
+	o.RetainStrategies = v
+}
+
+// GetRetainChunkBatchSize returns the RetainChunkBatchSize field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetRetainChunkBatchSize() int32 {
+	if o == nil || IsNil(o.RetainChunkBatchSize.Get()) {
+		var ret int32
+		return ret
+	}
+	return *o.RetainChunkBatchSize.Get()
+}
+
+// GetRetainChunkBatchSizeOk returns a tuple with the RetainChunkBatchSize field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetRetainChunkBatchSizeOk() (*int32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.RetainChunkBatchSize.Get(), o.RetainChunkBatchSize.IsSet()
+}
+
+// HasRetainChunkBatchSize returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasRetainChunkBatchSize() bool {
+	if o != nil && o.RetainChunkBatchSize.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetRetainChunkBatchSize gets a reference to the given NullableInt32 and assigns it to the RetainChunkBatchSize field.
+func (o *BankTemplateConfig) SetRetainChunkBatchSize(v int32) {
+	o.RetainChunkBatchSize.Set(&v)
+}
+// SetRetainChunkBatchSizeNil sets the value for RetainChunkBatchSize to be an explicit nil
+func (o *BankTemplateConfig) SetRetainChunkBatchSizeNil() {
+	o.RetainChunkBatchSize.Set(nil)
+}
+
+// UnsetRetainChunkBatchSize ensures that no value is present for RetainChunkBatchSize, not even an explicit nil
+func (o *BankTemplateConfig) UnsetRetainChunkBatchSize() {
+	o.RetainChunkBatchSize.Unset()
+}
+
+// GetMcpEnabledTools returns the McpEnabledTools field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetMcpEnabledTools() []string {
+	if o == nil {
+		var ret []string
+		return ret
+	}
+	return o.McpEnabledTools
+}
+
+// GetMcpEnabledToolsOk returns a tuple with the McpEnabledTools field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetMcpEnabledToolsOk() ([]string, bool) {
+	if o == nil || IsNil(o.McpEnabledTools) {
+		return nil, false
+	}
+	return o.McpEnabledTools, true
+}
+
+// HasMcpEnabledTools returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasMcpEnabledTools() bool {
+	if o != nil && !IsNil(o.McpEnabledTools) {
+		return true
+	}
+
+	return false
+}
+
+// SetMcpEnabledTools gets a reference to the given []string and assigns it to the McpEnabledTools field.
+func (o *BankTemplateConfig) SetMcpEnabledTools(v []string) {
+	o.McpEnabledTools = v
+}
+
+// GetConsolidationLlmBatchSize returns the ConsolidationLlmBatchSize field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetConsolidationLlmBatchSize() int32 {
+	if o == nil || IsNil(o.ConsolidationLlmBatchSize.Get()) {
+		var ret int32
+		return ret
+	}
+	return *o.ConsolidationLlmBatchSize.Get()
+}
+
+// GetConsolidationLlmBatchSizeOk returns a tuple with the ConsolidationLlmBatchSize field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetConsolidationLlmBatchSizeOk() (*int32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.ConsolidationLlmBatchSize.Get(), o.ConsolidationLlmBatchSize.IsSet()
+}
+
+// HasConsolidationLlmBatchSize returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasConsolidationLlmBatchSize() bool {
+	if o != nil && o.ConsolidationLlmBatchSize.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetConsolidationLlmBatchSize gets a reference to the given NullableInt32 and assigns it to the ConsolidationLlmBatchSize field.
+func (o *BankTemplateConfig) SetConsolidationLlmBatchSize(v int32) {
+	o.ConsolidationLlmBatchSize.Set(&v)
+}
+// SetConsolidationLlmBatchSizeNil sets the value for ConsolidationLlmBatchSize to be an explicit nil
+func (o *BankTemplateConfig) SetConsolidationLlmBatchSizeNil() {
+	o.ConsolidationLlmBatchSize.Set(nil)
+}
+
+// UnsetConsolidationLlmBatchSize ensures that no value is present for ConsolidationLlmBatchSize, not even an explicit nil
+func (o *BankTemplateConfig) UnsetConsolidationLlmBatchSize() {
+	o.ConsolidationLlmBatchSize.Unset()
+}
+
+// GetConsolidationSourceFactsMaxTokens returns the ConsolidationSourceFactsMaxTokens field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetConsolidationSourceFactsMaxTokens() int32 {
+	if o == nil || IsNil(o.ConsolidationSourceFactsMaxTokens.Get()) {
+		var ret int32
+		return ret
+	}
+	return *o.ConsolidationSourceFactsMaxTokens.Get()
+}
+
+// GetConsolidationSourceFactsMaxTokensOk returns a tuple with the ConsolidationSourceFactsMaxTokens field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetConsolidationSourceFactsMaxTokensOk() (*int32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.ConsolidationSourceFactsMaxTokens.Get(), o.ConsolidationSourceFactsMaxTokens.IsSet()
+}
+
+// HasConsolidationSourceFactsMaxTokens returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasConsolidationSourceFactsMaxTokens() bool {
+	if o != nil && o.ConsolidationSourceFactsMaxTokens.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetConsolidationSourceFactsMaxTokens gets a reference to the given NullableInt32 and assigns it to the ConsolidationSourceFactsMaxTokens field.
+func (o *BankTemplateConfig) SetConsolidationSourceFactsMaxTokens(v int32) {
+	o.ConsolidationSourceFactsMaxTokens.Set(&v)
+}
+// SetConsolidationSourceFactsMaxTokensNil sets the value for ConsolidationSourceFactsMaxTokens to be an explicit nil
+func (o *BankTemplateConfig) SetConsolidationSourceFactsMaxTokensNil() {
+	o.ConsolidationSourceFactsMaxTokens.Set(nil)
+}
+
+// UnsetConsolidationSourceFactsMaxTokens ensures that no value is present for ConsolidationSourceFactsMaxTokens, not even an explicit nil
+func (o *BankTemplateConfig) UnsetConsolidationSourceFactsMaxTokens() {
+	o.ConsolidationSourceFactsMaxTokens.Unset()
+}
+
+// GetConsolidationSourceFactsMaxTokensPerObservation returns the ConsolidationSourceFactsMaxTokensPerObservation field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetConsolidationSourceFactsMaxTokensPerObservation() int32 {
+	if o == nil || IsNil(o.ConsolidationSourceFactsMaxTokensPerObservation.Get()) {
+		var ret int32
+		return ret
+	}
+	return *o.ConsolidationSourceFactsMaxTokensPerObservation.Get()
+}
+
+// GetConsolidationSourceFactsMaxTokensPerObservationOk returns a tuple with the ConsolidationSourceFactsMaxTokensPerObservation field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetConsolidationSourceFactsMaxTokensPerObservationOk() (*int32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.ConsolidationSourceFactsMaxTokensPerObservation.Get(), o.ConsolidationSourceFactsMaxTokensPerObservation.IsSet()
+}
+
+// HasConsolidationSourceFactsMaxTokensPerObservation returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasConsolidationSourceFactsMaxTokensPerObservation() bool {
+	if o != nil && o.ConsolidationSourceFactsMaxTokensPerObservation.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetConsolidationSourceFactsMaxTokensPerObservation gets a reference to the given NullableInt32 and assigns it to the ConsolidationSourceFactsMaxTokensPerObservation field.
+func (o *BankTemplateConfig) SetConsolidationSourceFactsMaxTokensPerObservation(v int32) {
+	o.ConsolidationSourceFactsMaxTokensPerObservation.Set(&v)
+}
+// SetConsolidationSourceFactsMaxTokensPerObservationNil sets the value for ConsolidationSourceFactsMaxTokensPerObservation to be an explicit nil
+func (o *BankTemplateConfig) SetConsolidationSourceFactsMaxTokensPerObservationNil() {
+	o.ConsolidationSourceFactsMaxTokensPerObservation.Set(nil)
+}
+
+// UnsetConsolidationSourceFactsMaxTokensPerObservation ensures that no value is present for ConsolidationSourceFactsMaxTokensPerObservation, not even an explicit nil
+func (o *BankTemplateConfig) UnsetConsolidationSourceFactsMaxTokensPerObservation() {
+	o.ConsolidationSourceFactsMaxTokensPerObservation.Unset()
+}
+
+// GetMaxObservationsPerScope returns the MaxObservationsPerScope field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetMaxObservationsPerScope() int32 {
+	if o == nil || IsNil(o.MaxObservationsPerScope.Get()) {
+		var ret int32
+		return ret
+	}
+	return *o.MaxObservationsPerScope.Get()
+}
+
+// GetMaxObservationsPerScopeOk returns a tuple with the MaxObservationsPerScope field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetMaxObservationsPerScopeOk() (*int32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.MaxObservationsPerScope.Get(), o.MaxObservationsPerScope.IsSet()
+}
+
+// HasMaxObservationsPerScope returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasMaxObservationsPerScope() bool {
+	if o != nil && o.MaxObservationsPerScope.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetMaxObservationsPerScope gets a reference to the given NullableInt32 and assigns it to the MaxObservationsPerScope field.
+func (o *BankTemplateConfig) SetMaxObservationsPerScope(v int32) {
+	o.MaxObservationsPerScope.Set(&v)
+}
+// SetMaxObservationsPerScopeNil sets the value for MaxObservationsPerScope to be an explicit nil
+func (o *BankTemplateConfig) SetMaxObservationsPerScopeNil() {
+	o.MaxObservationsPerScope.Set(nil)
+}
+
+// UnsetMaxObservationsPerScope ensures that no value is present for MaxObservationsPerScope, not even an explicit nil
+func (o *BankTemplateConfig) UnsetMaxObservationsPerScope() {
+	o.MaxObservationsPerScope.Unset()
+}
+
+// GetReflectSourceFactsMaxTokens returns the ReflectSourceFactsMaxTokens field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetReflectSourceFactsMaxTokens() int32 {
+	if o == nil || IsNil(o.ReflectSourceFactsMaxTokens.Get()) {
+		var ret int32
+		return ret
+	}
+	return *o.ReflectSourceFactsMaxTokens.Get()
+}
+
+// GetReflectSourceFactsMaxTokensOk returns a tuple with the ReflectSourceFactsMaxTokens field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetReflectSourceFactsMaxTokensOk() (*int32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.ReflectSourceFactsMaxTokens.Get(), o.ReflectSourceFactsMaxTokens.IsSet()
+}
+
+// HasReflectSourceFactsMaxTokens returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasReflectSourceFactsMaxTokens() bool {
+	if o != nil && o.ReflectSourceFactsMaxTokens.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetReflectSourceFactsMaxTokens gets a reference to the given NullableInt32 and assigns it to the ReflectSourceFactsMaxTokens field.
+func (o *BankTemplateConfig) SetReflectSourceFactsMaxTokens(v int32) {
+	o.ReflectSourceFactsMaxTokens.Set(&v)
+}
+// SetReflectSourceFactsMaxTokensNil sets the value for ReflectSourceFactsMaxTokens to be an explicit nil
+func (o *BankTemplateConfig) SetReflectSourceFactsMaxTokensNil() {
+	o.ReflectSourceFactsMaxTokens.Set(nil)
+}
+
+// UnsetReflectSourceFactsMaxTokens ensures that no value is present for ReflectSourceFactsMaxTokens, not even an explicit nil
+func (o *BankTemplateConfig) UnsetReflectSourceFactsMaxTokens() {
+	o.ReflectSourceFactsMaxTokens.Unset()
+}
+
+// GetLlmGeminiSafetySettings returns the LlmGeminiSafetySettings field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetLlmGeminiSafetySettings() []interface{} {
+	if o == nil {
+		var ret []interface{}
+		return ret
+	}
+	return o.LlmGeminiSafetySettings
+}
+
+// GetLlmGeminiSafetySettingsOk returns a tuple with the LlmGeminiSafetySettings field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetLlmGeminiSafetySettingsOk() ([]interface{}, bool) {
+	if o == nil || IsNil(o.LlmGeminiSafetySettings) {
+		return nil, false
+	}
+	return o.LlmGeminiSafetySettings, true
+}
+
+// HasLlmGeminiSafetySettings returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasLlmGeminiSafetySettings() bool {
+	if o != nil && !IsNil(o.LlmGeminiSafetySettings) {
+		return true
+	}
+
+	return false
+}
+
+// SetLlmGeminiSafetySettings gets a reference to the given []interface{} and assigns it to the LlmGeminiSafetySettings field.
+func (o *BankTemplateConfig) SetLlmGeminiSafetySettings(v []interface{}) {
+	o.LlmGeminiSafetySettings = v
+}
+
+// GetRecallBudgetFunction returns the RecallBudgetFunction field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetRecallBudgetFunction() string {
+	if o == nil || IsNil(o.RecallBudgetFunction.Get()) {
+		var ret string
+		return ret
+	}
+	return *o.RecallBudgetFunction.Get()
+}
+
+// GetRecallBudgetFunctionOk returns a tuple with the RecallBudgetFunction field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetRecallBudgetFunctionOk() (*string, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.RecallBudgetFunction.Get(), o.RecallBudgetFunction.IsSet()
+}
+
+// HasRecallBudgetFunction returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasRecallBudgetFunction() bool {
+	if o != nil && o.RecallBudgetFunction.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetRecallBudgetFunction gets a reference to the given NullableString and assigns it to the RecallBudgetFunction field.
+func (o *BankTemplateConfig) SetRecallBudgetFunction(v string) {
+	o.RecallBudgetFunction.Set(&v)
+}
+// SetRecallBudgetFunctionNil sets the value for RecallBudgetFunction to be an explicit nil
+func (o *BankTemplateConfig) SetRecallBudgetFunctionNil() {
+	o.RecallBudgetFunction.Set(nil)
+}
+
+// UnsetRecallBudgetFunction ensures that no value is present for RecallBudgetFunction, not even an explicit nil
+func (o *BankTemplateConfig) UnsetRecallBudgetFunction() {
+	o.RecallBudgetFunction.Unset()
+}
+
+// GetRecallBudgetFixedLow returns the RecallBudgetFixedLow field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetRecallBudgetFixedLow() int32 {
+	if o == nil || IsNil(o.RecallBudgetFixedLow.Get()) {
+		var ret int32
+		return ret
+	}
+	return *o.RecallBudgetFixedLow.Get()
+}
+
+// GetRecallBudgetFixedLowOk returns a tuple with the RecallBudgetFixedLow field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetRecallBudgetFixedLowOk() (*int32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.RecallBudgetFixedLow.Get(), o.RecallBudgetFixedLow.IsSet()
+}
+
+// HasRecallBudgetFixedLow returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasRecallBudgetFixedLow() bool {
+	if o != nil && o.RecallBudgetFixedLow.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetRecallBudgetFixedLow gets a reference to the given NullableInt32 and assigns it to the RecallBudgetFixedLow field.
+func (o *BankTemplateConfig) SetRecallBudgetFixedLow(v int32) {
+	o.RecallBudgetFixedLow.Set(&v)
+}
+// SetRecallBudgetFixedLowNil sets the value for RecallBudgetFixedLow to be an explicit nil
+func (o *BankTemplateConfig) SetRecallBudgetFixedLowNil() {
+	o.RecallBudgetFixedLow.Set(nil)
+}
+
+// UnsetRecallBudgetFixedLow ensures that no value is present for RecallBudgetFixedLow, not even an explicit nil
+func (o *BankTemplateConfig) UnsetRecallBudgetFixedLow() {
+	o.RecallBudgetFixedLow.Unset()
+}
+
+// GetRecallBudgetFixedMid returns the RecallBudgetFixedMid field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetRecallBudgetFixedMid() int32 {
+	if o == nil || IsNil(o.RecallBudgetFixedMid.Get()) {
+		var ret int32
+		return ret
+	}
+	return *o.RecallBudgetFixedMid.Get()
+}
+
+// GetRecallBudgetFixedMidOk returns a tuple with the RecallBudgetFixedMid field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetRecallBudgetFixedMidOk() (*int32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.RecallBudgetFixedMid.Get(), o.RecallBudgetFixedMid.IsSet()
+}
+
+// HasRecallBudgetFixedMid returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasRecallBudgetFixedMid() bool {
+	if o != nil && o.RecallBudgetFixedMid.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetRecallBudgetFixedMid gets a reference to the given NullableInt32 and assigns it to the RecallBudgetFixedMid field.
+func (o *BankTemplateConfig) SetRecallBudgetFixedMid(v int32) {
+	o.RecallBudgetFixedMid.Set(&v)
+}
+// SetRecallBudgetFixedMidNil sets the value for RecallBudgetFixedMid to be an explicit nil
+func (o *BankTemplateConfig) SetRecallBudgetFixedMidNil() {
+	o.RecallBudgetFixedMid.Set(nil)
+}
+
+// UnsetRecallBudgetFixedMid ensures that no value is present for RecallBudgetFixedMid, not even an explicit nil
+func (o *BankTemplateConfig) UnsetRecallBudgetFixedMid() {
+	o.RecallBudgetFixedMid.Unset()
+}
+
+// GetRecallBudgetFixedHigh returns the RecallBudgetFixedHigh field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetRecallBudgetFixedHigh() int32 {
+	if o == nil || IsNil(o.RecallBudgetFixedHigh.Get()) {
+		var ret int32
+		return ret
+	}
+	return *o.RecallBudgetFixedHigh.Get()
+}
+
+// GetRecallBudgetFixedHighOk returns a tuple with the RecallBudgetFixedHigh field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetRecallBudgetFixedHighOk() (*int32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.RecallBudgetFixedHigh.Get(), o.RecallBudgetFixedHigh.IsSet()
+}
+
+// HasRecallBudgetFixedHigh returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasRecallBudgetFixedHigh() bool {
+	if o != nil && o.RecallBudgetFixedHigh.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetRecallBudgetFixedHigh gets a reference to the given NullableInt32 and assigns it to the RecallBudgetFixedHigh field.
+func (o *BankTemplateConfig) SetRecallBudgetFixedHigh(v int32) {
+	o.RecallBudgetFixedHigh.Set(&v)
+}
+// SetRecallBudgetFixedHighNil sets the value for RecallBudgetFixedHigh to be an explicit nil
+func (o *BankTemplateConfig) SetRecallBudgetFixedHighNil() {
+	o.RecallBudgetFixedHigh.Set(nil)
+}
+
+// UnsetRecallBudgetFixedHigh ensures that no value is present for RecallBudgetFixedHigh, not even an explicit nil
+func (o *BankTemplateConfig) UnsetRecallBudgetFixedHigh() {
+	o.RecallBudgetFixedHigh.Unset()
+}
+
+// GetRecallBudgetAdaptiveLow returns the RecallBudgetAdaptiveLow field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetRecallBudgetAdaptiveLow() float32 {
+	if o == nil || IsNil(o.RecallBudgetAdaptiveLow.Get()) {
+		var ret float32
+		return ret
+	}
+	return *o.RecallBudgetAdaptiveLow.Get()
+}
+
+// GetRecallBudgetAdaptiveLowOk returns a tuple with the RecallBudgetAdaptiveLow field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetRecallBudgetAdaptiveLowOk() (*float32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.RecallBudgetAdaptiveLow.Get(), o.RecallBudgetAdaptiveLow.IsSet()
+}
+
+// HasRecallBudgetAdaptiveLow returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasRecallBudgetAdaptiveLow() bool {
+	if o != nil && o.RecallBudgetAdaptiveLow.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetRecallBudgetAdaptiveLow gets a reference to the given NullableFloat32 and assigns it to the RecallBudgetAdaptiveLow field.
+func (o *BankTemplateConfig) SetRecallBudgetAdaptiveLow(v float32) {
+	o.RecallBudgetAdaptiveLow.Set(&v)
+}
+// SetRecallBudgetAdaptiveLowNil sets the value for RecallBudgetAdaptiveLow to be an explicit nil
+func (o *BankTemplateConfig) SetRecallBudgetAdaptiveLowNil() {
+	o.RecallBudgetAdaptiveLow.Set(nil)
+}
+
+// UnsetRecallBudgetAdaptiveLow ensures that no value is present for RecallBudgetAdaptiveLow, not even an explicit nil
+func (o *BankTemplateConfig) UnsetRecallBudgetAdaptiveLow() {
+	o.RecallBudgetAdaptiveLow.Unset()
+}
+
+// GetRecallBudgetAdaptiveMid returns the RecallBudgetAdaptiveMid field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetRecallBudgetAdaptiveMid() float32 {
+	if o == nil || IsNil(o.RecallBudgetAdaptiveMid.Get()) {
+		var ret float32
+		return ret
+	}
+	return *o.RecallBudgetAdaptiveMid.Get()
+}
+
+// GetRecallBudgetAdaptiveMidOk returns a tuple with the RecallBudgetAdaptiveMid field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetRecallBudgetAdaptiveMidOk() (*float32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.RecallBudgetAdaptiveMid.Get(), o.RecallBudgetAdaptiveMid.IsSet()
+}
+
+// HasRecallBudgetAdaptiveMid returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasRecallBudgetAdaptiveMid() bool {
+	if o != nil && o.RecallBudgetAdaptiveMid.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetRecallBudgetAdaptiveMid gets a reference to the given NullableFloat32 and assigns it to the RecallBudgetAdaptiveMid field.
+func (o *BankTemplateConfig) SetRecallBudgetAdaptiveMid(v float32) {
+	o.RecallBudgetAdaptiveMid.Set(&v)
+}
+// SetRecallBudgetAdaptiveMidNil sets the value for RecallBudgetAdaptiveMid to be an explicit nil
+func (o *BankTemplateConfig) SetRecallBudgetAdaptiveMidNil() {
+	o.RecallBudgetAdaptiveMid.Set(nil)
+}
+
+// UnsetRecallBudgetAdaptiveMid ensures that no value is present for RecallBudgetAdaptiveMid, not even an explicit nil
+func (o *BankTemplateConfig) UnsetRecallBudgetAdaptiveMid() {
+	o.RecallBudgetAdaptiveMid.Unset()
+}
+
+// GetRecallBudgetAdaptiveHigh returns the RecallBudgetAdaptiveHigh field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetRecallBudgetAdaptiveHigh() float32 {
+	if o == nil || IsNil(o.RecallBudgetAdaptiveHigh.Get()) {
+		var ret float32
+		return ret
+	}
+	return *o.RecallBudgetAdaptiveHigh.Get()
+}
+
+// GetRecallBudgetAdaptiveHighOk returns a tuple with the RecallBudgetAdaptiveHigh field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetRecallBudgetAdaptiveHighOk() (*float32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.RecallBudgetAdaptiveHigh.Get(), o.RecallBudgetAdaptiveHigh.IsSet()
+}
+
+// HasRecallBudgetAdaptiveHigh returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasRecallBudgetAdaptiveHigh() bool {
+	if o != nil && o.RecallBudgetAdaptiveHigh.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetRecallBudgetAdaptiveHigh gets a reference to the given NullableFloat32 and assigns it to the RecallBudgetAdaptiveHigh field.
+func (o *BankTemplateConfig) SetRecallBudgetAdaptiveHigh(v float32) {
+	o.RecallBudgetAdaptiveHigh.Set(&v)
+}
+// SetRecallBudgetAdaptiveHighNil sets the value for RecallBudgetAdaptiveHigh to be an explicit nil
+func (o *BankTemplateConfig) SetRecallBudgetAdaptiveHighNil() {
+	o.RecallBudgetAdaptiveHigh.Set(nil)
+}
+
+// UnsetRecallBudgetAdaptiveHigh ensures that no value is present for RecallBudgetAdaptiveHigh, not even an explicit nil
+func (o *BankTemplateConfig) UnsetRecallBudgetAdaptiveHigh() {
+	o.RecallBudgetAdaptiveHigh.Unset()
+}
+
+// GetRecallBudgetMin returns the RecallBudgetMin field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetRecallBudgetMin() int32 {
+	if o == nil || IsNil(o.RecallBudgetMin.Get()) {
+		var ret int32
+		return ret
+	}
+	return *o.RecallBudgetMin.Get()
+}
+
+// GetRecallBudgetMinOk returns a tuple with the RecallBudgetMin field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetRecallBudgetMinOk() (*int32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.RecallBudgetMin.Get(), o.RecallBudgetMin.IsSet()
+}
+
+// HasRecallBudgetMin returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasRecallBudgetMin() bool {
+	if o != nil && o.RecallBudgetMin.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetRecallBudgetMin gets a reference to the given NullableInt32 and assigns it to the RecallBudgetMin field.
+func (o *BankTemplateConfig) SetRecallBudgetMin(v int32) {
+	o.RecallBudgetMin.Set(&v)
+}
+// SetRecallBudgetMinNil sets the value for RecallBudgetMin to be an explicit nil
+func (o *BankTemplateConfig) SetRecallBudgetMinNil() {
+	o.RecallBudgetMin.Set(nil)
+}
+
+// UnsetRecallBudgetMin ensures that no value is present for RecallBudgetMin, not even an explicit nil
+func (o *BankTemplateConfig) UnsetRecallBudgetMin() {
+	o.RecallBudgetMin.Unset()
+}
+
+// GetRecallBudgetMax returns the RecallBudgetMax field value if set, zero value otherwise (both if not set or set to explicit null).
+func (o *BankTemplateConfig) GetRecallBudgetMax() int32 {
+	if o == nil || IsNil(o.RecallBudgetMax.Get()) {
+		var ret int32
+		return ret
+	}
+	return *o.RecallBudgetMax.Get()
+}
+
+// GetRecallBudgetMaxOk returns a tuple with the RecallBudgetMax field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
+func (o *BankTemplateConfig) GetRecallBudgetMaxOk() (*int32, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.RecallBudgetMax.Get(), o.RecallBudgetMax.IsSet()
+}
+
+// HasRecallBudgetMax returns a boolean if a field has been set.
+func (o *BankTemplateConfig) HasRecallBudgetMax() bool {
+	if o != nil && o.RecallBudgetMax.IsSet() {
+		return true
+	}
+
+	return false
+}
+
+// SetRecallBudgetMax gets a reference to the given NullableInt32 and assigns it to the RecallBudgetMax field.
+func (o *BankTemplateConfig) SetRecallBudgetMax(v int32) {
+	o.RecallBudgetMax.Set(&v)
+}
+// SetRecallBudgetMaxNil sets the value for RecallBudgetMax to be an explicit nil
+func (o *BankTemplateConfig) SetRecallBudgetMaxNil() {
+	o.RecallBudgetMax.Set(nil)
+}
+
+// UnsetRecallBudgetMax ensures that no value is present for RecallBudgetMax, not even an explicit nil
+func (o *BankTemplateConfig) UnsetRecallBudgetMax() {
+	o.RecallBudgetMax.Unset()
+}
+
 func (o BankTemplateConfig) MarshalJSON() ([]byte, error) {
 	toSerialize,err := o.ToMap()
 	if err != nil {
@@ -591,6 +1381,63 @@ func (o BankTemplateConfig) ToMap() (map[string]interface{}, error) {
 	if o.EntitiesAllowFreeForm.IsSet() {
 		toSerialize["entities_allow_free_form"] = o.EntitiesAllowFreeForm.Get()
 	}
+	if o.RetainDefaultStrategy.IsSet() {
+		toSerialize["retain_default_strategy"] = o.RetainDefaultStrategy.Get()
+	}
+	if o.RetainStrategies != nil {
+		toSerialize["retain_strategies"] = o.RetainStrategies
+	}
+	if o.RetainChunkBatchSize.IsSet() {
+		toSerialize["retain_chunk_batch_size"] = o.RetainChunkBatchSize.Get()
+	}
+	if o.McpEnabledTools != nil {
+		toSerialize["mcp_enabled_tools"] = o.McpEnabledTools
+	}
+	if o.ConsolidationLlmBatchSize.IsSet() {
+		toSerialize["consolidation_llm_batch_size"] = o.ConsolidationLlmBatchSize.Get()
+	}
+	if o.ConsolidationSourceFactsMaxTokens.IsSet() {
+		toSerialize["consolidation_source_facts_max_tokens"] = o.ConsolidationSourceFactsMaxTokens.Get()
+	}
+	if o.ConsolidationSourceFactsMaxTokensPerObservation.IsSet() {
+		toSerialize["consolidation_source_facts_max_tokens_per_observation"] = o.ConsolidationSourceFactsMaxTokensPerObservation.Get()
+	}
+	if o.MaxObservationsPerScope.IsSet() {
+		toSerialize["max_observations_per_scope"] = o.MaxObservationsPerScope.Get()
+	}
+	if o.ReflectSourceFactsMaxTokens.IsSet() {
+		toSerialize["reflect_source_facts_max_tokens"] = o.ReflectSourceFactsMaxTokens.Get()
+	}
+	if o.LlmGeminiSafetySettings != nil {
+		toSerialize["llm_gemini_safety_settings"] = o.LlmGeminiSafetySettings
+	}
+	if o.RecallBudgetFunction.IsSet() {
+		toSerialize["recall_budget_function"] = o.RecallBudgetFunction.Get()
+	}
+	if o.RecallBudgetFixedLow.IsSet() {
+		toSerialize["recall_budget_fixed_low"] = o.RecallBudgetFixedLow.Get()
+	}
+	if o.RecallBudgetFixedMid.IsSet() {
+		toSerialize["recall_budget_fixed_mid"] = o.RecallBudgetFixedMid.Get()
+	}
+	if o.RecallBudgetFixedHigh.IsSet() {
+		toSerialize["recall_budget_fixed_high"] = o.RecallBudgetFixedHigh.Get()
+	}
+	if o.RecallBudgetAdaptiveLow.IsSet() {
+		toSerialize["recall_budget_adaptive_low"] = o.RecallBudgetAdaptiveLow.Get()
+	}
+	if o.RecallBudgetAdaptiveMid.IsSet() {
+		toSerialize["recall_budget_adaptive_mid"] = o.RecallBudgetAdaptiveMid.Get()
+	}
+	if o.RecallBudgetAdaptiveHigh.IsSet() {
+		toSerialize["recall_budget_adaptive_high"] = o.RecallBudgetAdaptiveHigh.Get()
+	}
+	if o.RecallBudgetMin.IsSet() {
+		toSerialize["recall_budget_min"] = o.RecallBudgetMin.Get()
+	}
+	if o.RecallBudgetMax.IsSet() {
+		toSerialize["recall_budget_max"] = o.RecallBudgetMax.Get()
+	}
 	return toSerialize, nil
 }
 

hindsight-clients/go/model_bank_template_directive.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.3
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.