Merge branch 'main' into docs-chat-cloud-first

* docs(nemoclaw): prioritize Hindsight Cloud with callout banners

* feat(nemoclaw): default --api-url to Hindsight Cloud, make it optional

.env.example

@@ -2,7 +2,7 @@
 # Copy this file to .env and fill in your values
 
 # LLM Configuration (Required)
-# Supported providers: openai, groq, ollama, gemini, anthropic, lmstudio, vertexai, minimax, deepseek, volcano
+# Supported providers: openai, groq, ollama, gemini, anthropic, lmstudio, vertexai, minimax, deepseek, zai, volcano
 HINDSIGHT_API_LLM_PROVIDER=openai
 HINDSIGHT_API_LLM_API_KEY=your-api-key-here
 HINDSIGHT_API_LLM_MODEL=gpt-4o-mini
@@ -30,6 +30,11 @@ HINDSIGHT_API_LLM_BASE_URL=https://api.openai.com/v1
 # HINDSIGHT_API_LLM_API_KEY=your-deepseek-api-key
 # HINDSIGHT_API_LLM_MODEL=deepseek-v4-flash  # or deepseek-v4-pro / deepseek-chat / deepseek-reasoner
 
+# Example: z.ai configuration (Zhipu GLM series, https://z.ai)
+# HINDSIGHT_API_LLM_PROVIDER=zai
+# HINDSIGHT_API_LLM_API_KEY=your-zai-api-key
+# HINDSIGHT_API_LLM_MODEL=glm-4.5-flash  # or glm-4.5-air for the paid tier
+
 # Example: LM Studio local configuration (Qwen 2.5 32B recommended)
 # HINDSIGHT_API_LLM_PROVIDER=lmstudio
 # HINDSIGHT_API_LLM_API_KEY=lmstudio
@@ -49,6 +54,7 @@ HINDSIGHT_API_LOG_LEVEL=info
 
 # Database (Optional - uses embedded pg0 by default)
 # HINDSIGHT_API_DATABASE_URL=postgresql://user:pass@host:5432/db
+# HINDSIGHT_API_READ_DATABASE_URL=  # Optional read-replica URL. When set, recall queries (semantic, BM25, graph, temporal) flow through a separate pool against this URL, offloading the primary. Typically points to a read-only endpoint (CNPG's <cluster>-ro service or Aurora reader endpoint).
 # HINDSIGHT_API_MIGRATION_DATABASE_URL=  # Direct PostgreSQL URL for migrations (bypasses PgBouncer). Falls back to DATABASE_URL.
 # HINDSIGHT_API_DATABASE_SCHEMA=public  # PostgreSQL schema name (default: public)
 
@@ -59,12 +65,25 @@ HINDSIGHT_API_LOG_LEVEL=info
 # HINDSIGHT_API_VECTOR_EXTENSION=pgvectorscale  # Auto-detects pg_diskann on Azure
 
 # Embeddings Configuration (Optional - uses local by default)
-# Provider: "local" (default) or "tei" (HuggingFace Text Embeddings Inference)
+# Provider: "local" (default), "tei", "openai", "cohere", "google", "openrouter", "litellm", or "litellm-sdk"
 # HINDSIGHT_API_EMBEDDINGS_PROVIDER=local
 # For local provider:
 # HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL=BAAI/bge-small-en-v1.5
+# Optional for China network / restricted HF access:
+# HF_ENDPOINT=https://hf-mirror.com
 # For TEI provider:
 # HINDSIGHT_API_EMBEDDINGS_TEI_URL=http://localhost:8080
+# For OpenAI-compatible embeddings:
+# HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=sk-xxxx
+# HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL=text-embedding-3-small
+# HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL=https://api.openai.com/v1
+#
+# IMPORTANT: Embedding keys require provider-specific names:
+# HINDSIGHT_API_EMBEDDINGS_{PROVIDER}_{PARAMETER}
+# (for example, HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL).
+#
+# DeepSeek note: DeepSeek is supported for LLM calls, but not for embeddings.
+# If using DeepSeek as LLM provider, keep embeddings on local/openai/cohere/google/etc.
 
 # Reranker Configuration (Optional - uses local by default)
 # Provider: "local" (default) or "tei" (HuggingFace Text Embeddings Inference)
@@ -88,3 +107,15 @@ HINDSIGHT_API_LOG_LEVEL=info
 # Custom service name and environment (optional, defaults: hindsight-api, development)
 # HINDSIGHT_API_OTEL_SERVICE_NAME=hindsight-production
 # HINDSIGHT_API_OTEL_DEPLOYMENT_ENVIRONMENT=production
+
+# -----------------------------------------------------------------------------
+# Control Plane (Optional)
+# -----------------------------------------------------------------------------
+
+# Dataplane API URL - where the CP proxies requests to
+# HINDSIGHT_CP_DATAPLANE_API_URL=http://localhost:8888
+
+# Optional: Require a shared access key to view the Control Plane UI.
+# When set, visitors see a login page and must enter the key before
+# accessing the dashboard or any /api/* routes (except /api/health).
+# HINDSIGHT_CP_ACCESS_KEY=your-shared-secret-key

.github/workflows/perf-test.yml

@@ -25,10 +25,10 @@ on:
           - recall-with-observations
           - consolidation
         default: ""
-      locomo_max_conversations:
-        description: "LoComo max conversations (0 = skip, blank = all)"
-        type: number
-        default: 0
+      locomo_conversations:
+        description: "LoComo conversation IDs (space-separated). Blank = curated set (conv-26 conv-30 conv-43)."
+        type: string
+        default: ""
       locomo_skip:
         description: "Skip LoComo job"
         type: boolean
@@ -83,46 +83,36 @@ jobs:
       run: |
         cd hindsight-dev && uv sync --frozen --all-extras --index-strategy unsafe-best-match
 
-    - name: "Suite: retain"
-      if: inputs.suite == '' || inputs.suite == 'retain'
+    - name: Run perf-test
       run: |
+        SUITE_ARG=""
+        if [ -n "${{ inputs.suite }}" ]; then
+          SUITE_ARG="--suite ${{ inputs.suite }}"
+        fi
         ./scripts/benchmarks/run-perf-test.sh \
           --scale ${{ inputs.scale || 'large' }} \
-          --suite retain \
-          --output perf-results-retain.json
-
-    - name: "Suite: recall"
-      if: inputs.suite == '' || inputs.suite == 'recall'
-      run: |
-        ./scripts/benchmarks/run-perf-test.sh \
-          --scale ${{ inputs.scale || 'large' }} \
-          --suite recall \
-          --output perf-results-recall.json
-
-    - name: "Suite: recall-with-observations"
-      if: inputs.suite == '' || inputs.suite == 'recall-with-observations'
-      run: |
-        ./scripts/benchmarks/run-perf-test.sh \
-          --scale ${{ inputs.scale || 'large' }} \
-          --suite recall-with-observations \
-          --output perf-results-recall-with-observations.json
-
-    - name: "Suite: consolidation"
-      if: inputs.suite == '' || inputs.suite == 'consolidation'
-      run: |
-        ./scripts/benchmarks/run-perf-test.sh \
-          --scale ${{ inputs.scale || 'large' }} \
-          --suite consolidation \
-          --output perf-results-consolidation.json
+          $SUITE_ARG \
+          --output perf-results.json
 
     - name: Upload perf results
       if: always()
       uses: actions/upload-artifact@v7
       with:
         name: perf-results-${{ github.sha }}
-        path: hindsight-dev/perf-results-*.json
+        path: hindsight-dev/perf-results.json
         retention-days: 90
 
+    # Publish enriched results (perf JSON + commit metadata) to the dashboard
+    # repo's gh-pages branch. The static site at
+    #   https://vectorize-io.github.io/hindsight-continuous-performance-monitor/
+    # reads data/index.json + data/<run>.json and renders charts client-side.
+    - name: Publish to dashboard
+      if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+      env:
+        PERF_DASHBOARD_TOKEN: ${{ secrets.PERF_DASHBOARD_TOKEN }}
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: ./scripts/benchmarks/publish-perf-results.sh hindsight-dev/perf-results.json
+
   locomo:
     if: inputs.locomo_skip != true
     runs-on: ubuntu-latest
@@ -179,14 +169,20 @@ jobs:
         cd hindsight-dev && uv sync --frozen --all-extras --index-strategy unsafe-best-match
 
     - name: Run LoComo benchmark
+      # Curated 3-conversation subset (best/middle/worst by accuracy on the
+      # last successful full run): conv-26 (best), conv-30 (middle), conv-43
+      # (worst). Excludes conv-44, the bank with the largest unconsolidated
+      # set that has been pushing scheduled runs over the per-bank
+      # _wait_for_consolidation timeout. Override via workflow_dispatch with
+      # the locomo_conversations input.
       run: |
-        MAX_CONV_ARG=""
-        if [ "${{ inputs.locomo_max_conversations }}" != "0" ] && [ -n "${{ inputs.locomo_max_conversations }}" ]; then
-          MAX_CONV_ARG="--max-conversations ${{ inputs.locomo_max_conversations }}"
+        CONVERSATIONS="${{ inputs.locomo_conversations }}"
+        if [ -z "$CONVERSATIONS" ]; then
+          CONVERSATIONS="conv-26 conv-30 conv-43"
         fi
         uv run python hindsight-dev/benchmarks/locomo/locomo_benchmark.py \
           --wait-consolidation \
-          $MAX_CONV_ARG
+          --conversation $CONVERSATIONS
 
     - name: Upload LoComo results
       if: always()
@@ -195,3 +191,10 @@ jobs:
         name: locomo-results-${{ github.sha }}
         path: hindsight-dev/benchmarks/locomo/results/
         retention-days: 90
+
+    - name: Publish LoComo to dashboard
+      if: success() && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch')
+      env:
+        PERF_DASHBOARD_TOKEN: ${{ secrets.PERF_DASHBOARD_TOKEN }}
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: ./scripts/benchmarks/publish-locomo-results.sh hindsight-dev/benchmarks/locomo/results/benchmark_results.json

.github/workflows/release-integration.yml

@@ -117,7 +117,7 @@ jobs:
       working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
       run: |
         set +e
-        OUTPUT=$(npm publish --access public 2>&1)
+        OUTPUT=$(npm publish --access public --provenance 2>&1)
         EXIT_CODE=$?
         echo "$OUTPUT"
         if [ $EXIT_CODE -ne 0 ]; then

.github/workflows/release.yml

@@ -314,6 +314,7 @@ jobs:
     permissions:
       contents: read
       packages: write
+      id-token: write
     strategy:
       matrix:
         include:
@@ -410,6 +411,7 @@ jobs:
 
     # Build multi-platform and push to release tags
     - name: Build and push release images
+      id: build
       uses: docker/build-push-action@v7
       with:
         context: .
@@ -421,6 +423,31 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
 
+    - name: Install cosign
+      uses: sigstore/cosign-installer@v3
+
+    - name: Sign published images
+      env:
+        TAGS: ${{ steps.meta.outputs.tags }}
+        DIGEST: ${{ steps.build.outputs.digest }}
+      run: |
+        set -euo pipefail
+        refs=()
+        while IFS= read -r tag; do
+          [[ -z "${tag}" ]] && continue
+          refs+=("${tag}@${DIGEST}")
+        done <<< "${TAGS}"
+        cosign sign --yes "${refs[@]}"
+
+    - name: Verify signature on primary tag
+      env:
+        IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}
+        DIGEST: ${{ steps.build.outputs.digest }}
+      run: |
+        cosign verify "${IMAGE}@${DIGEST}" \
+          --certificate-identity-regexp "^https://github\.com/${{ github.repository }}/\.github/workflows/release\.yml@.*" \
+          --certificate-oidc-issuer https://token.actions.githubusercontent.com
+
   release-helm-chart:
     runs-on: ubuntu-latest
     permissions:

.github/workflows/sign-images.yml

@@ -0,0 +1,71 @@
+name: Sign published images
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to sign (without leading v, e.g. 0.6.0)'
+        required: true
+        type: string
+        default: '0.6.0'
+
+permissions:
+  contents: read
+  packages: write
+  id-token: write
+
+jobs:
+  sign:
+    name: Sign ${{ matrix.image }}:${{ inputs.version }}${{ matrix.suffix }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { image: hindsight-api, suffix: '' }
+          - { image: hindsight-api, suffix: '-slim' }
+          - { image: hindsight-control-plane, suffix: '' }
+          - { image: hindsight, suffix: '' }
+          - { image: hindsight, suffix: '-slim' }
+
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install cosign
+        uses: sigstore/cosign-installer@v3
+
+      - name: Resolve image digest
+        id: resolve
+        env:
+          IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}
+          TAG: ${{ inputs.version }}${{ matrix.suffix }}
+        run: |
+          set -euo pipefail
+          DIGEST=$(docker buildx imagetools inspect "${IMAGE}:${TAG}" --format '{{json .Manifest.Digest}}' | tr -d '"')
+          if [[ -z "${DIGEST}" || "${DIGEST}" != sha256:* ]]; then
+            echo "Failed to resolve digest for ${IMAGE}:${TAG} (got: ${DIGEST})" >&2
+            exit 1
+          fi
+          echo "Resolved ${IMAGE}:${TAG} -> ${DIGEST}"
+          echo "ref=${IMAGE}@${DIGEST}" >> "$GITHUB_OUTPUT"
+
+      - name: Sign image
+        env:
+          REF: ${{ steps.resolve.outputs.ref }}
+        run: cosign sign --yes "${REF}"
+
+      - name: Verify signature
+        env:
+          REF: ${{ steps.resolve.outputs.ref }}
+        run: |
+          cosign verify "${REF}" \
+            --certificate-identity-regexp "^https://github\.com/${{ github.repository }}/\.github/workflows/sign-images\.yml@.*" \
+            --certificate-oidc-issuer https://token.actions.githubusercontent.com

.github/workflows/test.yml

@@ -3,8 +3,6 @@ name: CI
 on:
   pull_request:
     branches: [ main ]
-  pull_request_review:
-    types: [ submitted ]
   workflow_dispatch:
 
 concurrency:
@@ -13,10 +11,6 @@ concurrency:
 
 jobs:
   detect-changes:
-    # Skip non-approved pull_request_review events
-    if: >-
-      github.event_name != 'pull_request_review' ||
-      github.event.review.state == 'approved'
     runs-on: ubuntu-latest
     permissions:
       pull-requests: read
@@ -47,24 +41,22 @@ jobs:
       integrations-llamaindex: ${{ steps.filter.outputs.integrations-llamaindex }}
       integrations-paperclip: ${{ steps.filter.outputs.integrations-paperclip }}
       integrations-opencode: ${{ steps.filter.outputs.integrations-opencode }}
+      integrations-n8n: ${{ steps.filter.outputs.integrations-n8n }}
       integrations-cloudflare-oauth-proxy: ${{ steps.filter.outputs.integrations-cloudflare-oauth-proxy }}
       integrations-lockfiles: ${{ steps.filter.outputs.integrations-lockfiles }}
       integrations-openai-agents: ${{ steps.filter.outputs.integrations-openai-agents }}
       integrations-pipecat: ${{ steps.filter.outputs.integrations-pipecat }}
       integrations-agentcore: ${{ steps.filter.outputs.integrations-agentcore }}
       integrations-smolagents: ${{ steps.filter.outputs.integrations-smolagents }}
+      integrations-dify: ${{ steps.filter.outputs.integrations-dify }}
       tools-agent-sdk: ${{ steps.filter.outputs.tools-agent-sdk }}
-      tools-self-driving-agents: ${{ steps.filter.outputs.tools-self-driving-agents }}
       dev: ${{ steps.filter.outputs.dev }}
       ci: ${{ steps.filter.outputs.ci }}
-      # Secrets are available for internal PRs, pull_request_review, and workflow_dispatch.
+      # Secrets are available for internal PRs and workflow_dispatch.
       # Fork PRs via pull_request event do NOT have access to secrets.
       has_secrets: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
     steps:
     - uses: actions/checkout@v6
-      with:
-        # For pull_request_review, checkout the PR head (not the base branch)
-        ref: ${{ github.event_name == 'pull_request_review' && github.event.pull_request.head.sha || '' }}
 
     - uses: dorny/paths-filter@v4
       id: filter
@@ -133,6 +125,8 @@ jobs:
             - 'hindsight-integrations/paperclip/**'
           integrations-opencode:
             - 'hindsight-integrations/opencode/**'
+          integrations-n8n:
+            - 'hindsight-integrations/n8n/**'
           integrations-cloudflare-oauth-proxy:
             - 'hindsight-integrations/cloudflare-oauth-proxy/**'
           integrations-lockfiles:
@@ -147,10 +141,10 @@ jobs:
             - 'hindsight-integrations/agentcore/**'
           integrations-smolagents:
             - 'hindsight-integrations/smolagents/**'
+          integrations-dify:
+            - 'hindsight-integrations/dify/**'
           tools-agent-sdk:
             - 'hindsight-tools/hindsight-agent-sdk/**'
-          tools-self-driving-agents:
-            - 'hindsight-tools/self-driving-agents/**'
           dev:
             - 'hindsight-dev/**'
           ci:
@@ -166,7 +160,6 @@ jobs:
   check-integration-lockfiles:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-lockfiles == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -182,7 +175,6 @@ jobs:
   build-api-python-versions:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -213,7 +205,6 @@ jobs:
   build-typescript-client:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.clients-ts == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -240,7 +231,6 @@ jobs:
   build-hindsight-all-npm:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.all-npm == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -270,7 +260,6 @@ jobs:
   build-openclaw-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-openclaw == 'true' ||
       needs.detect-changes.outputs.clients-ts == 'true' ||
@@ -326,7 +315,6 @@ jobs:
   smoke-openclaw-install:
     needs: [detect-changes, build-openclaw-integration]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-openclaw == 'true' ||
       needs.detect-changes.outputs.clients-ts == 'true' ||
@@ -384,7 +372,6 @@ jobs:
   test-claude-code-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-claude-code == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -410,7 +397,6 @@ jobs:
   test-codex-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-codex == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -436,7 +422,6 @@ jobs:
   build-ai-sdk-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-ai-sdk == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -467,7 +452,6 @@ jobs:
   test-ai-sdk-integration-deno:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-ai-sdk == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -499,7 +483,6 @@ jobs:
   test-opencode-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-opencode == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -527,10 +510,39 @@ jobs:
       working-directory: ./hindsight-integrations/opencode
       run: npm run build
 
+  test-n8n-integration:
+    needs: [detect-changes]
+    if: >-
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-n8n == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v6
+      with:
+        node-version: '22'
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/n8n
+      run: npm install --no-fund --no-audit
+
+    - name: Run tests
+      working-directory: ./hindsight-integrations/n8n
+      run: npm test
+
+    - name: Build
+      working-directory: ./hindsight-integrations/n8n
+      run: npm run build
+
   test-hindsight-agent-sdk:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.tools-agent-sdk == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -560,43 +572,9 @@ jobs:
     - name: Build
       run: npm run build --workspace=hindsight-tools/hindsight-agent-sdk
 
-  test-self-driving-agents:
-    needs: [detect-changes]
-    if: >-
-      github.event_name != 'pull_request_review' &&
-      (github.event_name == 'workflow_dispatch' ||
-      needs.detect-changes.outputs.tools-self-driving-agents == 'true' ||
-      needs.detect-changes.outputs.ci == 'true')
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        ref: ${{ github.event.pull_request.head.sha || '' }}
-
-    - name: Set up Node.js
-      uses: actions/setup-node@v6
-      with:
-        node-version: '22'
-        cache: 'npm'
-        cache-dependency-path: package-lock.json
-
-    - name: Install root workspace dependencies
-      run: npm ci
-
-    - name: Build hindsight-client (self-driving-agents dep)
-      run: npm run build --workspace=hindsight-clients/typescript
-
-    - name: Run tests
-      run: npm test --workspace=hindsight-tools/self-driving-agents
-
-    - name: Build
-      run: npm run build --workspace=hindsight-tools/self-driving-agents
-
   test-cloudflare-oauth-proxy-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-cloudflare-oauth-proxy == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -627,7 +605,6 @@ jobs:
   build-chat-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-chat == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -658,7 +635,6 @@ jobs:
   test-paperclip-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-paperclip == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -689,7 +665,6 @@ jobs:
   test-pipecat-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-pipecat == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -727,7 +702,6 @@ jobs:
   build-control-plane:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.control-plane == 'true' ||
       needs.detect-changes.outputs.clients-ts == 'true' ||
@@ -788,7 +762,6 @@ jobs:
   build-docs:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.docs == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -816,8 +789,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.cli == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -945,7 +917,6 @@ jobs:
   lint-helm-chart:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.helm == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -968,8 +939,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.docker == 'true' ||
       needs.detect-changes.outputs.control-plane == 'true' ||
@@ -1062,8 +1032,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
     runs-on: ubuntu-latest
@@ -1133,7 +1102,111 @@ jobs:
 
     - name: Run tests
       working-directory: ./hindsight-api-slim
-      run: uv run pytest tests -v
+      run: uv run pytest tests -v -m "not hs_llm_mat"
+
+  test-api-llm-acceptance:
+    needs: [detect-changes]
+    if: >-
+      needs.detect-changes.outputs.has_secrets == 'true' &&
+      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
+      github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.core == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - provider: vertexai
+            model: google/gemini-2.5-flash-lite
+          - provider: gemini
+            model: gemini-2.5-flash-lite
+            api_key_secret: GEMINI_API_KEY
+          - provider: openai
+            model: gpt-4.1-nano
+            api_key_secret: OPENAI_API_KEY
+          - provider: groq
+            model: openai/gpt-oss-20b
+            api_key_secret: GROQ_API_KEY
+          - provider: bedrock
+            model: us.amazon.nova-2-lite-v1:0
+          - provider: litellmrouter
+            model: gpt-4.1-nano
+            # Single-deployment chain over OpenAI — verifies the Router-backed
+            # call path works end-to-end. Built from secrets in the step below.
+    name: LLM acceptance (${{ matrix.provider }}/${{ matrix.model }})
+    env:
+      HINDSIGHT_API_LLM_PROVIDER: ${{ matrix.provider }}
+      HINDSIGHT_API_LLM_MODEL: ${{ matrix.model }}
+      HINDSIGHT_API_LLM_API_KEY: ${{ matrix.api_key_secret && secrets[matrix.api_key_secret] || '' }}
+      HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY: /tmp/gcp-credentials.json
+      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }}
+      HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Setup GCP credentials
+      run: |
+        printf '%s' '${{ secrets.GCP_VERTEXAI_CREDENTIALS }}' > /tmp/gcp-credentials.json
+        PROJECT_ID=$(jq -r '.project_id' /tmp/gcp-credentials.json)
+        echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version-file: ".python-version"
+
+    - name: Install dependencies
+      working-directory: ./hindsight-api-slim
+      run: uv sync --frozen --all-extras --index-strategy unsafe-best-match
+
+    - name: Cache HuggingFace models
+      uses: actions/cache@v5
+      with:
+        path: ~/.cache/huggingface
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-huggingface-
+
+    - name: Pre-download models
+      working-directory: ./hindsight-api-slim
+      run: |
+        uv run python -c "
+        from sentence_transformers import SentenceTransformer, CrossEncoder
+        SentenceTransformer('BAAI/bge-small-en-v1.5')
+        CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
+        "
+
+    - name: Build litellmrouter config
+      if: matrix.provider == 'litellmrouter'
+      env:
+        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        ROUTER_MODEL: ${{ matrix.model }}
+      run: |
+        cfg=$(jq -nc \
+          --arg model "openai/$ROUTER_MODEL" \
+          --arg key "$OPENAI_API_KEY" \
+          '{model_list: [{model_name: "default", litellm_params: {model: $model, api_key: $key}}]}')
+        echo "::add-mask::$cfg"
+        echo "HINDSIGHT_API_LLM_LITELLMROUTER_CONFIG=$cfg" >> "$GITHUB_ENV"
+
+    - name: Run LLM acceptance tests
+      working-directory: ./hindsight-api-slim
+      run: uv run pytest tests -v -m "hs_llm_mat" --timeout 600
 
   test-api-oracle:
     needs: [detect-changes]
@@ -1142,8 +1215,7 @@ jobs:
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
       contains(github.event.pull_request.labels.*.name, 'oracle-tests') &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
     runs-on: ubuntu-latest
@@ -1261,8 +1333,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.clients-python == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -1373,8 +1444,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.clients-ts == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -1503,8 +1573,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.clients-python == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -1663,8 +1732,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.clients-ts == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -1823,8 +1891,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.clients-ts == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -1945,7 +2012,6 @@ jobs:
   build-rust-cli-arm64:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.cli == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -1978,8 +2044,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.clients-rust == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -2094,8 +2159,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.clients-go == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -2208,8 +2272,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.integrations-openclaw == 'true' ||
       needs.detect-changes.outputs.embed == 'true' ||
@@ -2341,8 +2404,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.integration-tests == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -2447,7 +2509,6 @@ jobs:
   test-ag2-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-ag2 == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -2484,7 +2545,6 @@ jobs:
   test-smolagents-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-smolagents == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -2518,10 +2578,43 @@ jobs:
       working-directory: ./hindsight-integrations/smolagents
       run: uv run pytest tests -v
 
-  test-crewai-integration:
+  test-dify-integration:
     needs: [detect-changes]
     if: >-
       github.event_name != 'pull_request_review' &&
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-dify == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        # Dify plugin runtime targets Python 3.12 (see manifest.yaml)
+        python-version: '3.12'
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/dify
+      run: uv pip install --system -e . pytest pytest-mock
+
+    - name: Run tests
+      working-directory: ./hindsight-integrations/dify
+      run: pytest tests -v
+
+  test-crewai-integration:
+    needs: [detect-changes]
+    if: >-
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-crewai == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -2558,7 +2651,6 @@ jobs:
   test-litellm-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-litellm == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -2595,7 +2687,6 @@ jobs:
   test-pydantic-ai-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-pydantic-ai == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -2632,7 +2723,6 @@ jobs:
   test-llamaindex-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-llamaindex == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -2669,7 +2759,6 @@ jobs:
   test-openai-agents-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-openai-agents == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -2706,7 +2795,6 @@ jobs:
   test-agentcore-integration:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.integrations-agentcore == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -2744,8 +2832,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
     runs-on: ubuntu-latest
@@ -2813,8 +2900,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.embed == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -2881,8 +2967,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.embed == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -3059,8 +3144,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.hindsight-all == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -3144,8 +3228,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.clients-ts == 'true' ||
       needs.detect-changes.outputs.clients-python == 'true' ||
@@ -3297,8 +3380,7 @@ jobs:
     needs: [detect-changes]
     if: >-
       needs.detect-changes.outputs.has_secrets == 'true' &&
-      ((github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.dev == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -3380,8 +3462,9 @@ jobs:
         done
 
   verify-generated-files:
-    if: github.event_name != 'pull_request_review'
     runs-on: ubuntu-latest
+    env:
+      UV_FROZEN: "1"
     steps:
     - uses: actions/checkout@v6
       with:
@@ -3462,7 +3545,6 @@ jobs:
   check-openapi-compatibility:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.ci == 'true')
@@ -3514,7 +3596,6 @@ jobs:
   check-cli-coverage:
     needs: [detect-changes]
     if: >-
-      github.event_name != 'pull_request_review' &&
       (github.event_name == 'workflow_dispatch' ||
       needs.detect-changes.outputs.core == 'true' ||
       needs.detect-changes.outputs.cli == 'true' ||
@@ -3585,6 +3666,7 @@ jobs:
       - test-integration
       - test-ag2-integration
       - test-smolagents-integration
+      - test-dify-integration
       - test-crewai-integration
       - test-litellm-integration
       - test-pydantic-ai-integration
@@ -3595,7 +3677,6 @@ jobs:
       - test-embed-windows
       - test-hindsight-all
       - test-hindsight-agent-sdk
-      - test-self-driving-agents
       - test-doc-examples
       - test-upgrade
       - verify-generated-files
@@ -3684,4 +3765,4 @@ jobs:
                 issue_number: prNumber,
                 body,
               });
-            }
+            }

README.md

@@ -30,7 +30,7 @@ It eliminates the shortcomings of alternative techniques such as RAG and knowled
 
 Hindsight is the most accurate agent memory system ever tested according to benchmark performance. It has achieved state-of-the-art performance on the LongMemEval benchmark, widely used to assess memory system performance across a variety of conversational AI scenarios. The current reported performance of Hindsight and other agent memory solutions as of January 2026 is shown here:
 
-![Overview](./hindsight-docs/static/img/hindsight-bench.jpg)
+![Overview](./hindsight-docs/static/img/hindsight-benchmarks.png)
 
 The benchmark performance data for Hindsight has been independently reproduced by research collaborators at the Virginia Tech [Sanghani Center for Artificial Intelligence and Data Analytics](https://sanghani.cs.vt.edu/) and The Washington Post. Other scores are self-reported by software vendors.
 

deno.lock

@@ -42,6 +42,16 @@
   },
   "workspace": {
     "members": {
+      "hindsight-all-npm": {
+        "packageJson": {
+          "dependencies": [
+            "npm:@types/node@22",
+            "npm:tsup@^8.5.1",
+            "npm:typescript@^5.7.0",
+            "npm:vitest@^4.1.2"
+          ]
+        }
+      },
       "hindsight-clients/typescript": {
         "packageJson": {
           "dependencies": [
@@ -58,6 +68,7 @@
       "hindsight-control-plane": {
         "packageJson": {
           "dependencies": [
+            "npm:@chenglou/pretext@^0.0.3",
             "npm:@eslint/eslintrc@^3.3.3",
             "npm:@eslint/js@^9.39.2",
             "npm:@radix-ui/react-alert-dialog@^1.1.15",
@@ -91,11 +102,12 @@
             "npm:eslint@^9.39.1",
             "npm:[email protected]",
             "npm:next-themes@~0.4.6",
-            "npm:next@^16.1.6",
+            "npm:next@^16.1.7",
             "npm:postcss@^8.5.6",
             "npm:prettier@^3.7.4",
             "npm:react-chrono@^2.9.1",
             "npm:react-dom@^19.2.0",
+            "npm:react-is@^19.2.4",
             "npm:react-markdown@^10.1.0",
             "npm:react18-json-view@~0.2.9",
             "npm:react@^19.2.0",
@@ -133,6 +145,26 @@
             "npm:typescript@~5.6.2"
           ]
         }
+      },
+      "hindsight-tools/hindsight-agent-sdk": {
+        "packageJson": {
+          "dependencies": [
+            "npm:@vectorize-io/hindsight-client@~0.5.6",
+            "npm:typescript@^5.4.0",
+            "npm:vitest@^4.1.2"
+          ]
+        }
+      },
+      "hindsight-tools/self-driving-agents": {
+        "packageJson": {
+          "dependencies": [
+            "npm:@clack/prompts@^1.2.0",
+            "npm:@vectorize-io/hindsight-client@~0.5.6",
+            "npm:picocolors@^1.1.0",
+            "npm:typescript@^5.4.0",
+            "npm:vitest@^4.1.2"
+          ]
+        }
       }
     }
   }

docker/docker-compose/alloydb/docker-compose.yaml

@@ -0,0 +1,90 @@
+name: hindsight
+# Docker Compose file for Hindsight with AlloyDB Omni and ScaNN
+# Uses Google's free AlloyDB Omni container image: https://hub.docker.com/r/google/alloydbomni
+#
+# Usage:
+# docker compose -f docker/docker-compose/alloydb/docker-compose.yaml up -d
+#
+# Make sure to set the required environment variables before running:
+# - HINDSIGHT_DB_PASSWORD: password for the AlloyDB Omni/PostgreSQL user
+# - Configure LLM provider variables as needed (see the hindsight service below)
+#
+# Optional environment variables with defaults:
+# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
+# - HINDSIGHT_DB_VERSION: AlloyDB Omni image tag (default: 17)
+# - HINDSIGHT_DB_USER: database user (default: hindsight_user)
+# - HINDSIGHT_DB_NAME: database name (default: hindsight_db)
+
+services:
+  db:
+    image: google/alloydbomni:${HINDSIGHT_DB_VERSION:-17}
+    container_name: hindsight-db-alloydb
+    restart: always
+    ports:
+      - "5438:5432"
+    environment:
+      POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
+      POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:-hindsight_password}
+      POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
+    volumes:
+      - alloydb_data:/var/lib/postgresql/data
+    networks:
+      - hindsight-net
+
+  alloydb-init:
+    image: google/alloydbomni:${HINDSIGHT_DB_VERSION:-17}
+    depends_on:
+      - db
+    environment:
+      - PGPASSWORD=${HINDSIGHT_DB_PASSWORD:-hindsight_password}
+    command:
+      - bash
+      - -c
+      - |
+        echo 'Waiting for AlloyDB Omni to be ready...'
+        until pg_isready -h hindsight-db-alloydb -p 5432 -U ${HINDSIGHT_DB_USER:-hindsight_user}; do
+          echo 'AlloyDB Omni is unavailable - sleeping'
+          sleep 2
+        done
+        echo 'AlloyDB Omni is ready - creating ${HINDSIGHT_DB_NAME:-hindsight_db} database'
+        psql -h hindsight-db-alloydb -p 5432 -U ${HINDSIGHT_DB_USER:-hindsight_user} -c 'CREATE DATABASE ${HINDSIGHT_DB_NAME:-hindsight_db};' 2>/dev/null || echo 'Database already exists'
+        echo 'Creating vector and alloydb_scann extensions'
+        psql -h hindsight-db-alloydb -p 5432 -U ${HINDSIGHT_DB_USER:-hindsight_user} -d ${HINDSIGHT_DB_NAME:-hindsight_db} -c 'CREATE EXTENSION IF NOT EXISTS vector;'
+        psql -h hindsight-db-alloydb -p 5432 -U ${HINDSIGHT_DB_USER:-hindsight_user} -d ${HINDSIGHT_DB_NAME:-hindsight_db} -c 'CREATE EXTENSION IF NOT EXISTS alloydb_scann CASCADE;'
+        echo 'Database and extensions created successfully'
+    restart: "no"
+    networks:
+      - hindsight-net
+
+  hindsight:
+    image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
+    container_name: hindsight-app
+    ports:
+      - "8888:8888"
+      - "9999:9999"
+    environment:
+      # LLM Configuration
+      HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai}
+      HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key}
+
+      # Database Configuration
+      HINDSIGHT_API_DATABASE_URL: postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:-hindsight_password}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
+
+      # Vector and Text Search Extensions
+      HINDSIGHT_API_VECTOR_EXTENSION: scann
+      HINDSIGHT_API_TEXT_SEARCH_EXTENSION: native
+
+    depends_on:
+      db:
+        condition: service_started
+      alloydb-init:
+        condition: service_completed_successfully
+    networks:
+      - hindsight-net
+
+networks:
+  hindsight-net:
+    driver: bridge
+
+volumes:
+  alloydb_data:

docker/docker-compose/custom-models/Dockerfile

@@ -0,0 +1,34 @@
+# Example: custom Hindsight image with non-default local models baked in.
+#
+# Use this pattern in production when you run a non-default embedder or
+# reranker. Baking models into the image removes the runtime dependency on
+# HuggingFace and lets the container registry handle caching per node, so
+# you don't need a model-cache PVC.
+#
+# Built on top of the slim image so only the deps and models you actually
+# use end up in the final image.
+FROM ghcr.io/vectorize-io/hindsight:latest-slim
+
+# Install the local-ml deps required to load sentence-transformers /
+# cross-encoder models at runtime. Pinned ranges mirror hindsight-api-slim's
+# `local-ml` extra in hindsight-api-slim/pyproject.toml. Use `uv pip
+# install` against the image's venv explicitly: the slim image's venv was
+# created by `uv sync` and does not ship its own `pip`, so a bare
+# `pip install` would fall back to user site-packages and not be visible
+# to the runtime python.
+RUN uv pip install --python /app/api/.venv/bin/python --no-cache \
+    'sentence-transformers>=3.3.0' \
+    'transformers>=4.53.0' \
+    'torch>=2.6.0'
+
+# Pre-download the models you want to use. Replace these with your own.
+# The defaults bundled in the full image are BAAI/bge-small-en-v1.5 and
+# cross-encoder/ms-marco-MiniLM-L-6-v2; here we pick multilingual variants
+# as a concrete non-default example.
+ARG EMBEDDER=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+ARG RERANKER=cross-encoder/mmarco-mMiniLMv2-L12-H384-v1
+ENV HF_HUB_DOWNLOAD_TIMEOUT=600
+RUN python -c "\
+from sentence_transformers import SentenceTransformer, CrossEncoder; \
+SentenceTransformer('${EMBEDDER}'); \
+CrossEncoder('${RERANKER}')"

docker/docker-compose/custom-models/README.md

@@ -0,0 +1,81 @@
+# Hindsight with Custom Local Models
+
+Example Docker Compose setup that builds a Hindsight image with **non-default
+local embedder and reranker models baked in at build time**.
+
+This is the recommended pattern for production when you use a non-default
+local model: the container registry caches model layers per node, pod
+startup is deterministic, and you don't need a model-cache PVC (or any
+runtime dependency on HuggingFace).
+
+## When to use this
+
+- You override `HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL` or
+  `HINDSIGHT_API_RERANKER_LOCAL_MODEL` to a non-default model.
+- You want pod startup to be deterministic and offline-capable.
+- You'd otherwise reach for a Helm `modelCache` PVC just to avoid
+  re-downloading models.
+
+If you're using the **default** local models, the published full image
+(`ghcr.io/vectorize-io/hindsight:latest`) already bakes them in — you don't
+need this example.
+
+If you're using **external** providers (TEI, OpenAI, Cohere, ...) for
+embeddings and reranking, use the slim image directly — no models are
+needed in the image.
+
+## Quick start
+
+```bash
+export OPENAI_API_KEY=sk-xxx
+
+docker compose -f docker/docker-compose/custom-models/docker-compose.yaml up --build
+```
+
+- API: http://localhost:8888
+- Control Plane: http://localhost:9999
+
+## Using your own models
+
+Override the build args to bake different models:
+
+```bash
+docker compose -f docker/docker-compose/custom-models/docker-compose.yaml build \
+  --build-arg EMBEDDER=your-org/your-embedder \
+  --build-arg RERANKER=your-org/your-reranker
+```
+
+Then update the matching `HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL` and
+`HINDSIGHT_API_RERANKER_LOCAL_MODEL` values in `docker-compose.yaml` so the
+runtime points at the same model IDs.
+
+## Verifying the models are baked in
+
+`docker-compose.yaml` sets `HF_HUB_OFFLINE=1` and `TRANSFORMERS_OFFLINE=1`
+so that any attempt to download a model at runtime fails loudly instead of
+silently re-downloading. If the container starts and serves recall queries
+with these set, the models are correctly baked in.
+
+You can also inspect the image directly:
+
+```bash
+docker run --rm --entrypoint sh hindsight-custom-models-hindsight \
+  -c 'ls ~/.cache/huggingface/hub/'
+```
+
+## Why not a model-cache PVC?
+
+The Helm chart exposes an optional `api.persistence.modelCache` PVC for
+caching downloaded models across pod restarts. Compared to baking models
+into the image:
+
+- A PVC adds storage cost — one PVC per worker replica with
+  `volumeClaimTemplates`.
+- `ReadWriteOnce` (the default) pins pods to a node.
+- The PVC needs lifecycle management on `helm uninstall` / `helm upgrade`
+  — without `helm.sh/resource-policy: keep` it is deleted on uninstall;
+  with it, storage keeps billing forever until manually cleaned up.
+- Pod startup still depends on HuggingFace being reachable on first run.
+
+Image layers, by contrast, are pulled once per node and cached for free by
+the container runtime, with no orphaned-storage cleanup story.

docker/docker-compose/custom-models/docker-compose.yaml

@@ -0,0 +1,44 @@
+name: hindsight-custom-models
+# Example: run a custom Hindsight image with non-default local models baked
+# in at build time, so pod startup does not depend on HuggingFace at runtime.
+#
+# Quick start:
+#   export OPENAI_API_KEY=sk-xxx
+#   docker compose -f docker/docker-compose/custom-models/docker-compose.yaml up --build
+#
+# Required environment variables:
+# - OPENAI_API_KEY (or configure another LLM provider via HINDSIGHT_API_LLM_*)
+
+services:
+  hindsight:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      # Override at build time to bake different models:
+      #   docker compose build --build-arg EMBEDDER=your-org/your-embedder
+      args:
+        EMBEDDER: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+        RERANKER: cross-encoder/mmarco-mMiniLMv2-L12-H384-v1
+    container_name: hindsight-custom-models
+    ports:
+      - "8888:8888"
+      - "9999:9999"
+    environment:
+      HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai}
+      HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key}
+
+      # Point Hindsight at the models baked into the image above.
+      HINDSIGHT_API_EMBEDDINGS_PROVIDER: local
+      HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+      HINDSIGHT_API_RERANKER_PROVIDER: local
+      HINDSIGHT_API_RERANKER_LOCAL_MODEL: cross-encoder/mmarco-mMiniLMv2-L12-H384-v1
+
+      # Fail fast if a model is missing from the image instead of silently
+      # falling back to a HuggingFace download at runtime.
+      HF_HUB_OFFLINE: "1"
+      TRANSFORMERS_OFFLINE: "1"
+    volumes:
+      - pg_data:/home/hindsight/.pg0
+
+volumes:
+  pg_data:

docker/docker-compose/nginx/docker-compose.yml

@@ -51,6 +51,8 @@ services:
 
       # Control Plane config
       HINDSIGHT_CP_DATAPLANE_API_URL: http://localhost:8888
+      # Optional: Require a shared access key for Control Plane UI access
+      # HINDSIGHT_CP_ACCESS_KEY: your-secret-key
     volumes:
       # Persist embedded pg0 database
       - hindsight_data:/app/data

docker/standalone/Dockerfile

@@ -172,6 +172,10 @@ USER hindsight
 # "Permission denied" error when mounting a fresh root-owned volume.
 RUN mkdir -p /home/hindsight/.pg0
 
+# Make /home/hindsight traversable when running with --user UID:GID overrides
+# (default 0700 blocks traversal by non-owner UIDs needed for bind-mount ownership matching)
+RUN chmod 755 /home/hindsight
+
 ENV PATH="/app/api/.venv/bin:${PATH}"
 
 # Pre-download tiktoken encoding (ALWAYS - required for token counting even in air-gapped envs)
@@ -328,6 +332,10 @@ USER hindsight
 # "Permission denied" error when mounting a fresh root-owned volume.
 RUN mkdir -p /home/hindsight/.pg0
 
+# Make /home/hindsight traversable when running with --user UID:GID overrides
+# (default 0700 blocks traversal by non-owner UIDs needed for bind-mount ownership matching)
+RUN chmod 755 /home/hindsight
+
 ENV PATH="/app/api/.venv/bin:${PATH}"
 
 # Pre-download tiktoken encoding (ALWAYS - required for token counting even in air-gapped envs)

helm/hindsight/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: hindsight
 description: Hindsight helm chart
 type: application
-version: 0.5.6
-appVersion: "0.5.6"
+version: 0.6.2
+appVersion: "0.6.2"
 keywords:
   - ai
   - memory

helm/hindsight/values.yaml

@@ -70,6 +70,12 @@ api:
   # Persistent volume for local model cache (reranker, embeddings)
   # Models are downloaded to /home/hindsight/.cache on first use.
   # Without persistence, models are re-downloaded on every pod restart.
+  #
+  # For production, prefer baking models into a custom image instead of
+  # enabling this PVC: image layers are pulled once per node and cached
+  # for free, while a PVC adds storage cost, pins pods to a node
+  # (ReadWriteOnce), and needs lifecycle management on uninstall/upgrade.
+  # See docs: developer/installation#bundling-custom-models-in-a-custom-image
   persistence:
     modelCache:
       enabled: false
@@ -168,7 +174,10 @@ worker:
   # affinity: {}
 
   # Persistent volume for local model cache (reranker, embeddings)
-  # Uses volumeClaimTemplates since worker is a StatefulSet.
+  # Uses volumeClaimTemplates since worker is a StatefulSet — one PVC per
+  # replica. For production, prefer baking models into a custom image; see
+  # api.persistence.modelCache above and docs:
+  # developer/installation#bundling-custom-models-in-a-custom-image
   persistence:
     modelCache:
       enabled: false

hindsight-all-npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vectorize-io/hindsight-all",
-  "version": "0.5.6",
+  "version": "0.6.2",
   "description": "Node.js programmatic lifecycle manager for Hindsight — embeds a local hindsight daemon in a Node application. Pair with @vectorize-io/hindsight-client for memory operations.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

hindsight-all-slim/pyproject.toml

@@ -4,12 +4,12 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hindsight-all-slim"
-version = "0.5.6"
+version = "0.6.2"
 description = "Hindsight: Agent Memory That Works Like Human Memory - Slim All-in-One Bundle"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "hindsight-api-slim>=0.4.17",
+    "hindsight-api-slim==0.6.2",
     "hindsight-client>=0.0.7",
     "hindsight-embed>=0.1.0",
 ]

hindsight-all/pyproject.toml

@@ -4,12 +4,12 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-all"
-version = "0.5.6"
+version = "0.6.2"
 description = "Hindsight: Agent Memory That Works Like Human Memory - All-in-One Bundle"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "hindsight-api-slim[all]>=0.4.17",
+    "hindsight-api-slim[all]==0.6.2",
     "hindsight-client>=0.0.7",
     "hindsight-embed>=0.1.0",
 ]
@@ -21,7 +21,7 @@ hindsight-embed = { workspace = true }
 
 [project.optional-dependencies]
 local-llm = [
-    "hindsight-api-slim[local-llm]>=0.4.17",
+    "hindsight-api-slim[local-llm]==0.6.2",
 ]
 test = [
     "pytest>=7.0.0",

hindsight-api-slim/hindsight_api/__init__.py

@@ -46,4 +46,4 @@ __all__ = [
     "RemoteTEICrossEncoder",
     "LLMConfig",
 ]
-__version__ = "0.5.6"
+__version__ = "0.6.2"

hindsight-api-slim/hindsight_api/_vector_index.py

@@ -0,0 +1,170 @@
+"""Shared PostgreSQL vector-extension dispatch helpers."""
+
+from __future__ import annotations
+
+import logging
+
+from sqlalchemy import text
+from sqlalchemy.engine import Connection
+
+logger = logging.getLogger(__name__)
+
+# Extensions a user can set via HINDSIGHT_API_VECTOR_EXTENSION.
+CONFIGURABLE_EXTENSIONS = ("pgvector", "pgvectorscale", "vchord", "scann")
+
+# Extensions detect_vector_extension() can return. pg_diskann is a runtime-only
+# resolution from a configured "pgvectorscale" backend on Azure (uses a different
+# WITH clause), never a value the user sets directly.
+RESOLVED_EXTENSIONS = (*CONFIGURABLE_EXTENSIONS, "pg_diskann")
+
+# Backwards-compatible alias for older imports.
+VALID_EXTENSIONS = CONFIGURABLE_EXTENSIONS
+
+SCANN_MIN_ROWS_FOR_AUTO_INDEX = 10_000
+
+
+_EXTENSION_NAMES = {
+    "pgvector": "vector",
+    "pgvectorscale": "vectorscale",
+    "vchord": "vchord",
+    "scann": "alloydb_scann",
+}
+
+_INDEX_USING_CLAUSES = {
+    "pgvector": "USING hnsw (embedding vector_cosine_ops)",
+    "pgvectorscale": "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)",
+    "pg_diskann": "USING diskann (embedding vector_cosine_ops) WITH (max_neighbors = 50)",
+    "vchord": "USING vchordrq (embedding vector_l2_ops)",
+    "scann": "USING scann (embedding cosine) WITH (mode = 'AUTO')",
+}
+
+_INDEX_TYPE_KEYWORDS = {
+    "pgvector": "hnsw",
+    "pgvectorscale": "diskann",
+    "pg_diskann": "diskann",
+    "vchord": "vchordrq",
+    "scann": "scann",
+}
+
+_EXTENSION_INSTALL_SQL = {
+    "pgvector": ("CREATE EXTENSION IF NOT EXISTS vector",),
+    "pgvectorscale": (
+        "CREATE EXTENSION IF NOT EXISTS vector",
+        "CREATE EXTENSION IF NOT EXISTS vectorscale CASCADE",
+    ),
+    "vchord": ("CREATE EXTENSION IF NOT EXISTS vchord CASCADE",),
+    "scann": (
+        "CREATE EXTENSION IF NOT EXISTS vector",
+        "CREATE EXTENSION IF NOT EXISTS alloydb_scann CASCADE",
+    ),
+}
+
+_INSTALL_HINTS = {
+    "pgvector": "CREATE EXTENSION vector;",
+    "pgvectorscale": "CREATE EXTENSION vector; then CREATE EXTENSION vectorscale CASCADE; (or pg_diskann on Azure)",
+    "vchord": "CREATE EXTENSION vchord CASCADE;",
+    "scann": "CREATE EXTENSION vector; then CREATE EXTENSION alloydb_scann CASCADE;",
+}
+
+
+def validate_extension(name: str) -> str:
+    """Return a normalized configurable vector extension name or raise.
+
+    Used at the user-facing config boundary; pg_diskann is rejected here because
+    it is a detection-time alias, never a value the user sets directly.
+    """
+    ext = name.lower()
+    if ext not in CONFIGURABLE_EXTENSIONS:
+        valid = ", ".join(CONFIGURABLE_EXTENSIONS)
+        raise ValueError(f"Invalid vector_extension: {name}. Must be one of: {valid}")
+    return ext
+
+
+def _normalize_resolved(name: str) -> str:
+    """Normalize either a user-configurable or detect-time extension name."""
+    ext = name.lower()
+    if ext not in RESOLVED_EXTENSIONS:
+        valid = ", ".join(RESOLVED_EXTENSIONS)
+        raise ValueError(f"Unknown vector extension: {name}. Must be one of: {valid}")
+    return ext
+
+
+def pg_extension_name(ext: str) -> str:
+    """Return the PostgreSQL extension name for a configured vector backend."""
+    return _EXTENSION_NAMES[validate_extension(ext)]
+
+
+def index_using_clause(ext: str) -> str:
+    """Return the CREATE INDEX USING clause for the vector backend."""
+    return _INDEX_USING_CLAUSES[_normalize_resolved(ext)]
+
+
+def index_type_keyword(ext: str) -> str:
+    """Return the keyword that identifies this index type in pg_indexes.indexdef."""
+    return _INDEX_TYPE_KEYWORDS[_normalize_resolved(ext)]
+
+
+def minimum_rows_for_index(ext: str) -> int:
+    """Return the minimum populated embedding rows before creating this index type."""
+    return SCANN_MIN_ROWS_FOR_AUTO_INDEX if _normalize_resolved(ext) == "scann" else 0
+
+
+def should_defer_index_creation(ext: str, row_count: int) -> bool:
+    """Return True when index creation should wait for more embeddings."""
+    minimum_rows = minimum_rows_for_index(ext)
+    return minimum_rows > 0 and row_count < minimum_rows
+
+
+def uses_per_bank_vector_indexes(ext: str) -> bool:
+    """Return whether the backend should create per-bank partial vector indexes."""
+    return _normalize_resolved(ext) != "scann"
+
+
+def bootstrap_extension(conn: Connection, ext: str) -> None:
+    """Install the configured vector extension and any prerequisites if possible."""
+    normalized = validate_extension(ext)
+    for statement in _EXTENSION_INSTALL_SQL[normalized]:
+        conn.execute(text(statement))
+
+
+def detect_vector_extension(conn: Connection, vector_extension: str = "pgvector") -> str:
+    """Validate the configured vector extension exists and return the index backend."""
+    configured_ext = validate_extension(vector_extension)
+
+    if configured_ext == "pgvectorscale":
+        pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
+        if not pgvector_check:
+            raise RuntimeError(
+                "DiskANN (pgvectorscale/pg_diskann) requires pgvector to be installed. "
+                f"Install it with: {_INSTALL_HINTS['pgvectorscale']}"
+            )
+
+        vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar()
+        pg_diskann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_diskann'")).scalar()
+
+        if vectorscale_check:
+            logger.debug("Using vector extension: pgvectorscale (DiskANN)")
+            return "pgvectorscale"
+        if pg_diskann_check:
+            logger.debug("Using vector extension: pg_diskann (Azure DiskANN)")
+            return "pg_diskann"
+
+        raise RuntimeError(
+            "Configured vector extension 'pgvectorscale' not found. Install either:\n"
+            "  - pgvectorscale (open source): CREATE EXTENSION vectorscale CASCADE;\n"
+            "  - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
+        )
+
+    extension_name = pg_extension_name(configured_ext)
+    extension_check = conn.execute(
+        text("SELECT 1 FROM pg_extension WHERE extname = :extension_name"),
+        {"extension_name": extension_name},
+    ).scalar()
+    if not extension_check:
+        raise RuntimeError(
+            f"Configured vector extension '{configured_ext}' not found. "
+            f"Install it with: {_INSTALL_HINTS[configured_ext]}"
+        )
+
+    logger.debug("Using configured vector extension: %s", configured_ext)
+    return configured_ext

hindsight-api-slim/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py

@@ -26,52 +26,67 @@ depends_on: str | Sequence[str] | None = None
 
 def _detect_vector_extension() -> str:
     """
-    Detect or validate vector extension: 'pgvector', 'vchord', or 'pgvectorscale'.
+    Detect or validate vector extension for this immutable migration revision.
     Respects HINDSIGHT_API_VECTOR_EXTENSION env var if set.
     """
     conn = op.get_bind()
     vector_extension = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
 
-    # Validate configured extension is installed
     if vector_extension == "pgvectorscale":
-        # pgvectorscale/DiskANN requires pgvector
         pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
         if not pgvector_check:
             raise RuntimeError(
                 "DiskANN requires pgvector. Install with: CREATE EXTENSION vector; then vectorscale or pg_diskann CASCADE;"
             )
-        # Check for either vectorscale (open source) or pg_diskann (Azure)
         vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar()
         pg_diskann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_diskann'")).scalar()
 
         if vectorscale_check:
             return "pgvectorscale"
-        elif pg_diskann_check:
+        if pg_diskann_check:
             return "pg_diskann"
-        else:
-            raise RuntimeError(
-                "Configured vector extension 'pgvectorscale' not found. Install either:\n"
-                "  - pgvectorscale: CREATE EXTENSION vectorscale CASCADE;\n"
-                "  - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
-            )
-    elif vector_extension == "vchord":
+        raise RuntimeError(
+            "Configured vector extension 'pgvectorscale' not found. Install either:\n"
+            "  - pgvectorscale: CREATE EXTENSION vectorscale CASCADE;\n"
+            "  - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
+        )
+    if vector_extension == "vchord":
         vchord_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord'")).scalar()
         if not vchord_check:
             raise RuntimeError(
                 "Configured vector extension 'vchord' not found. Install it with: CREATE EXTENSION vchord CASCADE;"
             )
         return "vchord"
-    elif vector_extension == "pgvector":
+    if vector_extension == "scann":
+        scann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'alloydb_scann'")).scalar()
+        if not scann_check:
+            raise RuntimeError(
+                "Configured vector extension 'scann' not found. Install it with: CREATE EXTENSION alloydb_scann CASCADE;"
+            )
+        return "scann"
+    if vector_extension == "pgvector":
         pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
         if not pgvector_check:
             raise RuntimeError(
                 "Configured vector extension 'pgvector' not found. Install it with: CREATE EXTENSION vector;"
             )
         return "pgvector"
-    else:
-        raise ValueError(
-            f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {vector_extension}. Must be 'pgvector', 'vchord', or 'pgvectorscale'"
-        )
+    raise ValueError(
+        "Invalid HINDSIGHT_API_VECTOR_EXTENSION: "
+        f"{vector_extension}. Must be 'pgvector', 'vchord', 'pgvectorscale', or 'scann'"
+    )
+
+
+def _vector_index_using_clause(ext: str) -> str:
+    if ext == "pgvectorscale":
+        return "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)"
+    if ext == "pg_diskann":
+        return "USING diskann (embedding vector_cosine_ops) WITH (max_neighbors = 50)"
+    if ext == "vchord":
+        return "USING vchordrq (embedding vector_l2_ops)"
+    if ext == "scann":
+        return "USING scann (embedding cosine) WITH (mode = 'AUTO')"
+    return "USING hnsw (embedding vector_cosine_ops)"
 
 
 def _detect_text_search_extension() -> str:
@@ -313,36 +328,11 @@ def _pg_upgrade() -> None:
     )
     # Create vector index - conditional based on available extension
     vector_ext = _detect_vector_extension()
-
-    if vector_ext == "pgvectorscale":
-        # Use DiskANN index for pgvectorscale (disk-based, scalable)
-        op.execute("""
+    if vector_ext != "scann":
+        op.execute(f"""
             CREATE INDEX idx_memory_units_embedding ON memory_units
-            USING diskann (embedding vector_cosine_ops)
-            WITH (num_neighbors = 50)
+            {_vector_index_using_clause(vector_ext)}
         """)
-    elif vector_ext == "pg_diskann":
-        # Use DiskANN index for pg_diskann (Azure)
-        op.execute("""
-            CREATE INDEX idx_memory_units_embedding ON memory_units
-            USING diskann (embedding vector_cosine_ops)
-            WITH (max_neighbors = 50)
-        """)
-    elif vector_ext == "vchord":
-        # Use vchordrq index for vchord (supports high-dimensional embeddings)
-        op.execute("""
-            CREATE INDEX idx_memory_units_embedding ON memory_units
-            USING vchordrq (embedding vector_l2_ops)
-        """)
-    else:  # pgvector
-        # Use HNSW index for pgvector
-        op.create_index(
-            "idx_memory_units_embedding",
-            "memory_units",
-            ["embedding"],
-            postgresql_using="hnsw",
-            postgresql_ops={"embedding": "vector_cosine_ops"},
-        )
 
     # Create full-text search index on search_vector
     # Index type depends on text search backend

hindsight-api-slim/hindsight_api/alembic/versions/86f7a033d372_repair_mental_models_subtype_at_head.py

@@ -0,0 +1,117 @@
+"""Repair mental_models.subtype on databases stuck at m3rg3h3ad5f6
+
+Three production deployments reported `column "subtype" of relation
+"mental_models" does not exist` on `create_mental_model` even after their
+container reported `Database migrations completed successfully` and
+`alembic_version` advanced to `m3rg3h3ad5f6` (see issue #1553, #1553#1
+confirmations from @4Lienau and @khanhduyvt0101).
+
+Both `h3c4d5e6f7g8_mental_models_v4` and `d5y6z7a8b9c0_backfill_mental_models_subtype`
+were meant to ensure `subtype` exists, but on databases that came through the
+`reflections -> mental_models` rename chain *and* whose alembic_version
+advanced past `d5y6z7a8b9c0` along an alternate path during the divergent-heads
+reorganization, neither column-add actually fired. The result is a head-tagged
+database with a v3-shaped `mental_models` table missing six columns:
+``subtype``, ``description``, ``entity_id``, ``observations``, ``links``,
+``last_updated``.
+
+This migration sits at the current head (`m3rg3h3ad5f6`) so every affected
+deployment will pick it up on next container start. It mirrors the column-add
+block from `d5y6z7a8b9c0_backfill_mental_models_subtype` using
+``ADD COLUMN IF NOT EXISTS`` so it is a no-op on databases where the columns
+are already present.
+
+Revision ID: 86f7a033d372
+Revises: m3rg3h3ad5f6
+Create Date: 2026-05-14
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "86f7a033d372"
+down_revision: str | Sequence[str] | None = "m3rg3h3ad5f6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _pg_schema_prefix() -> str:
+    """Schema-qualifier for raw SQL on PG (multi-tenant search_path)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def _pg_upgrade() -> None:
+    """Idempotently ensure mental_models has the v4 column set.
+
+    Safe to re-apply on databases that already received the columns via
+    `h3c4d5e6f7g8_mental_models_v4` or `d5y6z7a8b9c0_backfill_mental_models_subtype` —
+    every column-add uses ``IF NOT EXISTS`` and the constraint is recreated
+    from scratch with the canonical v4 allowlist.
+    """
+    schema = _pg_schema_prefix()
+    bare_schema = schema.strip(".").strip('"') if schema else ""
+    schema_clause = f"AND table_schema = '{bare_schema}'" if bare_schema else ""
+
+    # Wrapped in a DO block so the existence check skips databases that
+    # predate the reflections -> mental_models rename chain (no table to
+    # repair). On those, every ALTER below would error.
+    op.execute(
+        f"""
+        DO $$
+        BEGIN
+            IF EXISTS (
+                SELECT 1 FROM information_schema.tables
+                WHERE table_name = 'mental_models'
+                  {schema_clause}
+            ) THEN
+                -- Add the six v4 columns idempotently.
+                ALTER TABLE {schema}mental_models
+                    ADD COLUMN IF NOT EXISTS subtype VARCHAR(32) NOT NULL DEFAULT 'structural';
+                ALTER TABLE {schema}mental_models
+                    ADD COLUMN IF NOT EXISTS description TEXT NOT NULL DEFAULT '';
+                ALTER TABLE {schema}mental_models
+                    ADD COLUMN IF NOT EXISTS entity_id UUID;
+                ALTER TABLE {schema}mental_models
+                    ADD COLUMN IF NOT EXISTS observations JSONB DEFAULT '{{"observations": []}}'::jsonb;
+                ALTER TABLE {schema}mental_models
+                    ADD COLUMN IF NOT EXISTS links VARCHAR[];
+                ALTER TABLE {schema}mental_models
+                    ADD COLUMN IF NOT EXISTS last_updated TIMESTAMP WITH TIME ZONE;
+
+                -- Recreate the CHECK constraint with the canonical v4 allowlist.
+                -- Existing rows with subtype = 'directive' (possible on databases
+                -- that ran the o0j1k2l3m4n5 directive-only path) are rewritten to
+                -- 'structural' first so the constraint add succeeds.
+                UPDATE {schema}mental_models SET subtype = 'structural' WHERE subtype = 'directive';
+
+                ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype;
+                ALTER TABLE {schema}mental_models
+                    ADD CONSTRAINT ck_mental_models_subtype
+                    CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'));
+
+                CREATE INDEX IF NOT EXISTS idx_mental_models_subtype
+                    ON {schema}mental_models(bank_id, subtype);
+            END IF;
+        END$$;
+        """
+    )
+
+
+def _pg_downgrade() -> None:
+    """No-op: dropping these columns would corrupt v4 application code."""
+    pass
+
+
+def upgrade() -> None:
+    # PG-only: Oracle's baseline (o1a2b3c4d5e6) creates mental_models with its
+    # own subtype shape (chk_mm_subtype IN ('directive', 'pinned')) and a
+    # different table topology, so this PG-shaped repair does not apply.
+    run_for_dialect(pg=_pg_upgrade)
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade)

hindsight-api-slim/hindsight_api/alembic/versions/a4b5c6d7e8f9_fix_per_bank_vector_index_type.py

@@ -11,7 +11,8 @@ was configured.
 
 This migration detects the mismatch and recreates the affected indexes with
 the correct type. Skipped entirely when the configured extension is pgvector
-(the default), since those indexes are already correct.
+(the default) or scann. ScaNN uses global vector indexes because empty or tiny
+per-bank indexes cannot be built safely on AlloyDB.
 """
 
 import os
@@ -39,39 +40,47 @@ def _get_schema_prefix() -> str:
     return f'"{schema}".' if schema else ""
 
 
-def _target_index_type() -> str | None:
-    """Return the target index type, or None if pgvector (no fix needed)."""
-    ext = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
+def _validate_extension(name: str) -> str:
+    ext = name.lower()
+    if ext not in {"pgvector", "pgvectorscale", "vchord", "scann"}:
+        raise ValueError(
+            f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {ext}. Must be 'pgvector', 'vchord', 'pgvectorscale', or 'scann'"
+        )
+    return ext
+
+
+def _index_type_keyword(ext: str) -> str:
     if ext == "pgvectorscale":
         return "diskann"
-    elif ext == "vchord":
+    if ext == "vchord":
         return "vchordrq"
-    return None
+    if ext == "scann":
+        return "scann"
+    return "hnsw"
 
 
-def _vector_index_using_clause() -> str:
-    """Return the USING clause based on the configured vector extension."""
-    ext = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
+def _vector_index_using_clause(ext: str) -> str:
     if ext == "pgvectorscale":
         return "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)"
-    elif ext == "vchord":
+    if ext == "vchord":
         return "USING vchordrq (embedding vector_l2_ops)"
-    else:
-        return "USING hnsw (embedding vector_cosine_ops)"
+    if ext == "scann":
+        return "USING scann (embedding cosine) WITH (mode = 'AUTO')"
+    return "USING hnsw (embedding vector_cosine_ops)"
 
 
 def _pg_upgrade() -> None:
-    target = _target_index_type()
-    if target is None:
-        # pgvector — indexes are already HNSW, nothing to fix
+    ext = _validate_extension(os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector"))
+    if ext in {"pgvector", "scann"}:
         return
+    target = _index_type_keyword(ext)
 
     bind = op.get_bind()
     schema_name = context.config.get_main_option("target_schema")
     schema = _get_schema_prefix()
     table_ref = f'"{schema_name}".memory_units' if schema_name else "memory_units"
     banks_ref = f'"{schema_name}".banks' if schema_name else "banks"
-    using_clause = _vector_index_using_clause()
+    using_clause = _vector_index_using_clause(ext)
     pg_schema = schema_name or "public"
 
     rows = bind.execute(text(f"SELECT bank_id, internal_id FROM {banks_ref}")).fetchall()  # noqa: S608
@@ -117,8 +126,8 @@ def _pg_upgrade() -> None:
 
 def _pg_downgrade() -> None:
     # Downgrade recreates indexes as HNSW (the original hardcoded behavior)
-    target = _target_index_type()
-    if target is None:
+    ext = _validate_extension(os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector"))
+    if ext in {"pgvector", "scann"}:
         return
 
     bind = op.get_bind()

hindsight-api-slim/hindsight_api/alembic/versions/b5d4e3f2a1c9_backfill_entity_cooccurrences_event_time.py

@@ -0,0 +1,91 @@
+"""Backfill entity_cooccurrences.last_cooccurred from memory_units event time
+
+Revision ID: b5d4e3f2a1c9
+Revises: o1a2b3c4d5e6
+Create Date: 2026-04-24
+
+The writer path in `entity_resolver.link_units_to_entities_batch` historically
+stamped `entity_cooccurrences.last_cooccurred` with `datetime.now(UTC)` at
+flush time, ignoring the source memory unit's event date. For normal online
+retains that's fine (now ≈ event time), but for any corpus that was
+backfilled in a single session — migrating from another memory system, for
+example — every co-occurrence collapsed to the import moment, which hid the
+underlying knowledge timeline from the dashboard's entity graph recency heat
+and from any downstream consumer of the column.
+
+The writer is fixed in the same change set to propagate the unit's event_date;
+this migration repairs historical rows by reading the true event time off
+`unit_entities × memory_units` (falling back to `created_at` when
+`mentioned_at` / `occurred_start` are NULL, so rows never regress).
+
+Oracle slot is intentionally absent: the Oracle baseline (`o1a2b3c4d5e6`)
+landed days before this fix, so any Oracle deployment runs the corrected
+writer against an effectively empty `entity_cooccurrences` — there is no
+historical residue on Oracle to repair. PG-only matches the asymmetry of
+the data, not negligence.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "b5d4e3f2a1c9"
+down_revision: str | Sequence[str] | None = "o1a2b3c4d5e6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def _pg_upgrade() -> None:
+    schema = _get_schema_prefix()
+    # Recompute last_cooccurred from the true event time per entity pair.
+    # COALESCE picks the first non-null of mentioned_at / occurred_start /
+    # created_at so banks without event-time metadata still see a sane value
+    # (equivalent to the pre-fix behaviour) instead of NULL.
+    #
+    # The self-join on `unit_entities` is O(k²) per memory_unit in the number
+    # of distinct entities mentioned (k). For typical units k is small (single
+    # digits), but a bank with units containing hundreds of entities and tens
+    # of millions of co-occurrence rows may want to run this off-hours — the
+    # whole UPDATE is one statement, so it locks every targeted ec row for
+    # the duration. The migration is one-time; subsequent online writes
+    # already carry event time via the writer fix.
+    op.execute(
+        f"""
+        UPDATE {schema}entity_cooccurrences ec
+        SET last_cooccurred = sub.event_time
+        FROM (
+            SELECT
+                LEAST(ue1.entity_id, ue2.entity_id) AS e1,
+                GREATEST(ue1.entity_id, ue2.entity_id) AS e2,
+                MAX(COALESCE(mu.mentioned_at, mu.occurred_start, mu.created_at)) AS event_time
+            FROM {schema}memory_units mu
+            JOIN {schema}unit_entities ue1 ON ue1.unit_id = mu.id
+            JOIN {schema}unit_entities ue2 ON ue2.unit_id = mu.id AND ue1.entity_id <> ue2.entity_id
+            GROUP BY 1, 2
+        ) sub
+        WHERE ec.entity_id_1 = sub.e1 AND ec.entity_id_2 = sub.e2
+        """
+    )
+
+
+def _pg_downgrade() -> None:
+    # No-op: the previous column value was `now()` at the time of write and
+    # isn't recoverable. Rolling back the code is sufficient — new writes will
+    # revert to the old behaviour for subsequent retains.
+    pass
+
+
+def upgrade() -> None:
+    run_for_dialect(pg=_pg_upgrade)  # oracle slot intentionally absent — see header
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade)

hindsight-api-slim/hindsight_api/alembic/versions/d5e6f7a8b9c0_add_bank_internal_id_and_per_bank_hnsw.py

@@ -6,10 +6,11 @@ Create Date: 2026-03-11
 
 This migration:
 1. Adds internal_id UUID column to banks (stable identifier for index naming)
-2. Drops the global vector index (competes with per-bank partial indexes)
-3. Creates per-(bank_id, fact_type) partial vector indexes for all existing banks
-   using the configured vector extension (HNSW for pgvector, DiskANN for
-   pgvectorscale, vchordrq for vchord).
+2. For non-ScaNN backends, drops the global vector index (competes with
+   per-bank partial indexes)
+3. For non-ScaNN backends, creates per-(bank_id, fact_type) partial vector
+   indexes for all existing banks using the configured vector extension
+   (HNSW for pgvector, DiskANN for pgvectorscale, vchordrq for vchord).
    (new banks get indexes created at bank-creation time via bank_utils.create_bank_vector_indexes)
 
 Why per-(bank, fact_type) indexes:
@@ -17,6 +18,8 @@ Why per-(bank, fact_type) indexes:
   clause, because the idx_memory_units_bank_id B-tree index always wins at planning time.
 - Per-(bank, fact_type) partial indexes have both predicates matching → planner selects them.
 - The global vector index competes for larger partitions (world, observation) and must be dropped.
+- AlloyDB ScaNN uses global vector indexes with filtered vector search instead
+  because empty or tiny per-bank indexes cannot be built safely.
 """
 
 import os
@@ -39,22 +42,30 @@ _FACT_TYPES: dict[str, str] = {
 }
 
 
+def _configured_vector_extension() -> str:
+    ext = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
+    if ext not in {"pgvector", "pgvectorscale", "vchord", "scann"}:
+        raise ValueError(
+            f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {ext}. Must be 'pgvector', 'vchord', 'pgvectorscale', or 'scann'"
+        )
+    return ext
+
+
+def _vector_index_using_clause(ext: str) -> str:
+    if ext == "pgvectorscale":
+        return "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)"
+    if ext == "vchord":
+        return "USING vchordrq (embedding vector_l2_ops)"
+    if ext == "scann":
+        return "USING scann (embedding cosine) WITH (mode = 'AUTO')"
+    return "USING hnsw (embedding vector_cosine_ops)"
+
+
 def _get_schema_prefix() -> str:
     schema = context.config.get_main_option("target_schema")
     return f'"{schema}".' if schema else ""
 
 
-def _vector_index_using_clause() -> str:
-    """Return the USING clause based on the configured vector extension."""
-    ext = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
-    if ext == "pgvectorscale":
-        return "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)"
-    elif ext == "vchord":
-        return "USING vchordrq (embedding vector_l2_ops)"
-    else:
-        return "USING hnsw (embedding vector_cosine_ops)"
-
-
 def _pg_upgrade() -> None:
     schema = _get_schema_prefix()
 
@@ -64,6 +75,14 @@ def _pg_upgrade() -> None:
     )
     op.execute(f"ALTER TABLE {schema}banks ADD CONSTRAINT banks_internal_id_unique UNIQUE (internal_id)")
 
+    ext = _configured_vector_extension()
+    if ext == "scann":
+        # ScaNN should keep/use a global vector index. Per-bank partial indexes
+        # are created while banks are empty and can fail AlloyDB's ScaNN build
+        # requirements, so this migration leaves vector index reconciliation to
+        # runtime ensure_vector_extension once enough rows exist.
+        return
+
     # 2. Drop any fact_type-only partial indexes that may exist from prior migrations
     #    (bank_id B-tree always wins over them when bank_id is in the WHERE clause)
     op.execute(f"DROP INDEX IF EXISTS {schema}idx_mu_emb_world")
@@ -79,7 +98,7 @@ def _pg_upgrade() -> None:
     schema_name = context.config.get_main_option("target_schema")
     table_ref = f'"{schema_name}".memory_units' if schema_name else "memory_units"
     banks_ref = f'"{schema_name}".banks' if schema_name else "banks"
-    using_clause = _vector_index_using_clause()
+    using_clause = _vector_index_using_clause(ext)
 
     rows = bind.execute(text(f"SELECT bank_id, internal_id FROM {banks_ref}")).fetchall()  # noqa: S608
     for row in rows:
@@ -109,7 +128,7 @@ def _pg_downgrade() -> None:
     rows = bind.execute(text(f"SELECT internal_id FROM {banks_ref}")).fetchall()  # noqa: S608
     for row in rows:
         internal_id = str(row[0]).replace("-", "")[:16]
-        for ft_short in _HNSW_FACT_TYPES.values():
+        for ft_short in _FACT_TYPES.values():
             idx_name = f"idx_mu_emb_{ft_short}_{internal_id}"
             bind.execute(text(f"DROP INDEX IF EXISTS {schema}{idx_name}"))
 

hindsight-api-slim/hindsight_api/alembic/versions/m3rg3h3ad5f6_merge_deferrable_fk_and_cooccurrence_backfill.py

@@ -0,0 +1,31 @@
+"""Merge divergent heads from deferrable FK and cooccurrence backfill
+
+Revision ID: m3rg3h3ad5f6
+Revises: 9f8e7d6c5b4a, b5d4e3f2a1c9
+Create Date: 2026-05-04
+"""
+
+from collections.abc import Sequence
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "m3rg3h3ad5f6"
+down_revision: tuple[str, ...] = ("9f8e7d6c5b4a", "b5d4e3f2a1c9")
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _pg_upgrade() -> None:
+    pass
+
+
+def _pg_downgrade() -> None:
+    pass
+
+
+def upgrade() -> None:
+    run_for_dialect(pg=_pg_upgrade)
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade)

hindsight-api-slim/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py

@@ -32,53 +32,65 @@ def _get_schema_prefix() -> str:
 
 
 def _detect_vector_extension() -> str:
-    """
-    Detect or validate vector extension: 'pgvector', 'vchord', or 'pgvectorscale'.
-    Respects HINDSIGHT_API_VECTOR_EXTENSION env var if set.
-    """
+    """Detect or validate vector extension for this immutable migration revision."""
     conn = op.get_bind()
     vector_extension = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
 
-    # Validate configured extension is installed
     if vector_extension == "pgvectorscale":
-        # pgvectorscale/DiskANN requires pgvector
         pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
         if not pgvector_check:
             raise RuntimeError(
                 "DiskANN requires pgvector. Install with: CREATE EXTENSION vector; then vectorscale or pg_diskann CASCADE;"
             )
-        # Check for either vectorscale (open source) or pg_diskann (Azure)
         vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar()
         pg_diskann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_diskann'")).scalar()
 
         if vectorscale_check:
             return "pgvectorscale"
-        elif pg_diskann_check:
+        if pg_diskann_check:
             return "pg_diskann"
-        else:
-            raise RuntimeError(
-                "Configured vector extension 'pgvectorscale' not found. Install either:\n"
-                "  - pgvectorscale: CREATE EXTENSION vectorscale CASCADE;\n"
-                "  - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
-            )
-    elif vector_extension == "vchord":
+        raise RuntimeError(
+            "Configured vector extension 'pgvectorscale' not found. Install either:\n"
+            "  - pgvectorscale: CREATE EXTENSION vectorscale CASCADE;\n"
+            "  - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
+        )
+    if vector_extension == "vchord":
         vchord_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord'")).scalar()
         if not vchord_check:
             raise RuntimeError(
                 "Configured vector extension 'vchord' not found. Install it with: CREATE EXTENSION vchord CASCADE;"
             )
         return "vchord"
-    elif vector_extension == "pgvector":
+    if vector_extension == "scann":
+        scann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'alloydb_scann'")).scalar()
+        if not scann_check:
+            raise RuntimeError(
+                "Configured vector extension 'scann' not found. Install it with: CREATE EXTENSION alloydb_scann CASCADE;"
+            )
+        return "scann"
+    if vector_extension == "pgvector":
         pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
         if not pgvector_check:
             raise RuntimeError(
                 "Configured vector extension 'pgvector' not found. Install it with: CREATE EXTENSION vector;"
             )
         return "pgvector"
-    else:
-        raise ValueError(
-            f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {vector_extension}. Must be 'pgvector', 'vchord', or 'pgvectorscale'"
-        )
+    raise ValueError(
+        "Invalid HINDSIGHT_API_VECTOR_EXTENSION: "
+        f"{vector_extension}. Must be 'pgvector', 'vchord', 'pgvectorscale', or 'scann'"
+    )
+
+
+def _vector_index_using_clause(ext: str) -> str:
+    if ext == "pgvectorscale":
+        return "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)"
+    if ext == "pg_diskann":
+        return "USING diskann (embedding vector_cosine_ops) WITH (max_neighbors = 50)"
+    if ext == "vchord":
+        return "USING vchordrq (embedding vector_l2_ops)"
+    if ext == "scann":
+        return "USING scann (embedding cosine) WITH (mode = 'AUTO')"
+    return "USING hnsw (embedding vector_cosine_ops)"
 
 
 def _detect_text_search_extension() -> str:
@@ -158,28 +170,12 @@ def _pg_upgrade() -> None:
     # Indexes for learnings
     op.execute(f"CREATE INDEX idx_learnings_bank_id ON {schema}learnings(bank_id)")
 
-    # Create vector index based on detected extension
-    if vector_ext == "pgvectorscale":
+    # Create vector index based on detected extension. ScaNN is deferred because
+    # this table is empty during migration and AlloyDB rejects empty ScaNN builds.
+    if vector_ext != "scann":
         op.execute(f"""
             CREATE INDEX idx_learnings_embedding ON {schema}learnings
-            USING diskann (embedding vector_cosine_ops)
-            WITH (num_neighbors = 50)
-        """)
-    elif vector_ext == "pg_diskann":
-        op.execute(f"""
-            CREATE INDEX idx_learnings_embedding ON {schema}learnings
-            USING diskann (embedding vector_cosine_ops)
-            WITH (max_neighbors = 50)
-        """)
-    elif vector_ext == "vchord":
-        op.execute(f"""
-            CREATE INDEX idx_learnings_embedding ON {schema}learnings
-            USING vchordrq (embedding vector_l2_ops)
-        """)
-    else:  # pgvector
-        op.execute(f"""
-            CREATE INDEX idx_learnings_embedding ON {schema}learnings
-            USING hnsw (embedding vector_cosine_ops)
+            {_vector_index_using_clause(vector_ext)}
         """)
 
     op.execute(f"CREATE INDEX idx_learnings_tags ON {schema}learnings USING GIN(tags)")
@@ -237,28 +233,12 @@ def _pg_upgrade() -> None:
     # Indexes for pinned_reflections
     op.execute(f"CREATE INDEX idx_pinned_reflections_bank_id ON {schema}pinned_reflections(bank_id)")
 
-    # Create vector index based on detected extension
-    if vector_ext == "pgvectorscale":
+    # Create vector index based on detected extension. ScaNN is deferred because
+    # this table is empty during migration and AlloyDB rejects empty ScaNN builds.
+    if vector_ext != "scann":
         op.execute(f"""
             CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
-            USING diskann (embedding vector_cosine_ops)
-            WITH (num_neighbors = 50)
-        """)
-    elif vector_ext == "pg_diskann":
-        op.execute(f"""
-            CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
-            USING diskann (embedding vector_cosine_ops)
-            WITH (max_neighbors = 50)
-        """)
-    elif vector_ext == "vchord":
-        op.execute(f"""
-            CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
-            USING vchordrq (embedding vector_l2_ops)
-        """)
-    else:  # pgvector
-        op.execute(f"""
-            CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
-            USING hnsw (embedding vector_cosine_ops)
+            {_vector_index_using_clause(vector_ext)}
         """)
 
     op.execute(f"CREATE INDEX idx_pinned_reflections_tags ON {schema}pinned_reflections USING GIN(tags)")

hindsight-api-slim/hindsight_api/api/http.py

@@ -997,7 +997,7 @@ class BackgroundResponse(BaseModel):
 
 
 class BankListItem(BaseModel):
-    """Bank list item with profile summary."""
+    """Bank list item with profile summary and stats."""
 
     bank_id: str
     name: str | None = None
@@ -1005,6 +1005,8 @@ class BankListItem(BaseModel):
     mission: str | None = None
     created_at: str | None = None
     updated_at: str | None = None
+    fact_count: int = 0
+    last_document_at: str | None = None
 
 
 class BankListResponse(BaseModel):
@@ -1021,6 +1023,8 @@ class BankListResponse(BaseModel):
                         "mission": "I am a software engineer helping my team ship quality code",
                         "created_at": "2024-01-15T10:30:00Z",
                         "updated_at": "2024-01-16T14:20:00Z",
+                        "fact_count": 156,
+                        "last_document_at": "2024-01-16T14:20:00Z",
                     }
                 ]
             }

hindsight-api-slim/hindsight_api/config.py

@@ -14,6 +14,7 @@ from typing import Any, Literal
 
 from dotenv import find_dotenv, load_dotenv
 
+from ._vector_index import validate_extension
 from .utils import mask_network_location
 
 # Load .env file, searching current and parent directories (overrides existing env vars)
@@ -121,6 +122,9 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]:
 # Environment variable names
 ENV_DATABASE_BACKEND = "HINDSIGHT_API_DATABASE_BACKEND"
 ENV_DATABASE_URL = "HINDSIGHT_API_DATABASE_URL"
+ENV_READ_DATABASE_URL = "HINDSIGHT_API_READ_DATABASE_URL"
+ENV_READ_DB_POOL_MIN_SIZE = "HINDSIGHT_API_READ_DB_POOL_MIN_SIZE"
+ENV_READ_DB_POOL_MAX_SIZE = "HINDSIGHT_API_READ_DB_POOL_MAX_SIZE"
 ENV_MIGRATION_DATABASE_URL = "HINDSIGHT_API_MIGRATION_DATABASE_URL"
 ENV_DATABASE_SCHEMA = "HINDSIGHT_API_DATABASE_SCHEMA"
 ENV_LLM_PROVIDER = "HINDSIGHT_API_LLM_PROVIDER"
@@ -135,11 +139,23 @@ ENV_LLM_TIMEOUT = "HINDSIGHT_API_LLM_TIMEOUT"
 ENV_LLM_GROQ_SERVICE_TIER = "HINDSIGHT_API_LLM_GROQ_SERVICE_TIER"
 ENV_LLM_OPENAI_SERVICE_TIER = "HINDSIGHT_API_LLM_OPENAI_SERVICE_TIER"
 ENV_LLM_EXTRA_BODY = "HINDSIGHT_API_LLM_EXTRA_BODY"
+ENV_LLM_DEFAULT_HEADERS = "HINDSIGHT_API_LLM_DEFAULT_HEADERS"
+
+# LiteLLM Router chain — provider-specific config consumed by the "litellmrouter"
+# provider. Each entry is a deployment; the Router tries them in declared order and
+# falls back to the next on transient errors (5xx, rate-limit, timeout).
+# Provider-scoped naming mirrors other provider-specific flags (e.g. llm_groq_*,
+# llm_vertexai_*). Note the single token "LITELLMROUTER" — keeping it one word
+# disambiguates from the embeddings/reranker LITELLM_* settings.
+ENV_LLM_LITELLMROUTER_CONFIG = "HINDSIGHT_API_LLM_LITELLMROUTER_CONFIG"
 
 # Defaults for service tiers
 DEFAULT_LLM_GROQ_SERVICE_TIER = "auto"  # "on_demand", "flex", or "auto"
 DEFAULT_LLM_OPENAI_SERVICE_TIER = None  # None (default) or "flex" (50% cheaper)
 DEFAULT_LLM_EXTRA_BODY = None  # None = no extra body params; JSON dict merged into OpenAI extra_body
+DEFAULT_LLM_DEFAULT_HEADERS = (
+    None  # None = no extra headers; JSON dict passed as default_headers to provider SDK clients
+)
 
 # Per-operation LLM configuration (optional, falls back to global LLM config)
 ENV_RETAIN_LLM_PROVIDER = "HINDSIGHT_API_RETAIN_LLM_PROVIDER"
@@ -151,6 +167,7 @@ ENV_RETAIN_LLM_MAX_RETRIES = "HINDSIGHT_API_RETAIN_LLM_MAX_RETRIES"
 ENV_RETAIN_LLM_INITIAL_BACKOFF = "HINDSIGHT_API_RETAIN_LLM_INITIAL_BACKOFF"
 ENV_RETAIN_LLM_MAX_BACKOFF = "HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF"
 ENV_RETAIN_LLM_TIMEOUT = "HINDSIGHT_API_RETAIN_LLM_TIMEOUT"
+ENV_RETAIN_LLM_LITELLMROUTER_CONFIG = "HINDSIGHT_API_RETAIN_LLM_LITELLMROUTER_CONFIG"
 
 ENV_REFLECT_LLM_PROVIDER = "HINDSIGHT_API_REFLECT_LLM_PROVIDER"
 ENV_REFLECT_LLM_API_KEY = "HINDSIGHT_API_REFLECT_LLM_API_KEY"
@@ -161,6 +178,7 @@ ENV_REFLECT_LLM_MAX_RETRIES = "HINDSIGHT_API_REFLECT_LLM_MAX_RETRIES"
 ENV_REFLECT_LLM_INITIAL_BACKOFF = "HINDSIGHT_API_REFLECT_LLM_INITIAL_BACKOFF"
 ENV_REFLECT_LLM_MAX_BACKOFF = "HINDSIGHT_API_REFLECT_LLM_MAX_BACKOFF"
 ENV_REFLECT_LLM_TIMEOUT = "HINDSIGHT_API_REFLECT_LLM_TIMEOUT"
+ENV_REFLECT_LLM_LITELLMROUTER_CONFIG = "HINDSIGHT_API_REFLECT_LLM_LITELLMROUTER_CONFIG"
 
 ENV_CONSOLIDATION_LLM_PROVIDER = "HINDSIGHT_API_CONSOLIDATION_LLM_PROVIDER"
 ENV_CONSOLIDATION_LLM_API_KEY = "HINDSIGHT_API_CONSOLIDATION_LLM_API_KEY"
@@ -171,6 +189,7 @@ ENV_CONSOLIDATION_LLM_MAX_RETRIES = "HINDSIGHT_API_CONSOLIDATION_LLM_MAX_RETRIES
 ENV_CONSOLIDATION_LLM_INITIAL_BACKOFF = "HINDSIGHT_API_CONSOLIDATION_LLM_INITIAL_BACKOFF"
 ENV_CONSOLIDATION_LLM_MAX_BACKOFF = "HINDSIGHT_API_CONSOLIDATION_LLM_MAX_BACKOFF"
 ENV_CONSOLIDATION_LLM_TIMEOUT = "HINDSIGHT_API_CONSOLIDATION_LLM_TIMEOUT"
+ENV_CONSOLIDATION_LLM_LITELLMROUTER_CONFIG = "HINDSIGHT_API_CONSOLIDATION_LLM_LITELLMROUTER_CONFIG"
 
 ENV_EMBEDDINGS_PROVIDER = "HINDSIGHT_API_EMBEDDINGS_PROVIDER"
 ENV_EMBEDDINGS_LOCAL_MODEL = "HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL"
@@ -451,6 +470,8 @@ PROVIDER_DEFAULT_MODELS = {
     "groq": "openai/gpt-oss-120b",
     "minimax": "MiniMax-M2.7",
     "deepseek": "deepseek-v4-flash",
+    "zai": "glm-4.5-flash",
+    "opencode-go": "deepseek-v4-flash",
     "ollama": "gemma3:12b",
     "llamacpp": "gemma-4-e2b-it",
     "lmstudio": "local-model",
@@ -488,7 +509,7 @@ DEFAULT_LLM_GEMINI_SAFETY_SETTINGS = None  # None = use Gemini default safety se
 
 DEFAULT_EMBEDDINGS_PROVIDER = "local"
 DEFAULT_EMBEDDINGS_LOCAL_MODEL = "BAAI/bge-small-en-v1.5"
-DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU = False  # Force CPU mode for local embeddings (avoids MPS/XPC issues on macOS)
+DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU = False  # Force CPU mode for local embeddings
 DEFAULT_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE = False  # Security: disabled by default, required for some models
 DEFAULT_EMBEDDINGS_OPENAI_MODEL = "text-embedding-3-small"
 DEFAULT_EMBEDDINGS_OPENAI_BATCH_SIZE = 100
@@ -499,7 +520,7 @@ DEFAULT_EMBEDDING_DIMENSION = 384
 
 DEFAULT_RERANKER_PROVIDER = "local"
 DEFAULT_RERANKER_LOCAL_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"
-DEFAULT_RERANKER_LOCAL_FORCE_CPU = False  # Force CPU mode for local reranker (avoids MPS/XPC issues on macOS)
+DEFAULT_RERANKER_LOCAL_FORCE_CPU = False  # Force CPU mode for local reranker
 DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT = 4  # Limit concurrent CPU-bound reranking to prevent thrashing
 DEFAULT_RERANKER_LOCAL_TRUST_REMOTE_CODE = (
     False  # Security: disabled by default, required for some models like jina-reranker-v2
@@ -529,8 +550,8 @@ DEFAULT_RERANKER_SILICONFLOW_BASE_URL = "https://api.siliconflow.cn/v1"
 
 DEFAULT_RERANKER_GOOGLE_MODEL = "semantic-ranker-default-004"
 
-# Vector extension (pgvector, vchord, or pgvectorscale)
-DEFAULT_VECTOR_EXTENSION = "pgvector"  # Options: "pgvector", "vchord", "pgvectorscale"
+# Vector extension (pgvector, vchord, pgvectorscale, or AlloyDB ScaNN)
+DEFAULT_VECTOR_EXTENSION = "pgvector"  # Options: "pgvector", "vchord", "pgvectorscale", "scann"
 
 # Text search extension (native PostgreSQL, vchord BM25, or Timescale pg_textsearch)
 DEFAULT_TEXT_SEARCH_EXTENSION = "native"  # Options: "native", "vchord", "pg_textsearch"
@@ -800,6 +821,24 @@ def _get_default_model_for_provider(provider: str) -> str:
     return PROVIDER_DEFAULT_MODELS.get(provider.lower(), DEFAULT_LLM_MODEL)
 
 
+def _parse_llm_router_config(env_var: str) -> dict | None:
+    """
+    Parse a LiteLLM Router configuration from a JSON env var.
+
+    The value is forwarded verbatim to ``litellm.Router(**config)``. We only
+    check that it parses as JSON; LiteLLM Router is authoritative about the
+    shape (``model_list``, ``fallbacks``, ``routing_strategy``, …). See
+    https://docs.litellm.ai/docs/routing.
+    """
+    raw = os.getenv(env_var, "").strip()
+    if not raw:
+        return None
+    try:
+        return json.loads(raw)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid {env_var}: invalid JSON: {e}") from e
+
+
 def _parse_default_bank_template(raw: str | None) -> dict | None:
     """
     Parse HINDSIGHT_API_DEFAULT_BANK_TEMPLATE as JSON.
@@ -827,9 +866,14 @@ class HindsightConfig:
     # Database
     database_backend: Literal["postgresql", "oracle"]
     database_url: str
+    # Optional read-replica URL for recall queries. When set, the engine opens
+    # a second pool and routes recall SELECTs through it.
+    read_database_url: str | None
+    read_db_pool_min_size: int
+    read_db_pool_max_size: int
     migration_database_url: str | None
     database_schema: str
-    vector_extension: str  # "pgvector" or "vchord"
+    vector_extension: str  # "pgvector", "vchord", "pgvectorscale", or "scann"
     text_search_extension: str  # "native" or "vchord"
 
     # LLM (default, used as fallback for per-operation config)
@@ -847,6 +891,15 @@ class HindsightConfig:
     llm_extra_body: (
         dict | None
     )  # Extra body params merged into OpenAI-compatible API calls (e.g. {"chat_template_kwargs": {"enable_thinking": true}})
+    llm_default_headers: (
+        dict | None
+    )  # Custom headers passed as default_headers to provider SDK clients (e.g. {"X-Component-Id": "hindsight"} for proxies / request tracing)
+
+    # LiteLLM Router chain (provider-specific; consumed by the "litellmrouter" provider).
+    # List of deployment dicts evaluated in order with fallback on transient errors.
+    # Each entry: {"provider": str, "model": str, "api_key": str | None, "base_url": str | None}.
+    # Treated as a credential field because entries embed api keys.
+    llm_litellmrouter_config: dict | None
 
     # Vertex AI configuration
     llm_vertexai_project_id: str | None
@@ -874,6 +927,7 @@ class HindsightConfig:
     retain_llm_initial_backoff: float | None
     retain_llm_max_backoff: float | None
     retain_llm_timeout: float | None
+    retain_llm_litellmrouter_config: dict | None
 
     reflect_llm_provider: str | None
     reflect_llm_api_key: str | None
@@ -884,6 +938,7 @@ class HindsightConfig:
     reflect_llm_initial_backoff: float | None
     reflect_llm_max_backoff: float | None
     reflect_llm_timeout: float | None
+    reflect_llm_litellmrouter_config: dict | None
 
     consolidation_llm_provider: str | None
     consolidation_llm_api_key: str | None
@@ -894,6 +949,7 @@ class HindsightConfig:
     consolidation_llm_initial_backoff: float | None
     consolidation_llm_max_backoff: float | None
     consolidation_llm_timeout: float | None
+    consolidation_llm_litellmrouter_config: dict | None
 
     # Embeddings
     embeddings_provider: str
@@ -1133,6 +1189,11 @@ class HindsightConfig:
         "retain_llm_api_key",
         "reflect_llm_api_key",
         "consolidation_llm_api_key",
+        # LiteLLM Router chains — entries embed api_keys and base_urls
+        "llm_litellmrouter_config",
+        "retain_llm_litellmrouter_config",
+        "reflect_llm_litellmrouter_config",
+        "consolidation_llm_litellmrouter_config",
         # Base URLs (could expose infrastructure)
         "llm_base_url",
         "retain_llm_base_url",
@@ -1270,11 +1331,7 @@ class HindsightConfig:
     def validate(self) -> None:
         """Validate configuration values and raise errors for invalid combinations."""
         # Validate vector_extension
-        valid_extensions = ("pgvector", "vchord", "pgvectorscale")
-        if self.vector_extension not in valid_extensions:
-            raise ValueError(
-                f"Invalid vector_extension: {self.vector_extension}. Must be one of: {', '.join(valid_extensions)}"
-            )
+        validate_extension(self.vector_extension)
 
         # Validate text_search_extension
         valid_text_search = ("native", "vchord", "pg_textsearch")
@@ -1352,6 +1409,9 @@ class HindsightConfig:
             # Database
             database_backend=os.getenv(ENV_DATABASE_BACKEND, DEFAULT_DATABASE_BACKEND).lower(),
             database_url=os.getenv(ENV_DATABASE_URL, DEFAULT_DATABASE_URL),
+            read_database_url=os.getenv(ENV_READ_DATABASE_URL) or None,
+            read_db_pool_min_size=int(os.getenv(ENV_READ_DB_POOL_MIN_SIZE, str(DEFAULT_DB_POOL_MIN_SIZE))),
+            read_db_pool_max_size=int(os.getenv(ENV_READ_DB_POOL_MAX_SIZE, str(DEFAULT_DB_POOL_MAX_SIZE))),
             migration_database_url=os.getenv(ENV_MIGRATION_DATABASE_URL) or None,
             database_schema=os.getenv(ENV_DATABASE_SCHEMA, DEFAULT_DATABASE_SCHEMA),
             vector_extension=os.getenv(ENV_VECTOR_EXTENSION, DEFAULT_VECTOR_EXTENSION).lower(),
@@ -1369,6 +1429,8 @@ class HindsightConfig:
             llm_groq_service_tier=os.getenv(ENV_LLM_GROQ_SERVICE_TIER, DEFAULT_LLM_GROQ_SERVICE_TIER),
             llm_openai_service_tier=os.getenv(ENV_LLM_OPENAI_SERVICE_TIER, DEFAULT_LLM_OPENAI_SERVICE_TIER),
             llm_extra_body=json.loads(os.getenv(ENV_LLM_EXTRA_BODY, "null")),
+            llm_default_headers=json.loads(os.getenv(ENV_LLM_DEFAULT_HEADERS, "null")),
+            llm_litellmrouter_config=_parse_llm_router_config(ENV_LLM_LITELLMROUTER_CONFIG),
             # Vertex AI
             llm_vertexai_project_id=os.getenv(ENV_LLM_VERTEXAI_PROJECT_ID) or DEFAULT_LLM_VERTEXAI_PROJECT_ID,
             llm_vertexai_region=os.getenv(ENV_LLM_VERTEXAI_REGION, DEFAULT_LLM_VERTEXAI_REGION),
@@ -1407,6 +1469,7 @@ class HindsightConfig:
             if os.getenv(ENV_RETAIN_LLM_MAX_BACKOFF)
             else None,
             retain_llm_timeout=float(os.getenv(ENV_RETAIN_LLM_TIMEOUT)) if os.getenv(ENV_RETAIN_LLM_TIMEOUT) else None,
+            retain_llm_litellmrouter_config=_parse_llm_router_config(ENV_RETAIN_LLM_LITELLMROUTER_CONFIG),
             reflect_llm_provider=os.getenv(ENV_REFLECT_LLM_PROVIDER) or None,
             reflect_llm_api_key=os.getenv(ENV_REFLECT_LLM_API_KEY) or None,
             reflect_llm_model=os.getenv(ENV_REFLECT_LLM_MODEL)
@@ -1431,6 +1494,7 @@ class HindsightConfig:
             reflect_llm_timeout=float(os.getenv(ENV_REFLECT_LLM_TIMEOUT))
             if os.getenv(ENV_REFLECT_LLM_TIMEOUT)
             else None,
+            reflect_llm_litellmrouter_config=_parse_llm_router_config(ENV_REFLECT_LLM_LITELLMROUTER_CONFIG),
             consolidation_llm_provider=os.getenv(ENV_CONSOLIDATION_LLM_PROVIDER) or None,
             consolidation_llm_api_key=os.getenv(ENV_CONSOLIDATION_LLM_API_KEY) or None,
             consolidation_llm_model=os.getenv(ENV_CONSOLIDATION_LLM_MODEL)
@@ -1455,6 +1519,7 @@ class HindsightConfig:
             consolidation_llm_timeout=float(os.getenv(ENV_CONSOLIDATION_LLM_TIMEOUT))
             if os.getenv(ENV_CONSOLIDATION_LLM_TIMEOUT)
             else None,
+            consolidation_llm_litellmrouter_config=_parse_llm_router_config(ENV_CONSOLIDATION_LLM_LITELLMROUTER_CONFIG),
             # Embeddings
             embeddings_provider=os.getenv(ENV_EMBEDDINGS_PROVIDER, DEFAULT_EMBEDDINGS_PROVIDER),
             embeddings_local_model=os.getenv(ENV_EMBEDDINGS_LOCAL_MODEL, DEFAULT_EMBEDDINGS_LOCAL_MODEL),
@@ -1880,6 +1945,8 @@ class HindsightConfig:
     def log_config(self) -> None:
         """Log the current configuration (without sensitive values)."""
         logger.info(f"Database: {mask_network_location(self.database_url)} (schema: {self.database_schema})")
+        if self.read_database_url:
+            logger.info(f"Read database (recall queries only): {mask_network_location(self.read_database_url)}")
         if self.migration_database_url:
             logger.info(f"Migration database: {mask_network_location(self.migration_database_url)}")
         logger.info(f"LLM: provider={self.llm_provider}, model={self.llm_model}")

hindsight-api-slim/hindsight_api/daemon.py

@@ -4,12 +4,20 @@ Daemon mode support for Hindsight API.
 Provides idle timeout for running as a background daemon.
 """
 
+from __future__ import annotations
+
 import asyncio
 import logging
 import os
+import platform
+import subprocess
 import sys
 import time
 from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import IO
 
 logger = logging.getLogger(__name__)
 
@@ -20,6 +28,12 @@ DEFAULT_IDLE_TIMEOUT = 0  # 0 = no auto-exit (hindsight-embed passes its own tim
 # Allow override via environment variable for profile-specific logs
 DAEMON_LOG_PATH = Path(os.getenv("HINDSIGHT_API_DAEMON_LOG", str(Path.home() / ".hindsight" / "daemon.log")))
 
+# Internal env var: set by daemonize() in the re-exec'd child so the child
+# skips re-exec and just redirects stdio.  Also set by hindsight-embed's
+# DaemonEmbedManager so the daemon launched via Popen skips re-exec entirely
+# (hindsight-embed's Popen already provides a clean, detached process).
+ENV_DAEMON_CHILD = "_HINDSIGHT_DAEMON_CHILD"
+
 
 class IdleTimeoutMiddleware:
     """ASGI middleware that tracks activity and exits after idle timeout."""
@@ -58,57 +72,105 @@ class IdleTimeoutMiddleware:
                 os.kill(os.getpid(), signal.SIGTERM)
 
 
-def daemonize():
+def _detach_popen_kwargs(log_handle: "IO[bytes]") -> dict:
+    """Cross-platform kwargs to spawn a subprocess detached from the caller.
+
+    On POSIX, ``start_new_session=True`` calls ``setsid(2)`` so the child
+    survives the parent's terminal.  On Windows we use
+    ``DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP``.
+
+    ``log_handle`` receives the child's stdout/stderr so output never leaks
+    into the parent's terminal.
     """
-    Fork the current process into a background daemon.
+    if platform.system() == "Windows":
+        detached_process = getattr(subprocess, "DETACHED_PROCESS", 0)
+        create_new_process_group = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
+        return {
+            "creationflags": detached_process | create_new_process_group,
+            "stdin": subprocess.DEVNULL,
+            "stdout": log_handle,
+            "stderr": subprocess.STDOUT,
+            "close_fds": True,
+        }
+    return {
+        "start_new_session": True,
+        "stdin": subprocess.DEVNULL,
+        "stdout": log_handle,
+        "stderr": log_handle,
+    }
 
-    Uses double-fork technique to properly detach from terminal.
 
-    On Windows there is no fork model: the spawning parent is expected to
-    detach us via `CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS` and to
-    redirect stdout/stderr to HINDSIGHT_API_DAEMON_LOG before exec. We
-    still ensure the log directory exists so that any file handlers set
-    up by the calling app have a valid target.
+def _redirect_stdio_to_log() -> None:
+    """Redirect stdin/stdout/stderr to the daemon log file.
+
+    Called in the daemon child process after re-exec.
     """
-    if sys.platform == "win32":
-        DAEMON_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
-        return
-
-    # First fork - detach from parent
-    try:
-        pid = os.fork()
-        if pid > 0:
-            sys.exit(0)
-    except OSError as e:
-        sys.stderr.write(f"fork #1 failed: {e}\n")
-        sys.exit(1)
-
-    # Decouple from parent environment
-    os.chdir("/")
-    os.setsid()
-    os.umask(0)
-
-    # Second fork - prevent zombie
-    pid = os.fork()
-    if pid > 0:
-        sys.exit(0)
-
-    # Redirect standard file descriptors to log file
     DAEMON_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
 
     sys.stdout.flush()
     sys.stderr.flush()
 
-    # Redirect stdin to /dev/null
-    with open("/dev/null", "r") as devnull:
+    with open(os.devnull, "r") as devnull:
         os.dup2(devnull.fileno(), sys.stdin.fileno())
 
-    # Redirect stdout/stderr to log file
     log_fd = open(DAEMON_LOG_PATH, "a")
     os.dup2(log_fd.fileno(), sys.stdout.fileno())
     os.dup2(log_fd.fileno(), sys.stderr.fileno())
 
 
+def daemonize():
+    """Detach the current process into a background daemon.
+
+    Uses ``subprocess.Popen`` (which maps to ``posix_spawn`` on macOS) to
+    re-exec the current command in a detached session.  This replaces the
+    traditional double-fork pattern because ``os.fork()`` without ``exec()``
+    corrupts Apple framework state (XPC, Metal/MPS, ObjC runtime) on macOS,
+    causing SIGBUS crashes when PyTorch uses the MPS backend.
+
+    The function has two code paths controlled by the ``_HINDSIGHT_DAEMON_CHILD``
+    environment variable:
+
+    * **Parent** (env var not set): re-exec the same command via Popen with
+      ``start_new_session=True``, stripping ``--daemon`` from argv and setting
+      ``_HINDSIGHT_DAEMON_CHILD=1``.  Then ``sys.exit(0)``.
+    * **Child** (env var set): redirect stdio to the daemon log file and return.
+      No fork, no re-exec.
+
+    On Windows there is no fork model: the spawning parent is expected to
+    detach us via ``CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS`` and to
+    redirect stdout/stderr to ``HINDSIGHT_API_DAEMON_LOG`` before exec.
+    We still ensure the log directory exists.
+    """
+    if sys.platform == "win32":
+        DAEMON_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
+        return
+
+    # If we are already the daemon child (re-exec'd by a previous daemonize()
+    # call, or launched by hindsight-embed with the env var set), just redirect
+    # stdio and return — no re-exec needed.
+    if os.environ.get(ENV_DAEMON_CHILD) == "1":
+        _redirect_stdio_to_log()
+        return
+
+    # --- Parent path: re-exec ourselves as a detached background process ---
+
+    # Build child command: same Python, same module entry point, all args
+    # except --daemon (replaced by the env var).
+    child_args = [a for a in sys.argv[1:] if a != "--daemon"]
+    cmd = [sys.executable, "-m", "hindsight_api.main"] + child_args
+
+    env = os.environ.copy()
+    env[ENV_DAEMON_CHILD] = "1"
+    env["HINDSIGHT_API_DAEMON_LOG"] = str(DAEMON_LOG_PATH)
+
+    DAEMON_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(DAEMON_LOG_PATH, "ab") as log_handle:
+        subprocess.Popen(cmd, env=env, **_detach_popen_kwargs(log_handle))
+
+    sys.exit(0)
+
+
 def check_daemon_running(port: int = DEFAULT_DAEMON_PORT) -> bool:
     """Check if a daemon is running and responsive on the given port."""
     import socket

hindsight-api-slim/hindsight_api/engine/cross_encoder.py

@@ -133,7 +133,7 @@ class LocalSTCrossEncoder(CrossEncoderModel):
                        Default: cross-encoder/ms-marco-MiniLM-L-6-v2
             max_concurrent: Maximum concurrent reranking calls (default: 2).
                            Higher values may cause CPU thrashing under load.
-            force_cpu: Force CPU mode (avoids MPS/XPC issues on macOS in daemon mode).
+            force_cpu: Force CPU mode for local inference.
                       Default: False
             trust_remote_code: Allow loading models with custom code (security risk).
                               Required for some models like jina-reranker-v2-base-multilingual.

hindsight-api-slim/hindsight_api/engine/db/optional_routines.py

@@ -0,0 +1,181 @@
+"""Registry of optional PostgreSQL server-side routines.
+
+Some hot paths (currently only the worker poller's per-cycle scan) can be
+sped up by a server-side PL/pgSQL routine that the API never installs
+itself — operators install it out-of-band (e.g. a Helm hook in
+hindsight-cloud) when they want the optimisation. When the routine is not
+installed, callers must fall back to a pure-Python implementation.
+
+This module centralises that pattern so we don't sprinkle ad-hoc
+``try / except`` blocks (which silently log a server-side error on every
+call) around the codebase. Each registered entry carries:
+
+* a ``schema`` and ``name`` (used to probe ``pg_proc``)
+* a ``contract`` describing the expected signature and return shape
+
+Bodies are deliberately not stored here. Hindsight never installs these
+routines, so a body checked into this repo would (a) drift from whatever
+operators actually deploy and (b) imply ownership we don't have. The
+contract is the entire API surface: any operator-supplied implementation
+that satisfies it is interchangeable.
+
+Probe behaviour:
+
+* On first ``is_installed()`` call per backend instance we issue a single
+  ``SELECT EXISTS(...) FROM pg_proc`` and cache the boolean result in
+  memory for the life of the process.
+* No TTL: if an operator installs a routine on a running cluster, workers
+  pick it up only after restart. This is intentional — these routines are
+  expected to be installed once at deploy time, and a probe-per-poll would
+  defeat the optimisation.
+* Non-PostgreSQL backends short-circuit to ``False`` without touching the
+  database, so callers can use the same code path for Oracle.
+
+PostgreSQL terminology note: ``CREATE FUNCTION ... RETURNS SETOF`` defines
+a *function* (invoked via ``SELECT``); ``CREATE PROCEDURE`` defines a
+*procedure* (invoked via ``CALL``). The SQL-standard umbrella term
+covering both is *routine*, and the system catalog (``pg_proc``) stores
+both. The module name uses "routine" so a future ``CREATE PROCEDURE``
+entry slots in without a rename.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .base import DatabaseBackend, DatabaseConnection
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class OptionalRoutine:
+    """One optional server-side routine Hindsight may call when present.
+
+    Hindsight never installs these — operators do, out-of-band. The fields
+    here describe the contract the API expects; any implementation that
+    matches is interchangeable.
+
+    Attributes:
+        name: Unqualified routine name as it appears in ``pg_proc.proname``.
+        schema: Schema the routine lives in (matched against
+            ``pg_namespace.nspname``).
+        contract: Free-form description of the expected signature,
+            arguments, return type, and any semantic constraints. Read
+            this before installing a custom implementation.
+    """
+
+    name: str
+    schema: str
+    contract: str
+
+
+# Registry of known optional routines.
+#
+# Add new entries here when a hot path grows a server-side optimisation.
+# Document the *contract* — not the implementation — so operator-supplied
+# variants stay interchangeable and we don't pretend to own SQL we never
+# install.
+SCHEMAS_WITH_PENDING_WORK = OptionalRoutine(
+    name="schemas_with_pending_work",
+    schema="public",
+    contract="""
+    Signature:  public.schemas_with_pending_work() RETURNS SETOF text
+
+    Called by the worker poller on every cycle to find schemas with
+    claimable async_operations rows. The poller then runs FOR UPDATE
+    SKIP LOCKED only against the returned schemas, so an empty set means
+    "nothing to do, skip the expensive claim query".
+
+    A schema is "claimable" iff at least one row matches:
+        status = 'pending' AND task_payload IS NOT NULL
+    in that schema's ``async_operations`` table.
+
+    Required semantics:
+      * No arguments.
+      * Returns a set of schema names (``text``); each must match a real
+        ``pg_namespace.nspname``. The poller passes them straight into
+        the claim query — anything that isn't a valid schema will fail.
+      * Operators choose the search scope (e.g. ``tenant_%`` only, or
+        include ``public``). A schema omitted from the scan will *never*
+        be serviced by the poller, so the implementation must cover
+        every schema that holds an ``async_operations`` table in that
+        deployment.
+      * Should be cheap and idempotent — called every poll cycle (~30s).
+
+    Fallback when the routine is absent: per-schema ``EXISTS`` queries
+    from Python (~4ms per schema). The server-side path is a single-
+    round-trip optimisation worth ~200ms in deployments with thousands
+    of tenant schemas; everything else works correctly without it.
+    """,
+)
+
+_REGISTRY: dict[str, OptionalRoutine] = {
+    SCHEMAS_WITH_PENDING_WORK.name: SCHEMAS_WITH_PENDING_WORK,
+}
+
+
+class OptionalRoutines:
+    """Per-backend cache of which optional routines are installed.
+
+    One instance per long-lived consumer (e.g. one per ``WorkerPoller``).
+    Probes ``pg_proc`` lazily on first lookup and caches the result in
+    memory until the process restarts.
+    """
+
+    def __init__(self, backend: DatabaseBackend) -> None:
+        self._backend = backend
+        self._cache: dict[str, bool] = {}
+
+    async def is_installed(self, conn: DatabaseConnection, routine_name: str) -> bool:
+        """Return True iff *routine_name* exists in ``pg_proc``.
+
+        On non-PostgreSQL backends always returns False without issuing a
+        query. Result is memoised for the life of this instance.
+        """
+        if self._backend.backend_type != "postgresql":
+            return False
+
+        cached = self._cache.get(routine_name)
+        if cached is not None:
+            return cached
+
+        routine = _REGISTRY.get(routine_name)
+        if routine is None:
+            raise KeyError(f"Unknown optional routine: {routine_name!r}")
+
+        exists = await conn.fetchval(
+            "SELECT EXISTS(SELECT 1 FROM pg_proc p "
+            "JOIN pg_namespace n ON p.pronamespace = n.oid "
+            "WHERE n.nspname = $1 AND p.proname = $2)",
+            routine.schema,
+            routine.name,
+        )
+        installed = bool(exists)
+        self._cache[routine_name] = installed
+        if installed:
+            logger.info(
+                "Optional PG routine %s.%s detected — using server-side path",
+                routine.schema,
+                routine.name,
+            )
+        else:
+            logger.debug(
+                "Optional PG routine %s.%s not installed — using fallback path",
+                routine.schema,
+                routine.name,
+            )
+        return installed
+
+    def invalidate(self, routine_name: str | None = None) -> None:
+        """Drop cached probe results (test helper).
+
+        Without an argument, clears the entire cache.
+        """
+        if routine_name is None:
+            self._cache.clear()
+        else:
+            self._cache.pop(routine_name, None)

hindsight-api-slim/hindsight_api/engine/embeddings.py

@@ -104,7 +104,7 @@ class LocalSTEmbeddings(Embeddings):
         Args:
             model_name: Name of the SentenceTransformer model to use.
                        Default: BAAI/bge-small-en-v1.5
-            force_cpu: Force CPU mode (avoids MPS/XPC issues on macOS in daemon mode).
+            force_cpu: Force CPU mode for local inference.
                       Default: False
             trust_remote_code: Allow loading models with custom code (security risk).
                               Required for some models with custom architectures.

hindsight-api-slim/hindsight_api/engine/entity_resolver.py

@@ -12,7 +12,7 @@ from collections import defaultdict
 from dataclasses import dataclass, field
 from datetime import UTC, datetime
 from difflib import SequenceMatcher
-from typing import Any
+from typing import Any, Final
 
 from .db_utils import acquire_with_retry
 from .memory_engine import fq_table
@@ -46,12 +46,38 @@ class _EntityStatAgg:
     max_date: datetime | None = None
 
 
+# Sentinel distinguishing "key not in dict" from "key present with value None".
+# Needed when merging event_date across duplicate unit rows: legacy two-tuple
+# callers surface `None`, which must not clobber a real datetime from another
+# caller for the same unit.
+_SENTINEL_MISSING: Final = object()
+
+
+def _later_date(a: datetime | None, b: datetime | None) -> datetime | None:
+    """Return whichever of ``a`` / ``b`` is later (None loses to any datetime).
+
+    Used to fold duplicate co-occurrence pairs across a retain batch: legacy
+    two-tuple callers surface ``None``, which must not clobber a real
+    datetime that arrived for the same pair from an aware caller.
+    """
+    if a is None:
+        return b
+    if b is None:
+        return a
+    return a if a > b else b
+
+
 @dataclass
 class _CooccurrencePair:
     """A (entity_id_1, entity_id_2) pair observed in a retain batch (for post-txn flush)."""
 
     entity_id_1: str
     entity_id_2: str
+    # When the two entities co-occurred in the source content. For real-time
+    # retains this is ~now; for backfilled corpora it's the historical event
+    # time, so the cooccurrence cache reflects the underlying knowledge
+    # timeline instead of collapsing to the import moment.
+    event_date: datetime | None = None
 
 
 # Load spaCy model (singleton)
@@ -143,11 +169,15 @@ class EntityResolver:
                 )
 
             if cooccurrences:
-                # Aggregate: count occurrences per (entity_id_1, entity_id_2) pair.
-                coo_agg: dict[tuple[str, str], int] = {}
+                # Aggregate per (entity_id_1, entity_id_2): count occurrences and
+                # keep the latest event_date we saw. Using GREATEST(...) in the SQL
+                # already handles merging against the existing row; here we fold
+                # the batch so executemany doesn't send the same pair twice.
+                coo_agg: dict[tuple[str, str], tuple[int, datetime | None]] = {}
                 for c in cooccurrences:
                     pair = (c.entity_id_1, c.entity_id_2)
-                    coo_agg[pair] = coo_agg.get(pair, 0) + 1
+                    prev_count, prev_date = coo_agg.get(pair, (0, None))
+                    coo_agg[pair] = (prev_count + 1, _later_date(prev_date, c.event_date))
 
                 now = datetime.now(UTC)
                 # Sort by (entity_id_1, entity_id_2) for consistent lock ordering.
@@ -161,7 +191,7 @@ class EntityResolver:
                         cooccurrence_count = {fq_table("entity_cooccurrences")}.cooccurrence_count + EXCLUDED.cooccurrence_count,
                         last_cooccurred    = GREATEST({fq_table("entity_cooccurrences")}.last_cooccurred, EXCLUDED.last_cooccurred)
                     """,
-                    sorted((e1, e2, count, now) for (e1, e2), count in coo_agg.items()),
+                    sorted((e1, e2, count, event_date or now) for (e1, e2), (count, event_date) in coo_agg.items()),
                 )
 
     @staticmethod
@@ -872,29 +902,45 @@ class EntityResolver:
             entity_id_2,
         )
 
-    async def link_units_to_entities_batch(self, unit_entity_pairs: list[tuple[str, str]], conn=None):
+    async def link_units_to_entities_batch(
+        self,
+        unit_entity_pairs: list[tuple[str, str]] | list[tuple[str, str, datetime | None]],
+        conn=None,
+    ):
         """
         Link multiple memory units to entities in batch (MUCH faster than sequential).
 
         Also updates co-occurrence cache for entities that appear in the same unit.
 
         Args:
-            unit_entity_pairs: List of (unit_id, entity_id) tuples
+            unit_entity_pairs: List of (unit_id, entity_id) or
+                (unit_id, entity_id, event_date) tuples. When `event_date` is
+                supplied, ``entity_cooccurrences.last_cooccurred`` for pairs
+                observed in that unit advances to the event time instead of
+                ``now()``, which matters for backfilled corpora where ingest
+                time is a single spike unrelated to the underlying timeline.
+                Legacy two-tuples remain accepted.
             conn: Optional connection to use (if None, acquires from pool)
         """
         if not unit_entity_pairs:
             return
 
+        # Normalize to 3-tuples internally so downstream code doesn't branch.
+        normalized: list[tuple[str, str, datetime | None]] = [
+            (t[0], t[1], t[2] if len(t) >= 3 else None)  # type: ignore[misc]
+            for t in unit_entity_pairs
+        ]
+
         if conn is None:
             async with acquire_with_retry(self.pool) as conn:
-                return await self._link_units_to_entities_batch_impl(conn, unit_entity_pairs)
+                return await self._link_units_to_entities_batch_impl(conn, normalized)
         else:
-            return await self._link_units_to_entities_batch_impl(conn, unit_entity_pairs)
+            return await self._link_units_to_entities_batch_impl(conn, normalized)
 
-    async def _link_units_to_entities_batch_impl(self, conn, unit_entity_pairs: list[tuple[str, str]]):
+    async def _link_units_to_entities_batch_impl(self, conn, unit_entity_pairs: list[tuple[str, str, datetime | None]]):
         # Sorted bulk insert to prevent deadlocks from inconsistent lock ordering
         # across concurrent transactions on the unit_entities unique index.
-        sorted_pairs = sorted(unit_entity_pairs)
+        sorted_pairs = sorted(unit_entity_pairs, key=lambda t: (t[0], t[1]))
         unit_ids = [p[0] for p in sorted_pairs]
         entity_ids = [p[1] for p in sorted_pairs]
 
@@ -905,28 +951,42 @@ class EntityResolver:
             entity_ids,
         )
 
-        # Build map of unit -> entities for co-occurrence calculation
-        # Use sets to avoid duplicate entities in the same unit
-        unit_to_entities = {}
-        for unit_id, entity_id in unit_entity_pairs:
-            if unit_id not in unit_to_entities:
-                unit_to_entities[unit_id] = set()
-            unit_to_entities[unit_id].add(entity_id)
+        # Build maps keyed by unit_id:
+        #   unit_to_entities: entity set per unit (for the co-occurrence cross-product)
+        #   unit_event_date:  event time per unit (propagated onto every pair from that unit)
+        # When a unit shows up more than once with conflicting event_dates (legacy
+        # callers passing None interleaved with aware callers), prefer the first
+        # non-None value so we don't accidentally erase an explicit timestamp.
+        unit_to_entities: dict[str, set[str]] = {}
+        unit_event_date: dict[str, datetime | None] = {}
+        for unit_id, entity_id, event_date in unit_entity_pairs:
+            unit_to_entities.setdefault(unit_id, set()).add(entity_id)
+            if event_date is not None and unit_event_date.get(unit_id) is None:
+                unit_event_date[unit_id] = event_date
+            elif unit_id not in unit_event_date:
+                unit_event_date[unit_id] = event_date
 
-        # Update co-occurrences for all pairs in each unit
-        cooccurrence_pairs = set()  # Use set to avoid duplicates
+        # Update co-occurrences for all pairs in each unit. Carry the unit's
+        # event_date onto every pair so the flush step can stamp
+        # `last_cooccurred` with the correct time.
+        cooccurrence_pairs: dict[tuple[str, str], datetime | None] = {}
         for unit_id, entity_ids in unit_to_entities.items():
-            entity_list = list(entity_ids)  # Convert set to list for iteration
-            # For each pair of entities in this unit, create co-occurrence
+            entity_list = list(entity_ids)
+            event_date = unit_event_date.get(unit_id)
             for i, entity_id_1 in enumerate(entity_list):
                 for entity_id_2 in entity_list[i + 1 :]:
-                    # Skip if same entity (shouldn't happen with set, but be safe)
                     if entity_id_1 == entity_id_2:
                         continue
-                    # Ensure consistent ordering (entity_id_1 < entity_id_2)
+                    # Canonical ordering (entity_id_1 < entity_id_2) matches the
+                    # entity_cooccurrences PK and check constraint.
                     if entity_id_1 > entity_id_2:
                         entity_id_1, entity_id_2 = entity_id_2, entity_id_1
-                    cooccurrence_pairs.add((entity_id_1, entity_id_2))
+                    key = (entity_id_1, entity_id_2)
+                    prev = cooccurrence_pairs.get(key, _SENTINEL_MISSING)
+                    if prev is _SENTINEL_MISSING:
+                        cooccurrence_pairs[key] = event_date
+                    else:
+                        cooccurrence_pairs[key] = _later_date(prev, event_date)
 
         # Accumulate co-occurrence pairs for post-transaction flush.
         # The actual INSERT/UPDATE is deferred to flush_pending_stats() to avoid
@@ -935,7 +995,8 @@ class EntityResolver:
         if cooccurrence_pairs:
             key = self._task_key()
             self._pending_cooccurrences.setdefault(key, []).extend(
-                _CooccurrencePair(entity_id_1=e1, entity_id_2=e2) for e1, e2 in cooccurrence_pairs
+                _CooccurrencePair(entity_id_1=e1, entity_id_2=e2, event_date=ed)
+                for (e1, e2), ed in cooccurrence_pairs.items()
             )
 
     async def get_units_by_entity(self, entity_id: str, limit: int = 100) -> list[str]:

hindsight-api-slim/hindsight_api/engine/llm_wrapper.py

@@ -129,6 +129,7 @@ _PROVIDERS_WITHOUT_API_KEY = frozenset(
         "none",
         "vertexai",
         "litellm",
+        "litellmrouter",
         "bedrock",
     }
 )
@@ -148,10 +149,12 @@ def create_llm_provider(
     groq_service_tier: str | None = None,
     openai_service_tier: str | None = None,
     extra_body: dict[str, Any] | None = None,
+    default_headers: dict[str, str] | None = None,
     vertexai_project_id: str | None = None,
     vertexai_region: str | None = None,
     vertexai_credentials: Any = None,
     gemini_safety_settings: list | None = None,
+    litellmrouter_config: dict[str, Any] | None = None,
 ) -> Any:  # Returns LLMInterface
     """
     Factory function to create the appropriate LLM provider implementation.
@@ -165,6 +168,9 @@ def create_llm_provider(
         groq_service_tier: Groq service tier (for Groq provider) - "on_demand", "flex", or "auto".
         openai_service_tier: OpenAI service tier (for OpenAI provider) - None (default) or "flex" (50% cheaper).
         extra_body: Extra body params merged into OpenAI-compatible API calls.
+        default_headers: Custom headers passed as ``default_headers`` to provider SDK clients
+            (used by operators routing through proxies / request-tracing middleware). Currently
+            wired into the Anthropic provider; other providers may opt in as needed.
         vertexai_project_id: Vertex AI project ID (for VertexAI provider).
         vertexai_region: Vertex AI region (for VertexAI provider).
         vertexai_credentials: Vertex AI credentials object (for VertexAI provider).
@@ -179,6 +185,7 @@ def create_llm_provider(
         CodexLLM,
         GeminiLLM,
         LiteLLMLLM,
+        LiteLLMRouterLLM,
         LlamaCppLLM,
         MockLLM,
         NoneLLM,
@@ -243,6 +250,7 @@ def create_llm_provider(
             base_url=base_url,
             model=model,
             reasoning_effort=reasoning_effort,
+            default_headers=default_headers,
         )
 
     elif provider_lower == "litellm":
@@ -254,6 +262,23 @@ def create_llm_provider(
             reasoning_effort=reasoning_effort,
         )
 
+    elif provider_lower == "litellmrouter":
+        if not litellmrouter_config:
+            raise ValueError(
+                "Provider 'litellmrouter' requires a config object. "
+                "Set HINDSIGHT_API_LLM_LITELLMROUTER_CONFIG (or the per-op variant) "
+                "to a JSON object accepted by litellm.Router. "
+                "See https://docs.litellm.ai/docs/routing."
+            )
+        return LiteLLMRouterLLM(
+            provider=provider,
+            api_key=api_key,
+            base_url=base_url,
+            model=model,
+            config=litellmrouter_config,
+            reasoning_effort=reasoning_effort,
+        )
+
     elif provider_lower == "bedrock":
         # Bedrock is a first-class alias backed by LiteLLM with auto-prefixed model names
         bedrock_model = model if model.startswith("bedrock/") else f"bedrock/{model}"
@@ -283,7 +308,18 @@ def create_llm_provider(
             extra_args=config.llamacpp_extra_args,
         )
 
-    elif provider_lower in ("openai", "groq", "ollama", "lmstudio", "minimax", "deepseek", "volcano", "openrouter"):
+    elif provider_lower in (
+        "openai",
+        "groq",
+        "ollama",
+        "lmstudio",
+        "minimax",
+        "deepseek",
+        "volcano",
+        "openrouter",
+        "zai",
+        "opencode-go",
+    ):
         return OpenAICompatibleLLM(
             provider=provider,
             api_key=api_key,
@@ -317,6 +353,8 @@ class LLMProvider:
         openai_service_tier: str | None = None,
         gemini_safety_settings: list | None = None,
         extra_body: dict[str, Any] | None = None,
+        default_headers: dict[str, str] | None = None,
+        litellmrouter_config: dict[str, Any] | None = None,
     ):
         """
         Initialize LLM provider.
@@ -331,12 +369,22 @@ class LLMProvider:
             openai_service_tier: OpenAI service tier (None or "flex") - from config.
             gemini_safety_settings: Safety settings for Gemini/VertexAI providers.
             extra_body: Extra body params merged into OpenAI-compatible API calls.
+            default_headers: Custom headers passed as ``default_headers`` to provider SDK clients.
+                Used by operators routing through proxies / request-tracing middleware. Falls
+                back to ``HindsightConfig.llm_default_headers`` (env: ``HINDSIGHT_API_LLM_DEFAULT_HEADERS``)
+                when ``None``.
+            litellmrouter_config: Provider-specific config for ``provider="litellmrouter"``.
+                JSON object passed verbatim to ``litellm.Router(**config)`` — see
+                https://docs.litellm.ai/docs/routing. Ignored unless ``provider == "litellmrouter"``.
+                When None and the provider is ``litellmrouter``, falls back to
+                ``HindsightConfig.llm_litellmrouter_config``.
         """
         self.provider = provider.lower()
         self.api_key = api_key
         self.base_url = base_url
         self.model = model
         self.reasoning_effort = reasoning_effort
+        self.litellmrouter_config = litellmrouter_config
         # Service tiers from hierarchical config (not env vars)
         self.groq_service_tier = groq_service_tier
         self.openai_service_tier = openai_service_tier
@@ -344,6 +392,17 @@ class LLMProvider:
         self.gemini_safety_settings = gemini_safety_settings
         # Extra body params for OpenAI-compatible providers (e.g. chat_template_kwargs)
         self.extra_body = extra_body
+        # Default headers passed to provider SDK clients (e.g. proxy auth, request tracing).
+        # Same pattern as ``gemini_safety_settings``: explicit override wins; otherwise read
+        # the static server-level default from ``HindsightConfig`` via ``_get_raw_config()``.
+        self.default_headers = default_headers
+        if self.default_headers is None:
+            from ..config import _get_raw_config
+
+            try:
+                self.default_headers = _get_raw_config().llm_default_headers
+            except Exception:
+                pass  # Config may not be initialized in test environments
 
         # Validate provider
         valid_providers = [
@@ -362,9 +421,12 @@ class LLMProvider:
             "minimax",
             "deepseek",
             "litellm",
+            "litellmrouter",
             "bedrock",
             "volcano",
             "openrouter",
+            "zai",
+            "opencode-go",
         ]
         if self.provider not in valid_providers:
             raise ValueError(f"Invalid LLM provider: {self.provider}. Must be one of: {', '.join(valid_providers)}")
@@ -383,6 +445,10 @@ class LLMProvider:
                 self.base_url = "https://api.deepseek.com"
             elif self.provider == "openrouter":
                 self.base_url = "https://openrouter.ai/api/v1"
+            elif self.provider == "zai":
+                self.base_url = "https://api.z.ai/api/coding/paas/v4"
+            elif self.provider == "opencode-go":
+                self.base_url = "https://opencode.ai/zen/go/v1"
 
         # Prepare Vertex AI config (if applicable)
         vertexai_project_id = None
@@ -438,6 +504,19 @@ class LLMProvider:
             except Exception:
                 pass  # Config may not be initialized in test environments
 
+        # For litellmrouter: prefer an explicit chain from the caller (per-op
+        # construction in MemoryEngine threads the right chain through). If the caller
+        # didn't supply one, fall back to the global ``llm_litellmrouter_config`` so
+        # ad-hoc constructions (e.g. ``LLMProvider.from_env()``) keep working.
+        router_config: dict[str, Any] | None = self.litellmrouter_config
+        if self.provider == "litellmrouter" and router_config is None:
+            from ..config import _get_raw_config
+
+            try:
+                router_config = _get_raw_config().llm_litellmrouter_config
+            except Exception:
+                router_config = None
+
         # Create provider implementation using factory
         self._provider_impl = create_llm_provider(
             provider=self.provider,
@@ -448,10 +527,12 @@ class LLMProvider:
             groq_service_tier=self.groq_service_tier,
             openai_service_tier=self.openai_service_tier,
             extra_body=self.extra_body,
+            default_headers=self.default_headers,
             vertexai_project_id=vertexai_project_id,
             vertexai_region=vertexai_region,
             vertexai_credentials=vertexai_credentials,
             gemini_safety_settings=self.gemini_safety_settings,
+            litellmrouter_config=router_config,
         )
 
         # Backward compatibility: Keep mock provider properties
@@ -759,13 +840,14 @@ class LLMProvider:
     def from_env(cls) -> "LLMProvider":
         """Create provider from environment variables using config.py constants."""
         from ..config import (
-            DEFAULT_LLM_MODEL,
             DEFAULT_LLM_PROVIDER,
             ENV_LLM_API_KEY,
             ENV_LLM_BASE_URL,
+            ENV_LLM_DEFAULT_HEADERS,
             ENV_LLM_EXTRA_BODY,
             ENV_LLM_MODEL,
             ENV_LLM_PROVIDER,
+            _get_default_model_for_provider,
         )
 
         provider = os.getenv(ENV_LLM_PROVIDER, DEFAULT_LLM_PROVIDER)
@@ -779,8 +861,9 @@ class LLMProvider:
             )
 
         base_url = os.getenv(ENV_LLM_BASE_URL, "")
-        model = os.getenv(ENV_LLM_MODEL, DEFAULT_LLM_MODEL)
+        model = os.getenv(ENV_LLM_MODEL) or _get_default_model_for_provider(provider)
         extra_body = json.loads(os.getenv(ENV_LLM_EXTRA_BODY, "null"))
+        default_headers = json.loads(os.getenv(ENV_LLM_DEFAULT_HEADERS, "null"))
 
         return cls(
             provider=provider,
@@ -789,6 +872,7 @@ class LLMProvider:
             model=model,
             reasoning_effort="low",
             extra_body=extra_body,
+            default_headers=default_headers,
         )
 
 

hindsight-api-slim/hindsight_api/engine/memory_engine.py

@@ -18,7 +18,7 @@ import uuid
 from collections.abc import Awaitable, Callable
 from dataclasses import dataclass
 from datetime import UTC, datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Literal, overload
+from typing import TYPE_CHECKING, Any, Literal, cast, overload
 
 import asyncpg
 import httpx
@@ -502,9 +502,15 @@ class MemoryEngine(MemoryEngineInterface):
         self._dialect: SQLDialect | None = None
         # Connection pool — set from backend.get_pool() for backward compatibility
         self._pool = None
+        self._read_backend: DatabaseBackend | None = None
+        self._read_database_url: str | None = (
+            config.read_database_url if self._database_backend_type == "postgresql" else None
+        )
         self._initialized = False
         self._pool_min_size = pool_min_size if pool_min_size is not None else config.db_pool_min_size
         self._pool_max_size = pool_max_size if pool_max_size is not None else config.db_pool_max_size
+        self._read_pool_min_size = config.read_db_pool_min_size
+        self._read_pool_max_size = config.read_db_pool_max_size
         self._db_command_timeout = db_command_timeout if db_command_timeout is not None else config.db_command_timeout
         self._db_acquire_timeout = db_acquire_timeout if db_acquire_timeout is not None else config.db_acquire_timeout
         self._db_statement_timeout = config.db_statement_timeout
@@ -539,6 +545,8 @@ class MemoryEngine(MemoryEngineInterface):
             base_url=memory_llm_base_url,
             model=memory_llm_model,
             extra_body=config.llm_extra_body,
+            default_headers=config.llm_default_headers,
+            litellmrouter_config=config.llm_litellmrouter_config,
         )
 
         # Store client and model for convenience (deprecated: use _llm_config.call() instead)
@@ -566,6 +574,8 @@ class MemoryEngine(MemoryEngineInterface):
             base_url=retain_base_url,
             model=retain_model,
             extra_body=config.llm_extra_body,
+            default_headers=config.llm_default_headers,
+            litellmrouter_config=config.retain_llm_litellmrouter_config or config.llm_litellmrouter_config,
         )
 
         # Reflect LLM config - for think/observe operations (can use lighter models)
@@ -588,6 +598,8 @@ class MemoryEngine(MemoryEngineInterface):
             base_url=reflect_base_url,
             model=reflect_model,
             extra_body=config.llm_extra_body,
+            default_headers=config.llm_default_headers,
+            litellmrouter_config=config.reflect_llm_litellmrouter_config or config.llm_litellmrouter_config,
         )
 
         # Consolidation LLM config - for mental model consolidation (can use efficient models)
@@ -610,6 +622,8 @@ class MemoryEngine(MemoryEngineInterface):
             base_url=consolidation_base_url,
             model=consolidation_model,
             extra_body=config.llm_extra_body,
+            default_headers=config.llm_default_headers,
+            litellmrouter_config=config.consolidation_llm_litellmrouter_config or config.llm_litellmrouter_config,
         )
 
         # Initialize cross-encoder reranker (cached for performance)
@@ -1644,11 +1658,16 @@ class MemoryEngine(MemoryEngineInterface):
                 # Parent doesn't exist (shouldn't happen)
                 return
 
-            # Get all sibling operations (including this one)
-            # This query runs in the same transaction, so it sees the current child's updated status
+            # Get all sibling operations (including this one).
+            # This query runs in the same transaction, so it sees the current
+            # child's updated status. Pull error_message too so a parent that
+            # fails can inherit a representative child reason -- otherwise
+            # downstream consumers (dashboards, alert filters) lose the actual
+            # cause once a batch has children. See the worker poller's
+            # _summarise_child_error_messages for the propagation rationale.
             siblings = await conn.fetch(
                 f"""
-                SELECT status
+                SELECT status, error_message
                 FROM {fq_table("async_operations")}
                 WHERE bank_id = $1
                 AND result_metadata::jsonb @> $2::jsonb
@@ -1672,7 +1691,12 @@ class MemoryEngine(MemoryEngineInterface):
             # All siblings are done - update parent status
             if any_failed:
                 new_status = "failed"
-                # Set parent error message to indicate child failure
+                # Set parent error message to indicate child failure. Inherit
+                # the most-common failed-child error_message rather than a
+                # generic string so downstream filters can attribute the
+                # cause correctly.
+                from hindsight_api.worker.poller import _summarise_child_error_messages
+
                 await conn.execute(
                     f"""
                     UPDATE {fq_table("async_operations")}
@@ -1681,7 +1705,7 @@ class MemoryEngine(MemoryEngineInterface):
                     """,
                     uuid.UUID(parent_operation_id),
                     new_status,
-                    "One or more sub-batches failed",
+                    _summarise_child_error_messages(siblings),
                 )
             elif all_completed:
                 new_status = "completed"
@@ -1925,6 +1949,23 @@ class MemoryEngine(MemoryEngineInterface):
         # These will be migrated to use self._backend.acquire() over time.
         self._pool = self._backend.get_pool()
 
+        if self._read_database_url:
+            logger.info(
+                f"Opening read backend against {mask_network_location(self._read_database_url)} for recall queries"
+            )
+            self._read_backend = create_database_backend(self._database_backend_type)
+            await self._read_backend.initialize(
+                self._read_database_url,
+                min_size=self._read_pool_min_size,
+                max_size=self._read_pool_max_size,
+                command_timeout=self._db_command_timeout,
+                acquire_timeout=self._db_acquire_timeout,
+                statement_cache_size=0,
+                init_callback=_init_connection,
+            )
+        else:
+            self._read_backend = self._backend
+
         # Initialize entity resolver with pool and configured lookup strategy
         self.entity_resolver = EntityResolver(
             self._backend,
@@ -2013,6 +2054,15 @@ class MemoryEngine(MemoryEngineInterface):
             await self.initialize()
         return self._pool
 
+    async def _get_read_backend(self) -> DatabaseBackend:
+        """Get the read-only backend (replica when configured, otherwise primary).
+
+        Writes MUST NOT be issued through this backend.
+        """
+        if not self._initialized:
+            await self.initialize()
+        return self._read_backend
+
     async def _get_backend(self) -> DatabaseBackend:
         """Get the database backend, auto-initializing if needed."""
         if not self._initialized:
@@ -2069,7 +2119,11 @@ class MemoryEngine(MemoryEngineInterface):
             await self._http_client.aclose()
             self._http_client = None
 
-        # Close database backend (shuts down pool)
+        if self._read_backend is not None and self._read_backend is not self._backend:
+            await self._read_backend.shutdown()
+        self._read_backend = None
+
+        # Close primary database backend (shuts down pool)
         if self._backend is not None:
             await self._backend.shutdown()
             self._backend = None
@@ -2294,6 +2348,12 @@ class MemoryEngine(MemoryEngineInterface):
             if result and result.contents is not None:
                 contents = result.contents
 
+        # Engine-owned copy: the orchestrator clears per-item "content" strings
+        # after building the document's combined text (memory pressure
+        # optimization, see retain/orchestrator.py). Without an internal copy
+        # those mutations leak back to the caller's dicts.
+        contents = cast(list[RetainContentDict], [dict(c) for c in contents])
+
         # Apply batch-level document_id to contents that don't have their own (backwards compatibility)
         if document_id:
             for item in contents:
@@ -2914,7 +2974,7 @@ class MemoryEngine(MemoryEngineInterface):
         if tracer:
             tracer.start()
 
-        backend = await self._get_backend()
+        backend = await self._get_read_backend()
         recall_start = time.time()
 
         # Buffer logs for clean output in concurrent scenarios.
@@ -2983,7 +3043,7 @@ class MemoryEngine(MemoryEngineInterface):
                     max_connections=effective_connection_budget,
                     operation_id=f"recall-{recall_id}",
                 ) as op:
-                    budgeted_pool = op.wrap_pool(self._backend)
+                    budgeted_pool = op.wrap_pool(backend)
                     parallel_start = time.time()
                     multi_result = await retrieve_all_fact_types_parallel(
                         budgeted_pool,
@@ -3572,26 +3632,21 @@ class MemoryEngine(MemoryEngineInterface):
                                     source_facts_dict[sid] = _make_source_fact(sid, r)
                                     total_source_tokens += fact_tokens
 
-            # Get entities for each fact if include_entities is requested
-            fact_entity_map = {}  # unit_id -> list of (entity_id, entity_name)
+            # Get entities for each fact if include_entities is requested.
+            # _entity_rows_for_units_sql resolves both direct unit_entities rows
+            # and observation-via-source-memory inheritance in a single query.
+            fact_entity_map = {}  # unit_id -> list of {entity_id, canonical_name}
             if include_entities and top_scored:
                 unit_ids = [uuid.UUID(sr.id) for sr in top_scored]
                 if unit_ids:
                     async with acquire_with_retry(backend) as entity_conn:
                         entity_rows = await entity_conn.fetch(
-                            f"""
-                            SELECT ue.unit_id, e.id as entity_id, e.canonical_name
-                            FROM {fq_table("unit_entities")} ue
-                            JOIN {fq_table("entities")} e ON ue.entity_id = e.id
-                            WHERE ue.unit_id = ANY($1::uuid[])
-                            """,
+                            self._entity_rows_for_units_sql(unit_ids_placeholder=1),
                             unit_ids,
                         )
                         for row in entity_rows:
                             unit_id = str(row["unit_id"])
-                            if unit_id not in fact_entity_map:
-                                fact_entity_map[unit_id] = []
-                            fact_entity_map[unit_id].append(
+                            fact_entity_map.setdefault(unit_id, []).append(
                                 {"entity_id": str(row["entity_id"]), "canonical_name": row["canonical_name"]}
                             )
 
@@ -3681,12 +3736,67 @@ class MemoryEngine(MemoryEngineInterface):
             )
 
         except Exception as e:
+            # Use repr(e) so exceptions with empty __str__ (e.g. raise SomeError())
+            # still emit a discriminating class+args string into operations.error_message.
             log_buffer.append(
-                f"[RECALL {recall_id}] ERROR after {time.time() - recall_start:.3f}s: {type(e).__name__}: {e}"
+                f"[RECALL {recall_id}] ERROR after {time.time() - recall_start:.3f}s: {type(e).__name__}: {e!r}"
             )
             if not quiet:
-                logger.error("\n" + "\n".join(log_buffer))
-            raise Exception(f"Failed to search memories: {type(e).__name__}: {e}")
+                logger.error("\n" + "\n".join(log_buffer), exc_info=True)
+            raise RuntimeError(f"Failed to search memories ({type(e).__name__}): {e!r}") from e
+
+    def _entity_rows_for_units_sql(self, unit_ids_placeholder: int) -> str:
+        """SQL SELECT producing ``(unit_id, entity_id, canonical_name)`` rows for
+        the given unit IDs.
+
+        Direct rows come from ``unit_entities``. Observations rarely carry
+        direct rows there; their entity association lives transitively through
+        their source memories (``source_memory_ids`` on PG, the
+        ``observation_sources`` junction on Oracle). When an observation has
+        no direct entity rows the SELECT inherits its source memories'
+        entities, so the result is the same set callers would get from
+        ``get_memory_unit``.
+
+        ``unit_ids_placeholder`` is the 1-based parameter index that holds the
+        ``uuid[]`` of unit IDs. The placeholder is referenced twice — both
+        sides of the UNION need it — so callers should not reuse the slot.
+        """
+        ue = fq_table("unit_entities")
+        ents = fq_table("entities")
+        mu = fq_table("memory_units")
+        p = unit_ids_placeholder
+
+        direct = (
+            f"SELECT ue.unit_id, e.id AS entity_id, e.canonical_name "
+            f"FROM {ue} ue "
+            f"JOIN {ents} e ON e.id = ue.entity_id "
+            f"WHERE ue.unit_id = ANY(${p}::uuid[])"
+        )
+
+        if self._backend.ops.uses_observation_sources_table:
+            os_t = fq_table("observation_sources")
+            inherited = (
+                f"SELECT os.observation_id AS unit_id, e.id AS entity_id, e.canonical_name "
+                f"FROM {os_t} os "
+                f"JOIN {ue} src_ue ON src_ue.unit_id = os.source_id "
+                f"JOIN {ents} e ON e.id = src_ue.entity_id "
+                f"WHERE os.observation_id = ANY(${p}::uuid[]) "
+                f"AND NOT EXISTS (SELECT 1 FROM {ue} d WHERE d.unit_id = os.observation_id)"
+            )
+        else:
+            inherited = (
+                f"SELECT obs.id AS unit_id, e.id AS entity_id, e.canonical_name "
+                f"FROM {mu} obs "
+                f"CROSS JOIN LATERAL unnest(obs.source_memory_ids) AS src_id "
+                f"JOIN {ue} src_ue ON src_ue.unit_id = src_id "
+                f"JOIN {ents} e ON e.id = src_ue.entity_id "
+                f"WHERE obs.id = ANY(${p}::uuid[]) "
+                f"AND obs.fact_type = 'observation' "
+                f"AND obs.source_memory_ids IS NOT NULL "
+                f"AND NOT EXISTS (SELECT 1 FROM {ue} d WHERE d.unit_id = obs.id)"
+            )
+
+        return f"({direct}) UNION ({inherited})"
 
     def _filter_by_token_budget(
         self, results: list[dict[str, Any]], max_tokens: int
@@ -5086,31 +5196,15 @@ class MemoryEngine(MemoryEngineInterface):
             if not row:
                 return None
 
-            # Get entity information
+            # Get entity information. _entity_rows_for_units_sql handles the
+            # observation→source_memory_ids inheritance fallback in SQL, so a
+            # single query covers direct rows and inherited ones.
             entities_rows = await conn.fetch(
-                f"""
-                SELECT e.canonical_name
-                FROM {fq_table("unit_entities")} ue
-                JOIN {fq_table("entities")} e ON ue.entity_id = e.id
-                WHERE ue.unit_id = $1
-                """,
-                row["id"],
+                self._entity_rows_for_units_sql(unit_ids_placeholder=1),
+                [row["id"]],
             )
             entities = [r["canonical_name"] for r in entities_rows]
 
-            # For observations with no direct entities, inherit from source memories
-            if not entities and row["fact_type"] == "observation" and row["source_memory_ids"]:
-                source_entities_rows = await conn.fetch(
-                    f"""
-                    SELECT DISTINCT e.canonical_name
-                    FROM {fq_table("unit_entities")} ue
-                    JOIN {fq_table("entities")} e ON ue.entity_id = e.id
-                    WHERE ue.unit_id = ANY($1::uuid[])
-                    """,
-                    row["source_memory_ids"],
-                )
-                entities = [r["canonical_name"] for r in source_entities_rows]
-
             result = {
                 "id": str(row["id"]),
                 "text": row["text"],

hindsight-api-slim/hindsight_api/engine/providers/__init__.py

@@ -9,6 +9,7 @@ from .claude_code_llm import ClaudeCodeLLM
 from .codex_llm import CodexLLM
 from .gemini_llm import GeminiLLM
 from .litellm_llm import LiteLLMLLM
+from .litellm_router_llm import LiteLLMRouterLLM
 from .llamacpp_llm import LlamaCppLLM
 from .mock_llm import MockLLM
 from .none_llm import NoneLLM
@@ -21,6 +22,7 @@ __all__ = [
     "GeminiLLM",
     "LlamaCppLLM",
     "LiteLLMLLM",
+    "LiteLLMRouterLLM",
     "MockLLM",
     "NoneLLM",
     "OpenAICompatibleLLM",

hindsight-api-slim/hindsight_api/engine/providers/anthropic_llm.py

@@ -37,6 +37,7 @@ class AnthropicLLM(LLMInterface):
         model: str,
         reasoning_effort: str = "low",
         timeout: float = 300.0,
+        default_headers: dict[str, str] | None = None,
         **kwargs: Any,
     ):
         """
@@ -49,6 +50,10 @@ class AnthropicLLM(LLMInterface):
             model: Model name (e.g., "claude-sonnet-4-20250514").
             reasoning_effort: Reasoning effort level (not used by Anthropic).
             timeout: Request timeout in seconds.
+            default_headers: Optional custom headers passed as ``default_headers`` to
+                the Anthropic SDK client. Used by operators routing through proxies
+                or request-tracing middleware. Sourced from ``llm_default_headers`` in
+                ``HindsightConfig`` (env: ``HINDSIGHT_API_LLM_DEFAULT_HEADERS``).
             **kwargs: Additional provider-specific parameters.
         """
         super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs)
@@ -60,11 +65,16 @@ class AnthropicLLM(LLMInterface):
         try:
             from anthropic import AsyncAnthropic
 
-            client_kwargs: dict[str, Any] = {"api_key": self.api_key}
+            # SDK retries disabled — wrapper-level retry loop in ``call`` handles
+            # backoff (mirrors ``OpenAICompatibleLLM`` so the two providers behave
+            # consistently).
+            client_kwargs: dict[str, Any] = {"api_key": self.api_key, "max_retries": 0}
             if self.base_url:
                 client_kwargs["base_url"] = self.base_url
             if timeout:
                 client_kwargs["timeout"] = timeout
+            if default_headers:
+                client_kwargs["default_headers"] = default_headers
 
             self._client = AsyncAnthropic(**client_kwargs)
             logger.info(f"Anthropic client initialized for model: {self.model}")

hindsight-api-slim/hindsight_api/engine/providers/claude_code_llm.py

@@ -12,6 +12,8 @@ import logging
 import time
 from typing import Any
 
+from pydantic import ValidationError
+
 from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError
 from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
 from hindsight_api.metrics import get_metrics_collector
@@ -278,6 +280,12 @@ class ClaudeCodeLLM(LLMInterface):
 
                 return result
 
+            except ValidationError:
+                # Pydantic schema validation failure — retrying with the same
+                # input won't produce a different schema.  Raise immediately
+                # instead of burning quota on identical calls (#1412).
+                raise
+
             except Exception as e:
                 last_exception = e
 

hindsight-api-slim/hindsight_api/engine/providers/codex_llm.py

@@ -4,14 +4,26 @@ OpenAI Codex LLM provider using ChatGPT Plus/Pro OAuth authentication.
 This provider enables using ChatGPT Plus/Pro subscriptions for API calls
 without separate OpenAI Platform API credits. It uses OAuth tokens from
 ~/.codex/auth.json and communicates with the ChatGPT backend API.
+
+Tokens are refreshed automatically: the provider decodes the access_token
+JWT's ``exp`` claim and proactively refreshes via
+``POST https://auth.openai.com/oauth/token`` ~60s before expiry. It also
+reactively refreshes once on a 401/403 from the Codex backend before giving
+up. The refresh request shape mirrors the canonical ``@openai/codex`` CLI
+implementation (codex-rs/login/src/auth/manager.rs on github.com/openai/codex)
+so that future server-side changes affect both clients identically.
 """
 
 import asyncio
+import base64
+import binascii
 import json
 import logging
 import os
+import tempfile
 import time
 import uuid
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
 
@@ -24,6 +36,36 @@ from hindsight_api.metrics import get_metrics_collector
 logger = logging.getLogger(__name__)
 
 
+# OAuth refresh endpoint and client id, mirrored from the canonical
+# ``@openai/codex`` CLI (codex-rs/login/src/auth/manager.rs on
+# github.com/openai/codex). The endpoint is overridable via env var so that
+# future Codex changes or staging environments can be pointed at without a
+# code change — same env var name the upstream CLI uses.
+_CODEX_REFRESH_TOKEN_URL = os.environ.get("CODEX_REFRESH_TOKEN_URL_OVERRIDE", "https://auth.openai.com/oauth/token")
+_CODEX_CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann"
+
+# Proactively refresh this many seconds before the JWT ``exp`` claim. The
+# upstream Codex CLI uses no skew (it refreshes at ``exp <= now``); the
+# extra window reduces races where a request leaves the client with a token
+# that the server has already declared expired by the time it arrives.
+_CODEX_TOKEN_REFRESH_SKEW_SECONDS = 60
+
+# OAuth error codes that the refresh endpoint returns when the refresh_token
+# itself is no longer usable. These are terminal — retrying refresh will not
+# succeed; the user must re-run ``codex auth login``.
+_CODEX_TERMINAL_REFRESH_ERROR_CODES = frozenset(
+    {"refresh_token_expired", "refresh_token_reused", "refresh_token_invalidated"}
+)
+
+
+class CodexRefreshExpiredError(RuntimeError):
+    """Raised when the Codex refresh_token itself is no longer valid.
+
+    The user must re-run ``codex auth login`` to obtain new credentials.
+    Callers should surface a clear remediation message and stop retrying.
+    """
+
+
 class CodexLLM(LLMInterface):
     """
     LLM provider using OpenAI Codex OAuth authentication.
@@ -44,9 +86,19 @@ class CodexLLM(LLMInterface):
         """Initialize Codex LLM provider."""
         super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs)
 
+        # Path is fixed at ~/.codex/auth.json — matches the upstream CLI.
+        # Storing it on self lets the refresh path re-read after another
+        # process (e.g. a sidecar) rotates the file out from under us.
+        self._auth_file = Path.home() / ".codex" / "auth.json"
+
+        # Single-flight refresh lock. Multiple concurrent requests racing
+        # toward an expired token should produce one network refresh, not N.
+        self._auth_lock = asyncio.Lock()
+
         # Load Codex OAuth credentials
         try:
             self.access_token, self.account_id = self._load_codex_auth()
+            self.refresh_token = self._load_codex_refresh_token()
             logger.info(f"Loaded Codex OAuth credentials for account: {self.account_id}")
         except Exception as e:
             raise RuntimeError(
@@ -108,6 +160,290 @@ class CodexLLM(LLMInterface):
 
         return access_token, account_id
 
+    def _load_codex_refresh_token(self) -> str | None:
+        """Load ``tokens.refresh_token`` from ``~/.codex/auth.json``.
+
+        Returns None when the auth file is unreadable or omits the field —
+        the provider still functions as a one-shot loader in that case, it
+        just can't refresh when the access_token expires. This deliberately
+        does not raise so that ``__init__`` keeps the existing failure mode
+        of raising only on missing ``access_token``.
+        """
+        try:
+            with open(self._auth_file) as f:
+                data = json.load(f)
+        except (OSError, json.JSONDecodeError) as e:
+            logger.warning(
+                f"Codex auth file unreadable when loading refresh_token: {type(e).__name__}. "
+                "Token refresh will not be available; the access_token in memory will be used until it expires."
+            )
+            return None
+        return data.get("tokens", {}).get("refresh_token")
+
+    @staticmethod
+    def _decode_jwt_exp_unixtime(token: str) -> int | None:
+        """Return the JWT ``exp`` claim as a unix timestamp, or None on parse failure.
+
+        ChatGPT/Codex access_tokens are JWTs whose payload includes ``exp``
+        (RFC 7519). We need the expiry to schedule proactive refresh — the
+        ``auth.json`` file does not persist a separate ``expires_at`` field
+        in the upstream CLI's shape, so decoding the JWT itself is the
+        canonical way to know when the token is stale.
+
+        We do not verify the signature — the server is the source of truth
+        on whether the token is actually accepted, and the only thing this
+        method affects is the *timing* of refresh, not whether to trust the
+        token contents.
+        """
+        try:
+            parts = token.split(".")
+            if len(parts) < 2:
+                return None
+            payload_b64 = parts[1]
+            # JWT uses base64url without padding. Re-pad before decoding.
+            padding = "=" * (-len(payload_b64) % 4)
+            payload_bytes = base64.urlsafe_b64decode(payload_b64 + padding)
+            payload = json.loads(payload_bytes.decode("utf-8"))
+            exp = payload.get("exp")
+            return int(exp) if exp is not None else None
+        except (ValueError, TypeError, json.JSONDecodeError, binascii.Error):
+            return None
+
+    def _token_is_stale(self, skew_seconds: int = _CODEX_TOKEN_REFRESH_SKEW_SECONDS) -> bool:
+        """True when the cached access_token is past expiry (with skew).
+
+        Returns False when expiry cannot be determined — we'd rather use a
+        possibly-expired token and recover via the reactive 401 path than
+        refresh aggressively on every request when ``exp`` parsing fails.
+        """
+        exp = self._decode_jwt_exp_unixtime(self.access_token)
+        if exp is None:
+            return False
+        return exp <= int(time.time()) + skew_seconds
+
+    def _persist_auth_atomic(self, updated_tokens: dict[str, Any]) -> None:
+        """Write the rotated tokens back to ``~/.codex/auth.json`` atomically.
+
+        Strategy: re-read the on-disk auth.json (so we don't clobber fields
+        another process may have added), patch ``tokens.*`` and
+        ``last_refresh``, write to a tempfile in the same directory with
+        mode 0600, then ``os.replace`` onto the target. ``os.replace`` is
+        atomic within the same filesystem on POSIX and Windows, so a
+        concurrent reader will see either the old file or the fully-written
+        new file — never a partial truncate, which is the upstream CLI's
+        worst-case race.
+
+        On non-Unix platforms the chmod is a best-effort no-op; the parent
+        directory permissions still bound access.
+        """
+        current: dict[str, Any]
+        try:
+            with open(self._auth_file) as f:
+                loaded = json.load(f)
+            # auth.json should always be a JSON object at the top level; if
+            # someone has hand-edited it into a non-object shape, fall back
+            # to the minimal default rather than crashing the refresh path.
+            current = loaded if isinstance(loaded, dict) else {"auth_mode": "chatgpt", "tokens": {}}
+        except (OSError, json.JSONDecodeError):
+            # If the file became unreadable between our last read and now,
+            # construct a minimal shape rather than refusing to persist.
+            current = {"auth_mode": "chatgpt", "tokens": {}}
+
+        existing_tokens = current.get("tokens")
+        tokens: dict[str, Any] = existing_tokens if isinstance(existing_tokens, dict) else {}
+        for key in ("access_token", "refresh_token", "id_token", "account_id"):
+            if key in updated_tokens and updated_tokens[key] is not None:
+                tokens[key] = updated_tokens[key]
+        current["tokens"] = tokens
+        current["last_refresh"] = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+        # Write to a sibling tempfile so the rename is same-filesystem.
+        parent = self._auth_file.parent
+        parent.mkdir(parents=True, exist_ok=True)
+        fd, tmp_path = tempfile.mkstemp(prefix=".auth.", suffix=".json.tmp", dir=str(parent))
+        try:
+            with os.fdopen(fd, "w") as f:
+                json.dump(current, f, indent=2)
+                f.flush()
+                os.fsync(f.fileno())
+            try:
+                os.chmod(tmp_path, 0o600)
+            except OSError:
+                pass  # best-effort on platforms that don't support chmod
+            os.replace(tmp_path, self._auth_file)
+        except Exception:
+            # Clean up the orphaned tempfile if rename fails.
+            try:
+                os.unlink(tmp_path)
+            except OSError:
+                pass
+            raise
+
+    async def _refresh_oauth_tokens(self, reason: str = "", *, force: bool = False) -> None:
+        """Refresh the OAuth access_token using the stored refresh_token.
+
+        Single-flight: serialized through ``self._auth_lock`` so concurrent
+        callers produce one network request. The first caller refreshes; the
+        rest wake up and observe that either (a) the in-memory token is no
+        longer stale (proactive case) or (b) the in-memory token has changed
+        since they entered (reactive case), and return without re-refreshing.
+
+        Args:
+            reason: Free-form string included in log lines for diagnostics.
+            force: When True, refresh even if the JWT exp claim looks fresh.
+                Used by the reactive 401 path — the server rejected the
+                token, so we cannot trust the JWT's self-reported expiry.
+
+        Raises:
+            CodexRefreshExpiredError: when the server returns a terminal
+                error code (refresh_token_expired/reused/invalidated) or any
+                401 on the refresh endpoint itself.
+            RuntimeError: for other refresh failures (network, 5xx, etc.).
+        """
+        # Capture the token we'd be refreshing BEFORE acquiring the lock so
+        # that we can detect mid-wait rotation by another coroutine.
+        token_before_lock = self.access_token
+        async with self._auth_lock:
+            if force:
+                # Reactive: skip only if another coroutine already rotated
+                # the token while we were waiting on the lock.
+                if self.access_token != token_before_lock:
+                    return
+            else:
+                # Proactive: skip if the token is no longer stale (the
+                # canonical "another coroutine refreshed first" check).
+                if not self._token_is_stale():
+                    return
+
+            if not self.refresh_token:
+                raise RuntimeError(
+                    "Codex access_token is expired but no refresh_token is available. "
+                    "Run 'codex auth login' to re-authenticate."
+                )
+
+            log_reason = f" ({reason})" if reason else ""
+            logger.info(f"Refreshing Codex OAuth access_token{log_reason}")
+
+            request_body = {
+                "client_id": _CODEX_CLIENT_ID,
+                "grant_type": "refresh_token",
+                "refresh_token": self.refresh_token,
+            }
+            try:
+                response = await self._client.post(
+                    _CODEX_REFRESH_TOKEN_URL,
+                    json=request_body,
+                    headers={"Content-Type": "application/json"},
+                    timeout=30.0,
+                )
+            except httpx.RequestError as e:
+                raise RuntimeError(f"Codex OAuth refresh network error: {type(e).__name__}") from e
+
+            if response.status_code == 401:
+                # Classify by ``error.code`` (or top-level ``error`` string) — same
+                # mapping as the upstream Rust CLI's request_chatgpt_token_refresh.
+                error_code = self._extract_oauth_error_code(response)
+                if error_code in _CODEX_TERMINAL_REFRESH_ERROR_CODES:
+                    raise CodexRefreshExpiredError(
+                        f"Codex refresh_token is permanently invalid (error.code={error_code}). "
+                        "Run 'codex auth login' to re-authenticate."
+                    )
+                # Unknown 401 — treat as terminal too, matching the upstream classification.
+                raise CodexRefreshExpiredError(
+                    f"Codex OAuth refresh returned 401 with unrecognized error code "
+                    f"({error_code or 'none'}). Run 'codex auth login' to re-authenticate."
+                )
+
+            if response.status_code >= 400:
+                # 5xx and other 4xx are transient/retryable from the caller's
+                # perspective; surface as RuntimeError without leaking the
+                # request body in logs.
+                raise RuntimeError(f"Codex OAuth refresh failed with HTTP {response.status_code}")
+
+            try:
+                body = response.json()
+            except json.JSONDecodeError as e:
+                raise RuntimeError(f"Codex OAuth refresh returned non-JSON body: {e}") from e
+
+            new_access = body.get("access_token")
+            if not new_access:
+                raise RuntimeError("Codex OAuth refresh returned no access_token")
+
+            # The refresh_token may rotate on each refresh — adopt the new
+            # one if the server sent it, otherwise keep the existing.
+            new_refresh = body.get("refresh_token") or self.refresh_token
+            new_id_token = body.get("id_token")
+
+            # Update in-memory state first so callers waiting on the lock
+            # see fresh credentials immediately, even if disk write fails.
+            self.access_token = new_access
+            self.refresh_token = new_refresh
+
+            persisted = {
+                "access_token": new_access,
+                "refresh_token": new_refresh,
+            }
+            if new_id_token:
+                persisted["id_token"] = new_id_token
+
+            try:
+                self._persist_auth_atomic(persisted)
+            except OSError as e:
+                # In-memory creds are valid; warn but don't fail the request
+                # path. Future process starts will fall back to the stale
+                # on-disk auth.json and immediately refresh.
+                logger.warning(
+                    f"Codex OAuth refresh succeeded but persisting auth.json failed: {type(e).__name__}. "
+                    "In-memory credentials are up to date; on-disk file is stale."
+                )
+
+            logger.info("Codex OAuth access_token refreshed successfully")
+
+    @staticmethod
+    def _extract_oauth_error_code(response: "httpx.Response") -> str | None:
+        """Pull the OAuth error code out of a 4xx response body, if present.
+
+        The refresh endpoint returns shapes like
+        ``{"error": "...", "error_code": "..."}`` or
+        ``{"error": {"code": "..."}}``. We don't fail the call if the body
+        is unparseable — the caller falls back to a generic "unknown" error.
+        """
+        try:
+            body = response.json()
+        except (json.JSONDecodeError, ValueError):
+            return None
+        if not isinstance(body, dict):
+            return None
+        # Shape 1: error is a nested object with "code"
+        err = body.get("error")
+        if isinstance(err, dict):
+            code = err.get("code")
+            if isinstance(code, str):
+                return code
+        # Shape 2: top-level error_code string
+        code = body.get("error_code")
+        if isinstance(code, str):
+            return code
+        # Shape 3: error is itself a string code
+        if isinstance(err, str):
+            return err
+        return None
+
+    async def _ensure_fresh_token(self) -> None:
+        """Refresh the access_token proactively if it is near or past expiry.
+
+        Called at the top of every API-bound method. Cheap when the token is
+        fresh (just decodes the JWT exp claim and returns).
+        """
+        if self._token_is_stale():
+            try:
+                await self._refresh_oauth_tokens(reason="proactive (token near expiry)")
+            except CodexRefreshExpiredError:
+                # Surface to the caller as the same RuntimeError shape the
+                # request loop has historically raised, so existing error
+                # handling paths keep working.
+                raise
+
     def _map_reasoning_effort(self, effort: str) -> str:
         """
         Map standard reasoning effort to Codex reasoning summary format.
@@ -189,6 +525,15 @@ class CodexLLM(LLMInterface):
         """Make API call to Codex backend with SSE streaming."""
         start_time = time.time()
 
+        # Proactively refresh the OAuth access_token if it's near expiry.
+        # Cheap when fresh: a JWT exp decode + comparison.
+        await self._ensure_fresh_token()
+
+        # Tracks whether we've already attempted a reactive refresh in
+        # response to a 401 from the backend. Set once on the first auth
+        # failure so we retry exactly once after refresh, not in a loop.
+        attempted_refresh_after_auth_error = False
+
         # Prepare system instructions
         system_instruction = ""
         user_messages = []
@@ -244,7 +589,12 @@ class CodexLLM(LLMInterface):
         url = f"{self.base_url}/codex/responses"
         last_exception = None
 
-        for attempt in range(max_retries + 1):
+        # Manual attempt tracking instead of ``for attempt in range(...)`` so
+        # that the reactive-refresh path can retry once without consuming a
+        # normal-retry budget slot. The refresh-retry is conceptually a
+        # separate auth-recovery attempt that shouldn't compete with backoff.
+        attempt = 0
+        while True:
             try:
                 response = await self._client.post(url, json=payload, headers=headers, timeout=120.0)
                 response.raise_for_status()
@@ -269,6 +619,7 @@ class CodexLLM(LLMInterface):
                             backoff = min(initial_backoff * (2**attempt), max_backoff)
                             await asyncio.sleep(backoff)
                             last_exception = e
+                            attempt += 1
                             continue
                         raise
 
@@ -332,8 +683,38 @@ class CodexLLM(LLMInterface):
                 last_exception = e
                 status_code = e.response.status_code
 
-                # Fast fail on auth errors
+                # Auth error: try one OAuth refresh + retry before giving up.
+                # The proactive refresh at the top of this method catches most
+                # expiries, but a token can also become invalid mid-request if
+                # another process rotates auth.json out from under us, or if
+                # the JWT exp claim is unparseable and we never knew it was
+                # stale. Reactive refresh is the safety net.
                 if status_code in (401, 403):
+                    if not attempted_refresh_after_auth_error:
+                        attempted_refresh_after_auth_error = True
+                        try:
+                            await self._refresh_oauth_tokens(
+                                reason=f"reactive (HTTP {status_code} from codex backend)",
+                                force=True,
+                            )
+                            # Rebuild the Authorization header with the new
+                            # token and retry without consuming a normal-retry
+                            # budget slot — this is a dedicated auth-recovery
+                            # attempt that shouldn't compete with backoff.
+                            headers["Authorization"] = f"Bearer {self.access_token}"
+                            logger.info("Codex auth refreshed after auth error; retrying request once")
+                            continue
+                        except CodexRefreshExpiredError as refresh_err:
+                            logger.error("Codex refresh_token is permanently invalid; cannot recover from auth error")
+                            raise RuntimeError(
+                                "Codex authentication failed and the refresh_token is no longer valid.\n"
+                                "Run 'codex auth login' to re-authenticate."
+                            ) from refresh_err
+                        except Exception as refresh_err:
+                            logger.error(
+                                f"Codex token refresh attempt failed: {type(refresh_err).__name__}: {refresh_err}"
+                            )
+                            # Fall through to the original raise below.
                     logger.error(f"Codex auth error (HTTP {status_code}): {e.response.text[:200]}")
                     raise RuntimeError(
                         "Codex authentication failed. Your OAuth token may have expired.\n"
@@ -349,6 +730,7 @@ class CodexLLM(LLMInterface):
                         f"Codex HTTP error {status_code} (attempt {attempt + 1}/{max_retries + 1}): {error_detail}"
                     )
                     await asyncio.sleep(backoff)
+                    attempt += 1
                     continue
                 else:
                     logger.error(
@@ -362,6 +744,7 @@ class CodexLLM(LLMInterface):
                     backoff = min(initial_backoff * (2**attempt), max_backoff)
                     logger.warning(f"Codex connection error (attempt {attempt + 1}/{max_retries + 1}): {e}")
                     await asyncio.sleep(backoff)
+                    attempt += 1
                     continue
                 else:
                     logger.error(f"Codex connection error after {max_retries + 1} attempts: {e}")
@@ -462,6 +845,11 @@ class CodexLLM(LLMInterface):
         """
         start_time = time.time()
 
+        # Proactively refresh the OAuth access_token if it's near expiry.
+        # Same rationale as in ``call()`` — keeps the request from leaving
+        # the client carrying a token that's already past ``exp``.
+        await self._ensure_fresh_token()
+
         # Prepare system instructions
         system_instruction = ""
         user_messages = []
@@ -534,9 +922,39 @@ class CodexLLM(LLMInterface):
         # Debug logging for troubleshooting
         logger.debug(f"Codex tool call request: url={url}, model={payload['model']}, tools={len(codex_tools)}")
 
+        # One reactive refresh attempt on auth failure, mirroring call().
+        # ``call_with_tools`` doesn't have a retry loop, so we hand-roll a
+        # single retry after refreshing the token. Any non-auth error still
+        # surfaces immediately to keep behavior identical for callers.
+        attempted_refresh_after_auth_error = False
+
         try:
             response = await self._client.post(url, json=payload, headers=headers, timeout=120.0)
 
+            if response.status_code in (401, 403) and not attempted_refresh_after_auth_error:
+                attempted_refresh_after_auth_error = True
+                try:
+                    await self._refresh_oauth_tokens(
+                        reason=f"reactive (HTTP {response.status_code} from codex backend in call_with_tools)",
+                        force=True,
+                    )
+                    headers["Authorization"] = f"Bearer {self.access_token}"
+                    logger.info("Codex auth refreshed after auth error; retrying tool-call request once")
+                    response = await self._client.post(url, json=payload, headers=headers, timeout=120.0)
+                except CodexRefreshExpiredError as refresh_err:
+                    logger.error(
+                        "Codex refresh_token is permanently invalid; cannot recover from auth error in tool-call path"
+                    )
+                    raise RuntimeError(
+                        "Codex authentication failed and the refresh_token is no longer valid.\n"
+                        "Run 'codex auth login' to re-authenticate."
+                    ) from refresh_err
+                except Exception as refresh_err:
+                    logger.error(
+                        f"Codex token refresh attempt failed in tool-call path: {type(refresh_err).__name__}: {refresh_err}"
+                    )
+                    # Fall through to the normal error path below.
+
             # Log response details on error
             if response.status_code != 200:
                 logger.error(f"Codex API error {response.status_code}: {response.text[:500]}")

hindsight-api-slim/hindsight_api/engine/providers/litellm_llm.py

@@ -103,12 +103,58 @@ class LiteLLMLLM(LLMInterface):
         if self.base_url:
             kwargs["api_base"] = self.base_url
         if max_completion_tokens is not None:
-            kwargs["max_completion_tokens"] = max_completion_tokens
+            kwargs["max_completion_tokens"] = self._cap_max_completion_tokens(max_completion_tokens)
         if temperature is not None:
             kwargs["temperature"] = temperature
 
         return kwargs
 
+    # ── per-model output-tokens cap (shared with Router subclass) ────────────
+    # Hindsight's defaults (e.g. retain_max_completion_tokens=64000) target
+    # high-capacity models. When a configured deployment supports fewer
+    # completion tokens (e.g. gpt-4.1-nano caps at 32768), the call would
+    # otherwise be rejected. Cap pre-emptively using LiteLLM's per-model
+    # registry so things work out of the box across the supported model set.
+
+    def _cap_max_completion_tokens(self, value: int) -> int:
+        cap = self._get_model_output_cap()
+        if cap and value > cap:
+            logger.debug("capping max_completion_tokens %d -> %d for model %s", value, cap, self.model)
+            return cap
+        return value
+
+    def _get_model_output_cap(self) -> int | None:
+        """Return the configured model's max output tokens, per LiteLLM's registry."""
+        try:
+            cap = self._litellm.get_max_tokens(self.model)
+            return int(cap) if cap else None
+        except Exception:
+            return None
+
+    # ── hooks for Router-style subclasses ────────────────────────────────────
+    # The retry+parse loop in call() / call_with_tools() is shared by every
+    # LiteLLM-backed provider. Subclasses override the small surface below to
+    # swap the completion fn (direct vs Router) and rename the deployment that
+    # actually answered the request.
+
+    @property
+    def _stage_label(self) -> str:
+        """Stage breadcrumb label — overridden by subclasses (e.g. ``litellmrouter``)."""
+        return "litellm"
+
+    async def _acompletion(self, **kwargs: Any) -> Any:
+        """Issue a chat completion. Subclasses override to route via ``litellm.Router``."""
+        return await self._litellm.acompletion(**kwargs)
+
+    def _resolve_completion_model(self, response: Any) -> str:
+        """
+        Return the model name to record in metrics/tracing.
+
+        For Router-backed providers this can differ from ``self.model`` — the Router
+        may pick a different deployment than the primary. Default: ``self.model``.
+        """
+        return self.model
+
     async def call(
         self,
         messages: list[dict[str, str]],
@@ -143,12 +189,13 @@ class LiteLLMLLM(LLMInterface):
 
         for attempt in range(max_retries + 1):
             if attempt > 0:
-                set_stage(f"llm.litellm.{scope}.attempt={attempt + 1}/{max_retries + 1}")
+                set_stage(f"llm.{self._stage_label}.{scope}.attempt={attempt + 1}/{max_retries + 1}")
             try:
-                response = await self._litellm.acompletion(**call_kwargs)
+                response = await self._acompletion(**call_kwargs)
 
                 content = response.choices[0].message.content or ""
                 finish_reason = response.choices[0].finish_reason
+                model_name = self._resolve_completion_model(response)
 
                 # Check for length-limited output
                 if finish_reason == "length":
@@ -184,7 +231,7 @@ class LiteLLMLLM(LLMInterface):
                 metrics = get_metrics_collector()
                 metrics.record_llm_call(
                     provider=self.provider,
-                    model=self.model,
+                    model=model_name,
                     scope=scope,
                     duration=duration,
                     input_tokens=input_tokens,
@@ -198,7 +245,7 @@ class LiteLLMLLM(LLMInterface):
                 span_recorder = get_span_recorder()
                 span_recorder.record_llm_call(
                     provider=self.provider,
-                    model=self.model,
+                    model=model_name,
                     scope=scope,
                     messages=messages,
                     response_content=_serialize_for_span(result),
@@ -211,7 +258,7 @@ class LiteLLMLLM(LLMInterface):
 
                 if duration > 10.0:
                     logger.info(
-                        f"slow llm call: scope={scope}, model={self.provider}/{self.model}, "
+                        f"slow llm call: scope={scope}, model={self.provider}/{model_name}, "
                         f"input_tokens={input_tokens}, output_tokens={output_tokens}, "
                         f"time={duration:.3f}s"
                     )
@@ -287,13 +334,14 @@ class LiteLLMLLM(LLMInterface):
         last_exception = None
         for attempt in range(max_retries + 1):
             if attempt > 0:
-                set_stage(f"llm.litellm.tools.attempt={attempt + 1}/{max_retries + 1}")
+                set_stage(f"llm.{self._stage_label}.tools.attempt={attempt + 1}/{max_retries + 1}")
             try:
-                response = await self._litellm.acompletion(**call_kwargs)
+                response = await self._acompletion(**call_kwargs)
 
                 message = response.choices[0].message
                 content = message.content
                 finish_reason = response.choices[0].finish_reason
+                model_name = self._resolve_completion_model(response)
 
                 # Extract tool calls
                 tool_calls: list[LLMToolCall] = []
@@ -319,7 +367,7 @@ class LiteLLMLLM(LLMInterface):
                 metrics = get_metrics_collector()
                 metrics.record_llm_call(
                     provider=self.provider,
-                    model=self.model,
+                    model=model_name,
                     scope=scope,
                     duration=duration,
                     input_tokens=input_tokens,
@@ -338,7 +386,7 @@ class LiteLLMLLM(LLMInterface):
                 )
                 span_recorder.record_llm_call(
                     provider=self.provider,
-                    model=self.model,
+                    model=model_name,
                     scope=scope,
                     messages=messages,
                     response_content=content,

hindsight-api-slim/hindsight_api/engine/providers/litellm_router_llm.py

@@ -0,0 +1,167 @@
+"""
+LiteLLM Router LLM provider — pure pass-through to ``litellm.Router``.
+
+The full configuration object is forwarded verbatim. We do not translate model
+names, infer fallbacks, validate shape, or introspect Router internals:
+whatever the user puts in ``HINDSIGHT_API_LLM_LITELLMROUTER_CONFIG`` becomes
+``Router(**config)``. If the shape is wrong, LiteLLM Router raises.
+
+The only Hindsight-imposed convention is that one entry in ``model_list``
+must have ``model_name: "default"`` — that's the entrypoint we issue
+completions against. Everything else (ordering, fallbacks, load-balancing,
+weighted picks, rate limits, retries, cooldowns) is whatever the user
+configures via LiteLLM's own keys.
+
+See https://docs.litellm.ai/docs/routing for the supported keys (``model_list``,
+``fallbacks``, ``context_window_fallbacks``, ``num_retries``, ``cooldown_time``,
+``routing_strategy``, ``allowed_fails``, …).
+
+The retry/parse/metrics loop is shared with ``LiteLLMLLM`` via inheritance:
+this class only overrides the completion fn, the call kwargs, and the model
+name reported in metrics.
+
+Example ``HINDSIGHT_API_LLM_LITELLMROUTER_CONFIG``::
+
+    {
+      "model_list": [
+        {"model_name": "default",  "litellm_params": {"model": "openai/gpt-4o-mini",       "api_key": "sk-..."}},
+        {"model_name": "fallback", "litellm_params": {"model": "anthropic/claude-sonnet-4", "api_key": "sk-ant-..."}}
+      ],
+      "fallbacks": [{"default": ["fallback"]}],
+      "num_retries": 0,
+      "cooldown_time": 60
+    }
+"""
+
+import logging
+from typing import Any
+
+from hindsight_api.engine.providers.litellm_llm import LiteLLMLLM
+
+logger = logging.getLogger(__name__)
+
+
+# Hindsight always issues completions against this ``model_name``. Users must
+# include at least one entry with ``model_name: "default"`` in their config's
+# ``model_list``; that entry is the entrypoint, and any other entries become
+# fallback / load-balance / weighted-pool members per the user's own
+# ``fallbacks`` / ``routing_strategy`` settings.
+_ENTRYPOINT_MODEL_NAME = "default"
+
+
+class LiteLLMRouterLLM(LiteLLMLLM):
+    """
+    LLM provider backed by ``litellm.Router``.
+
+    The full Router config is supplied by the caller. We pass it verbatim to
+    ``Router(**config)`` and route requests against the first ``model_list``
+    entry's ``model_name``. Inherits the retry/parse/metrics loop from
+    ``LiteLLMLLM``; only the completion fn and the call kwargs differ.
+    """
+
+    def __init__(
+        self,
+        provider: str,
+        api_key: str,
+        base_url: str,
+        model: str,
+        config: dict[str, Any],
+        reasoning_effort: str = "low",
+        timeout: float = 300.0,
+        **kwargs: Any,
+    ):
+        super().__init__(
+            provider=provider,
+            api_key=api_key,
+            base_url=base_url,
+            model=model,
+            reasoning_effort=reasoning_effort,
+            timeout=timeout,
+            **kwargs,
+        )
+        self.config = config
+
+        from litellm import Router
+
+        logging.getLogger("LiteLLM Router").setLevel(logging.WARNING)
+        # Pure pass-through: whatever the user gave goes straight to LiteLLM Router.
+        # If the shape is invalid, Router raises its own error — we don't pre-validate
+        # or introspect Router internals.
+        self._router = Router(**config)
+
+        # Pre-compute the most conservative output-tokens cap across every configured
+        # deployment so a single max_completion_tokens value works no matter which
+        # deployment Router picks. Uses LiteLLM's own per-model registry; unknown
+        # models contribute no cap. See LiteLLMLLM._cap_max_completion_tokens.
+        self._router_output_cap = self._compute_router_output_cap(config)
+
+        logger.info("LiteLLM Router initialized; entrypoint model_name=%r", _ENTRYPOINT_MODEL_NAME)
+
+    def _compute_router_output_cap(self, config: dict[str, Any]) -> int | None:
+        caps: list[int] = []
+        for deployment in (config.get("model_list") or []) if isinstance(config, dict) else []:
+            if not isinstance(deployment, dict):
+                continue
+            params = deployment.get("litellm_params") or {}
+            model_str = params.get("model") if isinstance(params, dict) else None
+            if not model_str:
+                continue
+            try:
+                cap = self._litellm.get_max_tokens(model_str)
+            except Exception:
+                cap = None
+            if cap:
+                caps.append(int(cap))
+        return min(caps) if caps else None
+
+    # ── overrides for the shared retry/parse loop ───────────────────────────
+
+    @property
+    def _stage_label(self) -> str:
+        return "litellmrouter"
+
+    async def _acompletion(self, **kwargs: Any) -> Any:
+        return await self._router.acompletion(**kwargs)
+
+    def _resolve_completion_model(self, response: Any) -> str:
+        hidden = getattr(response, "_hidden_params", None) or {}
+        return hidden.get("model") or _ENTRYPOINT_MODEL_NAME
+
+    def _get_model_output_cap(self) -> int | None:
+        return self._router_output_cap
+
+    def _build_common_kwargs(
+        self,
+        messages: list[dict[str, Any]],
+        max_completion_tokens: int | None = None,
+        temperature: float | None = None,
+    ) -> dict[str, Any]:
+        # Always issue against the entrypoint group; Router handles deployment selection,
+        # cross-group fallbacks, retries, cooldowns — whatever the user configured.
+        kwargs: dict[str, Any] = {
+            "model": _ENTRYPOINT_MODEL_NAME,
+            "messages": messages,
+        }
+        if max_completion_tokens is not None:
+            kwargs["max_completion_tokens"] = self._cap_max_completion_tokens(max_completion_tokens)
+        if temperature is not None:
+            kwargs["temperature"] = temperature
+        return kwargs
+
+    async def verify_connection(self) -> None:
+        from hindsight_api.engine.llm_interface import OutputTooLongError
+
+        try:
+            await self.call(
+                messages=[{"role": "user", "content": "test"}],
+                max_completion_tokens=50,
+                temperature=0.0,
+                scope="verification",
+                max_retries=0,
+            )
+            logger.info("LiteLLM Router connection verified successfully")
+        except OutputTooLongError:
+            logger.info("LiteLLM Router connection verified successfully (response truncated)")
+        except Exception as e:
+            logger.error(f"LiteLLM Router connection verification failed: {e}")
+            raise RuntimeError(f"Failed to verify LiteLLM Router connection: {e}") from e

hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py

@@ -1,5 +1,6 @@
 """
-OpenAI-compatible LLM provider supporting OpenAI, Groq, Ollama, LMStudio, MiniMax, and DeepSeek.
+OpenAI-compatible LLM provider supporting OpenAI, Groq, Ollama, LMStudio, MiniMax, DeepSeek,
+and Opencode Go.
 
 This provider handles all OpenAI API-compatible models including:
 - OpenAI: GPT-4, GPT-4o, GPT-5, o1, o3 (reasoning models)
@@ -8,6 +9,7 @@ This provider handles all OpenAI API-compatible models including:
 - LMStudio: Local models with OpenAI-compatible API
 - MiniMax: MiniMax-M2.7 models with 1M context window
 - DeepSeek: deepseek-v4-flash / deepseek-v4-pro / deepseek-chat / deepseek-reasoner via api.deepseek.com
+- Opencode Go: deepseek-v4-flash via https://opencode.ai/zen/go/v1
 
 Features:
 - Reasoning models with extended thinking (o1, o3, GPT-5 families)
@@ -232,6 +234,7 @@ class OpenAICompatibleLLM(LLMInterface):
     - LMStudio: Local models with OpenAI-compatible API
     - MiniMax: MiniMax-M2.7 models via OpenAI-compatible API (https://api.minimax.io/v1)
     - DeepSeek: deepseek-v4-flash / deepseek-v4-pro / deepseek-chat / deepseek-reasoner via https://api.deepseek.com
+    - opencode-go: deepseek-v4-flash via https://opencode.ai/zen/go/v1
     """
 
     def __init__(
@@ -250,7 +253,7 @@ class OpenAICompatibleLLM(LLMInterface):
         Initialize OpenAI-compatible LLM provider.
 
         Args:
-            provider: Provider name ("openai", "groq", "ollama", "lmstudio").
+            provider: Provider name ("openai", "groq", "ollama", "lmstudio", "opencode-go", etc.).
             api_key: API key (optional for ollama/lmstudio).
             base_url: Base URL for the API (uses defaults for groq/ollama/lmstudio if empty).
             model: Model name.
@@ -273,6 +276,8 @@ class OpenAICompatibleLLM(LLMInterface):
             "deepseek",
             "volcano",
             "openrouter",
+            "zai",
+            "opencode-go",
         ]
         if self.provider not in valid_providers:
             raise ValueError(f"OpenAICompatibleLLM only supports: {', '.join(valid_providers)}. Got: {self.provider}")
@@ -291,13 +296,29 @@ class OpenAICompatibleLLM(LLMInterface):
                 self.base_url = "https://api.deepseek.com"
             elif self.provider == "openrouter":
                 self.base_url = "https://openrouter.ai/api/v1"
+            elif self.provider == "zai":
+                self.base_url = "https://api.z.ai/api/coding/paas/v4"
+            elif self.provider == "opencode-go":
+                self.base_url = "https://opencode.ai/zen/go/v1"
 
         # For ollama/lmstudio, use dummy key if not provided
         if self.provider in ("ollama", "lmstudio") and not self.api_key:
             self.api_key = "local"
 
         # Validate API key for cloud providers
-        if self.provider in ("openai", "groq", "minimax", "deepseek", "openrouter") and not self.api_key:
+        if (
+            self.provider
+            in (
+                "openai",
+                "groq",
+                "minimax",
+                "deepseek",
+                "openrouter",
+                "zai",
+                "opencode-go",
+            )
+            and not self.api_key
+        ):
             raise ValueError(f"API key is required for {self.provider}")
 
         # Service tier configuration (from config, not env vars)
@@ -558,10 +579,11 @@ class OpenAICompatibleLLM(LLMInterface):
                     )
 
                     # Strip reasoning model thinking tags
-                    # Supports: <think>, <thinking>, <reasoning>, |startthink|/|endthink|
+                    # Supports: <think>, <thinking>, <thought>, <reasoning>, |startthink|/|endthink|
                     original_len = len(content)
                     content = re.sub(r"<think>.*?</think>", "", content, flags=re.DOTALL)
                     content = re.sub(r"<thinking>.*?</thinking>", "", content, flags=re.DOTALL)
+                    content = re.sub(r"<thought>.*?</thought>", "", content, flags=re.DOTALL)
                     content = re.sub(r"<reasoning>.*?</reasoning>", "", content, flags=re.DOTALL)
                     content = re.sub(r"\|startthink\|.*?\|endthink\|", "", content, flags=re.DOTALL)
                     content = content.strip()

hindsight-api-slim/hindsight_api/engine/reflect/agent.py

@@ -14,8 +14,6 @@ import re
 import time
 from typing import TYPE_CHECKING, Any, Awaitable, Callable
 
-import tiktoken
-
 from .models import DirectiveInfo, LLMCall, ReflectAgentResult, TokenUsageSummary, ToolCall
 from .prompts import (
     _extract_directive_rules,
@@ -23,6 +21,7 @@ from .prompts import (
     build_final_system_prompt,
     build_system_prompt_for_tools,
 )
+from .tokenization import count_cl100k_tokens
 from .tools_schema import get_reflect_tools
 
 
@@ -266,25 +265,22 @@ OUTPUT:"""
         return None, 0, 0
 
 
-_TIKTOKEN_ENCODING = tiktoken.get_encoding("cl100k_base")
-
-
 def _count_messages_tokens(messages: list[dict[str, Any]]) -> int:
     """Estimate the token count of the messages list using cl100k_base encoding."""
     total = 0
     for msg in messages:
         content = msg.get("content") or ""
         if isinstance(content, str):
-            total += len(_TIKTOKEN_ENCODING.encode(content))
+            total += count_cl100k_tokens(content)
         elif isinstance(content, list):
             for part in content:
                 if isinstance(part, dict) and isinstance(part.get("text"), str):
-                    total += len(_TIKTOKEN_ENCODING.encode(part["text"]))
+                    total += count_cl100k_tokens(part["text"])
         # Tool call arguments and results also count
         for tc in msg.get("tool_calls") or []:
             if isinstance(tc, dict):
                 func = tc.get("function", {})
-                total += len(_TIKTOKEN_ENCODING.encode(func.get("arguments", "")))
+                total += count_cl100k_tokens(func.get("arguments", ""))
     return total
 
 
@@ -672,7 +668,7 @@ async def run_reflect_agent(
                 # must respect max_tokens like the forced-final paths do. If it
                 # overshoots, run one extra capped call to rewrite it within
                 # the cap.
-                if max_tokens is not None and len(_TIKTOKEN_ENCODING.encode(answer)) > max_tokens:
+                if max_tokens is not None and count_cl100k_tokens(answer) > max_tokens:
                     rewrite_start = time.time()
                     rewritten, rewrite_usage = await llm_config.call(
                         messages=[

hindsight-api-slim/hindsight_api/engine/reflect/prompts.py

@@ -10,9 +10,7 @@ The reflect agent uses hierarchical retrieval:
 import json
 from typing import Any
 
-import tiktoken
-
-_TIKTOKEN_ENCODING = tiktoken.get_encoding("cl100k_base")
+from .tokenization import count_cl100k_tokens
 
 # Fraction of max_context_tokens reserved for tool results in the final synthesis prompt.
 # The remainder covers the system prompt, question, bank context, and output tokens.
@@ -453,7 +451,7 @@ def build_final_prompt(
             except (TypeError, ValueError):
                 output_str = str(output)
             block = f"\n### From {tool}:\n```json\n{output_str}\n```"
-            block_tokens = len(_TIKTOKEN_ENCODING.encode(block))
+            block_tokens = count_cl100k_tokens(block)
             if block_tokens > token_budget:
                 truncated = True
                 break

hindsight-api-slim/hindsight_api/engine/reflect/tokenization.py

@@ -0,0 +1,17 @@
+"""Token counting helpers for reflect prompts and agent control flow."""
+
+from functools import lru_cache
+
+import tiktoken
+
+
+@lru_cache(maxsize=1)
+def _get_cl100k_base_encoding() -> tiktoken.Encoding:
+    # tiktoken downloads this encoding on first lookup when it is not cached.
+    # Keep the lookup lazy so importing hindsight_api does not depend on network access.
+    return tiktoken.get_encoding("cl100k_base")
+
+
+def count_cl100k_tokens(text: str) -> int:
+    """Return the number of cl100k_base tokens in text."""
+    return len(_get_cl100k_base_encoding().encode(text))

hindsight-api-slim/hindsight_api/engine/reflect/tools.py

@@ -7,6 +7,7 @@ Implements hierarchical retrieval:
 3. recall - Raw facts as ground truth
 """
 
+import json
 import logging
 import uuid
 from dataclasses import replace
@@ -22,6 +23,21 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+def _document_metadata_from_retain_params(retain_params: Any) -> dict[str, Any] | None:
+    """Return document metadata stored under retain_params.metadata."""
+    if isinstance(retain_params, str):
+        try:
+            retain_params = json.loads(retain_params)
+        except json.JSONDecodeError:
+            return None
+
+    if not isinstance(retain_params, dict):
+        return None
+
+    metadata = retain_params.get("metadata")
+    return metadata if isinstance(metadata, dict) else None
+
+
 async def tool_search_mental_models(
     memory_engine: "MemoryEngine",
     conn: "Connection",
@@ -350,7 +366,7 @@ async def tool_expand(
     if all_doc_ids:
         docs = await conn.fetch(
             f"""
-            SELECT id, original_text, metadata, retain_params
+            SELECT id, original_text, retain_params
             FROM {fq_table("documents")}
             WHERE id = ANY($1) AND bank_id = $2
             """,
@@ -396,7 +412,7 @@ async def tool_expand(
                 item["document"] = {
                     "id": doc["id"],
                     "full_text": doc["original_text"],
-                    "metadata": doc["metadata"],
+                    "metadata": _document_metadata_from_retain_params(doc["retain_params"]),
                     "retain_params": doc["retain_params"],
                 }
         elif memory["document_id"] and depth == "document" and memory["document_id"] in doc_map:
@@ -405,7 +421,7 @@ async def tool_expand(
             item["document"] = {
                 "id": doc["id"],
                 "full_text": doc["original_text"],
-                "metadata": doc["metadata"],
+                "metadata": _document_metadata_from_retain_params(doc["retain_params"]),
                 "retain_params": doc["retain_params"],
             }
 

hindsight-api-slim/hindsight_api/engine/retain/bank_utils.py

@@ -10,6 +10,7 @@ from typing import TypedDict
 
 from pydantic import BaseModel, Field
 
+from ..._vector_index import index_using_clause, uses_per_bank_vector_indexes
 from ...config import get_config
 from ..db_utils import acquire_with_retry
 from ..memory_engine import fq_table, get_current_schema
@@ -35,15 +36,12 @@ def _bank_index_name(ft: str, internal_id: str) -> str:
     return f"idx_mu_emb_{_BANK_INDEX_FACT_TYPES[ft]}_{uid}"
 
 
-def _vector_index_clause() -> str:
-    """Return the USING clause for vector index creation based on the configured extension."""
+def _vector_index_clause() -> str | None:
+    """Return the USING clause for per-bank vector indexes, if this backend uses them."""
     ext = get_config().vector_extension
-    if ext == "pgvectorscale":
-        return "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)"
-    elif ext == "vchord":
-        return "USING vchordrq (embedding vector_l2_ops)"
-    else:  # pgvector (default)
-        return "USING hnsw (embedding vector_cosine_ops)"
+    if not uses_per_bank_vector_indexes(ext):
+        return None
+    return index_using_clause(ext)
 
 
 async def create_bank_vector_indexes(conn, bank_id: str, internal_id: str, ops=None) -> None:
@@ -52,20 +50,26 @@ async def create_bank_vector_indexes(conn, bank_id: str, internal_id: str, ops=N
     Respects the HINDSIGHT_API_VECTOR_EXTENSION config to use the appropriate
     index type (HNSW for pgvector, DiskANN for pgvectorscale, vchordrq for vchord).
 
-    Called immediately after the bank row is first inserted. Safe on empty banks
-    (index build is instant). Idempotent via CREATE INDEX IF NOT EXISTS.
+    AlloyDB ScaNN uses global vector indexes with filtered vector search; it
+    cannot safely create per-bank indexes at bank-creation time because new
+    banks have no embedding rows.
     bank_id is escaped for SQL literal safety (apostrophes doubled).
 
     On Oracle 23ai, this is a no-op — Oracle uses a single global vector index
     created during migrations. Partial indexes (WHERE clause) are not supported
     for Oracle vector indexes.
     """
+    index_clause = _vector_index_clause()
+    if index_clause is None:
+        logger.debug("Skipping per-bank vector indexes for configured backend")
+        return
+
     await ops.create_bank_vector_indexes(
         conn,
         fq_table("memory_units"),
         bank_id,
         internal_id,
-        _vector_index_clause(),
+        index_clause,
         _BANK_INDEX_FACT_TYPES,
     )
 
@@ -368,30 +372,49 @@ Merged mission:"""
 
 async def list_banks(pool) -> list:
     """
-    List all banks in the system.
+    List all banks in the system with summary stats.
 
     Args:
         pool: Database connection pool
 
     Returns:
-        List of dicts with bank_id, name, disposition, mission, created_at, updated_at
+        List of dicts with bank info and stats (document_count, fact_count, last_event_at)
     """
+    banks_table = fq_table("banks")
+    docs_table = fq_table("documents")
+    mu_table = fq_table("memory_units")
+
     async with acquire_with_retry(pool) as conn:
         rows = await conn.fetch(
             f"""
-            SELECT bank_id, name, disposition, mission, created_at, updated_at
-            FROM {fq_table("banks")}
-            ORDER BY updated_at DESC
+            SELECT
+                b.bank_id, b.name, b.disposition, b.mission,
+                b.created_at, b.updated_at,
+                COALESCE(m.fact_count, 0) AS fact_count,
+                d.last_document_at
+            FROM {banks_table} b
+            LEFT JOIN (
+                SELECT bank_id, MAX(created_at) AS last_document_at
+                FROM {docs_table}
+                GROUP BY bank_id
+            ) d ON d.bank_id = b.bank_id
+            LEFT JOIN (
+                SELECT bank_id, COUNT(*) AS fact_count
+                FROM {mu_table}
+                GROUP BY bank_id
+            ) m ON m.bank_id = b.bank_id
+            ORDER BY d.last_document_at DESC NULLS LAST, b.updated_at DESC
             """
         )
 
         result = []
         for row in rows:
-            # asyncpg returns JSONB as a string, so parse it
             disposition_data = row["disposition"]
             if isinstance(disposition_data, str):
                 disposition_data = json.loads(disposition_data)
 
+            last_doc = row["last_document_at"]
+
             result.append(
                 {
                     "bank_id": row["bank_id"],
@@ -400,6 +423,8 @@ async def list_banks(pool) -> list:
                     "mission": row["mission"] or "",
                     "created_at": row["created_at"].isoformat() if row["created_at"] else None,
                     "updated_at": row["updated_at"].isoformat() if row["updated_at"] else None,
+                    "fact_count": row["fact_count"],
+                    "last_document_at": last_doc.isoformat() if last_doc else None,
                 }
             )
 

hindsight-api-slim/hindsight_api/engine/retain/entity_labels.py

@@ -6,7 +6,7 @@ Defines a controlled vocabulary of key:value classification labels
 at retain time and stored as entities.
 """
 
-from typing import Literal
+from typing import Any, Literal
 
 from pydantic import BaseModel, Field, create_model
 
@@ -18,15 +18,28 @@ class LabelValue(BaseModel):
     description: str = ""
 
 
+class MapField(BaseModel):
+    """A field within a map-type entity label group. Supports recursion via type='map'."""
+
+    type: Literal["text", "value", "multi-values", "map"] = "text"
+    description: str = ""
+    values: list[LabelValue] = []
+    fields: dict[str, "MapField"] = {}
+
+
+MapField.model_rebuild()
+
+
 class LabelGroup(BaseModel):
     """A label group (dimension) with its type and allowed values."""
 
     key: str
     description: str = ""
-    type: Literal["value", "multi-values", "text"] = "value"
+    type: Literal["value", "multi-values", "text", "map"] = "value"
     optional: bool = True
     tag: bool = False
     values: list[LabelValue] = []
+    fields: dict[str, MapField] = {}
 
 
 class EntityLabelsConfig(BaseModel):
@@ -91,6 +104,71 @@ def _migrate_label_group(raw: dict) -> dict:
     return patched
 
 
+def _build_map_fields_model(fields: dict[str, MapField], model_name: str) -> type[BaseModel] | None:
+    """
+    Build a dynamic Pydantic model for a set of map fields (recursive).
+
+    Each field becomes a typed Pydantic field based on its type:
+    - text         → str | None
+    - value        → Literal[...] | None
+    - multi-values → list[Literal[...]]
+    - map          → list[NestedModel] (recursive)
+
+    Returns:
+        Dynamic Pydantic model class, or None if no fields defined
+    """
+    if not fields:
+        return None
+    model_fields: dict[str, Any] = {}
+    for field_name, map_field in fields.items():
+        description = map_field.description or field_name
+
+        if map_field.type == "map":
+            nested = _build_map_fields_model(map_field.fields, model_name + field_name.capitalize())
+            if nested is not None:
+                model_fields[field_name] = (
+                    list[nested],  # type: ignore[valid-type]
+                    Field(default_factory=list, description=description),
+                )
+        elif map_field.type == "text":
+            model_fields[field_name] = (str | None, Field(default=None, description=description))
+        else:
+            # value / multi-values — enum-constrained
+            if not map_field.values:
+                model_fields[field_name] = (str | None, Field(default=None, description=description))
+                continue
+            values = tuple(v.value for v in map_field.values if v.value)
+            if not values:
+                model_fields[field_name] = (str | None, Field(default=None, description=description))
+                continue
+            literal_type = Literal[values]  # type: ignore[valid-type]
+            if map_field.type == "multi-values":
+                model_fields[field_name] = (
+                    list[literal_type],  # type: ignore[valid-type]
+                    Field(default_factory=list, description=description),
+                )
+            else:
+                model_fields[field_name] = (
+                    literal_type | None,  # type: ignore[valid-type]
+                    Field(default=None, description=description),
+                )
+
+    if not model_fields:
+        return None
+    return create_model(model_name, **model_fields)
+
+
+def _build_map_entity_model(group: LabelGroup) -> type[BaseModel] | None:
+    """
+    Build a dynamic Pydantic model for a map-type entity label group.
+
+    Delegates to ``_build_map_fields_model`` which handles recursion.
+    """
+    # Capitalize group key for the model name (e.g., "person" → "Person")
+    model_name = group.key.capitalize() + "Entity"
+    return _build_map_fields_model(group.fields, model_name)
+
+
 def build_labels_model(labels_cfg: EntityLabelsConfig) -> type[BaseModel] | None:
     """
     Build a dynamic Pydantic model for structured label extraction.
@@ -100,6 +178,7 @@ def build_labels_model(labels_cfg: EntityLabelsConfig) -> type[BaseModel] | None
     - type="value",   optional=True      → Literal["v1","v2"] | None
     - type="value",   optional=False     → Literal["v1","v2"]  (required)
     - type="multi-values"                → list[Literal["v1","v2"]]
+    - type="map"                         → list[MapModel]  (structured entity)
 
     Args:
         labels_cfg: Parsed EntityLabelsConfig
@@ -113,7 +192,14 @@ def build_labels_model(labels_cfg: EntityLabelsConfig) -> type[BaseModel] | None
             continue
         description = group.description or group.key
 
-        if group.type == "text":
+        if group.type == "map":
+            map_model = _build_map_entity_model(group)
+            if map_model is not None:
+                fields[group.key] = (
+                    list[map_model],  # type: ignore[valid-type]
+                    Field(default_factory=list, description=description),
+                )
+        elif group.type == "text":
             # Free-form: any string value accepted, always optional
             fields[group.key] = (str | None, Field(default=None, description=description))
         else:
@@ -147,18 +233,35 @@ def build_labels_model(labels_cfg: EntityLabelsConfig) -> type[BaseModel] | None
     return create_model("Labels", **fields)
 
 
+def _is_map_label_entity(text_lower: str, prefix: str, fields: dict[str, MapField]) -> bool:
+    """Recursively check if text matches a map field path (e.g. 'person:address:city:...')."""
+    for field_name, map_field in fields.items():
+        field_prefix = f"{prefix}{field_name.lower()}:"
+        if map_field.type == "map" and map_field.fields:
+            if _is_map_label_entity(text_lower, field_prefix, map_field.fields):
+                return True
+        elif text_lower.startswith(field_prefix):
+            return True
+    return False
+
+
 def is_label_entity(text: str, labels_cfg: EntityLabelsConfig, labels_lookup: set[str]) -> bool:
     """
     Return True if entity text belongs to any configured label group.
 
     For enum groups: checks the pre-built lookup set.
     For text groups: checks that the text starts with a known key prefix.
+    For map groups: recursively checks ``key:field:...:value`` patterns.
     """
     if text.lower() in labels_lookup:
         return True
     for group in labels_cfg.attributes:
         if group.type == "text" and group.key and text.lower().startswith(f"{group.key.lower()}:"):
             return True
+        if group.type == "map" and group.key and group.fields:
+            prefix = f"{group.key.lower()}:"
+            if _is_map_label_entity(text.lower(), prefix, group.fields):
+                return True
     return False
 
 
@@ -186,8 +289,8 @@ def build_labels_lookup(labels_cfg: EntityLabelsConfig | list | None) -> set[str
 
     valid = set()
     for group in labels_cfg.attributes:
-        if group.type == "text":
-            continue  # No fixed vocabulary — all values accepted in post-processing
+        if group.type in ("text", "map"):
+            continue  # text: no fixed vocabulary; map: uses three-level key:field:value strings
         for v in group.values:
             if group.key and v.value:
                 valid.add(f"{group.key}:{v.value}".lower())

hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py

@@ -19,6 +19,7 @@ from ..llm_wrapper import LLMConfig, OutputTooLongError, sanitize_llm_output
 from ..response_models import TokenUsage
 from .entity_labels import (
     EntityLabelsConfig,
+    MapField,
     build_labels_lookup,
     build_labels_model,
     is_label_entity,
@@ -26,6 +27,49 @@ from .entity_labels import (
 )
 
 
+def _extract_map_entities(
+    entity_obj: dict,
+    fields: dict[str, MapField],
+    prefix: str,
+    validated_entities: "list[Entity]",
+    existing_texts_lower: set[str],
+) -> None:
+    """Recursively extract key:field:value entity strings from a map entity dict."""
+    for field_name, map_field in fields.items():
+        field_val = entity_obj.get(field_name)
+        if field_val is None or field_val == "":
+            continue
+        if map_field.type == "map" and map_field.fields:
+            # Nested map: recurse into each sub-entity
+            sub_list = field_val if isinstance(field_val, list) else [field_val]
+            for sub_obj in sub_list:
+                if isinstance(sub_obj, dict):
+                    _extract_map_entities(
+                        sub_obj,
+                        map_field.fields,
+                        f"{prefix}{field_name}:",
+                        validated_entities,
+                        existing_texts_lower,
+                    )
+        elif map_field.type == "multi-values":
+            vals = field_val if isinstance(field_val, list) else [field_val]
+            for v in vals:
+                if not isinstance(v, str) or not v.strip() or v.lower() in ("none", "null", "n/a"):
+                    continue
+                label_str = f"{prefix}{field_name}:{v.strip()}"
+                if label_str.lower() not in existing_texts_lower:
+                    validated_entities.append(Entity(text=label_str))
+                    existing_texts_lower.add(label_str.lower())
+        else:
+            # text or value — single string
+            if not isinstance(field_val, str) or not field_val.strip() or field_val.lower() in ("none", "null", "n/a"):
+                continue
+            label_str = f"{prefix}{field_name}:{field_val.strip()}"
+            if label_str.lower() not in existing_texts_lower:
+                validated_entities.append(Entity(text=label_str))
+                existing_texts_lower.add(label_str.lower())
+
+
 def _infer_temporal_date(fact_text: str, event_date: datetime | None) -> str | None:
     """
     Infer a temporal date from fact text when LLM didn't provide occurred_start.
@@ -731,6 +775,25 @@ Example: "Lost job → couldn't pay rent → moved apartment"
 - Fact 2: Moved apartment, causal_relations: [{target_index: 1, relation_type: "caused_by"}]"""
 
 
+def _append_map_fields_prompt(fields: dict[str, "MapField"], lines: list[str], indent: int = 4) -> None:
+    """Recursively append map field descriptions to the prompt lines."""
+    pad = " " * indent
+    for field_name, map_field in fields.items():
+        field_desc = f": {map_field.description}" if map_field.description else ""
+        if map_field.type == "map" and map_field.fields:
+            lines.append(f"{pad}• {field_name} (object){field_desc}")
+            _append_map_fields_prompt(map_field.fields, lines, indent + 4)
+        elif map_field.type == "multi-values":
+            vals = ", ".join(v.value for v in map_field.values if v.value)
+            type_hint = f"multi-values: {vals}" if vals else "multi-values"
+            lines.append(f"{pad}• {field_name} ({type_hint}){field_desc}")
+        elif map_field.type == "value" and map_field.values:
+            vals = ", ".join(v.value for v in map_field.values if v.value)
+            lines.append(f"{pad}• {field_name} (one of: {vals}){field_desc}")
+        else:
+            lines.append(f"{pad}• {field_name} (text){field_desc}")
+
+
 def _build_labels_prompt_section(labels_cfg: EntityLabelsConfig | list | None, free_form_entities: bool = True) -> str:
     """Build the entity labels classification section for the extraction prompt."""
     if labels_cfg is None:
@@ -763,7 +826,14 @@ def _build_labels_prompt_section(labels_cfg: EntityLabelsConfig | list | None, f
         "",
     ]
 
+    has_classification_attrs = False
+    has_map_attrs = False
+
     for attr in labels_cfg.attributes:
+        if attr.type == "map":
+            has_map_attrs = True
+            continue
+        has_classification_attrs = True
         if attr.type == "text":
             # Free-text: no predefined values — LLM writes any relevant string or null
             lines.append(f"- {attr.key} (free text or null): {attr.description}")
@@ -775,7 +845,31 @@ def _build_labels_prompt_section(labels_cfg: EntityLabelsConfig | list | None, f
                 lines.append(f'    • "{v.value}"{desc}')
         lines.append("")
 
-    lines.append("Only assign labels when clearly applicable. Leave null/empty if the fact does not match.")
+    if has_classification_attrs:
+        lines.append("Only assign labels when clearly applicable. Leave null/empty if the fact does not match.")
+        lines.append("")
+
+    # Add structured entity types (map groups)
+    if has_map_attrs:
+        lines.append("")
+        lines.append("══════════════════════════════════════════════════════════════════════════")
+        lines.append("STRUCTURED ENTITY TYPES")
+        lines.append("══════════════════════════════════════════════════════════════════════════")
+        lines.append("")
+        lines.append("For each fact, extract structured entities into the corresponding list field in 'labels'.")
+        lines.append("Each structured entity type has defined fields. Return a list of objects, one per entity found.")
+        lines.append("")
+        for attr in labels_cfg.attributes:
+            if attr.type != "map" or not attr.fields:
+                continue
+            desc = f": {attr.description}" if attr.description else ""
+            lines.append(f"- {attr.key}{desc}")
+            _append_map_fields_prompt(attr.fields, lines, indent=4)
+            lines.append("")
+        lines.append(
+            "Only extract structured entities when clearly present in the text. Leave the list empty if none found."
+        )
+
     return "\n".join(lines)
 
 
@@ -1164,6 +1258,19 @@ async def _extract_facts_from_chunk(
                             value = labels_data.get(group.key)
                             if not value:
                                 continue
+                            # Map-type groups: recursively extract key:field:value strings
+                            if group.type == "map" and group.fields:
+                                entities_list = value if isinstance(value, list) else [value]
+                                for entity_obj in entities_list:
+                                    if isinstance(entity_obj, dict):
+                                        _extract_map_entities(
+                                            entity_obj,
+                                            group.fields,
+                                            f"{group.key}:",
+                                            validated_entities,
+                                            existing_texts_lower,
+                                        )
+                                continue
                             values_list = value if isinstance(value, list) else [value]
                             for v in values_list:
                                 if not isinstance(v, str) or not v.strip() or v.lower() in ("none", "null", "n/a"):
@@ -1275,14 +1382,9 @@ async def _extract_facts_from_chunk(
                     f"     (current value: {config.retain_max_completion_tokens}, must be > RETAIN_CHUNK_SIZE={config.retain_chunk_size})"
                 ) from e
 
-            if "json_validate_failed" in str(e):
-                logger.warning(
-                    f"          [1.3.{chunk_index + 1}] Attempt {attempt + 1}/{llm_max_retries} failed with JSON validation error: {e}"
-                )
-                if attempt < llm_max_retries - 1:
-                    logger.info(f"          [1.3.{chunk_index + 1}] Retrying...")
-                    continue
-            # If it's not a JSON validation error or we're out of retries, re-raise
+            # Don't retry json_validate_failed here — the inner provider
+            # loop already retried the 400 error.  Re-entering the LLM call
+            # with the same input just multiplies wasted calls.
             raise
 
     # If we exhausted all retries, raise the last error or a descriptive fallback
@@ -1455,47 +1557,25 @@ async def extract_facts_from_text(
             f"chunk_size={config.retain_chunk_size:,}) - starting parallel LLM extraction"
         )
 
-    # Per-chunk retry wrapper: each chunk gets up to MAX_CHUNK_RETRIES attempts.
-    # This handles transient LLM failures (timeouts, rate limits, malformed responses)
-    # without discarding the entire batch. If a chunk still fails after all retries,
-    # the ENTIRE retain fails — we do not accept partial extraction.
-    MAX_CHUNK_RETRIES = 3
-    CHUNK_RETRY_BASE_DELAY = 2.0  # seconds, doubles each retry
-
-    async def _extract_chunk_with_retry(chunk: str, chunk_index: int) -> tuple:
-        """Extract facts from a single chunk with retries on failure."""
-        last_exception = None
-        for attempt in range(MAX_CHUNK_RETRIES):
-            try:
-                return await _extract_facts_with_auto_split(
-                    chunk=chunk,
-                    chunk_index=chunk_index,
-                    total_chunks=len(chunks),
-                    event_date=event_date,
-                    context=context,
-                    llm_config=llm_config,
-                    config=config,
-                    agent_name=agent_name,
-                    metadata=metadata,
-                )
-            except Exception as e:
-                last_exception = e
-                if attempt < MAX_CHUNK_RETRIES - 1:
-                    delay = CHUNK_RETRY_BASE_DELAY * (2**attempt)
-                    logger.warning(
-                        f"Chunk {chunk_index}/{len(chunks)} extraction failed "
-                        f"(attempt {attempt + 1}/{MAX_CHUNK_RETRIES}): "
-                        f"{type(e).__name__}. Retrying in {delay:.0f}s..."
-                    )
-                    await asyncio.sleep(delay)
-                else:
-                    logger.error(
-                        f"Chunk {chunk_index}/{len(chunks)} extraction failed after "
-                        f"{MAX_CHUNK_RETRIES} attempts: {type(e).__name__}: {e}"
-                    )
-        raise last_exception
-
-    tasks = [_extract_chunk_with_retry(chunk, i) for i, chunk in enumerate(chunks)]
+    # Transient LLM failures (timeouts, rate limits) are already retried inside
+    # the provider's inner loop.  Content-quality retries (malformed facts) are
+    # handled by the middle loop in _extract_facts_from_chunk.  Adding a third
+    # retry layer here would multiply wasted calls on deterministic failures
+    # (see https://github.com/vectorize-io/hindsight/issues/1412).
+    tasks = [
+        _extract_facts_with_auto_split(
+            chunk=chunk,
+            chunk_index=i,
+            total_chunks=len(chunks),
+            event_date=event_date,
+            context=context,
+            llm_config=llm_config,
+            config=config,
+            agent_name=agent_name,
+            metadata=metadata,
+        )
+        for i, chunk in enumerate(chunks)
+    ]
 
     # return_exceptions=True so we can collect all results even if some chunks
     # exhausted their retries. We check for failures below and fail the retain
@@ -1521,8 +1601,8 @@ async def extract_facts_from_text(
         # hasn't committed yet. The worker poller will retry the entire task.
         failed_summary = ", ".join(f"chunk {idx}: {type(err).__name__}" for idx, err in failed_chunks[:5])
         raise RuntimeError(
-            f"Fact extraction failed: {len(failed_chunks)}/{len(chunks)} chunks failed "
-            f"after {MAX_CHUNK_RETRIES} retries each. First failures: {failed_summary}"
+            f"Fact extraction failed: {len(failed_chunks)}/{len(chunks)} chunks failed. "
+            f"First failures: {failed_summary}"
         )
 
     return all_facts, chunk_metadata, total_usage
@@ -1863,6 +1943,19 @@ async def extract_facts_from_contents_batch_api(
                         value = labels_data.get(group.key)
                         if not value:
                             continue
+                        # Map-type groups: recursively extract key:field:value strings
+                        if group.type == "map" and group.fields:
+                            entities_list = value if isinstance(value, list) else [value]
+                            for entity_obj in entities_list:
+                                if isinstance(entity_obj, dict):
+                                    _extract_map_entities(
+                                        entity_obj,
+                                        group.fields,
+                                        f"{group.key}:",
+                                        validated_entities,
+                                        existing_texts_lower,
+                                    )
+                            continue
                         values_list = value if isinstance(value, list) else [value]
                         for v in values_list:
                             if not isinstance(v, str) or not v.strip() or v.lower() in ("none", "null", "n/a"):

hindsight-api-slim/hindsight_api/engine/retain/link_utils.py

@@ -405,8 +405,10 @@ async def build_entity_links_from_resolved(
         # Insert unit-entity links (used in fallback path where Phase 2 didn't do this)
         substep_start = time.time()
         unit_entity_pairs = []
-        for idx, (unit_id, _local_idx, _fact_date) in enumerate(entity_to_unit):
-            unit_entity_pairs.append((unit_id, resolved_entity_ids[idx]))
+        for idx, (unit_id, _local_idx, fact_date) in enumerate(entity_to_unit):
+            # Propagate the unit's fact_date so entity_cooccurrences.last_cooccurred
+            # reflects the event timeline, not the ingest moment.
+            unit_entity_pairs.append((unit_id, resolved_entity_ids[idx], fact_date))
 
         await entity_resolver.link_units_to_entities_batch(unit_entity_pairs, conn=conn)
         _log(

hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py

@@ -284,10 +284,12 @@ async def _insert_facts_and_links(
         )
         # Update semantic_ann_links with remapped IDs for Phase 2
         semantic_ann_links = remapped_semantic
-        # INSERT unit_entities (FK to memory_units, must be in transaction)
+        # INSERT unit_entities (FK to memory_units, must be in transaction).
+        # Pass fact_date alongside so entity_cooccurrences.last_cooccurred
+        # tracks the event timeline, not the ingest moment.
         unit_entity_pairs = [
-            (unit_id, resolved_entity_ids[idx])
-            for idx, (unit_id, _local_idx, _fact_date) in enumerate(remapped_entity_to_unit)
+            (unit_id, resolved_entity_ids[idx], fact_date)
+            for idx, (unit_id, _local_idx, fact_date) in enumerate(remapped_entity_to_unit)
         ]
         await entity_resolver.link_units_to_entities_batch(unit_entity_pairs, conn=conn)
         log_buffer.append(f"  Insert unit_entities: {len(unit_entity_pairs)} pairs in {time.time() - step_start:.3f}s")
@@ -680,6 +682,14 @@ async def retain_batch(
         all_pre_chunks.extend(content_chunks)
         chunk_to_content.extend([content_idx] * len(content_chunks))
 
+    # Memory: after chunking, the original content bodies in RetainContent are
+    # no longer needed (all_pre_chunks holds the working set). Clear them so
+    # Python can reclaim the (potentially multi-MB) strings.
+    # Note: contents_dicts["content"] is still needed briefly for hash computation
+    # inside _streaming_retain_batch, but gets cleared there after use.
+    for content in contents:
+        content.content = ""
+
     total_pre_chunks = len(all_pre_chunks)
     num_batches = (total_pre_chunks + chunk_batch_size - 1) // chunk_batch_size if total_pre_chunks > 0 else 1
     log_buffer.append(
@@ -882,9 +892,15 @@ async def _streaming_retain_batch(
     # the producer can skip already-extracted chunks to avoid duplicate work.
     existing_chunk_hashes: set[str] = set()
     combined_content = "\n".join([c.get("content", "") for c in contents_dicts])
+    # Memory: contents_dicts content strings are now captured in combined_content.
+    # Clear them from the dicts to release the per-item copies (can be multi-MB each).
+    for d in contents_dicts:
+        d.pop("content", None)
     # Sanitize before hashing to match what handle_document_tracking stores
     sanitized_content = fact_extraction._sanitize_text(combined_content) or ""
     new_content_hash = hashlib.sha256(sanitized_content.encode()).hexdigest()
+    # Memory: sanitized_content is only needed for the hash; free it immediately.
+    sanitized_content = ""
     is_recovery = False
 
     try:
@@ -969,12 +985,17 @@ async def _streaming_retain_batch(
                 schema,
             )
             await chunk_queue.put((global_idx, content, extracted, processed, chunk_meta, usage))
+            # Memory: release the chunk text from the shared list now that it's
+            # been extracted and queued. The queued RetainContent holds its own copy.
+            all_pre_chunks[global_idx] = ""
 
         tasks: list[asyncio.Task] = []
         skipped_total = 0
         for i, chunk_text in enumerate(all_pre_chunks):
             chunk_hash = chunk_storage.compute_chunk_hash(chunk_text)
             if chunk_hash in existing_chunk_hashes:
+                # Memory: skipped chunks aren't needed either.
+                all_pre_chunks[i] = ""
                 skipped_total += 1
                 continue
             tasks.append(asyncio.create_task(_extract_one(i, chunk_text)))
@@ -1035,6 +1056,9 @@ async def _streaming_retain_batch(
         is_last: bool,
     ) -> None:
         """Run Phase 1 + Phase 2 + Phase 3 for a batch of pre-extracted chunks."""
+        # Allow clearing combined_content after the no-facts skip path runs
+        # doc tracking — see the assignment further below.
+        nonlocal combined_content
         # Combine results from individual chunk extractions
         batch_contents: list[RetainContent] = []
         batch_extracted: list = []
@@ -1106,6 +1130,10 @@ async def _streaming_retain_batch(
                                 ops=pool.ops,
                             )
                         doc_tracking_done[0] = True
+                        # Memory: combined_content has been persisted; release
+                        # it now so the rest of the consumer loop doesn't pin
+                        # a multi-MB string. Nothing reads it after tracking.
+                        combined_content = ""
                         log_buffer.append(f"[streaming] Document {effective_doc_id} tracked (0 facts in first batch)")
             log_buffer.append(
                 f"[streaming] Consumer batch {consumer_batch_idx + 1}: "
@@ -1119,6 +1147,9 @@ async def _streaming_retain_batch(
         )
 
         async def _run_mini_batch_db_work() -> None:
+            # Allow clearing combined_content after the doc-tracking call so
+            # subsequent batches don't carry the per-document text in memory.
+            nonlocal combined_content
             entity_resolver.discard_pending_stats()
             mb_start = time.time()
 
@@ -1210,6 +1241,10 @@ async def _streaming_retain_batch(
                             )
                             log_buffer.append(f"[streaming] Document {effective_doc_id} tracked (full content)")
                         doc_tracking_done[0] = True
+                        # Memory: combined_content is no longer needed after
+                        # this first-batch tracking call. Release it so the
+                        # remaining consumer batches don't pin the string.
+                        combined_content = ""
                     else:
                         # --- Later batches: verify we still own the document ---
                         # If another request took over (cascade-deleted our doc and
@@ -1291,6 +1326,14 @@ async def _streaming_retain_batch(
         else:
             await _run_mini_batch_db_work()
 
+        # Memory: after DB write, clear the batch-local lists that hold extracted
+        # facts and embedding vectors. These can be large (384 floats per fact ×
+        # thousands of facts) and are no longer needed after commit.
+        batch_contents.clear()
+        batch_extracted.clear()
+        batch_processed.clear()
+        batch_chunk_meta.clear()
+
     # ---------------------------------------------------------------------------
     # Check if facts are already committed (recovery from previous crash).
     # If so, skip extraction+writes and jump straight to final ANN pass.
@@ -1376,6 +1419,9 @@ async def _streaming_retain_batch(
                             ops=pool.ops,
                         )
                     doc_tracking_done[0] = True
+                    # Memory: combined_content has been persisted and won't be
+                    # read again — release the per-document text now.
+                    combined_content = ""
                     log_buffer.append(f"[streaming] Document {effective_doc_id} tracked (no facts extracted)")
 
         # Mark facts as committed in operation metadata (crash recovery checkpoint)

hindsight-api-slim/hindsight_api/engine/search/tags.py

@@ -191,21 +191,21 @@ class TagGroupLeaf(BaseModel):
 class TagGroupAnd(BaseModel):
     """Compound AND group: all child filters must match."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(populate_by_name=True, serialize_by_alias=True)
     filters: list[TagGroup] = Field(alias="and")
 
 
 class TagGroupOr(BaseModel):
     """Compound OR group: at least one child filter must match."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(populate_by_name=True, serialize_by_alias=True)
     filters: list[TagGroup] = Field(alias="or")
 
 
 class TagGroupNot(BaseModel):
     """Compound NOT group: child filter must NOT match."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(populate_by_name=True, serialize_by_alias=True)
     filter: TagGroup = Field(alias="not")
 
 

hindsight-api-slim/hindsight_api/engine/sql/postgresql.py

@@ -185,9 +185,8 @@ class PostgreSQLDialect(SQLDialect):
         extra_where: str = "",
     ) -> str:
         if text_search_extension == "vchord":
-            bm25_score_expr = (
-                f"search_vector <&> to_bm25query('idx_memory_units_text_search', tokenize({text_param}, 'llmlingua2'))"
-            )
+            # <&> returns a distance (lower = more relevant), negate for score
+            bm25_score_expr = f"-(search_vector <&> to_bm25query('idx_memory_units_text_search', tokenize({text_param}, 'llmlingua2')))"
             bm25_order_by = f"{bm25_score_expr} DESC"
             bm25_where_filter = ""
         elif text_search_extension == "pg_textsearch":

hindsight-api-slim/hindsight_api/main.py

@@ -28,6 +28,7 @@ from .config import DEFAULT_WORKERS, ENV_HOST, ENV_WORKERS, HindsightConfig, _ge
 from .daemon import (
     DEFAULT_DAEMON_PORT,
     DEFAULT_IDLE_TIMEOUT,
+    ENV_DAEMON_CHILD,
     IdleTimeoutMiddleware,
     daemonize,
 )
@@ -150,8 +151,15 @@ def main():
 
     args = parser.parse_args()
 
-    # Daemon mode handling
-    if args.daemon:
+    # Daemon mode handling.
+    # is_daemon_child is True when we are the re-exec'd child spawned by
+    # daemonize() or by hindsight-embed's DaemonEmbedManager.  The child
+    # does not have --daemon in its argv, but must still behave as a daemon
+    # (resolve host/port, enable idle timeout, suppress banner, etc.).
+    is_daemon_child = os.environ.get(ENV_DAEMON_CHILD) == "1"
+    is_daemon = args.daemon or is_daemon_child
+
+    if is_daemon:
         args.host, args.port = resolve_daemon_host_port(
             args_host=args.host,
             args_port=args.port,
@@ -159,12 +167,13 @@ def main():
             config_port=config.port,
         )
 
-        # Fork into background
-        # No lockfile needed - port binding prevents duplicate daemons
+        # Detach into background (parent re-execs and exits; child redirects
+        # stdio to log file).  No lockfile needed — port binding prevents
+        # duplicate daemons.
         daemonize()
 
     # Print banner (not in daemon mode)
-    if not args.daemon:
+    if not is_daemon:
         print()
         print_banner()
 
@@ -173,7 +182,7 @@ def main():
     if args.log_level != config.log_level:
         config = dataclasses.replace(config, host=args.host, port=args.port, log_level=args.log_level)
     config.configure_logging()
-    if not args.daemon:
+    if not is_daemon:
         config.log_config()
 
     # Register cleanup handlers
@@ -222,7 +231,7 @@ def main():
 
     # Wrap with idle timeout middleware in daemon mode
     idle_middleware = None
-    if args.daemon:
+    if is_daemon:
         idle_middleware = IdleTimeoutMiddleware(app, idle_timeout=args.idle_timeout)
         app = idle_middleware
 
@@ -277,7 +286,7 @@ def main():
         uvicorn_config["ssl_certfile"] = args.ssl_certfile
 
     # Print startup info (not in daemon mode)
-    if not args.daemon:
+    if not is_daemon:
         from .banner import print_startup_info
 
         print_startup_info(

hindsight-api-slim/hindsight_api/mcp_tools.py

@@ -12,6 +12,7 @@ from datetime import datetime, timezone
 from typing import Any, Callable
 
 from fastmcp import FastMCP
+from pydantic import TypeAdapter
 
 from hindsight_api import MemoryEngine
 from hindsight_api.config import (
@@ -21,9 +22,12 @@ from hindsight_api.config import (
 from hindsight_api.engine.audit import AuditEntry, AuditLogger
 from hindsight_api.engine.memory_engine import Budget
 from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES
+from hindsight_api.engine.search.tags import TagGroup
 from hindsight_api.extensions import OperationValidationError
 from hindsight_api.models import RequestContext
 
+_TAG_GROUP_LIST_ADAPTER = TypeAdapter(list[TagGroup])
+
 # All tools available in the system (explicit list — no wildcards).
 # Defined here (shared module) to avoid circular imports with api/mcp.py.
 _ALL_TOOLS: frozenset[str] = frozenset(
@@ -773,6 +777,7 @@ def _register_recall(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig)
             types: list[str] | None = None,
             tags: list[str] | None = None,
             tags_match: str = "any",
+            tag_groups: list[dict] | None = None,
             query_timestamp: str | None = None,
             bank_id: str | None = None,
         ) -> str | dict:
@@ -782,8 +787,12 @@ def _register_recall(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig)
                 max_tokens: Maximum tokens to return in results (default: 4096)
                 budget: Search budget - 'low', 'mid', or 'high' (default: 'high'). Higher budgets search more thoroughly.
                 types: Fact types to include (e.g., ['world', 'experience']). Default: all types.
-                tags: Optional tags to filter results by (e.g., ['project:alpha'])
+                tags: Optional tags to filter results by (e.g., ['project:alpha']). Mutually exclusive with tag_groups.
                 tags_match: How to match tags - 'any' (match any tag) or 'all' (match all tags). Default: 'any'
+                tag_groups: Compound tag filter using boolean groups (AND-ed together). Each group is a leaf
+                    {"tags": [...], "match": "any_strict"} or compound {"and": [...]}, {"or": [...]}, {"not": {...}}.
+                    Example: [{"not": {"tags": ["closeout"], "match": "any_strict"}}] excludes memories tagged closeout.
+                    Mutually exclusive with tags.
                 query_timestamp: Temporal context for the query (ISO format, e.g., '2024-01-15T10:30:00Z'). Helps retrieve time-relevant memories.
                 bank_id: Optional bank to search in (defaults to session bank). Use for cross-bank operations.
             """
@@ -792,6 +801,11 @@ def _register_recall(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig)
                 if target_bank is None:
                     return "Error: No bank_id configured"
 
+                if tags is not None and tag_groups is not None:
+                    raise ValueError(
+                        "'tags' and 'tag_groups' are mutually exclusive. Use 'tag_groups' for compound filtering."
+                    )
+
                 budget_map = {"low": Budget.LOW, "mid": Budget.MID, "high": Budget.HIGH}
                 budget_enum = budget_map.get(budget.lower(), Budget.HIGH)
                 fact_types = types if types is not None else list(VALID_RECALL_FACT_TYPES)
@@ -807,6 +821,8 @@ def _register_recall(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig)
                 if tags is not None:
                     recall_kwargs["tags"] = tags
                     recall_kwargs["tags_match"] = tags_match
+                if tag_groups is not None:
+                    recall_kwargs["tag_groups"] = _TAG_GROUP_LIST_ADAPTER.validate_python(tag_groups)
                 if query_timestamp is not None:
                     recall_kwargs["question_date"] = parse_timestamp(query_timestamp)
 
@@ -832,6 +848,7 @@ def _register_recall(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig)
             types: list[str] | None = None,
             tags: list[str] | None = None,
             tags_match: str = "any",
+            tag_groups: list[dict] | None = None,
             query_timestamp: str | None = None,
         ) -> dict:
             """
@@ -840,8 +857,12 @@ def _register_recall(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig)
                 max_tokens: Maximum tokens to return in results (default: 4096)
                 budget: Search budget - 'low', 'mid', or 'high' (default: 'high'). Higher budgets search more thoroughly.
                 types: Fact types to include (e.g., ['world', 'experience']). Default: all types.
-                tags: Optional tags to filter results by (e.g., ['project:alpha'])
+                tags: Optional tags to filter results by (e.g., ['project:alpha']). Mutually exclusive with tag_groups.
                 tags_match: How to match tags - 'any' (match any tag) or 'all' (match all tags). Default: 'any'
+                tag_groups: Compound tag filter using boolean groups (AND-ed together). Each group is a leaf
+                    {"tags": [...], "match": "any_strict"} or compound {"and": [...]}, {"or": [...]}, {"not": {...}}.
+                    Example: [{"not": {"tags": ["closeout"], "match": "any_strict"}}] excludes memories tagged closeout.
+                    Mutually exclusive with tags.
                 query_timestamp: Temporal context for the query (ISO format, e.g., '2024-01-15T10:30:00Z'). Helps retrieve time-relevant memories.
             """
             try:
@@ -849,6 +870,11 @@ def _register_recall(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig)
                 if target_bank is None:
                     return {"error": "No bank_id configured", "results": []}
 
+                if tags is not None and tag_groups is not None:
+                    raise ValueError(
+                        "'tags' and 'tag_groups' are mutually exclusive. Use 'tag_groups' for compound filtering."
+                    )
+
                 budget_map = {"low": Budget.LOW, "mid": Budget.MID, "high": Budget.HIGH}
                 budget_enum = budget_map.get(budget.lower(), Budget.HIGH)
                 fact_types = types if types is not None else list(VALID_RECALL_FACT_TYPES)
@@ -864,6 +890,8 @@ def _register_recall(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig)
                 if tags is not None:
                     recall_kwargs["tags"] = tags
                     recall_kwargs["tags_match"] = tags_match
+                if tag_groups is not None:
+                    recall_kwargs["tag_groups"] = _TAG_GROUP_LIST_ADAPTER.validate_python(tag_groups)
                 if query_timestamp is not None:
                     recall_kwargs["question_date"] = parse_timestamp(query_timestamp)
 

hindsight-api-slim/hindsight_api/migrations.py

@@ -27,6 +27,15 @@ from alembic.config import Config
 from alembic.script.revision import ResolutionError
 from sqlalchemy import Connection, create_engine, text
 
+from ._vector_index import (
+    bootstrap_extension,
+    detect_vector_extension,
+    index_type_keyword,
+    index_using_clause,
+    minimum_rows_for_index,
+    should_defer_index_creation,
+    uses_per_bank_vector_indexes,
+)
 from .db_url import is_oracle_url, to_libpq_url
 from .utils import mask_network_location
 
@@ -44,66 +53,28 @@ _alembic_lock = threading.Lock()
 
 
 def _detect_vector_extension(conn, vector_extension: str = "pgvector") -> str:
-    """
-    Validate vector extension: 'pgvector', 'vchord', or 'pgvectorscale'.
+    """Validate configured vector extension and preserve Azure DiskANN detection."""
+    return detect_vector_extension(conn, vector_extension)
 
-    Args:
-        conn: SQLAlchemy connection object
-        vector_extension: Configured extension ("pgvector", "vchord", or "pgvectorscale")
 
-    Returns:
-        "pgvector", "vchord", "pgvectorscale", or "pg_diskann"
-
-    Raises:
-        RuntimeError: If configured extension is not installed
-    """
-    # Verify the configured extension is installed
-    if vector_extension == "pgvectorscale":
-        # pgvectorscale/DiskANN requires pgvector to be installed first
-        pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
-        if not pgvector_check:
-            raise RuntimeError(
-                "DiskANN (pgvectorscale/pg_diskann) requires pgvector to be installed. "
-                "Install it with: CREATE EXTENSION vector; then CREATE EXTENSION vectorscale CASCADE; (or pg_diskann on Azure)"
-            )
-
-        # Check for either vectorscale (open source) or pg_diskann (Azure)
-        vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar()
-        pg_diskann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_diskann'")).scalar()
-
-        if vectorscale_check:
-            logger.debug("Using vector extension: pgvectorscale (DiskANN)")
-            return "pgvectorscale"
-        elif pg_diskann_check:
-            logger.debug("Using vector extension: pg_diskann (Azure DiskANN)")
-            return "pg_diskann"  # Return distinct name for parameter handling
-        else:
-            raise RuntimeError(
-                "Configured vector extension 'pgvectorscale' not found. "
-                "Install either:\n"
-                "  - pgvectorscale (open source): CREATE EXTENSION vectorscale CASCADE;\n"
-                "  - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
-            )
-    elif vector_extension == "vchord":
-        vchord_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord'")).scalar()
-        if not vchord_check:
-            raise RuntimeError(
-                "Configured vector extension 'vchord' not found. Install it with: CREATE EXTENSION vchord CASCADE;"
-            )
-        logger.debug("Using configured vector extension: vchord")
-        return "vchord"
-    elif vector_extension == "pgvector":
-        pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
-        if not pgvector_check:
-            raise RuntimeError(
-                "Configured vector extension 'pgvector' not found. Install it with: CREATE EXTENSION vector;"
-            )
-        logger.debug("Using configured vector extension: pgvector")
-        return "pgvector"
-    else:
-        raise ValueError(
-            f"Invalid vector_extension: {vector_extension}. Must be 'pgvector', 'vchord', or 'pgvectorscale'"
-        )
+def _drop_per_bank_vector_indexes(conn: Connection, schema_name: str) -> None:
+    """Drop per-bank partial memory_units vector indexes after global ScaNN is ready."""
+    rows = conn.execute(
+        text("""
+            SELECT indexname
+            FROM pg_indexes
+            WHERE schemaname = :schema_name
+              AND tablename = 'memory_units'
+              AND indexname LIKE 'idx_mu_emb_%'
+              AND indexdef LIKE '%embedding%'
+        """),
+        {"schema_name": schema_name},
+    ).fetchall()
+    # DDL identifiers cannot be passed as bound parameters, so escape inline.
+    safe_schema = schema_name.replace('"', '""')
+    for row in rows:
+        safe_index = row[0].replace('"', '""')
+        conn.execute(text(f'DROP INDEX IF EXISTS "{safe_schema}"."{safe_index}"'))
 
 
 def _get_schema_lock_id(schema: str) -> int:
@@ -364,47 +335,8 @@ def run_migrations(
                                 "Please install it with: CREATE EXTENSION vector;"
                             ) from e
 
-                # If using pgvectorscale, ensure vectorscale extension is also installed
                 vector_extension = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
-                if vector_extension == "pgvectorscale":
-                    logger.debug("Checking pgvectorscale (vectorscale) extension availability...")
-
-                    vectorscale_check = conn.execute(
-                        text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")
-                    ).scalar()
-
-                    if vectorscale_check:
-                        logger.info("pgvectorscale extension already installed")
-                    else:
-                        # Extension doesn't exist - try to install
-                        logger.info("pgvectorscale extension not found, attempting to install...")
-                        try:
-                            conn.execute(text("CREATE EXTENSION vectorscale CASCADE"))
-                            conn.commit()
-                            logger.info("pgvectorscale extension installed successfully")
-                        except Exception as e:
-                            # Installation failed - check one more time in case another process installed it
-                            conn.rollback()
-                            vectorscale_recheck = conn.execute(
-                                text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")
-                            ).fetchone()
-
-                            if vectorscale_recheck:
-                                logger.warning(
-                                    "Could not install pgvectorscale extension (permission denied?), "
-                                    "but extension exists. Continuing..."
-                                )
-                            else:
-                                # Extension truly doesn't exist and we can't install it
-                                logger.error(
-                                    f"pgvectorscale extension is not installed and cannot be installed: {e}. "
-                                    f"Please ensure pgvectorscale is installed by a database administrator. "
-                                    f"See: https://github.com/timescale/pgvectorscale#installation"
-                                )
-                                raise RuntimeError(
-                                    "pgvectorscale extension is required but not installed. "
-                                    "Please install it with: CREATE EXTENSION vectorscale CASCADE;"
-                                ) from e
+                bootstrap_extension(conn, vector_extension)
 
                 # Commit any pending transaction on the advisory-lock connection
                 # before running migrations.  Some code paths above (e.g., the
@@ -545,7 +477,10 @@ def _migrate_table_embedding_dimension(
 
     logger.info(f"Altering {table_name}.embedding column dimension from {current_dim} to {required_dimension}")
 
-    # Drop existing vector index (works for both HNSW and vchordrq)
+    # Drop existing vector index (works for HNSW, DiskANN, vchordrq, and ScaNN)
+    # The EXCEPTION block handles 'could not open relation with OID' errors that
+    # occur when concurrent sessions drop schemas (e.g. pytest-xdist workers),
+    # invalidating pg_indexes OID references mid-cursor-iteration.
     conn.execute(
         text(f"""
             DO $$
@@ -555,11 +490,14 @@ def _migrate_table_embedding_dimension(
                     SELECT indexname FROM pg_indexes
                     WHERE schemaname = '{schema_name}'
                       AND tablename = '{table_name}'
-                      AND (indexdef LIKE '%hnsw%' OR indexdef LIKE '%vchordrq%' OR indexdef LIKE '%diskann%')
+                      AND (indexdef LIKE '%hnsw%' OR indexdef LIKE '%vchordrq%' OR indexdef LIKE '%diskann%' OR indexdef LIKE '%scann%')
                       AND indexdef LIKE '%embedding%'
                 LOOP
                     EXECUTE 'DROP INDEX IF EXISTS {schema_name}.' || idx_name;
                 END LOOP;
+            EXCEPTION WHEN internal_error THEN
+                -- Stale OID from concurrent schema drop; nothing to drop anyway
+                NULL;
             END $$;
         """)
     )
@@ -570,41 +508,34 @@ def _migrate_table_embedding_dimension(
     conn.commit()
 
     # Recreate index with appropriate type based on detected extension
-    if vector_ext == "pgvectorscale":
-        conn.execute(
-            text(f"""
-                CREATE INDEX IF NOT EXISTS idx_{table_name}_embedding_diskann
-                ON {schema_name}.{table_name}
-                USING diskann (embedding vector_cosine_ops)
-                WITH (num_neighbors = 50)
-            """)
+    if vector_ext == "pgvector" and required_dimension > 2000:
+        raise RuntimeError(
+            f"Embedding dimension {required_dimension} exceeds pgvector HNSW index limit of 2000. "
+            f"Use an embedding model with <= 2000 dimensions, or switch to a vector extension "
+            f"that supports higher dimensions (e.g., pgvectorscale/DiskANN or AlloyDB ScaNN)."
         )
-        logger.info(f"Created DiskANN index on {table_name} for {required_dimension}-dimensional embeddings")
-    elif vector_ext == "vchord":
-        conn.execute(
-            text(f"""
-                CREATE INDEX IF NOT EXISTS idx_{table_name}_embedding_vchordrq
-                ON {schema_name}.{table_name}
-                USING vchordrq (embedding vector_l2_ops)
-            """)
+
+    index_type = index_type_keyword(vector_ext)
+    if should_defer_index_creation(vector_ext, row_count):
+        minimum_rows = minimum_rows_for_index(vector_ext)
+        logger.warning(
+            "Skipping %s index recreation on %s: AlloyDB ScaNN AUTO indexes need at least %s populated "
+            "embedding rows; table currently has %s",
+            vector_ext,
+            table_name,
+            minimum_rows,
+            row_count,
         )
-        logger.info(f"Created vchordrq index on {table_name} for {required_dimension}-dimensional embeddings")
-    else:  # pgvector
-        if required_dimension > 2000:
-            raise RuntimeError(
-                f"Embedding dimension {required_dimension} exceeds pgvector HNSW index limit of 2000. "
-                f"Use an embedding model with <= 2000 dimensions, or switch to a vector extension "
-                f"that supports higher dimensions (e.g., pgvectorscale/DiskANN)."
-            )
-        conn.execute(
-            text(f"""
-                CREATE INDEX IF NOT EXISTS idx_{table_name}_embedding_hnsw
-                ON {schema_name}.{table_name}
-                USING hnsw (embedding vector_cosine_ops)
-                WITH (m = 16, ef_construction = 64)
-            """)
-        )
-        logger.info(f"Created HNSW index on {table_name} for {required_dimension}-dimensional embeddings")
+        return
+
+    conn.execute(
+        text(f"""
+            CREATE INDEX IF NOT EXISTS idx_{table_name}_embedding_{index_type}
+            ON {schema_name}.{table_name}
+            {index_using_clause(vector_ext)}
+        """)
+    )
+    logger.info(f"Created {index_type} index on {table_name} for {required_dimension}-dimensional embeddings")
     conn.commit()
 
     logger.info(f"Successfully changed {table_name}.embedding dimension to {required_dimension}")
@@ -628,7 +559,7 @@ def ensure_embedding_dimension(
         database_url: SQLAlchemy database URL
         required_dimension: The embedding dimension required by the model
         schema: Target PostgreSQL schema name (None for public)
-        vector_extension: Configured vector extension ("pgvector" or "vchord")
+        vector_extension: Configured vector extension ("pgvector", "vchord", "pgvectorscale", or "scann")
 
     Raises:
         RuntimeError: If dimension mismatch with existing data
@@ -676,7 +607,7 @@ def ensure_vector_extension(
 
     Args:
         database_url: SQLAlchemy database URL
-        vector_extension: Configured vector extension ("pgvector" or "vchord")
+        vector_extension: Configured vector extension ("pgvector", "vchord", "pgvectorscale", or "scann")
         schema: Target PostgreSQL schema name (None for public)
 
     Raises:
@@ -697,13 +628,7 @@ def ensure_vector_extension(
             ("pinned_reflections", "idx_pinned_reflections_embedding"),
         ]
 
-        # Determine target index type
-        if target_ext in ("pgvectorscale", "pg_diskann"):
-            target_index_type = "diskann"
-        elif target_ext == "vchord":
-            target_index_type = "vchordrq"
-        else:
-            target_index_type = "hnsw"
+        target_index_type = index_type_keyword(target_ext)
 
         mismatched_tables = []
         tables_with_data = []
@@ -724,6 +649,10 @@ def ensure_vector_extension(
                 logger.debug(f"Table {table_name} does not exist in schema '{schema_name}', skipping")
                 continue
 
+            row_count = conn.execute(
+                text(f"SELECT COUNT(*) FROM {schema_name}.{table_name} WHERE embedding IS NOT NULL")
+            ).scalar()
+
             # Check current index type by querying pg_indexes
             current_index_info = conn.execute(
                 text("""
@@ -737,30 +666,33 @@ def ensure_vector_extension(
             ).fetchone()
 
             if not current_index_info:
-                # Check whether per-bank partial HNSW indexes already cover this table
-                # (created by the bank_utils lifecycle — no global index needed in that case)
-                per_bank_index_count = conn.execute(
-                    text("""
-                        SELECT COUNT(*)
-                        FROM pg_indexes
-                        WHERE schemaname = :schema
-                          AND tablename = :table_name
-                          AND indexname LIKE 'idx_mu_emb_%'
-                    """),
-                    {"schema": schema_name, "table_name": table_name},
-                ).scalar()
-                if per_bank_index_count and per_bank_index_count > 0:
-                    logger.debug(
-                        f"No global embedding index on {table_name}, but {per_bank_index_count} "
-                        f"per-bank partial HNSW indexes exist — skipping global index creation"
-                    )
-                    continue
-                logger.warning(f"No embedding index found for {table_name}, will create it")
-                mismatched_tables.append((table_name, index_name, None))
+                if table_name == "memory_units" and uses_per_bank_vector_indexes(target_ext):
+                    # Check whether per-bank partial vector indexes already cover this table
+                    # (created by the bank_utils lifecycle — no global index needed in that case)
+                    per_bank_index_count = conn.execute(
+                        text("""
+                            SELECT COUNT(*)
+                            FROM pg_indexes
+                            WHERE schemaname = :schema
+                              AND tablename = :table_name
+                              AND indexname LIKE 'idx_mu_emb_%'
+                        """),
+                        {"schema": schema_name, "table_name": table_name},
+                    ).scalar()
+                    if per_bank_index_count and per_bank_index_count > 0:
+                        logger.debug(
+                            f"No global embedding index on {table_name}, but {per_bank_index_count} "
+                            f"per-bank partial vector indexes exist — skipping global index creation"
+                        )
+                        continue
+                logger.warning(f"No embedding index found for {table_name}, will create it if safe")
+                mismatched_tables.append((table_name, index_name, None, row_count))
                 continue
 
             indexdef = current_index_info[0].lower()
-            if "diskann" in indexdef:
+            if "scann" in indexdef:
+                current_index_type = "scann"
+            elif "diskann" in indexdef:
                 current_index_type = "diskann"
             elif "vchordrq" in indexdef:
                 current_index_type = "vchordrq"
@@ -775,30 +707,32 @@ def ensure_vector_extension(
                 logger.info(
                     f"Index type mismatch on {table_name}: current={current_index_type}, target={target_index_type}"
                 )
-                mismatched_tables.append((table_name, index_name, current_index_type))
+                mismatched_tables.append((table_name, index_name, current_index_type, row_count))
 
-                # Check if table has data
-                row_count = conn.execute(
-                    text(f"SELECT COUNT(*) FROM {schema_name}.{table_name} WHERE embedding IS NOT NULL")
-                ).scalar()
-
-                if row_count > 0:
-                    tables_with_data.append((table_name, row_count))
+                if row_count > 0 and target_ext != "scann":
+                    tables_with_data.append((table_name, row_count, current_index_type))
             else:
                 logger.debug(f"Index type OK for {table_name}: {current_index_type}")
+                if target_ext == "scann" and table_name == "memory_units":
+                    _drop_per_bank_vector_indexes(conn, schema_name)
+                    conn.commit()
 
         # If no mismatches, we're done
         if not mismatched_tables:
             logger.debug(f"All vector indexes match configured extension: {target_ext}")
             return
 
-        # If there's data in any mismatched table, raise error
+        # If there's data in any non-ScaNN mismatched table, raise error
         if tables_with_data:
-            table_list = ", ".join([f"{table}({count} rows)" for table, count in tables_with_data])
+            table_list = ", ".join([f"{table}({count} rows)" for table, count, _ in tables_with_data])
+            current_index_type = tables_with_data[0][2]
             # Map index type back to extension name for error message
-            current_ext_name = {"diskann": "pgvectorscale", "vchordrq": "vchord", "hnsw": "pgvector"}.get(
-                current_index_type, current_index_type
-            )
+            current_ext_name = {
+                "diskann": "pgvectorscale",
+                "vchordrq": "vchord",
+                "hnsw": "pgvector",
+                "scann": "scann",
+            }.get(current_index_type, current_index_type)
 
             raise RuntimeError(
                 f"Cannot change vector extension from {current_index_type} to {target_index_type}: "
@@ -809,46 +743,28 @@ def ensure_vector_extension(
                 f"  2. Use the current vector extension (set HINDSIGHT_API_VECTOR_EXTENSION='{current_ext_name}')"
             )
 
-        # Tables are empty, safe to recreate indexes
-        logger.info(f"Recreating vector indexes for {target_ext}")
+        logger.info(f"Reconciling vector indexes for {target_ext}")
+
+        for table_name, index_name, current_type, row_count in mismatched_tables:
+            if should_defer_index_creation(target_ext, row_count):
+                minimum_rows = minimum_rows_for_index(target_ext)
+                logger.warning(
+                    "Skipping %s index creation on %s: AlloyDB ScaNN AUTO indexes need at least %s populated "
+                    "embedding rows; table currently has %s",
+                    target_ext,
+                    table_name,
+                    minimum_rows,
+                    row_count,
+                )
+                continue
 
-        for table_name, index_name, current_type in mismatched_tables:
             # Drop existing index if it exists
             if current_type:
                 logger.info(f"Dropping {current_type} index on {table_name}")
                 conn.execute(text(f"DROP INDEX IF EXISTS {schema_name}.{index_name}"))
 
             # Create new index with appropriate type
-            if target_ext == "pgvectorscale":
-                logger.info(f"Creating DiskANN index on {table_name} (pgvectorscale)")
-                conn.execute(
-                    text(f"""
-                        CREATE INDEX IF NOT EXISTS {index_name}
-                        ON {schema_name}.{table_name}
-                        USING diskann (embedding vector_cosine_ops)
-                        WITH (num_neighbors = 50)
-                    """)
-                )
-            elif target_ext == "pg_diskann":
-                logger.info(f"Creating DiskANN index on {table_name} (pg_diskann/Azure)")
-                conn.execute(
-                    text(f"""
-                        CREATE INDEX IF NOT EXISTS {index_name}
-                        ON {schema_name}.{table_name}
-                        USING diskann (embedding vector_cosine_ops)
-                        WITH (max_neighbors = 50)
-                    """)
-                )
-            elif target_ext == "vchord":
-                logger.info(f"Creating vchordrq index on {table_name}")
-                conn.execute(
-                    text(f"""
-                        CREATE INDEX IF NOT EXISTS {index_name}
-                        ON {schema_name}.{table_name}
-                        USING vchordrq (embedding vector_l2_ops)
-                    """)
-                )
-            else:  # pgvector
+            if target_ext == "pgvector":
                 # Check embedding dimension — pgvector HNSW indexes only support up to 2000 dims
                 embed_dim = conn.execute(
                     text("""
@@ -865,20 +781,22 @@ def ensure_vector_extension(
                     raise RuntimeError(
                         f"Embedding dimension {embed_dim} on {table_name} exceeds pgvector HNSW index limit of 2000. "
                         f"Use an embedding model with <= 2000 dimensions, or switch to a vector extension "
-                        f"that supports higher dimensions (e.g., pgvectorscale/DiskANN)."
+                        f"that supports higher dimensions (e.g., pgvectorscale/DiskANN or AlloyDB ScaNN)."
                     )
-                logger.info(f"Creating HNSW index on {table_name}")
-                conn.execute(
-                    text(f"""
-                        CREATE INDEX IF NOT EXISTS {index_name}
-                        ON {schema_name}.{table_name}
-                        USING hnsw (embedding vector_cosine_ops)
-                        WITH (m = 16, ef_construction = 64)
-                    """)
-                )
+
+            logger.info(f"Creating {target_index_type} index on {table_name}")
+            conn.execute(
+                text(f"""
+                    CREATE INDEX IF NOT EXISTS {index_name}
+                    ON {schema_name}.{table_name}
+                    {index_using_clause(target_ext)}
+                """)
+            )
+            if target_ext == "scann" and table_name == "memory_units":
+                _drop_per_bank_vector_indexes(conn, schema_name)
 
         conn.commit()
-        logger.info(f"Successfully migrated vector indexes to {target_ext}")
+        logger.info(f"Successfully reconciled vector indexes for {target_ext}")
 
 
 def ensure_text_search_extension(

hindsight-api-slim/hindsight_api/worker/main.py

@@ -16,6 +16,7 @@ import signal
 import socket
 import sys
 import warnings
+from collections.abc import Callable
 
 from ..config import get_config
 from ..engine.task_backend import WorkerTaskBackend
@@ -31,6 +32,26 @@ os.environ["TOKENIZERS_PARALLELISM"] = "false"
 logger = logging.getLogger(__name__)
 
 
+def _install_shutdown_signal_handlers(
+    loop: asyncio.AbstractEventLoop,
+    handler: Callable[[], None],
+) -> bool:
+    """Register SIGINT/SIGTERM handlers on the asyncio loop.
+
+    Returns True when handlers were installed via ``loop.add_signal_handler``.
+    Returns False on platforms (Windows ProactorEventLoop) where asyncio
+    does not implement signal handlers; the caller falls back to Python's
+    default SIGINT behavior, which still terminates the process on Ctrl+C
+    but loses the in-loop two-stage graceful shutdown.
+    """
+    try:
+        loop.add_signal_handler(signal.SIGINT, handler)
+        loop.add_signal_handler(signal.SIGTERM, handler)
+    except NotImplementedError:
+        return False
+    return True
+
+
 def create_worker_app(poller: WorkerPoller, memory):
     """Create a minimal FastAPI app for worker metrics and health."""
     from fastapi import FastAPI
@@ -243,6 +264,7 @@ def main():
         # Setup signal handlers for graceful shutdown using asyncio
         shutdown_requested = asyncio.Event()
         force_exit = False
+        async_handlers_installed = False
 
         loop = asyncio.get_event_loop()
 
@@ -253,17 +275,26 @@ def main():
                 print("\nReceived second signal, forcing immediate exit...")
                 force_exit = True
                 # Restore default handler so third signal kills process
-                loop.remove_signal_handler(signal.SIGINT)
-                loop.remove_signal_handler(signal.SIGTERM)
+                if async_handlers_installed:
+                    loop.remove_signal_handler(signal.SIGINT)
+                    loop.remove_signal_handler(signal.SIGTERM)
                 sys.exit(1)
             else:
                 print("\nReceived shutdown signal, initiating graceful shutdown...")
                 print("(Press Ctrl+C again to force immediate exit)")
                 shutdown_requested.set()
 
-        # Use asyncio's signal handlers which work properly with the event loop
-        loop.add_signal_handler(signal.SIGINT, signal_handler)
-        loop.add_signal_handler(signal.SIGTERM, signal_handler)
+        async_handlers_installed = _install_shutdown_signal_handlers(loop, signal_handler)
+        if not async_handlers_installed:
+            # Windows ProactorEventLoop: asyncio.add_signal_handler is Unix-only
+            # and raises NotImplementedError. Default Python SIGINT handler still
+            # terminates the worker on Ctrl+C, just without the two-stage path.
+            print(
+                f"WARN: asyncio signal handlers unavailable on this platform "
+                f"({sys.platform}); graceful two-stage shutdown disabled, "
+                f"default Python SIGINT handler remains active.",
+                flush=True,
+            )
 
         # Create uvicorn config and server
         uvicorn_config = uvicorn.Config(

hindsight-api-slim/hindsight_api/worker/poller.py

@@ -14,7 +14,8 @@ import json
 import logging
 import time
 import traceback
-from collections.abc import Awaitable, Callable
+from collections import Counter
+from collections.abc import Awaitable, Callable, Iterable
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any
 
@@ -37,6 +38,36 @@ STUCK_STACK_INITIAL_THRESHOLD_S = 300
 STUCK_STACK_MAX_THRESHOLD_S = 3600 * 6  # cap doubling at 6h
 
 
+def _summarise_child_error_messages(siblings: "Iterable[Any]") -> str:
+    """Pick a representative error message for a parent whose children failed.
+
+    Used when a batch_retain parent transitions to 'failed' because at least
+    one child sub-batch failed. Without this, the parent gets a generic
+    "One or more sub-batches failed" string and any consumer that reasons
+    about errors via error_message (dashboards, alert filters, log
+    aggregators) loses the actual cause -- a class of failures that all
+    share the same root reason at the child level becomes indistinguishable
+    at the parent level.
+
+    Strategy: pick the most common non-empty error_message among failed
+    siblings. If they all failed for the same reason (the common case), the
+    parent inherits that reason verbatim. If they vary, the most-common one
+    is still a useful representative. Falls back to the legacy generic
+    string when no failed sibling carries an error_message at all.
+    """
+    failed_errors: list[str] = []
+    for s in siblings:
+        if s["status"] != "failed":
+            continue
+        msg = (s["error_message"] or "").strip()
+        if msg:
+            failed_errors.append(msg)
+    if not failed_errors:
+        return "One or more sub-batches failed"
+    most_common, _count = Counter(failed_errors).most_common(1)[0]
+    return most_common
+
+
 @dataclass
 class ActiveTaskInfo:
     """Tracking info for an in-flight worker task.
@@ -137,6 +168,11 @@ class WorkerPoller:
         self._slot_reservations: dict[str, int] = (
             slot_reservations if slot_reservations is not None else {"consolidation": 2}
         )
+        # Cache of which optional PG routines are installed on the server
+        # (probed once, memoised for the life of the poller).
+        from ..engine.db.optional_routines import OptionalRoutines
+
+        self._optional_routines = OptionalRoutines(self._backend)
         self._shutdown = asyncio.Event()
         self._current_tasks: set[asyncio.Task] = set()
         self._in_flight_count = 0
@@ -162,46 +198,22 @@ class WorkerPoller:
     async def _scan_active_schemas(self, schemas: list[str | None]) -> set[str | None]:
         """Find which schemas have pending work.
 
-        Tries a server-side PL/pgSQL function first (single DB round-trip,
-        ~200ms for 1400+ schemas). Falls back to per-schema Python EXISTS
-        queries if the function is not installed (~4ms each).
+        Prefers a server-side PL/pgSQL routine (single DB round-trip,
+        ~200ms for 1400+ schemas) when ``public.schemas_with_pending_work()``
+        is installed. The presence check goes through
+        ``OptionalRoutines.is_installed`` which probes ``pg_proc`` once and
+        caches the result, so we don't generate a server-side error on
+        every poll cycle when the routine isn't installed.
 
-        The server-side function should be installed in the ``public``
-        schema as::
-
-            CREATE OR REPLACE FUNCTION public.schemas_with_pending_work()
-            RETURNS SETOF text AS $$
-            DECLARE
-              r RECORD; has_work BOOLEAN;
-            BEGIN
-              FOR r IN SELECT nspname FROM pg_namespace
-                       WHERE nspname LIKE 'tenant_%' LOOP
-                BEGIN
-                  EXECUTE format(
-                    'SELECT EXISTS(SELECT 1 FROM %I.async_operations '
-                    'WHERE status = ''pending'' '
-                    'AND task_payload IS NOT NULL LIMIT 1)',
-                    r.nspname) INTO has_work;
-                  IF has_work THEN RETURN NEXT r.nspname; END IF;
-                EXCEPTION WHEN OTHERS THEN NULL;
-                END;
-              END LOOP;
-            END $$ LANGUAGE plpgsql STABLE;
-
-        In hindsight-cloud deployments this is installed by a Helm hook
-        job alongside ``total_pending_tasks()``.
+        Falls back to per-schema Python EXISTS queries (~4ms each) on
+        non-PostgreSQL backends or when the routine isn't installed. See
+        ``hindsight_api.engine.db.optional_routines`` for the canonical
+        install SQL.
         """
         async with self._backend.acquire() as conn:
-            # The schemas_with_pending_work() PL/pgSQL function is a
-            # PostgreSQL-specific optimisation installed by Helm hooks in
-            # hindsight-cloud. Skip on non-PG backends to avoid constant
-            # ORA-00904 / syntax errors on every poll cycle.
-            if self._backend.backend_type == "postgresql":
-                try:
-                    rows = await conn.fetch("SELECT * FROM schemas_with_pending_work()")
-                    return {r[0] for r in rows}
-                except Exception:
-                    pass
+            if await self._optional_routines.is_installed(conn, "schemas_with_pending_work"):
+                rows = await conn.fetch("SELECT * FROM public.schemas_with_pending_work()")
+                return {r[0] for r in rows}
 
             # Fallback: per-schema EXISTS checks from Python
             active: set[str | None] = set()
@@ -514,10 +526,14 @@ class WorkerPoller:
             if not parent_row:
                 return
 
-            # Check whether all siblings are done
+            # Check whether all siblings are done. Pull error_message too so a
+            # parent that fails can inherit a representative child reason --
+            # otherwise the parent's error_message is generic ("One or more
+            # sub-batches failed") and downstream consumers (dashboards, alerts,
+            # filters) lose the actual cause once a batch has children.
             siblings = await conn.fetch(
                 f"""
-                SELECT status FROM {table}
+                SELECT status, error_message FROM {table}
                 WHERE bank_id = $1
                   AND result_metadata::jsonb @> $2::jsonb
                 """,
@@ -536,7 +552,7 @@ class WorkerPoller:
                     WHERE operation_id = $1
                     """,
                     uuid.UUID(parent_operation_id),
-                    "One or more sub-batches failed",
+                    _summarise_child_error_messages(siblings),
                 )
             else:
                 await conn.execute(
@@ -959,15 +975,30 @@ class WorkerPoller:
             if len(processing_info) > 10:
                 processing_str += f" +{len(processing_info) - 10} more"
 
-            # Get global stats from DB
+            # Get global stats from DB — scope the heavy COUNT/GROUP BY
+            # queries to schemas that actually have work. With N tenants the
+            # full fanout is 2*N queries every PROGRESS_LOG_INTERVAL; scoping
+            # via the routine (or per-schema EXISTS fallback) reduces this to
+            # 2*active_schemas which is typically << N.
             schemas = await self._get_schemas()
+            total_schema_count = len(schemas)
+
+            # Schemas with pending async_operations (uses server-side
+            # routine when installed, falls back to per-schema EXISTS).
+            schemas_with_pending = await self._scan_active_schemas(schemas)
+
+            # Also include schemas that have in-flight tasks on this worker
+            # so the "processing" worker_id GROUP BY still reports correctly.
+            schemas_with_active_tasks = {info.schema for info in active_tasks.values()}
+            schemas_to_query = schemas_with_pending | schemas_with_active_tasks
+
             global_pending = 0
             all_worker_counts: dict[str, int] = {}
             # operation_type -> aggregated bucket counts across schemas
             pending_breakdown: dict[str, dict[str, int]] = {}
 
             async with self._backend.acquire() as conn:
-                for schema in schemas:
+                for schema in schemas_to_query:
                     table = fq_table("async_operations", schema)
 
                     # Bucket pending rows by the same predicates the claim query
@@ -976,20 +1007,24 @@ class WorkerPoller:
                     # retry backoff, etc.).
                     # Use SUM(CASE WHEN ...) instead of COUNT(*) FILTER (WHERE ...)
                     # for Oracle compatibility — FILTER is PG-specific.
-                    breakdown_rows = await conn.fetch(
-                        f"""
-                        SELECT
-                            operation_type,
-                            COUNT(*) AS total,
-                            SUM(CASE WHEN task_payload IS NULL THEN 1 ELSE 0 END) AS payload_null,
-                            SUM(CASE WHEN next_retry_at IS NOT NULL AND next_retry_at > now()
-                                THEN 1 ELSE 0 END) AS retry_blocked,
-                            SUM(CASE WHEN worker_id IS NOT NULL THEN 1 ELSE 0 END) AS assigned
-                        FROM {table}
-                        WHERE status = 'pending'
-                        GROUP BY operation_type
-                        """
-                    )
+                    try:
+                        breakdown_rows = await conn.fetch(
+                            f"""
+                            SELECT
+                                operation_type,
+                                COUNT(*) AS total,
+                                SUM(CASE WHEN task_payload IS NULL THEN 1 ELSE 0 END) AS payload_null,
+                                SUM(CASE WHEN next_retry_at IS NOT NULL AND next_retry_at > now()
+                                    THEN 1 ELSE 0 END) AS retry_blocked,
+                                SUM(CASE WHEN worker_id IS NOT NULL THEN 1 ELSE 0 END) AS assigned
+                            FROM {table}
+                            WHERE status = 'pending'
+                            GROUP BY operation_type
+                            """
+                        )
+                    except Exception:
+                        # Schema may be partially provisioned (table missing).
+                        breakdown_rows = []
                     for br in breakdown_rows:
                         op_type = br["operation_type"] or "unknown"
                         bucket = pending_breakdown.setdefault(
@@ -1001,14 +1036,17 @@ class WorkerPoller:
                         bucket["assigned"] += br["assigned"]
                         global_pending += br["total"]
 
-                    worker_rows = await conn.fetch(
-                        f"""
-                        SELECT worker_id, COUNT(*) as count
-                        FROM {table}
-                        WHERE status = 'processing'
-                        GROUP BY worker_id
-                        """
-                    )
+                    try:
+                        worker_rows = await conn.fetch(
+                            f"""
+                            SELECT worker_id, COUNT(*) as count
+                            FROM {table}
+                            WHERE status = 'processing'
+                            GROUP BY worker_id
+                            """
+                        )
+                    except Exception:
+                        worker_rows = []
                     for wr in worker_rows:
                         wid = wr["worker_id"] or "unknown"
                         all_worker_counts[wid] = all_worker_counts.get(wid, 0) + wr["count"]
@@ -1024,14 +1062,19 @@ class WorkerPoller:
             pool_str = self._format_pool_stats()
             proc_str = self._format_proc_stats()
 
-            # Display None as "default" in logs
-            schemas_str = ", ".join(s if s else "default" for s in schemas)
+            queried_count = len(schemas_to_query)
+            # Display queried schemas (cap at 20 for readability)
+            queried_list = sorted(s if s else "default" for s in schemas_to_query)
+            schemas_str = ", ".join(queried_list[:20])
+            if len(queried_list) > 20:
+                schemas_str += f" +{len(queried_list) - 20} more"
             logger.info(
                 f"[WORKER_STATS] worker={self._worker_id} "
                 f"slots={in_flight}/{self._max_slots} | "
                 f"reserved: [{reserved_str}] | "
                 f"shared={tasks_in_shared}/{shared_pool_size}(avail={shared_available}) | "
-                f"global: pending={global_pending} (schemas: {schemas_str}) | "
+                f"global: pending={global_pending} "
+                f"(queried={queried_count}/{total_schema_count} schemas: {schemas_str}) | "
                 f"others: {others_str} | "
                 f"pool: {pool_str} | "
                 f"proc: {proc_str} | "

hindsight-api-slim/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-api-slim"
-version = "0.5.6"
+version = "0.6.2"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -47,14 +47,14 @@ dependencies = [
     "anthropic>=0.40.0",
     "typer>=0.9.0",
     "cohere>=5.0.0",
-    "litellm>=1.83.0", # 1.82.7/1.82.8 had a supply chain compromise (yanked); 1.83.0+ also fixes GHSA-jjhc-v7c2-5hh6 / GHSA-53mr-6c8q-9789
+    "litellm>=1.83.14", # 1.82.7/1.82.8 had a supply chain compromise (yanked); 1.83.0+ also fixes GHSA-jjhc-v7c2-5hh6 / GHSA-53mr-6c8q-9789 / GHSA-pq44-5pcq-4r5g / GHSA-8cjq-wjmh-q42r
     "markitdown[pdf,docx,pptx,xlsx,xls]>=0.1.4", # File to markdown conversion
     "obstore>=0.4.0", # S3/GCS/Azure object storage client (Rust-backed)
     "winloop>=0.1.0; sys_platform == 'win32'",
     "uvloop>=0.22.1; sys_platform != 'win32'",
     # Transitive dependency security fixes
     "pyasn1>=0.6.3", # DoS vulnerability fix
-    "urllib3>=2.6.3", # Decompression-bomb safeguards bypass fix
+    "urllib3>=2.7.0", # Decompression-bomb safeguards bypass + sensitive header forwarding fixes
     "langchain-core>=1.2.22", # Path traversal in legacy load_prompt functions fix
     "langsmith>=0.6.3", # SSRF via tracing header injection fix
     "protobuf>=6.33.5", # JSON recursion depth bypass fix
@@ -93,7 +93,7 @@ local-llm = [
     "huggingface-hub>=0.20.0",
 ]
 embedded-db = [
-    "pg0-embedded>=0.13.0",
+    "pg0-embedded>=0.14.0",
 ]
 oracle = [
     "oracledb>=2.5.0",
@@ -141,6 +141,7 @@ log_cli_date_format = "%Y-%m-%d %H:%M:%S"
 addopts = "--timeout 300 -n 8 --dist loadgroup --durations=10 -v"
 markers = [
     "oracle: Oracle 23ai integration tests (require ORACLE_TEST_DSN env var)",
+    "hs_llm_mat: LLM minimum acceptance tests — run in CI matrix across multiple providers",
 ]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"

hindsight-api-slim/tests/test_codex_oauth_refresh.py

@@ -0,0 +1,535 @@
+"""Tests for Codex OAuth token refresh (issue #1637).
+
+The Codex provider was originally a startup-only credential loader: it read
+``~/.codex/auth.json`` once and used the cached access_token forever. These
+tests pin the new automatic-refresh behavior:
+
+- ``refresh_token`` is now actually loaded from auth.json.
+- The provider proactively refreshes ~60s before the JWT ``exp`` claim.
+- It reactively refreshes once on a 401/403 from the Codex backend.
+- The OAuth refresh request shape mirrors the canonical ``@openai/codex``
+  CLI (POST https://auth.openai.com/oauth/token, JSON body with hardcoded
+  client_id, grant_type=refresh_token).
+- Terminal error codes (refresh_token_expired/reused/invalidated) raise a
+  permanent error and do not loop.
+- Concurrent callers serialize through a single-flight lock.
+- ``auth.json`` is persisted atomically via tempfile+rename with mode 0600.
+
+Tests construct ``CodexLLM`` with ``_load_codex_auth`` mocked, then drive
+JWT exp / network / persistence paths through targeted patches.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import json
+import os
+import stat
+import sys
+import time
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import httpx
+import pytest
+
+from hindsight_api.engine.providers.codex_llm import (
+    _CODEX_CLIENT_ID,
+    _CODEX_REFRESH_TOKEN_URL,
+    CodexLLM,
+    CodexRefreshExpiredError,
+)
+
+
+def _make_jwt(exp_unixtime: int | None) -> str:
+    """Build a minimal JWT-shaped token with the given ``exp`` claim.
+
+    Signature segment is a placeholder — we don't verify, we only decode
+    the payload to read ``exp``.
+    """
+    header = base64.urlsafe_b64encode(json.dumps({"alg": "none"}).encode()).rstrip(b"=").decode()
+    payload_dict: dict[str, object] = {}
+    if exp_unixtime is not None:
+        payload_dict["exp"] = exp_unixtime
+    payload = base64.urlsafe_b64encode(json.dumps(payload_dict).encode()).rstrip(b"=").decode()
+    signature = "sig"
+    return f"{header}.{payload}.{signature}"
+
+
+def _build_llm(refresh_token: str | None = "rt-initial", access_token: str | None = None) -> CodexLLM:
+    """Construct a CodexLLM with patched auth-file reads."""
+    if access_token is None:
+        access_token = _make_jwt(int(time.time()) + 3600)  # fresh by default
+    with (
+        patch.object(CodexLLM, "_load_codex_auth", return_value=(access_token, "acct-123")),
+        patch.object(CodexLLM, "_load_codex_refresh_token", return_value=refresh_token),
+    ):
+        return CodexLLM(
+            provider="openai-codex",
+            api_key="ignored",
+            base_url="https://chatgpt.com/backend-api",
+            model="gpt-5.4-mini",
+        )
+
+
+# ---------------------------------------------------------------------------
+# JWT exp decode
+# ---------------------------------------------------------------------------
+
+
+def test_jwt_exp_decode_returns_int_for_valid_token():
+    token = _make_jwt(1_800_000_000)
+    assert CodexLLM._decode_jwt_exp_unixtime(token) == 1_800_000_000
+
+
+def test_jwt_exp_decode_returns_none_when_exp_missing():
+    token = _make_jwt(None)
+    assert CodexLLM._decode_jwt_exp_unixtime(token) is None
+
+
+def test_jwt_exp_decode_returns_none_for_malformed_token():
+    assert CodexLLM._decode_jwt_exp_unixtime("not.a.real.jwt") is None
+    assert CodexLLM._decode_jwt_exp_unixtime("only-one-segment") is None
+    assert CodexLLM._decode_jwt_exp_unixtime("a.!!notbase64!!.c") is None
+
+
+# ---------------------------------------------------------------------------
+# Staleness
+# ---------------------------------------------------------------------------
+
+
+def test_token_is_stale_true_when_expired():
+    expired = _make_jwt(int(time.time()) - 60)
+    llm = _build_llm(access_token=expired)
+    assert llm._token_is_stale() is True
+
+
+def test_token_is_stale_true_within_skew_window():
+    # 30s before expiry, default skew is 60s → should be considered stale.
+    soon = _make_jwt(int(time.time()) + 30)
+    llm = _build_llm(access_token=soon)
+    assert llm._token_is_stale() is True
+
+
+def test_token_is_stale_false_when_far_from_expiry():
+    far = _make_jwt(int(time.time()) + 3600)
+    llm = _build_llm(access_token=far)
+    assert llm._token_is_stale() is False
+
+
+def test_token_is_stale_false_when_exp_unparseable():
+    # When we can't decide, we'd rather use a possibly-expired token and
+    # recover via the reactive 401 path than refresh aggressively.
+    llm = _build_llm(access_token="opaque-token-no-jwt-structure")
+    assert llm._token_is_stale() is False
+
+
+# ---------------------------------------------------------------------------
+# refresh_token loading
+# ---------------------------------------------------------------------------
+
+
+def test_refresh_token_loaded_from_auth_file(tmp_path: Path):
+    auth_file = tmp_path / "auth.json"
+    auth_file.write_text(
+        json.dumps(
+            {
+                "auth_mode": "chatgpt",
+                "tokens": {
+                    "access_token": "at",
+                    "refresh_token": "rt-from-disk",
+                    "account_id": "acct",
+                },
+            }
+        )
+    )
+    with patch.object(CodexLLM, "_load_codex_auth", return_value=("at", "acct")):
+        llm = CodexLLM(
+            provider="openai-codex",
+            api_key="ignored",
+            base_url="https://chatgpt.com/backend-api",
+            model="gpt-5.4-mini",
+        )
+    # Now point the auth_file at our tmp file and reload.
+    llm._auth_file = auth_file
+    assert llm._load_codex_refresh_token() == "rt-from-disk"
+
+
+def test_refresh_token_returns_none_when_field_absent(tmp_path: Path):
+    auth_file = tmp_path / "auth.json"
+    auth_file.write_text(json.dumps({"auth_mode": "chatgpt", "tokens": {"access_token": "at"}}))
+    llm = _build_llm()
+    llm._auth_file = auth_file
+    assert llm._load_codex_refresh_token() is None
+
+
+def test_refresh_token_returns_none_when_file_missing(tmp_path: Path):
+    llm = _build_llm()
+    llm._auth_file = tmp_path / "definitely-not-here.json"
+    assert llm._load_codex_refresh_token() is None
+
+
+# ---------------------------------------------------------------------------
+# Atomic persistence
+# ---------------------------------------------------------------------------
+
+
+def test_persist_auth_atomic_writes_mode_0600_and_preserves_fields(tmp_path: Path):
+    auth_file = tmp_path / "auth.json"
+    auth_file.write_text(
+        json.dumps(
+            {
+                "OPENAI_API_KEY": None,
+                "auth_mode": "chatgpt",
+                "tokens": {
+                    "access_token": "old",
+                    "refresh_token": "rt-old",
+                    "account_id": "acct-keep",
+                    "id_token": {"email": "[email protected]"},
+                },
+                "last_refresh": "2026-01-01T00:00:00Z",
+            }
+        )
+    )
+
+    llm = _build_llm()
+    llm._auth_file = auth_file
+    llm._persist_auth_atomic({"access_token": "new", "refresh_token": "rt-new"})
+
+    written = json.loads(auth_file.read_text())
+    assert written["tokens"]["access_token"] == "new"
+    assert written["tokens"]["refresh_token"] == "rt-new"
+    # Untouched fields are preserved (account_id, id_token, auth_mode).
+    assert written["tokens"]["account_id"] == "acct-keep"
+    assert written["tokens"]["id_token"] == {"email": "[email protected]"}
+    assert written["auth_mode"] == "chatgpt"
+    # last_refresh got bumped to a new ISO-8601 UTC timestamp.
+    assert written["last_refresh"] != "2026-01-01T00:00:00Z"
+    assert written["last_refresh"].endswith("Z")
+
+    if sys.platform != "win32":
+        mode = stat.S_IMODE(auth_file.stat().st_mode)
+        assert mode == 0o600, f"expected 0600, got {oct(mode)}"
+
+
+def test_persist_auth_atomic_does_not_leak_tempfile_on_success(tmp_path: Path):
+    auth_file = tmp_path / "auth.json"
+    auth_file.write_text(json.dumps({"tokens": {"access_token": "old"}}))
+
+    llm = _build_llm()
+    llm._auth_file = auth_file
+    llm._persist_auth_atomic({"access_token": "new"})
+
+    # No sibling tempfile should remain — atomic rename consumed it.
+    siblings = [p.name for p in tmp_path.iterdir()]
+    assert siblings == ["auth.json"], f"unexpected leftover files: {siblings}"
+
+
+# ---------------------------------------------------------------------------
+# _refresh_oauth_tokens — request shape, in-memory update, rotation
+# ---------------------------------------------------------------------------
+
+
+def _refresh_response(status_code: int, body: dict | str) -> MagicMock:
+    response = MagicMock()
+    response.status_code = status_code
+    if isinstance(body, dict):
+        response.json.return_value = body
+        response.text = json.dumps(body)
+    else:
+        response.json.side_effect = json.JSONDecodeError("nope", body, 0)
+        response.text = body
+    return response
+
+
+@pytest.mark.asyncio
+async def test_refresh_sends_canonical_request_shape(tmp_path: Path):
+    """POST JSON body with client_id + grant_type=refresh_token + refresh_token."""
+    expired = _make_jwt(int(time.time()) - 60)
+    llm = _build_llm(refresh_token="rt-current", access_token=expired)
+    llm._auth_file = tmp_path / "auth.json"
+    llm._auth_file.write_text(json.dumps({"tokens": {"access_token": "x", "refresh_token": "rt-current"}}))
+
+    fresh_access = _make_jwt(int(time.time()) + 3600)
+    refresh_resp = _refresh_response(200, {"access_token": fresh_access, "refresh_token": "rt-rotated"})
+
+    with patch.object(llm._client, "post", new_callable=AsyncMock, return_value=refresh_resp) as mock_post:
+        await llm._refresh_oauth_tokens()
+
+    call_args = mock_post.call_args
+    assert call_args.args[0] == _CODEX_REFRESH_TOKEN_URL
+    assert call_args.kwargs["headers"]["Content-Type"] == "application/json"
+    assert call_args.kwargs["json"] == {
+        "client_id": _CODEX_CLIENT_ID,
+        "grant_type": "refresh_token",
+        "refresh_token": "rt-current",
+    }
+
+
+@pytest.mark.asyncio
+async def test_refresh_updates_in_memory_credentials(tmp_path: Path):
+    expired = _make_jwt(int(time.time()) - 60)
+    llm = _build_llm(refresh_token="rt-old", access_token=expired)
+    llm._auth_file = tmp_path / "auth.json"
+    llm._auth_file.write_text(json.dumps({"tokens": {"access_token": "x", "refresh_token": "rt-old"}}))
+
+    new_access = _make_jwt(int(time.time()) + 3600)
+    refresh_resp = _refresh_response(200, {"access_token": new_access, "refresh_token": "rt-new"})
+
+    with patch.object(llm._client, "post", new_callable=AsyncMock, return_value=refresh_resp):
+        await llm._refresh_oauth_tokens()
+
+    assert llm.access_token == new_access
+    assert llm.refresh_token == "rt-new"
+
+
+@pytest.mark.asyncio
+async def test_refresh_keeps_existing_refresh_token_when_server_omits_one(tmp_path: Path):
+    """If the OAuth response has no ``refresh_token`` field, keep the one we have."""
+    expired = _make_jwt(int(time.time()) - 60)
+    llm = _build_llm(refresh_token="rt-keep", access_token=expired)
+    llm._auth_file = tmp_path / "auth.json"
+    llm._auth_file.write_text(json.dumps({"tokens": {"access_token": "x", "refresh_token": "rt-keep"}}))
+
+    new_access = _make_jwt(int(time.time()) + 3600)
+    refresh_resp = _refresh_response(200, {"access_token": new_access})
+
+    with patch.object(llm._client, "post", new_callable=AsyncMock, return_value=refresh_resp):
+        await llm._refresh_oauth_tokens()
+
+    assert llm.refresh_token == "rt-keep"
+
+
+@pytest.mark.asyncio
+async def test_refresh_raises_permanent_error_on_terminal_oauth_code(tmp_path: Path):
+    expired = _make_jwt(int(time.time()) - 60)
+    llm = _build_llm(refresh_token="rt-stale", access_token=expired)
+    llm._auth_file = tmp_path / "auth.json"
+    llm._auth_file.write_text(json.dumps({"tokens": {"access_token": "x", "refresh_token": "rt-stale"}}))
+
+    bad_resp = _refresh_response(401, {"error": {"code": "refresh_token_expired"}})
+
+    with patch.object(llm._client, "post", new_callable=AsyncMock, return_value=bad_resp):
+        with pytest.raises(CodexRefreshExpiredError):
+            await llm._refresh_oauth_tokens()
+
+
+@pytest.mark.asyncio
+async def test_refresh_raises_permanent_error_on_unknown_401(tmp_path: Path):
+    """Any 401 from the refresh endpoint is treated as permanent — matches upstream Rust classification."""
+    expired = _make_jwt(int(time.time()) - 60)
+    llm = _build_llm(refresh_token="rt-stale", access_token=expired)
+    llm._auth_file = tmp_path / "auth.json"
+    llm._auth_file.write_text(json.dumps({"tokens": {"access_token": "x", "refresh_token": "rt-stale"}}))
+
+    bad_resp = _refresh_response(401, {"error": "something_else"})
+
+    with patch.object(llm._client, "post", new_callable=AsyncMock, return_value=bad_resp):
+        with pytest.raises(CodexRefreshExpiredError):
+            await llm._refresh_oauth_tokens()
+
+
+@pytest.mark.asyncio
+async def test_refresh_raises_runtime_error_on_5xx(tmp_path: Path):
+    """5xx is transient from the caller's perspective — surface as RuntimeError, not CodexRefreshExpiredError."""
+    expired = _make_jwt(int(time.time()) - 60)
+    llm = _build_llm(refresh_token="rt-current", access_token=expired)
+    llm._auth_file = tmp_path / "auth.json"
+    llm._auth_file.write_text(json.dumps({"tokens": {"access_token": "x", "refresh_token": "rt-current"}}))
+
+    bad_resp = _refresh_response(503, "service unavailable")
+
+    with patch.object(llm._client, "post", new_callable=AsyncMock, return_value=bad_resp):
+        with pytest.raises(RuntimeError) as exc_info:
+            await llm._refresh_oauth_tokens()
+    assert not isinstance(exc_info.value, CodexRefreshExpiredError)
+
+
+@pytest.mark.asyncio
+async def test_refresh_does_not_log_token_values(tmp_path: Path, caplog):
+    expired = _make_jwt(int(time.time()) - 60)
+    secret_rt = "rt-DO-NOT-LEAK-THIS"
+    llm = _build_llm(refresh_token=secret_rt, access_token=expired)
+    llm._auth_file = tmp_path / "auth.json"
+    llm._auth_file.write_text(json.dumps({"tokens": {"access_token": "x", "refresh_token": secret_rt}}))
+
+    new_access = _make_jwt(int(time.time()) + 3600)
+    refresh_resp = _refresh_response(200, {"access_token": new_access, "refresh_token": "rt-also-secret"})
+
+    with patch.object(llm._client, "post", new_callable=AsyncMock, return_value=refresh_resp):
+        with caplog.at_level("DEBUG"):
+            await llm._refresh_oauth_tokens()
+
+    log_text = "\n".join(record.getMessage() for record in caplog.records)
+    assert secret_rt not in log_text
+    assert new_access not in log_text
+    assert "rt-also-secret" not in log_text
+
+
+# ---------------------------------------------------------------------------
+# Single-flight under concurrent callers
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_concurrent_ensure_fresh_token_calls_produce_one_refresh(tmp_path: Path):
+    expired = _make_jwt(int(time.time()) - 60)
+    llm = _build_llm(refresh_token="rt", access_token=expired)
+    llm._auth_file = tmp_path / "auth.json"
+    llm._auth_file.write_text(json.dumps({"tokens": {"access_token": "x", "refresh_token": "rt"}}))
+
+    new_access = _make_jwt(int(time.time()) + 3600)
+    call_count = 0
+
+    async def fake_post(*args, **kwargs):
+        nonlocal call_count
+        call_count += 1
+        # Simulate non-zero refresh latency so concurrent callers actually queue.
+        await asyncio.sleep(0.01)
+        return _refresh_response(200, {"access_token": new_access, "refresh_token": "rt-new"})
+
+    with patch.object(llm._client, "post", new=fake_post):
+        await asyncio.gather(*(llm._ensure_fresh_token() for _ in range(10)))
+
+    assert call_count == 1, f"expected 1 network refresh under contention, got {call_count}"
+
+
+# ---------------------------------------------------------------------------
+# Reactive 401 retry on the request path
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_call_reactively_refreshes_on_401_and_retries(tmp_path: Path):
+    """A backend 401 triggers one refresh + retry instead of immediately raising."""
+    fresh = _make_jwt(int(time.time()) + 3600)  # not stale; the 401 is the trigger
+    llm = _build_llm(refresh_token="rt", access_token=fresh)
+    llm._auth_file = tmp_path / "auth.json"
+    llm._auth_file.write_text(json.dumps({"tokens": {"access_token": fresh, "refresh_token": "rt"}}))
+
+    new_access = _make_jwt(int(time.time()) + 3600)
+
+    # First post → 401 (backend rejects the token). After refresh, second post → 200.
+    success_resp = MagicMock()
+    success_resp.status_code = 200
+    success_resp.raise_for_status.return_value = None
+
+    fail_response = MagicMock()
+    fail_response.status_code = 401
+    fail_response.text = "unauthorized"
+    fail_exc = httpx.HTTPStatusError("401", request=MagicMock(), response=fail_response)
+    success_resp.raise_for_status = MagicMock(return_value=None)
+
+    post_responses = [fail_exc, success_resp]
+
+    async def fake_post(*args, **kwargs):
+        item = post_responses.pop(0)
+        if isinstance(item, Exception):
+            raise item
+        return item
+
+    refresh_resp = _refresh_response(200, {"access_token": new_access, "refresh_token": "rt-new"})
+
+    call_count = {"refresh": 0, "post": 0}
+
+    async def counting_post(url, **kwargs):
+        if url == _CODEX_REFRESH_TOKEN_URL:
+            call_count["refresh"] += 1
+            return refresh_resp
+        call_count["post"] += 1
+        # First backend call fails with 401 wrapped in an HTTPStatusError-style response,
+        # second succeeds.
+        if call_count["post"] == 1:
+            raise httpx.HTTPStatusError("401", request=MagicMock(), response=fail_response)
+        return success_resp
+
+    with (
+        patch.object(llm._client, "post", new=counting_post),
+        patch.object(llm, "_parse_sse_stream", new_callable=AsyncMock, return_value="ok"),
+    ):
+        result = await llm.call(
+            messages=[{"role": "user", "content": "ping"}],
+            max_retries=0,
+            initial_backoff=0.0,
+            max_backoff=0.0,
+        )
+
+    assert result == "ok"
+    assert call_count["refresh"] == 1
+    assert call_count["post"] == 2  # one 401, one success after refresh
+    assert llm.access_token == new_access
+
+
+@pytest.mark.asyncio
+async def test_call_proactively_refreshes_when_token_is_stale(tmp_path: Path):
+    """A near-expiry token triggers refresh BEFORE the request is sent."""
+    expired = _make_jwt(int(time.time()) - 60)
+    llm = _build_llm(refresh_token="rt", access_token=expired)
+    llm._auth_file = tmp_path / "auth.json"
+    llm._auth_file.write_text(json.dumps({"tokens": {"access_token": expired, "refresh_token": "rt"}}))
+
+    new_access = _make_jwt(int(time.time()) + 3600)
+    refresh_resp = _refresh_response(200, {"access_token": new_access, "refresh_token": "rt-new"})
+
+    success_resp = MagicMock()
+    success_resp.status_code = 200
+    success_resp.raise_for_status.return_value = None
+
+    call_order: list[str] = []
+
+    async def fake_post(url, **kwargs):
+        if url == _CODEX_REFRESH_TOKEN_URL:
+            call_order.append("refresh")
+            return refresh_resp
+        call_order.append("backend")
+        # Assert that by the time the backend is called, the new token is in use.
+        assert kwargs["headers"]["Authorization"] == f"Bearer {new_access}"
+        return success_resp
+
+    with (
+        patch.object(llm._client, "post", new=fake_post),
+        patch.object(llm, "_parse_sse_stream", new_callable=AsyncMock, return_value="ok"),
+    ):
+        await llm.call(
+            messages=[{"role": "user", "content": "ping"}],
+            max_retries=0,
+            initial_backoff=0.0,
+            max_backoff=0.0,
+        )
+
+    assert call_order == ["refresh", "backend"], "expected proactive refresh BEFORE the backend call"
+
+
+@pytest.mark.asyncio
+async def test_call_does_not_refresh_when_token_is_fresh(tmp_path: Path):
+    fresh = _make_jwt(int(time.time()) + 3600)
+    llm = _build_llm(refresh_token="rt", access_token=fresh)
+    llm._auth_file = tmp_path / "auth.json"
+    llm._auth_file.write_text(json.dumps({"tokens": {"access_token": fresh, "refresh_token": "rt"}}))
+
+    success_resp = MagicMock()
+    success_resp.status_code = 200
+    success_resp.raise_for_status.return_value = None
+
+    call_count = {"refresh": 0, "backend": 0}
+
+    async def fake_post(url, **kwargs):
+        if url == _CODEX_REFRESH_TOKEN_URL:
+            call_count["refresh"] += 1
+            raise AssertionError("refresh endpoint should not be hit for a fresh token")
+        call_count["backend"] += 1
+        return success_resp
+
+    with (
+        patch.object(llm._client, "post", new=fake_post),
+        patch.object(llm, "_parse_sse_stream", new_callable=AsyncMock, return_value="ok"),
+    ):
+        await llm.call(
+            messages=[{"role": "user", "content": "ping"}],
+            max_retries=0,
+            initial_backoff=0.0,
+            max_backoff=0.0,
+        )
+
+    assert call_count == {"refresh": 0, "backend": 1}

hindsight-api-slim/tests/test_config_validation.py

@@ -128,6 +128,74 @@ def test_log_config_masks_database_urls(caplog):
     assert "postgresql://***:***@db-admin:5432/hindsight_db" in log_output
 
 
+def test_read_database_url_defaults_to_none_when_unset(monkeypatch):
+    """Without HINDSIGHT_API_READ_DATABASE_URL, the field is None — engine
+    will alias the read backend to the primary, preserving today's
+    single-pool behaviour byte-for-bit. This is the most important guarantee
+    of the change: zero-config means zero behaviour change.
+    """
+    from hindsight_api.config import HindsightConfig
+
+    monkeypatch.delenv("HINDSIGHT_API_READ_DATABASE_URL", raising=False)
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    config = HindsightConfig.from_env()
+    assert config.read_database_url is None
+
+
+def test_read_database_url_is_loaded_when_set(monkeypatch):
+    """When HINDSIGHT_API_READ_DATABASE_URL is set, the value flows into
+    config so MemoryEngine.initialize() will open a second backend against
+    that URL for recall queries.
+    """
+    from hindsight_api.config import HindsightConfig
+
+    read_url = "postgresql://reader:[email protected]:5432/hindsight"
+    monkeypatch.setenv("HINDSIGHT_API_READ_DATABASE_URL", read_url)
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    config = HindsightConfig.from_env()
+    assert config.read_database_url == read_url
+
+
+def test_read_database_url_empty_string_is_treated_as_unset(monkeypatch):
+    """Helm sometimes renders an unset env var as the empty string. Treat it
+    the same as unset so deployments that conditionally set the var don't
+    accidentally try to open a pool against `''`.
+    """
+    from hindsight_api.config import HindsightConfig
+
+    monkeypatch.setenv("HINDSIGHT_API_READ_DATABASE_URL", "")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    config = HindsightConfig.from_env()
+    assert config.read_database_url is None
+
+
+def test_log_config_masks_read_database_url(monkeypatch, caplog):
+    """Read-replica URL credentials must be masked in startup logs, same as
+    the primary URL.
+    """
+    from hindsight_api.config import HindsightConfig
+
+    monkeypatch.setenv("HINDSIGHT_API_DATABASE_URL", "postgresql://hindsight:pw@primary:5432/db")
+    monkeypatch.setenv("HINDSIGHT_API_READ_DATABASE_URL", "postgresql://reader:replica-secret@replica:5432/db")
+    monkeypatch.setenv("HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS", "64000")
+    monkeypatch.setenv("HINDSIGHT_API_RETAIN_CHUNK_SIZE", "3000")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    caplog.set_level(logging.INFO, logger="hindsight_api.config")
+
+    config = HindsightConfig.from_env()
+    config.log_config()
+
+    log_output = "\n".join(record.getMessage() for record in caplog.records)
+    assert "reader" not in log_output
+    assert "replica-secret" not in log_output
+    assert "Read database" in log_output
+    assert "postgresql://***:***@replica:5432/db" in log_output
+
+
 # Note: The BadRequestError wrapping is implemented in fact_extraction.py
 # but requires a complex integration test setup. The functionality is
 # straightforward: when a BadRequestError containing keywords like

hindsight-api-slim/tests/test_consolidation.py

@@ -335,6 +335,7 @@ class TestConsolidationIntegration:
         # Cleanup
         await memory.delete_bank(bank_id, request_context=request_context)
 
+    @pytest.mark.hs_llm_mat
     @pytest.mark.asyncio
     async def test_consolidation_merges_only_redundant_facts(self, memory: MemoryEngine, request_context):
         """Test that consolidation only merges truly redundant facts.

hindsight-api-slim/tests/test_custom_embedding_dimension.py

@@ -69,15 +69,76 @@ def create_isolated_schema(db_url: str, schema_name: str, dimension: int | None
 
     # Adjust embedding dimension if specified
     if dimension is not None:
-        ensure_embedding_dimension(db_url, dimension, schema=schema_name)
+        _ensure_embedding_dimension_with_retry(db_url, dimension, schema=schema_name)
 
 
 def drop_schema(db_url: str, schema_name: str):
-    """Drop an isolated schema."""
+    """Drop an isolated schema.
+
+    Retries once on InternalError (e.g. 'could not open relation with OID')
+    which can happen when pg0 has concurrent connections referencing the schema.
+    """
     engine = create_engine(db_url)
-    with engine.connect() as conn:
-        conn.execute(text(f"DROP SCHEMA IF EXISTS {schema_name} CASCADE"))
-        conn.commit()
+    for attempt in range(2):
+        try:
+            with engine.connect() as conn:
+                conn.execute(text(f"DROP SCHEMA IF EXISTS {schema_name} CASCADE"))
+                conn.commit()
+            return
+        except Exception:
+            if attempt == 0:
+                import time
+
+                time.sleep(0.5)
+            # Best-effort teardown — don't fail the test over cleanup issues
+
+
+def _ensure_embedding_dimension_with_retry(db_url: str, dimension: int, schema: str):
+    """Wrapper around ensure_embedding_dimension with OID race retry.
+
+    pg0 with concurrent xdist workers can cause 'could not open relation with OID'
+    when one worker's DROP SCHEMA CASCADE invalidates pg_indexes references mid-query.
+    """
+    import time
+
+    for attempt in range(3):
+        try:
+            ensure_embedding_dimension(db_url, dimension, schema=schema)
+            return
+        except Exception as e:
+            if "could not open relation with OID" in str(e) and attempt < 2:
+                time.sleep(0.5)
+                continue
+            raise
+
+
+def _assert_raises_runtime_error_with_retry(
+    db_url: str,
+    dimension: int,
+    schema: str,
+    expected_messages: list[str],
+):
+    """Assert ensure_embedding_dimension raises RuntimeError, retrying on transient OID errors.
+
+    Concurrent xdist workers can cause 'could not open relation with OID' errors
+    that mask the expected RuntimeError. This retries to give the system a chance to
+    reach the actual dimension-mismatch check.
+    """
+    import time
+
+    for attempt in range(3):
+        try:
+            ensure_embedding_dimension(db_url, dimension, schema=schema)
+            raise AssertionError("Expected RuntimeError but ensure_embedding_dimension succeeded")
+        except RuntimeError as e:
+            for msg in expected_messages:
+                assert msg in str(e), f"Expected '{msg}' in error message, got: {e}"
+            return
+        except Exception as e:
+            if "could not open relation with OID" in str(e) and attempt < 2:
+                time.sleep(0.5)
+                continue
+            raise
 
 
 def get_column_dimension(db_url: str, schema: str = "public", table: str = "memory_units") -> int | None:
@@ -188,7 +249,7 @@ class TestEmbeddingDimension:
         assert initial_dim == 384, f"Expected 384, got {initial_dim}"
 
         # Call ensure_embedding_dimension with matching dimension
-        ensure_embedding_dimension(db_url, 384, schema=schema)
+        _ensure_embedding_dimension_with_retry(db_url, 384, schema=schema)
 
         # Dimension should still be 384
         assert get_column_dimension(db_url, schema) == 384
@@ -202,14 +263,14 @@ class TestEmbeddingDimension:
         assert get_row_count(db_url, schema) == 0
 
         # Change dimension to 768
-        ensure_embedding_dimension(db_url, 768, schema=schema)
+        _ensure_embedding_dimension_with_retry(db_url, 768, schema=schema)
 
         # Verify dimension changed
         new_dim = get_column_dimension(db_url, schema)
         assert new_dim == 768, f"Expected 768, got {new_dim}"
 
         # Change back to 384 for other tests
-        ensure_embedding_dimension(db_url, 384, schema=schema)
+        _ensure_embedding_dimension_with_retry(db_url, 384, schema=schema)
         assert get_column_dimension(db_url, schema) == 384
 
     def test_dimension_change_blocked_with_data(self, dimension_test_schema):
@@ -223,12 +284,12 @@ class TestEmbeddingDimension:
         insert_test_embedding(db_url, schema, 384)
         assert get_row_count(db_url, schema) == 1
 
-        # Try to change dimension - should raise error
-        with pytest.raises(RuntimeError) as exc_info:
-            ensure_embedding_dimension(db_url, 768, schema=schema)
-
-        assert "Cannot change embedding dimension" in str(exc_info.value)
-        assert "1 rows with embeddings" in str(exc_info.value)
+        # Try to change dimension - should raise RuntimeError.
+        # Retry on transient OID errors from concurrent xdist schema drops.
+        _assert_raises_runtime_error_with_retry(
+            db_url, 768, schema,
+            expected_messages=["Cannot change embedding dimension", "1 rows with embeddings"],
+        )
 
         # Dimension should be unchanged
         assert get_column_dimension(db_url, schema) == 384
@@ -243,7 +304,7 @@ class TestEmbeddingDimension:
         initial_dim = get_column_dimension(db_url, schema, table="mental_models")
         assert initial_dim == 384, f"Expected 384, got {initial_dim}"
 
-        ensure_embedding_dimension(db_url, 384, schema=schema)
+        _ensure_embedding_dimension_with_retry(db_url, 384, schema=schema)
 
         assert get_column_dimension(db_url, schema, table="mental_models") == 384
 
@@ -253,12 +314,12 @@ class TestEmbeddingDimension:
 
         clear_mental_model_embeddings(db_url, schema)
 
-        ensure_embedding_dimension(db_url, 768, schema=schema)
+        _ensure_embedding_dimension_with_retry(db_url, 768, schema=schema)
 
         assert get_column_dimension(db_url, schema, table="mental_models") == 768
 
         # Change back for other tests
-        ensure_embedding_dimension(db_url, 384, schema=schema)
+        _ensure_embedding_dimension_with_retry(db_url, 384, schema=schema)
         assert get_column_dimension(db_url, schema, table="mental_models") == 384
 
     def test_mental_models_dimension_change_blocked_with_data(self, dimension_test_schema):
@@ -268,11 +329,12 @@ class TestEmbeddingDimension:
         clear_mental_model_embeddings(db_url, schema)
         insert_test_mental_model_embedding(db_url, schema, 384)
 
-        with pytest.raises(RuntimeError) as exc_info:
-            ensure_embedding_dimension(db_url, 768, schema=schema)
-
-        assert "Cannot change embedding dimension" in str(exc_info.value)
-        assert "mental_models" in str(exc_info.value)
+        # Try to change dimension - should raise RuntimeError.
+        # Retry on transient OID errors from concurrent xdist schema drops.
+        _assert_raises_runtime_error_with_retry(
+            db_url, 768, schema,
+            expected_messages=["Cannot change embedding dimension", "mental_models"],
+        )
 
         assert get_column_dimension(db_url, schema, table="mental_models") == 384
 

hindsight-api-slim/tests/test_daemonize.py

@@ -0,0 +1,88 @@
+"""Tests for daemonize() — subprocess.Popen re-exec instead of os.fork()."""
+
+import sys
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+def test_daemonize_parent_reexecs_via_popen(monkeypatch, tmp_path):
+    """Parent path: daemonize() must spawn a child via subprocess.Popen and exit."""
+    monkeypatch.setattr(sys, "platform", "linux")
+    monkeypatch.delenv("_HINDSIGHT_DAEMON_CHILD", raising=False)
+    monkeypatch.setattr(sys, "argv", ["hindsight-api", "--daemon", "--port", "9999"])
+
+    log_path = tmp_path / "daemon.log"
+    monkeypatch.setattr("hindsight_api.daemon.DAEMON_LOG_PATH", log_path)
+
+    captured: dict = {}
+
+    def fake_popen(cmd, **kwargs):
+        captured["cmd"] = cmd
+        captured["kwargs"] = kwargs
+        proc = MagicMock()
+        proc.pid = 99999
+        return proc
+
+    with (
+        patch("hindsight_api.daemon.subprocess.Popen", side_effect=fake_popen),
+        pytest.raises(SystemExit) as exc_info,
+    ):
+        from hindsight_api.daemon import daemonize
+
+        daemonize()
+
+    assert exc_info.value.code == 0
+
+    # Verify child command does NOT contain --daemon
+    assert "--daemon" not in captured["cmd"]
+    # Verify it uses the module entry point
+    assert "-m" in captured["cmd"]
+    assert "hindsight_api.main" in captured["cmd"]
+    # Verify remaining args are preserved
+    assert "--port" in captured["cmd"]
+    assert "9999" in captured["cmd"]
+
+    # Verify env has the daemon child marker
+    env = captured["kwargs"]["env"]
+    assert env["_HINDSIGHT_DAEMON_CHILD"] == "1"
+
+    # Verify detach kwargs
+    kwargs = captured["kwargs"]
+    assert kwargs.get("start_new_session") is True
+
+
+def test_daemonize_child_does_not_reexec(monkeypatch, tmp_path):
+    """Child path: when _HINDSIGHT_DAEMON_CHILD=1, daemonize() does NOT call
+    Popen — it only redirects stdio."""
+    monkeypatch.setattr(sys, "platform", "linux")
+    monkeypatch.setenv("_HINDSIGHT_DAEMON_CHILD", "1")
+
+    log_path = tmp_path / "daemon.log"
+    monkeypatch.setattr("hindsight_api.daemon.DAEMON_LOG_PATH", log_path)
+
+    with (
+        patch("hindsight_api.daemon.subprocess.Popen") as mock_popen,
+        patch("hindsight_api.daemon._redirect_stdio_to_log") as mock_redirect,
+    ):
+        from hindsight_api.daemon import daemonize
+
+        daemonize()
+        mock_popen.assert_not_called()
+        mock_redirect.assert_called_once()
+
+
+def test_daemonize_windows_noop(monkeypatch, tmp_path):
+    """On Windows, daemonize() just creates the log directory."""
+    monkeypatch.setattr(sys, "platform", "win32")
+
+    log_path = tmp_path / "subdir" / "daemon.log"
+    monkeypatch.setattr("hindsight_api.daemon.DAEMON_LOG_PATH", log_path)
+
+    with patch("hindsight_api.daemon.subprocess.Popen") as mock_popen:
+        from hindsight_api.daemon import daemonize
+
+        daemonize()
+        mock_popen.assert_not_called()
+
+    assert log_path.parent.exists()

hindsight-api-slim/tests/test_entity_labels.py

@@ -17,7 +17,10 @@ from hindsight_api.engine.retain.entity_labels import (
     EntityLabelsConfig,
     LabelGroup,
     LabelValue,
+    MapField,
     build_labels_lookup,
+    build_labels_model,
+    is_label_entity,
     parse_entity_labels,
 )
 
@@ -1118,3 +1121,776 @@ async def test_retain_extracts_free_values_label(memory, request_context):
         )
     finally:
         await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_retain_extracts_map_type_entities(memory, request_context):
+    """
+    End-to-end: retain content with a map-type entity_labels group.
+    Verify that structured entity fields are extracted as key:field:value entity strings.
+    """
+    from hindsight_api.engine.memory_engine import fq_table
+
+    bank_id = f"test-labels-map-{uuid.uuid4().hex[:8]}"
+    try:
+        await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
+
+        # Configure a map-type entity label
+        await memory._config_resolver.update_bank_config(
+            bank_id=bank_id,
+            updates={
+                "entity_labels": [
+                    {
+                        "key": "person",
+                        "type": "map",
+                        "description": "A person mentioned in the text",
+                        "fields": {
+                            "name": {"type": "text", "description": "Full name of the person"},
+                            "role": {"type": "text", "description": "Job title or role"},
+                            "organization": {"type": "text", "description": "Company or organization"},
+                        },
+                    }
+                ],
+                "entities_allow_free_form": False,  # map entities only
+            },
+            context=request_context,
+        )
+
+        unit_ids = await memory.retain_async(
+            bank_id=bank_id,
+            content=(
+                "Alice Johnson is a Senior Software Engineer at Google. "
+                "She leads the search infrastructure team and has been with the company for 5 years."
+            ),
+            request_context=request_context,
+        )
+
+        assert len(unit_ids) > 0, "Should have extracted at least one fact"
+
+        async with memory._pool.acquire() as conn:
+            rows = await conn.fetch(
+                f"""
+                SELECT e.canonical_name
+                FROM {fq_table("unit_entities")} ue
+                JOIN {fq_table("entities")} e ON e.id = ue.entity_id
+                WHERE ue.unit_id = ANY($1::uuid[])
+                """,
+                [u for u in unit_ids],
+            )
+
+        entity_names = {r["canonical_name"].lower() for r in rows}
+        # Should have person:name:* entity
+        name_entities = {n for n in entity_names if n.startswith("person:name:")}
+        assert len(name_entities) > 0, (
+            f"Expected at least one person:name:* entity. Got: {entity_names}"
+        )
+        # Name should contain "alice" somewhere
+        assert any("alice" in n for n in name_entities), (
+            f"Expected person:name entity containing 'alice'. Got: {name_entities}"
+        )
+        # Should have person:organization:* entity mentioning google
+        org_entities = {n for n in entity_names if n.startswith("person:organization:")}
+        assert len(org_entities) > 0, (
+            f"Expected at least one person:organization:* entity. Got: {entity_names}"
+        )
+        assert any("google" in n for n in org_entities), (
+            f"Expected person:organization entity containing 'google'. Got: {org_entities}"
+        )
+        # In labels-only mode, free-form entities should be absent
+        non_person_entities = {n for n in entity_names if not n.startswith("person:")}
+        assert len(non_person_entities) == 0, (
+            f"Free-form entities should not appear in labels-only mode. Got: {non_person_entities}"
+        )
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+# ─── map-type entity labels ──────────────────────────────────────────────────
+
+
+def test_parse_entity_labels_map_type():
+    """Map-type label group with fields is parsed correctly."""
+    raw = [
+        {
+            "key": "person",
+            "type": "map",
+            "description": "A person entity",
+            "fields": {
+                "name": {"type": "text", "description": "Full name"},
+                "role": {"type": "text", "description": "Job title"},
+                "organization": {"type": "text", "description": "Company"},
+            },
+        }
+    ]
+    result = parse_entity_labels(raw)
+    assert result is not None
+    assert len(result.attributes) == 1
+    group = result.attributes[0]
+    assert group.key == "person"
+    assert group.type == "map"
+    assert len(group.fields) == 3
+    assert "name" in group.fields
+    assert group.fields["name"].description == "Full name"
+
+
+def test_parse_entity_labels_map_type_dict_format():
+    """Map-type label group via dict format."""
+    raw = {
+        "attributes": [
+            {
+                "key": "company",
+                "type": "map",
+                "fields": {
+                    "name": {"type": "text"},
+                    "industry": {"type": "text"},
+                },
+            }
+        ]
+    }
+    result = parse_entity_labels(raw)
+    assert result is not None
+    assert result.attributes[0].type == "map"
+    assert len(result.attributes[0].fields) == 2
+
+
+def test_build_labels_model_map_type():
+    """Map-type groups produce list[MapModel] fields in the Labels model."""
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="person",
+                type="map",
+                description="A person",
+                fields={
+                    "name": MapField(description="Full name"),
+                    "role": MapField(description="Job title"),
+                },
+            ),
+        ]
+    )
+    model = build_labels_model(labels_cfg)
+    assert model is not None
+    schema = model.model_json_schema()
+    assert "person" in schema["properties"]
+    # Should be an array of objects
+    person_prop = schema["properties"]["person"]
+    assert person_prop["type"] == "array"
+
+
+def test_build_labels_model_mixed_map_and_value():
+    """Both map-type and value-type groups coexist in the same Labels model."""
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="person",
+                type="map",
+                description="A person",
+                fields={
+                    "name": MapField(description="Full name"),
+                },
+            ),
+            LabelGroup(
+                key="topic",
+                type="value",
+                values=[LabelValue(value="math"), LabelValue(value="science")],
+            ),
+        ]
+    )
+    model = build_labels_model(labels_cfg)
+    assert model is not None
+    schema = model.model_json_schema()
+    assert "person" in schema["properties"]
+    assert "topic" in schema["properties"]
+
+
+def test_build_labels_model_map_type_no_fields():
+    """Map-type group with no fields produces no field in the model."""
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(key="empty", type="map", fields={}),
+        ]
+    )
+    model = build_labels_model(labels_cfg)
+    assert model is None
+
+
+def test_build_labels_lookup_skips_map_type():
+    """Map-type groups should not contribute to the two-level lookup set."""
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="person",
+                type="map",
+                fields={"name": MapField()},
+            ),
+            LabelGroup(
+                key="topic",
+                type="value",
+                values=[LabelValue(value="math")],
+            ),
+        ]
+    )
+    lookup = build_labels_lookup(labels_cfg)
+    assert "topic:math" in lookup
+    # No map-type entries in the lookup
+    assert not any("person" in v for v in lookup)
+
+
+def test_is_label_entity_map_type():
+    """Three-level key:field:value strings are recognized as label entities."""
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="person",
+                type="map",
+                fields={
+                    "name": MapField(),
+                    "role": MapField(),
+                },
+            ),
+        ]
+    )
+    lookup = build_labels_lookup(labels_cfg)
+    assert is_label_entity("person:name:Alice", labels_cfg, lookup)
+    assert is_label_entity("person:role:Engineer", labels_cfg, lookup)
+    assert is_label_entity("Person:Name:Alice", labels_cfg, lookup)  # case insensitive
+    assert not is_label_entity("person:unknown_field:value", labels_cfg, lookup)
+    assert not is_label_entity("person:Alice", labels_cfg, lookup)  # two-level, not map
+    assert not is_label_entity("Alice", labels_cfg, lookup)
+
+
+def test_build_labels_prompt_section_map_type():
+    """Map-type groups appear in the STRUCTURED ENTITY TYPES prompt section."""
+    from hindsight_api.engine.retain.fact_extraction import _build_labels_prompt_section
+
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="person",
+                type="map",
+                description="A person entity",
+                fields={
+                    "name": MapField(description="Full name"),
+                    "role": MapField(description="Job title"),
+                },
+            ),
+        ]
+    )
+    result = _build_labels_prompt_section(labels_cfg)
+    assert "STRUCTURED ENTITY TYPES" in result
+    assert "person" in result
+    assert "name" in result
+    assert "role" in result
+    assert "Full name" in result
+
+
+def test_build_labels_prompt_section_mixed():
+    """Mixed map-type and value-type groups both appear in the prompt."""
+    from hindsight_api.engine.retain.fact_extraction import _build_labels_prompt_section
+
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="topic",
+                type="value",
+                description="Subject area",
+                values=[LabelValue(value="math")],
+            ),
+            LabelGroup(
+                key="person",
+                type="map",
+                description="A person",
+                fields={"name": MapField(description="Full name")},
+            ),
+        ]
+    )
+    result = _build_labels_prompt_section(labels_cfg)
+    assert "CLASSIFICATION ATTRIBUTES" in result
+    assert "topic" in result
+    assert "STRUCTURED ENTITY TYPES" in result
+    assert "person" in result
+
+
+def test_map_entity_post_processing():
+    """Map-type labels are converted to key:field:value entity strings."""
+    from hindsight_api.engine.retain.fact_extraction import Entity, _extract_map_entities
+
+    fields = {
+        "name": MapField(type="text"),
+        "role": MapField(type="text"),
+        "organization": MapField(type="text"),
+    }
+    entity_obj = {"name": "Alice", "role": "Senior Engineer", "organization": "Acme Corp"}
+
+    validated: list[Entity] = []
+    existing: set[str] = set()
+    _extract_map_entities(entity_obj, fields, "person:", validated, existing)
+
+    texts = {e.text for e in validated}
+    assert "person:name:Alice" in texts
+    assert "person:role:Senior Engineer" in texts
+    assert "person:organization:Acme Corp" in texts
+
+
+def test_map_entity_post_processing_null_fields_skipped():
+    """Null/empty fields in map entities are skipped."""
+    from hindsight_api.engine.retain.fact_extraction import Entity, _extract_map_entities
+
+    fields = {
+        "name": MapField(type="text"),
+        "role": MapField(type="text"),
+    }
+    entity_obj = {"name": "Bob", "role": None}
+
+    validated: list[Entity] = []
+    existing: set[str] = set()
+    _extract_map_entities(entity_obj, fields, "person:", validated, existing)
+
+    texts = {e.text for e in validated}
+    assert "person:name:Bob" in texts
+    assert len(texts) == 1  # role was null, so only name
+
+
+def test_map_entity_post_processing_multiple_entities():
+    """Multiple map entities in a single fact produce separate entity strings."""
+    from hindsight_api.engine.retain.fact_extraction import Entity, _extract_map_entities
+
+    fields = {
+        "name": MapField(type="text"),
+        "role": MapField(type="text"),
+    }
+
+    validated: list[Entity] = []
+    existing: set[str] = set()
+    _extract_map_entities({"name": "Alice", "role": "Engineer"}, fields, "person:", validated, existing)
+    _extract_map_entities({"name": "Bob", "role": "Manager"}, fields, "person:", validated, existing)
+
+    texts = {e.text for e in validated}
+    assert "person:name:Alice" in texts
+    assert "person:role:Engineer" in texts
+    assert "person:name:Bob" in texts
+    assert "person:role:Manager" in texts
+    assert len(texts) == 4
+
+
+# ─── recursive map-type entity labels ────────────────────────────────────────
+
+
+def test_parse_entity_labels_recursive_map():
+    """Nested map fields parse correctly."""
+    raw = [
+        {
+            "key": "person",
+            "type": "map",
+            "fields": {
+                "name": {"type": "text"},
+                "address": {
+                    "type": "map",
+                    "fields": {
+                        "city": {"type": "text", "description": "City name"},
+                        "country": {"type": "text"},
+                    },
+                },
+            },
+        }
+    ]
+    result = parse_entity_labels(raw)
+    assert result is not None
+    group = result.attributes[0]
+    assert group.fields["address"].type == "map"
+    assert "city" in group.fields["address"].fields
+    assert group.fields["address"].fields["city"].description == "City name"
+
+
+def test_build_labels_model_recursive_map():
+    """Nested map fields produce nested list[Model] in the schema."""
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="person",
+                type="map",
+                fields={
+                    "name": MapField(type="text"),
+                    "address": MapField(
+                        type="map",
+                        fields={
+                            "city": MapField(type="text"),
+                            "country": MapField(type="text"),
+                        },
+                    ),
+                },
+            ),
+        ]
+    )
+    model = build_labels_model(labels_cfg)
+    assert model is not None
+    schema = model.model_json_schema()
+    person_prop = schema["properties"]["person"]
+    assert person_prop["type"] == "array"
+
+
+def test_is_label_entity_recursive_map():
+    """Deeply nested key:field:subfield:value strings are recognized."""
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="person",
+                type="map",
+                fields={
+                    "name": MapField(type="text"),
+                    "address": MapField(
+                        type="map",
+                        fields={
+                            "city": MapField(type="text"),
+                            "country": MapField(type="text"),
+                        },
+                    ),
+                },
+            ),
+        ]
+    )
+    lookup = build_labels_lookup(labels_cfg)
+    assert is_label_entity("person:name:Alice", labels_cfg, lookup)
+    assert is_label_entity("person:address:city:New York", labels_cfg, lookup)
+    assert is_label_entity("person:address:country:US", labels_cfg, lookup)
+    assert not is_label_entity("person:address:zip:12345", labels_cfg, lookup)
+    assert not is_label_entity("person:address:New York", labels_cfg, lookup)
+
+
+def test_recursive_map_post_processing():
+    """Nested map entities produce deeply-joined key:field:subfield:value strings."""
+    from hindsight_api.engine.retain.fact_extraction import Entity, _extract_map_entities
+
+    fields = {
+        "name": MapField(type="text"),
+        "address": MapField(
+            type="map",
+            fields={
+                "city": MapField(type="text"),
+                "country": MapField(type="text"),
+            },
+        ),
+    }
+
+    entity_obj = {
+        "name": "Alice",
+        "address": [{"city": "New York", "country": "US"}],
+    }
+
+    validated: list[Entity] = []
+    existing: set[str] = set()
+    _extract_map_entities(entity_obj, fields, "person:", validated, existing)
+
+    texts = {e.text for e in validated}
+    assert "person:name:Alice" in texts
+    assert "person:address:city:New York" in texts
+    assert "person:address:country:US" in texts
+    assert len(texts) == 3
+
+
+def test_map_field_with_enum_values():
+    """Map fields with value/multi-values types constrain extraction."""
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="person",
+                type="map",
+                fields={
+                    "name": MapField(type="text"),
+                    "department": MapField(
+                        type="value",
+                        values=[LabelValue(value="engineering"), LabelValue(value="sales")],
+                    ),
+                },
+            ),
+        ]
+    )
+    model = build_labels_model(labels_cfg)
+    assert model is not None
+    schema = model.model_json_schema()
+    # The model should exist and have the person field
+    assert "person" in schema["properties"]
+
+
+def test_build_labels_prompt_section_recursive_map():
+    """Nested map fields appear indented in the prompt."""
+    from hindsight_api.engine.retain.fact_extraction import _build_labels_prompt_section
+
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="person",
+                type="map",
+                description="A person",
+                fields={
+                    "name": MapField(type="text", description="Full name"),
+                    "address": MapField(
+                        type="map",
+                        description="Home address",
+                        fields={
+                            "city": MapField(type="text", description="City name"),
+                        },
+                    ),
+                },
+            ),
+        ]
+    )
+    result = _build_labels_prompt_section(labels_cfg)
+    assert "name (text)" in result
+    assert "address (object)" in result
+    assert "city (text)" in result
+
+
+# ─── map fields with value/multi-values types ────────────────────────────────
+
+
+def test_map_field_value_post_processing():
+    """Map field with type='value' extracts a single enum entity string."""
+    from hindsight_api.engine.retain.fact_extraction import Entity, _extract_map_entities
+
+    fields = {
+        "name": MapField(type="text"),
+        "department": MapField(
+            type="value",
+            values=[LabelValue(value="engineering"), LabelValue(value="sales")],
+        ),
+    }
+    entity_obj = {"name": "Alice", "department": "engineering"}
+
+    validated: list[Entity] = []
+    existing: set[str] = set()
+    _extract_map_entities(entity_obj, fields, "person:", validated, existing)
+
+    texts = {e.text for e in validated}
+    assert "person:name:Alice" in texts
+    assert "person:department:engineering" in texts
+    assert len(texts) == 2
+
+
+def test_map_field_multi_values_post_processing():
+    """Map field with type='multi-values' extracts one entity per value."""
+    from hindsight_api.engine.retain.fact_extraction import Entity, _extract_map_entities
+
+    fields = {
+        "name": MapField(type="text"),
+        "skills": MapField(
+            type="multi-values",
+            values=[LabelValue(value="python"), LabelValue(value="go"), LabelValue(value="rust")],
+        ),
+    }
+    entity_obj = {"name": "Alice", "skills": ["python", "rust"]}
+
+    validated: list[Entity] = []
+    existing: set[str] = set()
+    _extract_map_entities(entity_obj, fields, "person:", validated, existing)
+
+    texts = {e.text for e in validated}
+    assert "person:name:Alice" in texts
+    assert "person:skills:python" in texts
+    assert "person:skills:rust" in texts
+    assert "person:skills:go" not in texts
+    assert len(texts) == 3
+
+
+def test_map_field_multi_values_null_skipped():
+    """Null/sentinel values in multi-values are skipped."""
+    from hindsight_api.engine.retain.fact_extraction import Entity, _extract_map_entities
+
+    fields = {
+        "tags": MapField(type="multi-values"),
+    }
+    entity_obj = {"tags": ["valid", "none", "null", "", "  ", "n/a", "also_valid"]}
+
+    validated: list[Entity] = []
+    existing: set[str] = set()
+    _extract_map_entities(entity_obj, fields, "item:", validated, existing)
+
+    texts = {e.text for e in validated}
+    assert "item:tags:valid" in texts
+    assert "item:tags:also_valid" in texts
+    assert len(texts) == 2
+
+
+def test_nested_map_with_enum_fields_post_processing():
+    """Nested map containing value/multi-values fields produces correct paths."""
+    from hindsight_api.engine.retain.fact_extraction import Entity, _extract_map_entities
+
+    fields = {
+        "name": MapField(type="text"),
+        "job": MapField(
+            type="map",
+            fields={
+                "title": MapField(type="text"),
+                "level": MapField(
+                    type="value",
+                    values=[LabelValue(value="junior"), LabelValue(value="senior")],
+                ),
+                "languages": MapField(
+                    type="multi-values",
+                    values=[LabelValue(value="python"), LabelValue(value="java")],
+                ),
+            },
+        ),
+    }
+    entity_obj = {
+        "name": "Bob",
+        "job": [{"title": "Engineer", "level": "senior", "languages": ["python", "java"]}],
+    }
+
+    validated: list[Entity] = []
+    existing: set[str] = set()
+    _extract_map_entities(entity_obj, fields, "person:", validated, existing)
+
+    texts = {e.text for e in validated}
+    assert "person:name:Bob" in texts
+    assert "person:job:title:Engineer" in texts
+    assert "person:job:level:senior" in texts
+    assert "person:job:languages:python" in texts
+    assert "person:job:languages:java" in texts
+    assert len(texts) == 5
+
+
+def test_is_label_entity_map_with_enum_fields():
+    """Entity strings from map fields with value/multi-values types are recognized."""
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="person",
+                type="map",
+                fields={
+                    "name": MapField(type="text"),
+                    "department": MapField(
+                        type="value",
+                        values=[LabelValue(value="engineering")],
+                    ),
+                    "skills": MapField(type="multi-values"),
+                },
+            ),
+        ]
+    )
+    lookup = build_labels_lookup(labels_cfg)
+    assert is_label_entity("person:name:Alice", labels_cfg, lookup)
+    assert is_label_entity("person:department:engineering", labels_cfg, lookup)
+    assert is_label_entity("person:skills:python", labels_cfg, lookup)
+    assert not is_label_entity("person:unknown:value", labels_cfg, lookup)
+
+
+def test_is_label_entity_nested_map_with_enum():
+    """Deeply nested paths with value/multi-values fields are recognized."""
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="person",
+                type="map",
+                fields={
+                    "job": MapField(
+                        type="map",
+                        fields={
+                            "level": MapField(type="value"),
+                            "languages": MapField(type="multi-values"),
+                        },
+                    ),
+                },
+            ),
+        ]
+    )
+    lookup = build_labels_lookup(labels_cfg)
+    assert is_label_entity("person:job:level:senior", labels_cfg, lookup)
+    assert is_label_entity("person:job:languages:python", labels_cfg, lookup)
+    assert not is_label_entity("person:job:salary:100k", labels_cfg, lookup)
+
+
+def test_map_field_enum_schema_generation():
+    """Map fields with value/multi-values generate correct JSON schema constraints."""
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="person",
+                type="map",
+                fields={
+                    "name": MapField(type="text"),
+                    "department": MapField(
+                        type="value",
+                        values=[LabelValue(value="engineering"), LabelValue(value="sales")],
+                    ),
+                    "skills": MapField(
+                        type="multi-values",
+                        values=[LabelValue(value="python"), LabelValue(value="go")],
+                    ),
+                },
+            ),
+        ]
+    )
+    model = build_labels_model(labels_cfg)
+    assert model is not None
+    schema = model.model_json_schema()
+    # Resolve $defs to find the person entity schema
+    person_ref = schema["properties"]["person"]["items"]
+    if "$ref" in person_ref:
+        ref_name = person_ref["$ref"].split("/")[-1]
+        person_schema = schema["$defs"][ref_name]
+    else:
+        person_schema = person_ref
+    props = person_schema["properties"]
+    # department should be an enum
+    assert "department" in props
+    assert "enum" in props["department"] or "anyOf" in props["department"]
+    # skills should be an array
+    assert "skills" in props
+    assert props["skills"]["type"] == "array"
+
+
+def test_prompt_section_map_with_all_field_types():
+    """Prompt includes correct type hints for value/multi-values/map fields."""
+    from hindsight_api.engine.retain.fact_extraction import _build_labels_prompt_section
+
+    labels_cfg = EntityLabelsConfig(
+        attributes=[
+            LabelGroup(
+                key="person",
+                type="map",
+                description="A person",
+                fields={
+                    "name": MapField(type="text", description="Full name"),
+                    "department": MapField(
+                        type="value",
+                        description="Department",
+                        values=[LabelValue(value="eng"), LabelValue(value="sales")],
+                    ),
+                    "skills": MapField(
+                        type="multi-values",
+                        description="Skills list",
+                        values=[LabelValue(value="python"), LabelValue(value="go")],
+                    ),
+                    "address": MapField(
+                        type="map",
+                        description="Home address",
+                        fields={"city": MapField(type="text")},
+                    ),
+                },
+            ),
+        ]
+    )
+    result = _build_labels_prompt_section(labels_cfg)
+    assert "name (text)" in result
+    assert "one of: eng, sales" in result
+    assert "multi-values: python, go" in result
+    assert "address (object)" in result
+    assert "city (text)" in result
+
+
+def test_duplicate_entity_strings_deduplicated():
+    """Same entity string from multiple nested objects is only added once."""
+    from hindsight_api.engine.retain.fact_extraction import Entity, _extract_map_entities
+
+    fields = {
+        "name": MapField(type="text"),
+    }
+    # Two entities with the same name
+    validated: list[Entity] = []
+    existing: set[str] = set()
+    _extract_map_entities({"name": "Alice"}, fields, "person:", validated, existing)
+    _extract_map_entities({"name": "Alice"}, fields, "person:", validated, existing)
+
+    texts = [e.text for e in validated]
+    assert texts == ["person:name:Alice"]  # only once

hindsight-api-slim/tests/test_entity_resolver.py

@@ -326,3 +326,88 @@ class TestOracleFuzzyEntityResolution:
         # The candidate IDs should be passed as bind parameter
         cooc_bind_args = mock_conn.fetch.call_args_list[1].args[1:]
         assert "eid-1" in cooc_bind_args[0], "Co-occurrence query must receive candidate IDs"
+
+
+@pytest.mark.asyncio
+async def test_link_units_carries_event_date_into_cooccurrences(pg0_db_url):
+    """
+    `link_units_to_entities_batch` must propagate each unit's event_date onto the
+    accumulated _CooccurrencePair entries, so flush_pending_stats() stamps
+    entity_cooccurrences.last_cooccurred with the event time instead of "now".
+
+    This protects banks that were backfilled from another memory system —
+    without it, every pair collapses to the import moment and the UI's entity
+    graph recency heat loses the underlying knowledge timeline.
+    """
+    resolved_url = await resolve_database_url(pg0_db_url)
+    backend = create_database_backend("postgresql")
+    await backend.initialize(resolved_url, min_size=1, max_size=2, command_timeout=30)
+    bank_id = f"test-cooccurrence-evt-{uuid.uuid4().hex[:8]}"
+    resolver = EntityResolver(pool=backend, entity_lookup="full")
+
+    historical = datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc)
+
+    try:
+        async with backend.acquire() as conn:
+            unit_id = await conn.fetchval(
+                """
+                INSERT INTO memory_units
+                    (bank_id, text, fact_type, mentioned_at, occurred_start, created_at)
+                VALUES ($1, 'paired-entity unit', 'experience', $2, $2, now())
+                RETURNING id
+                """,
+                bank_id,
+                historical,
+            )
+            e1 = await conn.fetchval(
+                "INSERT INTO entities (bank_id, canonical_name, first_seen, last_seen, mention_count) "
+                "VALUES ($1, 'alpha', $2, $2, 1) RETURNING id",
+                bank_id,
+                historical,
+            )
+            e2 = await conn.fetchval(
+                "INSERT INTO entities (bank_id, canonical_name, first_seen, last_seen, mention_count) "
+                "VALUES ($1, 'beta', $2, $2, 1) RETURNING id",
+                bank_id,
+                historical,
+            )
+
+            await resolver.link_units_to_entities_batch(
+                [(str(unit_id), str(e1), historical), (str(unit_id), str(e2), historical)],
+                conn=conn,
+            )
+
+        # Flush accumulates to entity_cooccurrences on a fresh connection, as
+        # the production flush runs post-transaction to avoid lock contention.
+        await resolver.flush_pending_stats()
+
+        async with backend.acquire() as conn:
+            row = await conn.fetchrow(
+                """
+                SELECT last_cooccurred
+                FROM entity_cooccurrences
+                WHERE entity_id_1 = LEAST($1::uuid, $2::uuid)
+                  AND entity_id_2 = GREATEST($1::uuid, $2::uuid)
+                """,
+                e1,
+                e2,
+            )
+        assert row is not None, "co-occurrence row should exist"
+        assert row["last_cooccurred"] == historical, (
+            f"expected last_cooccurred == {historical}, got {row['last_cooccurred']}"
+        )
+    finally:
+        async with backend.acquire() as conn:
+            await conn.execute(
+                "DELETE FROM unit_entities WHERE unit_id IN (SELECT id FROM memory_units WHERE bank_id = $1)", bank_id
+            )
+            await conn.execute(
+                "DELETE FROM entity_cooccurrences WHERE entity_id_1 IN "
+                "(SELECT id FROM entities WHERE bank_id = $1) "
+                "OR entity_id_2 IN "
+                "(SELECT id FROM entities WHERE bank_id = $1)",
+                bank_id,
+            )
+            await conn.execute("DELETE FROM memory_units WHERE bank_id = $1", bank_id)
+            await conn.execute("DELETE FROM entities WHERE bank_id = $1", bank_id)
+        await backend.shutdown()

hindsight-api-slim/tests/test_fact_extraction_quality.py

@@ -305,6 +305,7 @@ Family is the most important thing to her.
             f"Found: {found_evaluative}"
         )
 
+    @pytest.mark.hs_llm_mat
     @pytest.mark.asyncio
     async def test_comprehensive_multi_dimension(self):
         """Test a realistic scenario with multiple dimensions in one fact."""
@@ -336,7 +337,21 @@ I prefer presenting in person rather than virtually because I can read the room
         has_emotional = any(term in all_facts_text for term in [
             "thrilled", "positive feedback", "positive", "feedback", "enthusiastic"
         ])
-        assert has_emotional, "Should preserve emotional dimension"
+
+        # Check preference - should capture the in-person vs virtual preference
+        has_preference = any(term in all_facts_text for term in [
+            "prefer", "rather than", "in person", "in-person", "virtually",
+            "read the room", "face-to-face", "face to face", "remote",
+        ])
+
+        # MAT bar: at least one of emotional or preferential must be preserved.
+        # Smaller models (e.g. nova-2-lite) may compress both sentences into a
+        # single fact that only captures one dimension — that's acceptable for
+        # a minimum-acceptance test.
+        assert has_emotional or has_preference, (
+            f"Should preserve at least one of emotional or preferential dimension. "
+            f"Extracted facts: {all_facts_text}"
+        )
 
         # Check no vague temporal terms
         prohibited_terms = ["recently", "soon", "lately"]
@@ -344,12 +359,6 @@ I prefer presenting in person rather than virtually because I can read the room
         assert len(found_prohibited) == 0, \
             f"Should NOT use vague temporal terms. Found: {found_prohibited}"
 
-        # Check preference - should capture the in-person vs virtual preference
-        has_preference = any(term in all_facts_text for term in [
-            "prefer", "rather than", "in person", "virtually", "read the room"
-        ])
-        assert has_preference, "Should preserve preferential dimension"
-
 
 # =============================================================================
 # TEMPORAL CONVERSION TESTS

hindsight-api-slim/tests/test_http_api_integration.py

@@ -158,6 +158,14 @@ async def test_full_api_workflow(api_client, test_bank_id):
     assert "total_nodes" in stats
     assert stats["total_nodes"] > 0
 
+    # Verify bank list returns stats (fact_count, last_document_at)
+    response = await api_client.get("/v1/default/banks")
+    assert response.status_code == 200
+    banks_after = response.json()["banks"]
+    our_bank = next(b for b in banks_after if b["bank_id"] == test_bank_id)
+    assert our_bank["fact_count"] > 0, "fact_count should reflect retained memories"
+    assert our_bank["last_document_at"] is not None, "last_document_at should be set after retain"
+
     # List memory units
     response = await api_client.get(
         f"/v1/default/banks/{test_bank_id}/memories/list",

hindsight-api-slim/tests/test_llm_provider.py

@@ -1,60 +1,37 @@
 """
-Test LLM provider with different models using actual Hindsight memory operations.
+LLM Minimum Acceptance Tests — provider API surface.
 
-Tests validate that providers work correctly with:
-1. Retain (memory ingestion with fact extraction)
-2. Reflect (memory retrieval with tool calling)
-3. Mental models (consolidated knowledge generation)
+Validates that a given LLM provider/model works correctly with Hindsight's
+low-level LLM API methods: plain text, structured output, and tool calling.
+
+The provider/model under test comes from HINDSIGHT_API_LLM_PROVIDER /
+HINDSIGHT_API_LLM_MODEL env vars, which are set by the CI matrix in the
+test-api-llm-acceptance job.
+
+These tests are excluded from the regular test-api CI job via the
+hs_llm_mat marker.
 """
+
 import os
 from datetime import datetime
+
 import pytest
+
 from hindsight_api.engine.llm_wrapper import LLMProvider
 from hindsight_api.engine.utils import extract_facts
 from hindsight_api.engine.search.think_utils import reflect
 
+pytestmark = pytest.mark.hs_llm_mat
 
-# Model matrix: (provider, model)
-MODEL_MATRIX = [
-    # OpenAI models
-    ("openai", "gpt-4o-mini"),
-    ("openai", "gpt-4.1-mini"),
-    ("openai", "gpt-4.1-nano"),
-    ("openai", "gpt-5-mini"),
-    ("openai", "gpt-5-nano"),
-    ("openai", "gpt-5"),
-    ("openai", "gpt-5.2"),
-    # Anthropic models
-    ("anthropic", "claude-sonnet-4-20250514"),
-    ("anthropic", "claude-opus-4-5-20251101"),
-    ("anthropic", "claude-haiku-4-20250514"),
-    # Groq models
-    ("groq", "openai/gpt-oss-120b"),
-    ("groq", "openai/gpt-oss-20b"),
-    # DeepSeek models
-    ("deepseek", "deepseek-v4-flash"),
-    ("deepseek", "deepseek-chat"),
-    # Gemini models
-    ("gemini", "gemini-2.5-flash"),
-    ("gemini", "gemini-2.5-flash-lite"),
-    ("gemini", "gemini-3.1-pro-preview"),
-    ("gemini", "gemini-3.1-flash-lite-preview"),
-    # Ollama models (local)
-    ("ollama", "gemma3:12b"),
-    ("ollama", "gemma3:1b"),
-    # Claude Code (uses Claude Agent SDK with Claude models)
-    ("claude-code", "claude-sonnet-4-20250514"),
-    # OpenAI Codex (uses MCP with Codex-specific models)
-    ("openai-codex", "gpt-5.4-mini"),
-    # Bedrock models (via LiteLLM)
-    ("bedrock", "us.amazon.nova-2-lite-v1:0"),
-    # Mock provider (for testing)
-    ("mock", "mock"),
-]
+_PROVIDER = os.environ.get("HINDSIGHT_API_LLM_PROVIDER", "")
+_MODEL = os.environ.get("HINDSIGHT_API_LLM_MODEL", "")
 
-
-def get_api_key_for_provider(provider: str) -> str | None:
-    """Get API key for provider from environment variables."""
+def _get_api_key() -> str:
+    """Get API key from HINDSIGHT_API_LLM_API_KEY (CI) or provider-specific env var."""
+    key = os.environ.get("HINDSIGHT_API_LLM_API_KEY", "")
+    if key:
+        return key
+    # Fallback to provider-specific env vars for local dev
     provider_key_map = {
         "openai": "OPENAI_API_KEY",
         "anthropic": "ANTHROPIC_API_KEY",
@@ -62,50 +39,24 @@ def get_api_key_for_provider(provider: str) -> str | None:
         "gemini": "GEMINI_API_KEY",
         "deepseek": "DEEPSEEK_API_KEY",
     }
-    env_var = provider_key_map.get(provider)
-    return os.getenv(env_var) if env_var else None
+    env_var = provider_key_map.get(_PROVIDER, "")
+    return os.environ.get(env_var, "") if env_var else ""
 
 
-def should_skip_provider(provider: str, model: str = "") -> tuple[bool, str]:
-    """Check if provider should be skipped and return reason."""
-    # Never skip mock provider
-    if provider == "mock":
-        return False, ""
-
-    # Skip claude-code and openai-codex in CI (require local auth)
-    if os.getenv("CI") and provider in ("claude-code", "openai-codex"):
-        return True, f"{provider} not available in CI (requires local authentication)"
-
-    # Skip Ollama in CI (no models available)
-    if provider == "ollama" and os.getenv("CI"):
-        return True, "Ollama not available in CI"
-
-    # Skip Ollama gemma models (don't support tool calling)
-    if provider == "ollama" and "gemma" in model.lower():
-        return True, f"Ollama {model} does not support tool calling"
-
-    # Bedrock needs AWS credentials
-    if provider == "bedrock":
-        if not os.getenv("AWS_ACCESS_KEY_ID"):
-            return True, "No AWS credentials available (set AWS_ACCESS_KEY_ID)"
-        return False, ""
-
-    # Other providers need an API key
-    if provider not in ("ollama", "claude-code", "openai-codex", "mock"):
-        api_key = get_api_key_for_provider(provider)
-        if not api_key:
-            return True, f"No API key available (set {provider.upper()}_API_KEY)"
-
-    return False, ""
+def _make_llm() -> LLMProvider:
+    return LLMProvider(
+        provider=_PROVIDER,
+        api_key=_get_api_key(),
+        base_url=os.environ.get("HINDSIGHT_API_LLM_BASE_URL", ""),
+        model=_MODEL,
+    )
 
 
-@pytest.mark.parametrize("provider,model", MODEL_MATRIX)
 @pytest.mark.asyncio
-@pytest.mark.timeout(300)  # Increase timeout for slow models like groq gpt-oss-120b
-async def test_llm_provider_api_methods(provider: str, model: str):
+@pytest.mark.timeout(300)
+async def test_llm_api_methods():
     """
     Test all LLM API methods used by Hindsight at runtime.
-    This validates that the provider correctly implements the LLMInterface.
 
     Tests:
     1. verify_connection() - Connection verification
@@ -113,184 +64,108 @@ async def test_llm_provider_api_methods(provider: str, model: str):
     3. call() with response_format - Structured output (used in fact extraction)
     4. call_with_tools() - Tool calling (used in reflect agent)
     """
-    # Skip mock provider - it's a test stub, not a real LLM implementation
-    if provider == "mock":
-        pytest.skip("Mock provider is a test stub, not a real LLM")
-
-    should_skip, reason = should_skip_provider(provider, model)
-    if should_skip:
-        pytest.skip(f"Skipping {provider}/{model}: {reason}")
-
-    api_key = get_api_key_for_provider(provider)
-
-    llm = LLMProvider(
-        provider=provider,
-        api_key=api_key or "",
-        base_url="",
-        model=model,
-    )
-
-    print(f"\n{provider}/{model} - API methods test:")
+    llm = _make_llm()
 
     # Test 1: verify_connection()
-    try:
-        await llm.verify_connection()
-        print("  ✓ verify_connection()")
-    except Exception as e:
-        pytest.fail(f"{provider}/{model} verify_connection() failed: {e}")
+    await llm.verify_connection()
 
     # Test 2: call() with plain text
-    try:
-        response = await llm.call(
-            messages=[
-                {"role": "system", "content": "You are a helpful assistant."},
-                {"role": "user", "content": "What is 2+2? Answer in one word."},
-            ],
-            max_completion_tokens=50,
-        )
-        assert response is not None, "call() returned None"
-        assert len(response) > 0, "call() returned empty string"
-        print(f"  ✓ call() plain text: {response[:50]}")
-    except Exception as e:
-        pytest.fail(f"{provider}/{model} call() plain text failed: {e}")
+    response = await llm.call(
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "What is 2+2? Answer in one word."},
+        ],
+        max_completion_tokens=50,
+    )
+    assert response is not None, "call() returned None"
+    assert len(response) > 0, "call() returned empty string"
 
     # Test 3: call() with response_format (structured output)
-    # Skip for models that don't support structured output
-    skip_structured_output = (provider == "groq" and "gpt-oss-120b" in model.lower())
-    if skip_structured_output:
-        print(f"  ⊘ call() structured output: skipped (model doesn't support response_format)")
-    else:
-        try:
-            from pydantic import BaseModel
+    from pydantic import BaseModel
 
-            class TestResponse(BaseModel):
-                answer: str
-                confidence: str
+    class TestResponse(BaseModel):
+        answer: str
+        confidence: str
 
-            response = await llm.call(
-                messages=[
-                    {"role": "system", "content": "You are a math assistant."},
-                    {"role": "user", "content": "What is the capital of France?"},
-                ],
-                response_format=TestResponse,
-                max_completion_tokens=100,
-            )
-            assert isinstance(response, TestResponse), f"Expected TestResponse, got {type(response)}"
-            assert hasattr(response, "answer"), "Structured output missing 'answer' field"
-            assert hasattr(response, "confidence"), "Structured output missing 'confidence' field"
-            print(f"  ✓ call() structured output: answer={response.answer}, confidence={response.confidence}")
-        except Exception as e:
-            pytest.fail(f"{provider}/{model} call() structured output failed: {e}")
+    structured = await llm.call(
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "What is the capital of France?"},
+        ],
+        response_format=TestResponse,
+        max_completion_tokens=100,
+    )
+    assert isinstance(structured, TestResponse), f"Expected TestResponse, got {type(structured)}"
+    assert structured.answer, "Structured output missing 'answer'"
+    assert structured.confidence, "Structured output missing 'confidence'"
 
     # Test 4: call_with_tools() (tool calling)
-    try:
-        tools = [
-            {
-                "type": "function",
-                "function": {
-                    "name": "get_weather",
-                    "description": "Get the weather for a location",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "location": {"type": "string", "description": "City name"},
-                            "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
-                        },
-                        "required": ["location"],
+    tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get the weather for a location",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "location": {"type": "string", "description": "City name"},
+                        "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
                     },
+                    "required": ["location"],
                 },
-            }
-        ]
+            },
+        }
+    ]
 
-        result = await llm.call_with_tools(
-            messages=[
-                {"role": "system", "content": "You are a helpful assistant with access to tools."},
-                {"role": "user", "content": "What's the weather like in Paris?"},
-            ],
-            tools=tools,
-            max_completion_tokens=500,  # Increased from 200 to give models enough space for tool calls
-        )
-
-        assert result is not None, "call_with_tools() returned None"
-        assert hasattr(result, "tool_calls"), "Result missing 'tool_calls' attribute"
-
-        # Nano models may hit token limits before making tool calls - that's acceptable
-        is_nano_model = "nano" in model.lower()
-        if is_nano_model and len(result.tool_calls) == 0:
-            # Check if it hit length limit (expected for nano models)
-            if hasattr(result, "finish_reason") and result.finish_reason == "length":
-                print(f"  ✓ call_with_tools(): nano model hit token limit (expected)")
-            else:
-                pytest.fail(f"Nano model made 0 tool calls but didn't hit length limit (finish_reason={getattr(result, 'finish_reason', 'unknown')})")
-        else:
-            assert len(result.tool_calls) > 0, f"Expected at least 1 tool call, got {len(result.tool_calls)}"
-
-            # Verify tool call structure
-            tool_call = result.tool_calls[0]
-            assert hasattr(tool_call, "name"), "Tool call missing 'name'"
-            assert hasattr(tool_call, "arguments"), "Tool call missing 'arguments'"
-            assert tool_call.name == "get_weather", f"Expected 'get_weather', got '{tool_call.name}'"
-            assert "location" in tool_call.arguments, "Tool call arguments missing 'location'"
-
-            print(f"  ✓ call_with_tools(): {tool_call.name}({tool_call.arguments})")
-    except Exception as e:
-        pytest.fail(f"{provider}/{model} call_with_tools() failed: {e}")
-
-
-@pytest.mark.parametrize("provider,model", MODEL_MATRIX)
-@pytest.mark.asyncio
-@pytest.mark.timeout(600)  # 600s: some providers (e.g., bedrock via litellm) need extra time for fact extraction
-async def test_llm_provider_memory_operations(provider: str, model: str):
-    """
-    Test LLM provider with actual memory operations: fact extraction and reflect.
-    All models must pass this test.
-    """
-    # Skip mock provider - it's a test stub, not designed for real operations
-    if provider == "mock":
-        pytest.skip("Mock provider is a test stub, not designed for real operations")
-
-    should_skip, reason = should_skip_provider(provider, model)
-    if should_skip:
-        pytest.skip(f"Skipping {provider}/{model}: {reason}")
-
-    api_key = get_api_key_for_provider(provider)
-
-    llm = LLMProvider(
-        provider=provider,
-        api_key=api_key or "",
-        base_url="",
-        model=model,
+    result = await llm.call_with_tools(
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant with access to tools."},
+            {"role": "user", "content": "What's the weather like in Paris?"},
+        ],
+        tools=tools,
+        max_completion_tokens=500,
     )
 
-    # Test 1: Fact extraction (structured output)
+    assert result is not None, "call_with_tools() returned None"
+    assert hasattr(result, "tool_calls"), "Result missing 'tool_calls' attribute"
+    assert len(result.tool_calls) > 0, f"Expected at least 1 tool call, got {len(result.tool_calls)}"
+
+    tool_call = result.tool_calls[0]
+    assert tool_call.name == "get_weather", f"Expected 'get_weather', got '{tool_call.name}'"
+    assert "location" in tool_call.arguments, "Tool call arguments missing 'location'"
+
+
+@pytest.mark.asyncio
+@pytest.mark.timeout(600)
+async def test_llm_memory_operations():
+    """
+    Test fact extraction and reflect with the configured LLM provider.
+    """
+    llm = _make_llm()
+
+    # Fact extraction (structured output)
     test_text = """
     User: I just got back from my trip to Paris last week. The Eiffel Tower was amazing!
     Assistant: That sounds wonderful! How long were you there?
     User: About 5 days. I also visited the Louvre and saw the Mona Lisa.
     """
-    event_date = datetime(2024, 12, 10)
 
     facts, chunks = await extract_facts(
         text=test_text,
-        event_date=event_date,
+        event_date=datetime(2024, 12, 10),
         context="Travel conversation",
         llm_config=llm,
     )
 
-    print(f"\n{provider}/{model} - Fact extraction:")
-    print(f"  Extracted {len(facts)} facts from {len(chunks)} chunks")
+    assert facts is not None, "fact extraction returned None"
+    assert len(facts) > 0, "should extract at least one fact"
+
     for fact in facts:
-        print(f"  - {fact.fact}")
+        assert fact.fact, "fact missing text"
+        assert fact.fact_type in ["world", "experience"], f"invalid fact_type: {fact.fact_type}"
 
-    assert facts is not None, f"{provider}/{model} fact extraction returned None"
-    assert len(facts) > 0, f"{provider}/{model} should extract at least one fact"
-
-    # Verify facts have required fields
-    for fact in facts:
-        assert fact.fact, f"{provider}/{model} fact missing text"
-        assert fact.fact_type in ["world", "experience"], f"{provider}/{model} invalid fact_type: {fact.fact_type}"
-
-    # Test 2: Reflect (actual reflect function)
+    # Reflect
     response = await reflect(
         llm_config=llm,
         query="What was the highlight of my Paris trip?",
@@ -307,120 +182,5 @@ async def test_llm_provider_memory_operations(provider: str, model: str):
         name="Traveler",
     )
 
-    print(f"\n{provider}/{model} - Reflect response:")
-    print(f"  {response[:200]}...")
-
-    assert response is not None, f"{provider}/{model} reflect returned None"
-    assert len(response) > 10, f"{provider}/{model} reflect response too short"
-
-
-@pytest.mark.parametrize("provider,model", [
-    ("claude-code", "claude-sonnet-4-20250514"),
-    ("openai-codex", "gpt-5.4-mini"),
-])
-@pytest.mark.asyncio
-async def test_llm_provider_consolidation(memory_no_llm_verify, request_context, provider: str, model: str):
-    """
-    Test LLM provider with consolidation (automatic mental model generation from observations).
-    This validates that the provider can generate synthesized knowledge from raw memories.
-
-    This test is limited to claude-code and codex since they're the critical providers
-    that needed tool calling fixes for reflect and consolidation operations.
-    """
-    should_skip, reason = should_skip_provider(provider, model)
-    if should_skip:
-        pytest.skip(f"Skipping {provider}/{model}: {reason}")
-
-    # Use provider-specific LLM for this test
-    api_key = get_api_key_for_provider(provider)
-    memory_no_llm_verify._consolidation_llm = LLMProvider(
-        provider=provider,
-        api_key=api_key or "",
-        base_url="",
-        model=model,
-    )
-    # Also need retain LLM for ingesting data
-    memory_no_llm_verify._retain_llm = memory_no_llm_verify._consolidation_llm
-
-    test_bank_id = f"llm_test_consolidation_{provider}_{model}_{datetime.now().timestamp()}"
-
-    # Enable observations for this bank
-    from hindsight_api.config import _get_raw_config
-    config = _get_raw_config()
-    original_value = config.enable_observations
-    config.enable_observations = True
-
-    try:
-        # Retain memories to consolidate
-        test_content = """
-        Bob prefers functional programming with Rust and Haskell.
-        He emphasizes immutability and pure functions in code reviews.
-        Bob advocates for type safety and compile-time guarantees.
-        He avoids mutable state and prefers declarative code patterns.
-        """
-
-        await memory_no_llm_verify.retain_async(
-            bank_id=test_bank_id,
-            content=test_content,
-            context="Team coding preferences",
-            event_date=datetime(2024, 12, 1),
-            request_context=request_context,
-        )
-
-        print(f"\n{provider}/{model} - Consolidation test:")
-
-        # Run consolidation to generate observations (mental models)
-        from hindsight_api.engine.consolidation.consolidator import run_consolidation_job
-
-        result = await run_consolidation_job(
-            memory_engine=memory_no_llm_verify,
-            bank_id=test_bank_id,
-            request_context=request_context,
-        )
-
-        print(f"  Processed: {result.get('memories_processed', 0)} memories")
-        print(f"  Created: {result.get('observations_created', 0)} observations")
-        print(f"  Updated: {result.get('observations_updated', 0)} observations")
-
-        # Verify consolidation ran successfully
-        assert result["status"] in ["success", "no_new_memories"], f"{provider}/{model} consolidation failed"
-
-        # If observations were created, verify they contain relevant content
-        if result.get("observations_created", 0) > 0:
-            observations = await memory_no_llm_verify.list_mental_models_consolidated(
-                bank_id=test_bank_id,
-                request_context=request_context,
-            )
-
-            assert len(observations) > 0, f"{provider}/{model} consolidation created 0 observations"
-
-            # Check first observation contains relevant information
-            obs_content = observations[0].get("content", "").lower()
-            relevant_terms = ["bob", "functional", "rust", "immutab", "type"]
-            matches = [term for term in relevant_terms if term in obs_content]
-
-            print(f"  Observation preview: {observations[0].get('content', '')[:200]}...")
-            print(f"  Found {len(matches)} relevant terms: {matches}")
-
-            assert len(matches) >= 2, (
-                f"{provider}/{model} consolidated observation doesn't contain relevant info. "
-                f"Expected at least 2 of {relevant_terms}, found {len(matches)}: {matches}"
-            )
-
-    finally:
-        # Restore original config
-        config.enable_observations = original_value
-
-
-# NOTE: The tests above validate the critical Hindsight operations:
-#
-# test_llm_provider_memory_operations (ALL providers):
-#   - Fact extraction (retain): tests structured output generation
-#   - Reflect: tests memory retrieval and reasoning (uses tool calling for claude-code/codex)
-#
-# test_llm_provider_consolidation (claude-code and codex only):
-#   - Consolidation: tests automatic mental model generation from observations
-#   - Requires MemoryEngine fixture with working LLM (from .env or env vars)
-#   - Run your local LLM server OR set HINDSIGHT_API_LLM_PROVIDER/API_KEY/MODEL env vars
-#
-# For full end-to-end integration tests using the HTTP API, see tests/test_http_api_integration.py
+    assert response is not None, "reflect returned None"
+    assert len(response) > 10, "reflect response too short"

hindsight-api-slim/tests/test_llm_router_provider.py

@@ -0,0 +1,345 @@
+"""
+Tests for the LiteLLM Router LLM provider — config parsing, factory dispatch,
+and the Router-backed call paths (plain text, structured output, tool calls,
+retry on transient failure).
+
+The provider is a thin pass-through to ``litellm.Router``. The chain config
+shape mirrors LiteLLM's API; we don't translate model names or impose
+fallbacks. See https://docs.litellm.ai/docs/routing.
+"""
+
+import json
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from pydantic import BaseModel
+
+from hindsight_api.config import (
+    ENV_CONSOLIDATION_LLM_LITELLMROUTER_CONFIG,
+    ENV_LLM_LITELLMROUTER_CONFIG,
+    ENV_LLM_PROVIDER,
+    ENV_REFLECT_LLM_LITELLMROUTER_CONFIG,
+    ENV_RETAIN_LLM_LITELLMROUTER_CONFIG,
+    HindsightConfig,
+    _parse_llm_router_config,
+)
+from hindsight_api.engine.llm_wrapper import create_llm_provider
+from hindsight_api.engine.providers.litellm_router_llm import LiteLLMRouterLLM
+
+
+@pytest.fixture
+def two_step_config() -> dict[str, Any]:
+    """Raw LiteLLM Router config: two deployments wired for ordered fallback.
+
+    Hindsight always issues completions against ``model_name="default"``;
+    additional groups become fallback / load-balance pool members per the
+    user's ``fallbacks`` / ``routing_strategy`` settings.
+    """
+    return {
+        "model_list": [
+            {
+                "model_name": "default",
+                "litellm_params": {
+                    "model": "openai/MiniMax-M2.7",
+                    "api_key": "sk-primary",
+                    "api_base": "https://api.minimax.io/v1",
+                },
+            },
+            {
+                "model_name": "fallback",
+                "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-fallback"},
+            },
+        ],
+        "fallbacks": [{"default": ["fallback"]}],
+        "num_retries": 0,
+    }
+
+
+@pytest.fixture
+def mock_router_response() -> MagicMock:
+    response = MagicMock()
+    choice = MagicMock()
+    choice.message.content = "ok"
+    choice.message.tool_calls = None
+    choice.finish_reason = "stop"
+    response.choices = [choice]
+    response.usage.prompt_tokens = 12
+    response.usage.completion_tokens = 3
+    response._hidden_params = {"model": "openai/gpt-4o-mini"}
+    return response
+
+
+# --- config parsing ----------------------------------------------------------
+
+
+class TestParseRouterConfig:
+    def test_unset_returns_none(self, monkeypatch):
+        monkeypatch.delenv(ENV_LLM_LITELLMROUTER_CONFIG, raising=False)
+        assert _parse_llm_router_config(ENV_LLM_LITELLMROUTER_CONFIG) is None
+
+    def test_empty_string_returns_none(self, monkeypatch):
+        monkeypatch.setenv(ENV_LLM_LITELLMROUTER_CONFIG, "  ")
+        assert _parse_llm_router_config(ENV_LLM_LITELLMROUTER_CONFIG) is None
+
+    def test_valid_config_passes_through(self, monkeypatch, two_step_config):
+        """Whatever the user provides round-trips verbatim — no translation."""
+        monkeypatch.setenv(ENV_LLM_LITELLMROUTER_CONFIG, json.dumps(two_step_config))
+        assert _parse_llm_router_config(ENV_LLM_LITELLMROUTER_CONFIG) == two_step_config
+
+    def test_invalid_json(self, monkeypatch):
+        monkeypatch.setenv(ENV_LLM_LITELLMROUTER_CONFIG, "{not json")
+        with pytest.raises(ValueError, match="invalid JSON"):
+            _parse_llm_router_config(ENV_LLM_LITELLMROUTER_CONFIG)
+
+    def test_no_shape_validation(self, monkeypatch):
+        """We don't validate the shape — anything that parses as JSON gets passed through.
+
+        LiteLLM Router is authoritative for shape errors; we let them surface at
+        Router construction time rather than pre-validating.
+        """
+        # A list, a string, an object with junk keys — all accepted by the parser.
+        monkeypatch.setenv(ENV_LLM_LITELLMROUTER_CONFIG, json.dumps([{"hello": "world"}]))
+        assert _parse_llm_router_config(ENV_LLM_LITELLMROUTER_CONFIG) == [{"hello": "world"}]
+        monkeypatch.setenv(ENV_LLM_LITELLMROUTER_CONFIG, json.dumps({"only": "garbage"}))
+        assert _parse_llm_router_config(ENV_LLM_LITELLMROUTER_CONFIG) == {"only": "garbage"}
+
+
+class TestFromEnvLoadsConfig:
+    def test_loaded_when_provider_is_litellmrouter(self, monkeypatch, two_step_config):
+        monkeypatch.setenv(ENV_LLM_PROVIDER, "litellmrouter")
+        monkeypatch.setenv(ENV_LLM_LITELLMROUTER_CONFIG, json.dumps(two_step_config))
+        cfg = HindsightConfig.from_env()
+        assert cfg.llm_provider == "litellmrouter"
+        assert cfg.llm_litellmrouter_config == two_step_config
+
+    def test_unset_keeps_default_provider(self, monkeypatch):
+        monkeypatch.setenv(ENV_LLM_PROVIDER, "openai")
+        monkeypatch.setenv("HINDSIGHT_API_LLM_API_KEY", "sk-primary")
+        monkeypatch.delenv(ENV_LLM_LITELLMROUTER_CONFIG, raising=False)
+        cfg = HindsightConfig.from_env()
+        assert cfg.llm_provider == "openai"
+        assert cfg.llm_litellmrouter_config is None
+
+    def test_per_op_configs_independent(self, monkeypatch):
+        """Per-op env vars populate per-op fields without touching the default."""
+        retain_config = {
+            "model_list": [{"model_name": "r", "litellm_params": {"model": "openai/retain", "api_key": "rk"}}]
+        }
+        reflect_config = {
+            "model_list": [{"model_name": "f", "litellm_params": {"model": "anthropic/claude", "api_key": "ak"}}]
+        }
+        consol_config = {
+            "model_list": [{"model_name": "c", "litellm_params": {"model": "openai/consol", "api_key": "ck"}}]
+        }
+        monkeypatch.setenv(ENV_LLM_PROVIDER, "openai")
+        monkeypatch.setenv("HINDSIGHT_API_LLM_API_KEY", "sk-primary")
+        monkeypatch.setenv(ENV_RETAIN_LLM_LITELLMROUTER_CONFIG, json.dumps(retain_config))
+        monkeypatch.setenv(ENV_REFLECT_LLM_LITELLMROUTER_CONFIG, json.dumps(reflect_config))
+        monkeypatch.setenv(ENV_CONSOLIDATION_LLM_LITELLMROUTER_CONFIG, json.dumps(consol_config))
+        cfg = HindsightConfig.from_env()
+        assert cfg.llm_litellmrouter_config is None
+        assert cfg.retain_llm_litellmrouter_config == retain_config
+        assert cfg.reflect_llm_litellmrouter_config == reflect_config
+        assert cfg.consolidation_llm_litellmrouter_config == consol_config
+
+
+# --- factory dispatch --------------------------------------------------------
+
+
+class TestFactoryDispatch:
+    def test_router_provider_requires_config(self):
+        with pytest.raises(ValueError, match="config object"):
+            create_llm_provider(
+                provider="litellmrouter",
+                api_key="",
+                base_url="",
+                model="unused",
+                reasoning_effort="low",
+                litellmrouter_config=None,
+            )
+
+    def test_router_provider_returns_router_impl(self, two_step_config):
+        with patch.dict("sys.modules", {"litellm": MagicMock()}):
+            with patch(
+                "hindsight_api.engine.providers.litellm_router_llm.LiteLLMRouterLLM.__init__",
+                return_value=None,
+            ) as mock_init:
+                impl = create_llm_provider(
+                    provider="litellmrouter",
+                    api_key="",
+                    base_url="",
+                    model="unused",
+                    reasoning_effort="low",
+                    litellmrouter_config=two_step_config,
+                )
+                assert isinstance(impl, LiteLLMRouterLLM)
+                _, kwargs = mock_init.call_args
+                assert kwargs["config"] == two_step_config
+
+
+# --- Router-backed call paths ------------------------------------------------
+
+
+def _make_router_provider(config: dict[str, Any], mock_router: Any) -> LiteLLMRouterLLM:
+    """Construct a LiteLLMRouterLLM with the inner Router replaced by a mock."""
+    fake_litellm = MagicMock()
+    fake_litellm.Router = MagicMock(return_value=mock_router)
+    with patch.dict("sys.modules", {"litellm": fake_litellm}):
+        # Bypass the heavy ctor chain by injecting state directly.
+        provider = LiteLLMRouterLLM.__new__(LiteLLMRouterLLM)
+        provider.provider = "litellmrouter"
+        provider.api_key = ""
+        provider.base_url = ""
+        provider.model = "unused"
+        provider.reasoning_effort = "low"
+        provider.timeout = 300.0
+        provider.config = config
+        provider._litellm = fake_litellm
+        provider._router = mock_router
+        provider._router_output_cap = None  # tests that exercise the cap override this directly
+        return provider
+
+
+class TestRouterCall:
+    @pytest.mark.asyncio
+    async def test_plain_text_call_targets_default_entrypoint(self, two_step_config, mock_router_response):
+        mock_router = MagicMock()
+        mock_router.acompletion = AsyncMock(return_value=mock_router_response)
+        provider = _make_router_provider(two_step_config, mock_router)
+
+        result = await provider.call(
+            messages=[{"role": "user", "content": "hi"}],
+            max_completion_tokens=50,
+            max_retries=0,
+        )
+        assert result == "ok"
+        # Hindsight always issues against model_name="default"; Router handles fallback,
+        # load-balancing, and routing strategy from there.
+        kwargs = mock_router.acompletion.await_args.kwargs
+        assert kwargs["model"] == "default"
+
+    @pytest.mark.asyncio
+    async def test_structured_output(self, two_step_config):
+        class MySchema(BaseModel):
+            answer: str
+
+        response = MagicMock()
+        choice = MagicMock()
+        choice.message.content = '{"answer": "42"}'
+        choice.message.tool_calls = None
+        choice.finish_reason = "stop"
+        response.choices = [choice]
+        response.usage.prompt_tokens = 5
+        response.usage.completion_tokens = 5
+        response._hidden_params = {"model": "openai/gpt-4o-mini"}
+
+        mock_router = MagicMock()
+        mock_router.acompletion = AsyncMock(return_value=response)
+        provider = _make_router_provider(two_step_config, mock_router)
+
+        result = await provider.call(
+            messages=[{"role": "user", "content": "q"}],
+            response_format=MySchema,
+            max_retries=0,
+        )
+        assert isinstance(result, MySchema)
+        assert result.answer == "42"
+
+    @pytest.mark.asyncio
+    async def test_retry_on_transient_then_success(self, two_step_config, mock_router_response):
+        mock_router = MagicMock()
+        # First call raises a 503-style error, second call returns ok.
+        mock_router.acompletion = AsyncMock(side_effect=[Exception("503 Service Unavailable"), mock_router_response])
+        provider = _make_router_provider(two_step_config, mock_router)
+
+        result = await provider.call(
+            messages=[{"role": "user", "content": "hi"}],
+            max_retries=2,
+            initial_backoff=0.0,
+            max_backoff=0.0,
+        )
+        assert result == "ok"
+        assert mock_router.acompletion.await_count == 2
+
+    @pytest.mark.asyncio
+    async def test_auth_error_does_not_retry(self, two_step_config):
+        mock_router = MagicMock()
+        mock_router.acompletion = AsyncMock(side_effect=Exception("401 Unauthorized: bad key"))
+        provider = _make_router_provider(two_step_config, mock_router)
+
+        with pytest.raises(Exception, match="401"):
+            await provider.call(
+                messages=[{"role": "user", "content": "hi"}],
+                max_retries=5,
+                initial_backoff=0.0,
+            )
+        assert mock_router.acompletion.await_count == 1
+
+    @pytest.mark.asyncio
+    async def test_caps_max_completion_tokens_to_litellm_registry(self, two_step_config, mock_router_response):
+        """Cap max_completion_tokens to the most conservative deployment limit.
+
+        Hindsight's defaults (e.g. retain_max_completion_tokens=64000) target
+        high-capacity models. When a configured deployment has a smaller cap
+        (gpt-4.1-nano = 32768), the call would otherwise be rejected — apply
+        the cap silently using LiteLLM's per-model registry.
+        """
+        mock_router = MagicMock()
+        mock_router.acompletion = AsyncMock(return_value=mock_router_response)
+        provider = _make_router_provider(two_step_config, mock_router)
+        provider._router_output_cap = 32768  # what _compute_router_output_cap would yield
+
+        await provider.call(
+            messages=[{"role": "user", "content": "hi"}],
+            max_completion_tokens=64000,  # over the cap
+            max_retries=0,
+        )
+        kwargs = mock_router.acompletion.await_args.kwargs
+        assert kwargs["max_completion_tokens"] == 32768
+
+    @pytest.mark.asyncio
+    async def test_no_cap_when_litellm_registry_has_no_data(self, two_step_config, mock_router_response):
+        """If LiteLLM doesn't know any of the deployment models, pass the requested value through."""
+        mock_router = MagicMock()
+        mock_router.acompletion = AsyncMock(return_value=mock_router_response)
+        provider = _make_router_provider(two_step_config, mock_router)
+        provider._router_output_cap = None
+
+        await provider.call(
+            messages=[{"role": "user", "content": "hi"}],
+            max_completion_tokens=64000,
+            max_retries=0,
+        )
+        kwargs = mock_router.acompletion.await_args.kwargs
+        assert kwargs["max_completion_tokens"] == 64000
+
+    @pytest.mark.asyncio
+    async def test_call_with_tools(self, two_step_config):
+        response = MagicMock()
+        choice = MagicMock()
+        choice.message.content = None
+        tool_call = MagicMock()
+        tool_call.id = "call_1"
+        tool_call.function.name = "lookup"
+        tool_call.function.arguments = '{"q": "x"}'
+        choice.message.tool_calls = [tool_call]
+        choice.finish_reason = "tool_calls"
+        response.choices = [choice]
+        response.usage.prompt_tokens = 5
+        response.usage.completion_tokens = 2
+        response._hidden_params = {"model": "openai/gpt-4o-mini"}
+
+        mock_router = MagicMock()
+        mock_router.acompletion = AsyncMock(return_value=response)
+        provider = _make_router_provider(two_step_config, mock_router)
+
+        result = await provider.call_with_tools(
+            messages=[{"role": "user", "content": "use tool"}],
+            tools=[{"type": "function", "function": {"name": "lookup", "parameters": {}}}],
+            max_retries=0,
+        )
+        assert result.content is None
+        assert len(result.tool_calls) == 1
+        assert result.tool_calls[0].name == "lookup"
+        assert result.tool_calls[0].arguments == {"q": "x"}

hindsight-api-slim/tests/test_mcp_tools.py

@@ -949,6 +949,51 @@ class TestRecallNewParams:
         call_kwargs = mock_memory.recall_async.call_args.kwargs
         assert call_kwargs["question_date"] == datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
 
+    async def test_recall_with_tag_groups_negative_filter(self, mock_memory):
+        """tag_groups with NOT should pass through to engine after Pydantic validation."""
+        from hindsight_api.engine.search.tags import TagGroupNot
+
+        mcp = _make_mcp_server(mock_memory, {"recall"})
+        await _tools(mcp)["recall"].fn(
+            query="test",
+            tag_groups=[{"not": {"tags": ["closeout"], "match": "any_strict"}}],
+        )
+        call_kwargs = mock_memory.recall_async.call_args.kwargs
+        assert "tag_groups" in call_kwargs
+        assert len(call_kwargs["tag_groups"]) == 1
+        group = call_kwargs["tag_groups"][0]
+        assert isinstance(group, TagGroupNot)
+        assert group.filter.tags == ["closeout"]
+
+    async def test_recall_without_tag_groups_no_kwarg(self, mock_memory):
+        """tag_groups omitted should not appear in engine kwargs."""
+        mcp = _make_mcp_server(mock_memory, {"recall"})
+        await _tools(mcp)["recall"].fn(query="test")
+        call_kwargs = mock_memory.recall_async.call_args.kwargs
+        assert "tag_groups" not in call_kwargs
+
+    async def test_recall_tags_and_tag_groups_mutually_exclusive(self, mock_memory):
+        """Passing both tags and tag_groups returns an error and does not call engine."""
+        mcp = _make_mcp_server(mock_memory, {"recall"})
+        result = await _tools(mcp)["recall"].fn(
+            query="test",
+            tags=["project:x"],
+            tag_groups=[{"tags": ["closeout"], "match": "any_strict"}],
+        )
+        assert "mutually exclusive" in result
+        mock_memory.recall_async.assert_not_called()
+
+    async def test_recall_tag_groups_single_bank(self, mock_memory):
+        """tag_groups should also work in single-bank mode."""
+        mcp = _make_mcp_server(mock_memory, {"recall"}, include_bank_id=False)
+        await _tools(mcp)["recall"].fn(
+            query="test",
+            tag_groups=[{"tags": ["scope:work"], "match": "all_strict"}],
+        )
+        call_kwargs = mock_memory.recall_async.call_args.kwargs
+        assert "tag_groups" in call_kwargs
+        assert len(call_kwargs["tag_groups"]) == 1
+
 
 @pytest.mark.asyncio
 class TestReflectNewParams:

hindsight-api-slim/tests/test_mental_model_trigger_tag_groups.py

@@ -0,0 +1,48 @@
+from hindsight_api.api.http import MentalModelTrigger
+from hindsight_api.engine.search.tags import TagGroupOr
+
+
+def test_mental_model_trigger_model_dump_preserves_or_tag_group():
+    trigger = MentalModelTrigger.model_validate(
+        {
+            "tag_groups": [
+                {
+                    "or": [
+                        {"tags": ["ns:a"], "match": "all_strict"},
+                        {"tags": ["ns:b"], "match": "all_strict"},
+                    ]
+                }
+            ]
+        }
+    )
+
+    dumped = trigger.model_dump()
+
+    assert dumped["tag_groups"] == [
+        {
+            "or": [
+                {"tags": ["ns:a"], "match": "all_strict"},
+                {"tags": ["ns:b"], "match": "all_strict"},
+            ]
+        }
+    ]
+
+
+def test_mental_model_trigger_or_tag_group_survives_storage_round_trip():
+    trigger = MentalModelTrigger.model_validate(
+        {
+            "tag_groups": [
+                {
+                    "or": [
+                        {"tags": ["ns:a"], "match": "all_strict"},
+                        {"tags": ["ns:b"], "match": "all_strict"},
+                    ]
+                }
+            ]
+        }
+    )
+
+    round_tripped = MentalModelTrigger.model_validate(trigger.model_dump())
+
+    assert isinstance(round_tripped.tag_groups[0], TagGroupOr)
+    assert round_tripped.model_dump()["tag_groups"] == trigger.model_dump()["tag_groups"]

hindsight-api-slim/tests/test_openai_compatible_response_hardening.py

@@ -1,5 +1,5 @@
-from unittest.mock import AsyncMock, MagicMock, patch
 from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 from pydantic import BaseModel
@@ -52,6 +52,24 @@ async def test_json_object_call_adds_json_hint_to_user_message():
     assert sent_messages[0]["content"].startswith("Return valid json only.")
 
 
+@pytest.mark.asyncio
+async def test_json_object_call_strips_gemma_thought_tags_before_parsing():
+    llm = _llm()
+    create = AsyncMock(
+        return_value=_response(content='<thought>\nI should return a compact JSON object.\n</thought>\n{"ok": true}')
+    )
+    llm._client.chat.completions.create = create
+
+    with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector"):
+        result = await llm.call(
+            messages=[{"role": "user", "content": "Return whether this worked."}],
+            response_format=SimpleJsonResponse,
+            max_retries=0,
+        )
+
+    assert result.ok is True
+
+
 @pytest.mark.asyncio
 async def test_error_payload_with_no_choices_raises_clear_provider_error_without_retry():
     llm = _llm()

hindsight-api-slim/tests/test_opencode_go_provider.py

@@ -0,0 +1,80 @@
+"""Tests for the opencode-go OpenAI-compatible LLM provider."""
+
+import pytest
+
+
+def test_opencode_go_config_has_expected_default_model(monkeypatch):
+    """HindsightConfig should default opencode-go to the DeepSeek v4 flash model."""
+    from hindsight_api.config import PROVIDER_DEFAULT_MODELS, HindsightConfig, clear_config_cache
+
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "opencode-go")
+    monkeypatch.delenv("HINDSIGHT_API_LLM_MODEL", raising=False)
+    clear_config_cache()
+
+    try:
+        assert PROVIDER_DEFAULT_MODELS["opencode-go"] == "deepseek-v4-flash"
+        config = HindsightConfig.from_env()
+        assert config.llm_provider == "opencode-go"
+        assert config.llm_model == "deepseek-v4-flash"
+    finally:
+        clear_config_cache()
+
+
+def test_opencode_go_llm_provider_from_env_has_expected_default_model(monkeypatch):
+    """LLMProvider.from_env should use the opencode-go provider default model."""
+    from hindsight_api.config import clear_config_cache
+    from hindsight_api.engine.llm_wrapper import LLMProvider
+
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "opencode-go")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_API_KEY", "test-key")
+    monkeypatch.delenv("HINDSIGHT_API_LLM_MODEL", raising=False)
+    monkeypatch.delenv("HINDSIGHT_API_LLM_BASE_URL", raising=False)
+    clear_config_cache()
+
+    try:
+        llm = LLMProvider.from_env()
+        assert llm.provider == "opencode-go"
+        assert llm.model == "deepseek-v4-flash"
+        assert llm.base_url == "https://opencode.ai/zen/go/v1"
+    finally:
+        clear_config_cache()
+
+
+def test_opencode_go_requires_api_key_like_zai():
+    """opencode-go is a cloud provider and should require an API key."""
+    from hindsight_api.engine.llm_wrapper import requires_api_key
+
+    assert requires_api_key("opencode-go") is True
+
+
+def test_opencode_go_uses_openai_compatible_provider_with_default_base_url():
+    """The provider factory should route opencode-go to OpenAICompatibleLLM."""
+    from hindsight_api.engine.llm_wrapper import LLMProvider
+    from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
+
+    llm = LLMProvider(
+        provider="opencode-go",
+        api_key="test-key",
+        base_url="",
+        model="deepseek-v4-flash",
+    )
+
+    assert llm.provider == "opencode-go"
+    assert llm.model == "deepseek-v4-flash"
+    assert llm.base_url == "https://opencode.ai/zen/go/v1"
+    assert not llm.base_url.endswith("/")
+    assert isinstance(llm._provider_impl, OpenAICompatibleLLM)
+    assert llm._provider_impl.base_url == "https://opencode.ai/zen/go/v1"
+
+
+def test_opencode_go_rejects_missing_api_key():
+    """opencode-go should fail fast without an API key, matching zai behavior."""
+    from hindsight_api.engine.llm_wrapper import LLMProvider
+
+    with pytest.raises(ValueError, match="API key is required for opencode-go"):
+        LLMProvider(
+            provider="opencode-go",
+            api_key="",
+            base_url="",
+            model="deepseek-v4-flash",
+        )

hindsight-api-slim/tests/test_openrouter_null_content.py

@@ -10,13 +10,12 @@ retry hits the same unhandled error so the entire retry budget is wasted.
 See https://github.com/vectorize-io/hindsight/issues/1334.
 """
 
-import json
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 from pydantic import BaseModel
 
-from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
+from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM, ProviderResponseError
 
 
 class _Response(BaseModel):
@@ -33,25 +32,38 @@ def _make_llm() -> OpenAICompatibleLLM:
 
 
 def _make_chat_response(content: str | None) -> MagicMock:
+    """Build a mock that matches the shape expected by _first_choice_or_error.
+
+    Key fields that must be explicitly set (not left as auto-MagicMock):
+    - response.error = None          (otherwise truthy MagicMock triggers error path)
+    - response.model_dump()          (returns dict without 'error' key)
+    - choice.message.tool_calls/refusal  (otherwise truthy MagicMock in error msg)
+    """
+    choice = MagicMock()
+    choice.finish_reason = "stop"
+    choice.message.content = content
+    choice.message.tool_calls = None
+    choice.message.refusal = None
+
     response = MagicMock()
+    response.error = None
+    response.model_dump.return_value = {}
     response.usage.prompt_tokens = 10
     response.usage.completion_tokens = 0 if content is None else 5
     response.usage.total_tokens = 10 if content is None else 15
-    response.choices[0].finish_reason = "stop"
-    response.choices[0].message.content = content
-    response.choices[0].message.tool_calls = None
+    response.choices = [choice]
     return response
 
 
 @pytest.mark.asyncio
 async def test_null_content_raises_after_retries_exhausted():
-    """All retries return null content -> JSONDecodeError, not TypeError."""
+    """All retries return null content -> ProviderResponseError, not TypeError."""
     llm = _make_llm()
 
     with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create:
         mock_create.return_value = _make_chat_response(None)
 
-        with pytest.raises(json.JSONDecodeError):
+        with pytest.raises(ProviderResponseError, match="empty message content"):
             await llm.call(
                 messages=[{"role": "user", "content": "extract facts"}],
                 response_format=_Response,

hindsight-api-slim/tests/test_read_backend.py

@@ -0,0 +1,135 @@
+"""Tests for the optional read-only backend for recall queries."""
+
+from __future__ import annotations
+
+import pytest
+import pytest_asyncio
+
+from hindsight_api import MemoryEngine
+from hindsight_api.engine.task_backend import SyncTaskBackend
+
+
+def _make_engine(pg0_db_url: str, embeddings, cross_encoder, query_analyzer) -> MemoryEngine:
+    """Build a MemoryEngine for a single test. Tiny pool, no migrations,
+    SyncTaskBackend so async tasks resolve inline. Mirrors conftest's
+    ``memory`` fixture but lets each test build its own with custom env.
+    """
+    return MemoryEngine(
+        db_url=pg0_db_url,
+        memory_llm_provider="none",  # No LLM calls in these tests
+        memory_llm_api_key="unused",
+        memory_llm_model="unused",
+        embeddings=embeddings,
+        cross_encoder=cross_encoder,
+        query_analyzer=query_analyzer,
+        pool_min_size=1,
+        pool_max_size=2,
+        run_migrations=False,
+        task_backend=SyncTaskBackend(),
+    )
+
+
+@pytest_asyncio.fixture
+async def engine_no_read_url(monkeypatch, pg0_db_url, embeddings, cross_encoder, query_analyzer):
+    """Engine built with READ_DATABASE_URL explicitly unset."""
+    from hindsight_api.config import clear_config_cache
+
+    monkeypatch.delenv("HINDSIGHT_API_READ_DATABASE_URL", raising=False)
+    clear_config_cache()
+
+    mem = _make_engine(pg0_db_url, embeddings, cross_encoder, query_analyzer)
+    await mem.initialize()
+    yield mem
+    try:
+        await mem.close()
+    except Exception:
+        pass
+    clear_config_cache()
+
+
+@pytest_asyncio.fixture
+async def engine_with_read_url(monkeypatch, pg0_db_url, embeddings, cross_encoder, query_analyzer):
+    """Engine built with READ_DATABASE_URL set to the same DB. The engine
+    can't tell it's the same server — it just sees a second URL and opens
+    a second backend, which is what we want to verify.
+    """
+    from hindsight_api.config import clear_config_cache
+
+    monkeypatch.setenv("HINDSIGHT_API_READ_DATABASE_URL", pg0_db_url)
+    clear_config_cache()
+
+    mem = _make_engine(pg0_db_url, embeddings, cross_encoder, query_analyzer)
+    await mem.initialize()
+    yield mem
+    try:
+        await mem.close()
+    except Exception:
+        pass
+    clear_config_cache()
+
+
+@pytest.mark.asyncio
+async def test_read_backend_aliases_primary_when_url_unset(engine_no_read_url):
+    """Without HINDSIGHT_API_READ_DATABASE_URL, _read_backend is _backend.
+    This is the back-compat invariant — every call site that uses
+    _get_read_backend() resolves to the same object _get_backend() returns,
+    so nothing observable changes.
+    """
+    assert engine_no_read_url._read_backend is engine_no_read_url._backend
+
+
+@pytest.mark.asyncio
+async def test_read_backend_is_separate_instance_when_url_set(engine_with_read_url):
+    """With HINDSIGHT_API_READ_DATABASE_URL set, a SECOND backend object is
+    created. Even if both URLs point at the same DB (as in this test), the
+    two backends own independent connection pools — connections taken from
+    one don't drain the other, and shutting one down doesn't close the
+    other's pool.
+    """
+    primary = engine_with_read_url._backend
+    read = engine_with_read_url._read_backend
+    assert read is not primary
+    # Both backends are independently initialized (each owns a pool)
+    assert primary.get_pool() is not None
+    assert read.get_pool() is not None
+    assert primary.get_pool() is not read.get_pool()
+
+
+@pytest.mark.asyncio
+async def test_get_read_backend_returns_read_backend(engine_with_read_url):
+    """The accessor used by recall (`_get_read_backend`) returns the dedicated
+    read backend, not the primary. Without this, the env var would have no
+    effect.
+    """
+    backend = await engine_with_read_url._get_read_backend()
+    assert backend is engine_with_read_url._read_backend
+    assert backend is not engine_with_read_url._backend
+
+
+@pytest.mark.asyncio
+async def test_get_read_backend_returns_primary_when_unset(engine_no_read_url):
+    """The accessor falls through to the primary when no read URL is set,
+    so callers don't need to handle a None case.
+    """
+    backend = await engine_no_read_url._get_read_backend()
+    assert backend is engine_no_read_url._backend
+
+
+@pytest.mark.asyncio
+async def test_close_terminates_distinct_read_backend(engine_with_read_url):
+    """When the read backend is distinct, close() must shut it down too,
+    not just the primary. Otherwise we leak the read pool when the engine
+    is recycled (e.g. across pytest sessions or in app shutdown).
+    """
+    primary_before = engine_with_read_url._backend
+    read_before = engine_with_read_url._read_backend
+    assert read_before is not primary_before
+
+    await engine_with_read_url.close()
+
+    # Both backends should be cleared on close. The exact post-close state
+    # is "primary _backend cleared, _read_backend cleared". The shutdown
+    # method on the read backend is called — we verify by checking the
+    # engine no longer references either.
+    assert engine_with_read_url._backend is None
+    assert engine_with_read_url._read_backend is None

hindsight-api-slim/tests/test_recall_error_propagation.py

@@ -0,0 +1,56 @@
+"""Tests that recall_async surfaces a non-opaque error message when the
+underlying retrieval pipeline raises an exception with empty __str__.
+
+Regression for issue #1384: ``raise Exception(f"Failed to search memories: ...{e}")``
+collapsed to ``Failed to search memories: `` for any exception whose __str__()
+returns blank, dropping the original class name and traceback chain.
+"""
+
+from unittest.mock import patch
+
+import pytest
+
+from hindsight_api import MemoryEngine, RequestContext
+
+RC = RequestContext(tenant_id="default")
+
+
+class _SilentError(Exception):
+    """Mimics asyncpg.exceptions.ConnectionDoesNotExistError() and similar
+    exceptions whose __str__() returns blank when raised with no args."""
+
+
+async def _raise_silent(*_args, **_kwargs):
+    raise _SilentError()
+
+
+async def test_recall_async_error_preserves_original(memory_no_llm_verify: MemoryEngine):
+    engine = memory_no_llm_verify
+    bank_id = "test-error-propagation"
+    await engine.get_bank_profile(bank_id, request_context=RC)
+
+    try:
+        with patch(
+            "hindsight_api.engine.memory_engine.embedding_utils.generate_embeddings_batch",
+            side_effect=_raise_silent,
+        ):
+            with pytest.raises(RuntimeError) as excinfo:
+                await engine.recall_async(
+                    bank_id=bank_id,
+                    query="anything",
+                    request_context=RC,
+                )
+
+        # The wrapping message must include the original exception class name —
+        # the symptom in #1384 was an empty trailer like "Failed to search memories: ".
+        message = str(excinfo.value)
+        assert "Failed to search memories" in message
+        assert "_SilentError" in message, (
+            f"wrapper message dropped the original exception class: {message!r}"
+        )
+
+        # `from e` chain must be preserved so worker logs / debuggers can walk
+        # back to the real cause.
+        assert isinstance(excinfo.value.__cause__, _SilentError)
+    finally:
+        await engine.delete_bank(bank_id, request_context=RC)

hindsight-api-slim/tests/test_recall_observation_entities.py

@@ -0,0 +1,214 @@
+"""Recall projects entities for observations through source_memory_ids.
+
+Observations don't carry rows in `unit_entities`; their entity association
+lives transitively via `memory_units.source_memory_ids`. The per-memory
+endpoint (`get_memory_unit`) follows that chain, but recall used to query
+`unit_entities` directly and silently dropped entities for every observation
+result, even when `include_entities=True` was set.
+
+This test seeds an observation linked through `source_memory_ids` to a fact
+with entities, runs an observation-only recall, and asserts both the
+per-result `entities` field and the top-level aggregate map carry the
+inherited entities.
+
+No LLM required.
+"""
+
+import uuid
+
+import pytest
+import pytest_asyncio
+
+from hindsight_api import MemoryEngine, RequestContext
+from hindsight_api.engine.retain import embedding_utils
+
+# Tests in this file insert memory_units with shared hardcoded UUIDs and
+# memory_units.id is a global PK; share an xdist group so parallel workers
+# don't collide on the same row IDs.
+pytestmark = pytest.mark.xdist_group("recall_observation_entities")
+
+ID_FACT = "11111111-0000-0000-0000-000000000001"
+ID_OBS_INHERITED = "11111111-0000-0000-0000-000000000002"
+ID_OBS_DIRECT = "11111111-0000-0000-0000-000000000003"
+
+RC = RequestContext(tenant_id="default")
+
+
+def _to_str(emb: list[float]) -> str:
+    return "[" + ",".join(str(v) for v in emb) + "]"
+
+
+@pytest_asyncio.fixture
+async def seeded(memory_no_llm_verify: MemoryEngine):
+    engine = memory_no_llm_verify
+    bank_id = f"test-recall-obs-ent-{uuid.uuid4().hex[:8]}"
+    await engine.get_bank_profile(bank_id, request_context=RC)
+
+    embeddings = await embedding_utils.generate_embeddings_batch(
+        engine.embeddings,
+        [
+            "HeadClaw waitlist tracked in Google Sheets",
+            "HeadClaw users sign up via the waitlist",
+            "Reddit thread mentions HeadClaw waitlist signups",
+        ],
+    )
+
+    pool = await engine._get_pool()
+    async with pool.acquire() as conn:
+        await conn.execute(
+            "DELETE FROM memory_units WHERE id IN ($1, $2, $3)",
+            ID_FACT,
+            ID_OBS_INHERITED,
+            ID_OBS_DIRECT,
+        )
+        await conn.execute("DELETE FROM entities WHERE bank_id = $1", bank_id)
+
+        # Source fact with two entities. The observation that lacks direct
+        # entity rows must inherit both through source_memory_ids.
+        headclaw_id = await conn.fetchval(
+            """
+            INSERT INTO entities (bank_id, canonical_name, mention_count)
+            VALUES ($1, $2, 1) RETURNING id
+            """,
+            bank_id,
+            "HeadClaw",
+        )
+        waitlist_id = await conn.fetchval(
+            """
+            INSERT INTO entities (bank_id, canonical_name, mention_count)
+            VALUES ($1, $2, 1) RETURNING id
+            """,
+            bank_id,
+            "waitlist users",
+        )
+        # Independent entity attached directly to the second observation —
+        # exercises the existing direct-link path so we don't regress it.
+        reddit_id = await conn.fetchval(
+            """
+            INSERT INTO entities (bank_id, canonical_name, mention_count)
+            VALUES ($1, $2, 1) RETURNING id
+            """,
+            bank_id,
+            "Reddit",
+        )
+
+        await conn.execute(
+            """
+            INSERT INTO memory_units (id, bank_id, text, fact_type, embedding, event_date)
+            VALUES ($1, $2, $3, 'world', $4::vector, now())
+            """,
+            ID_FACT,
+            bank_id,
+            "HeadClaw waitlist tracked in Google Sheets",
+            _to_str(embeddings[0]),
+        )
+        await conn.execute(
+            """
+            INSERT INTO unit_entities (unit_id, entity_id) VALUES ($1, $2), ($1, $3)
+            """,
+            ID_FACT,
+            headclaw_id,
+            waitlist_id,
+        )
+
+        # Observation with NO direct unit_entities — must inherit HeadClaw +
+        # waitlist users from source_memory_ids.
+        await conn.execute(
+            """
+            INSERT INTO memory_units (
+                id, bank_id, text, fact_type, embedding, event_date,
+                source_memory_ids, history, proof_count
+            )
+            VALUES ($1, $2, $3, 'observation', $4::vector, now(), $5::uuid[], '[]'::jsonb, 1)
+            """,
+            ID_OBS_INHERITED,
+            bank_id,
+            "HeadClaw users sign up via the waitlist",
+            _to_str(embeddings[1]),
+            [ID_FACT],
+        )
+
+        # Observation with a DIRECT unit_entities link — must keep its own entity.
+        await conn.execute(
+            """
+            INSERT INTO memory_units (
+                id, bank_id, text, fact_type, embedding, event_date,
+                source_memory_ids, history, proof_count
+            )
+            VALUES ($1, $2, $3, 'observation', $4::vector, now(), NULL, '[]'::jsonb, 1)
+            """,
+            ID_OBS_DIRECT,
+            bank_id,
+            "Reddit thread mentions HeadClaw waitlist signups",
+            _to_str(embeddings[2]),
+        )
+        await conn.execute(
+            "INSERT INTO unit_entities (unit_id, entity_id) VALUES ($1, $2)",
+            ID_OBS_DIRECT,
+            reddit_id,
+        )
+
+    yield engine, bank_id
+
+    await engine.delete_bank(bank_id, request_context=RC)
+
+
+@pytest.mark.asyncio
+async def test_recall_includes_inherited_entities_for_observations(seeded):
+    """Observation-only recall must surface entities inherited from source memories."""
+    engine, bank_id = seeded
+
+    result = await engine.recall_async(
+        bank_id=bank_id,
+        query="HeadClaw waitlist users",
+        fact_type=["observation"],
+        request_context=RC,
+        max_tokens=4000,
+        include_entities=True,
+        max_entity_tokens=2000,
+    )
+
+    by_id = {str(r.id): r for r in result.results}
+    assert ID_OBS_INHERITED in by_id, f"Expected inherited-entity observation in results, got {list(by_id)}"
+    assert ID_OBS_DIRECT in by_id, f"Expected direct-entity observation in results, got {list(by_id)}"
+
+    inherited = by_id[ID_OBS_INHERITED].entities or []
+    assert "HeadClaw" in inherited, f"Observation entities must inherit through source_memory_ids; got {inherited}"
+    assert "waitlist users" in inherited, f"All source entities should propagate; got {inherited}"
+
+    direct = by_id[ID_OBS_DIRECT].entities or []
+    assert "Reddit" in direct, f"Direct unit_entities link must still project on observation results; got {direct}"
+
+    assert result.entities is not None, "Top-level entities map must populate when include_entities=True"
+    aggregate_names = set(result.entities.keys())
+    assert {"HeadClaw", "waitlist users", "Reddit"}.issubset(aggregate_names), (
+        f"Top-level entities map must include inherited + direct entities; got {aggregate_names}"
+    )
+
+
+@pytest.mark.asyncio
+async def test_get_memory_unit_inherits_observation_entities(seeded):
+    """get_memory_unit shares the recall helper, so observation inheritance
+    must keep working through the per-memory endpoint as well.
+    """
+    engine, bank_id = seeded
+
+    inherited = await engine.get_memory_unit(
+        memory_id=ID_OBS_INHERITED,
+        bank_id=bank_id,
+        request_context=RC,
+    )
+    assert inherited is not None
+    assert set(inherited["entities"]) >= {"HeadClaw", "waitlist users"}, (
+        f"Observation must inherit source-memory entities; got {inherited['entities']}"
+    )
+
+    direct = await engine.get_memory_unit(
+        memory_id=ID_OBS_DIRECT,
+        bank_id=bank_id,
+        request_context=RC,
+    )
+    assert direct is not None
+    assert "Reddit" in direct["entities"], (
+        f"Direct unit_entities link must still resolve via get_memory_unit; got {direct['entities']}"
+    )

hindsight-api-slim/tests/test_reflect_tiktoken_lazy_load.py

@@ -0,0 +1,41 @@
+import importlib
+import sys
+from unittest.mock import MagicMock, patch
+
+
+def _drop_reflect_modules() -> None:
+    for name in list(sys.modules):
+        if name == "hindsight_api.engine.reflect" or name.startswith("hindsight_api.engine.reflect."):
+            sys.modules.pop(name)
+
+
+def test_reflect_import_does_not_load_tiktoken_encoding():
+    _drop_reflect_modules()
+
+    with patch("tiktoken.get_encoding") as get_encoding:
+        reflect = importlib.import_module("hindsight_api.engine.reflect")
+
+    get_encoding.assert_not_called()
+    assert reflect.run_reflect_agent is not None
+
+
+def test_reflect_token_counting_loads_tiktoken_encoding_when_used():
+    _drop_reflect_modules()
+    fake_encoding = MagicMock()
+    fake_encoding.encode.side_effect = lambda text: text.split()
+
+    with patch("tiktoken.get_encoding", return_value=fake_encoding) as get_encoding:
+        agent = importlib.import_module("hindsight_api.engine.reflect.agent")
+        prompts = importlib.import_module("hindsight_api.engine.reflect.prompts")
+
+        count = agent._count_messages_tokens([{"role": "user", "content": "one two"}])
+        final_prompt = prompts.build_final_prompt(
+            query="What happened?",
+            context_history=[{"tool": "recall", "output": {"answer": "three four"}}],
+            bank_profile={"name": "test"},
+            max_context_tokens=1000,
+        )
+
+    assert count == 2
+    assert "three four" in final_prompt
+    get_encoding.assert_called_once_with("cl100k_base")

hindsight-api-slim/tests/test_reflect_tools.py

@@ -0,0 +1,120 @@
+"""Regression tests for reflect tool helpers."""
+
+import re
+import uuid
+
+import pytest
+
+from hindsight_api.engine.reflect.tools import _document_metadata_from_retain_params, tool_expand
+
+
+class _FakeReflectConnection:
+    """Tiny asyncpg-like connection for tool_expand query behavior."""
+
+    def __init__(self, bank_id: str, memory_id: uuid.UUID, document_id: str, chunk_id: str | None) -> None:
+        self.bank_id = bank_id
+        self.memory_id = memory_id
+        self.document_id = document_id
+        self.chunk_id = chunk_id
+
+    async def fetch(self, query: str, *args):
+        normalized_query = re.sub(r"\s+", " ", query).strip()
+
+        if "FROM public.memory_units" in normalized_query:
+            return [
+                {
+                    "id": self.memory_id,
+                    "text": "The user prefers test-first bug fixes.",
+                    "chunk_id": self.chunk_id,
+                    "document_id": self.document_id,
+                    "fact_type": "experience",
+                    "context": "preference",
+                }
+            ]
+
+        if "FROM public.chunks" in normalized_query:
+            if self.chunk_id is None:
+                return []
+            return [
+                {
+                    "chunk_id": self.chunk_id,
+                    "chunk_text": "The user prefers test-first bug fixes.",
+                    "chunk_index": 0,
+                    "document_id": self.document_id,
+                }
+            ]
+
+        if "FROM public.documents" in normalized_query:
+            select_clause = normalized_query.split(" FROM ", 1)[0]
+            assert " metadata," not in f" {select_clause},", (
+                "tool_expand must not query documents.metadata; that column was removed and "
+                "document metadata now lives in retain_params.metadata"
+            )
+            return [
+                {
+                    "id": self.document_id,
+                    "original_text": "The user prefers test-first bug fixes.",
+                    "retain_params": {"metadata": {"source": "regression-test"}},
+                }
+            ]
+
+        raise AssertionError(f"Unexpected query: {normalized_query}")
+
+
+@pytest.mark.asyncio
+async def test_tool_expand_document_depth_reads_metadata_from_retain_params() -> None:
+    """Document expansion must work after documents.metadata has been dropped."""
+    bank_id = "test-reflect-expand-retain-params-metadata"
+    memory_id = uuid.uuid4()
+    document_id = "doc-reflect-expand"
+    chunk_id = "chunk-reflect-expand"
+    conn = _FakeReflectConnection(bank_id, memory_id, document_id, chunk_id)
+
+    result = await tool_expand(
+        conn=conn,
+        bank_id=bank_id,
+        memory_ids=[str(memory_id)],
+        depth="document",
+    )
+
+    assert result["count"] == 1
+    document = result["results"][0]["document"]
+    assert document["metadata"] == {"source": "regression-test"}
+    assert document["retain_params"] == {"metadata": {"source": "regression-test"}}
+
+
+@pytest.mark.asyncio
+async def test_tool_expand_document_depth_without_chunk_reads_metadata_from_retain_params() -> None:
+    """Direct document expansion follows the same metadata source contract."""
+    bank_id = "test-reflect-expand-direct-retain-params-metadata"
+    memory_id = uuid.uuid4()
+    document_id = "doc-reflect-expand-direct"
+    conn = _FakeReflectConnection(bank_id, memory_id, document_id, chunk_id=None)
+
+    result = await tool_expand(
+        conn=conn,
+        bank_id=bank_id,
+        memory_ids=[str(memory_id)],
+        depth="document",
+    )
+
+    assert result["count"] == 1
+    document = result["results"][0]["document"]
+    assert document["metadata"] == {"source": "regression-test"}
+    assert document["retain_params"] == {"metadata": {"source": "regression-test"}}
+
+
+def test_document_metadata_from_retain_params_accepts_json_strings() -> None:
+    """asyncpg JSONB codecs may return retain_params as a dict or JSON string."""
+    retain_params = '{"metadata": {"source": "json-string"}}'
+
+    assert _document_metadata_from_retain_params(retain_params) == {"source": "json-string"}
+
+
+@pytest.mark.parametrize(
+    "retain_params",
+    [None, [], "not json", {"metadata": ["not", "a", "dict"]}],
+)
+def test_document_metadata_from_retain_params_ignores_invalid_values(retain_params) -> None:
+    """Malformed retain_params should not break reflect expansion."""
+    assert _document_metadata_from_retain_params(retain_params) is None

hindsight-api-slim/tests/test_reflections.py

@@ -401,6 +401,7 @@ class TestRecallWithObservationsAndMentalModels:
 class TestReflectUsesMentalModels:
     """Test that reflect searches and uses mental models when available."""
 
+    @pytest.mark.hs_llm_mat
     @pytest.mark.asyncio
     async def test_reflect_searches_mental_models_when_available(self, memory: MemoryEngine, request_context):
         """Test that reflect uses search_mental_models when the bank has mental models.

hindsight-api-slim/tests/test_retain.py

@@ -13,6 +13,7 @@ from hindsight_api.engine.memory_engine import Budget
 logger = logging.getLogger(__name__)
 
 
+@pytest.mark.hs_llm_mat
 @pytest.mark.asyncio
 async def test_retain_with_chunks(memory, request_context):
     """

hindsight-api-slim/tests/test_vector_index.py

@@ -0,0 +1,109 @@
+from pathlib import Path
+
+from hindsight_api._vector_index import (
+    SCANN_MIN_ROWS_FOR_AUTO_INDEX,
+    bootstrap_extension,
+    index_type_keyword,
+    index_using_clause,
+    pg_extension_name,
+    should_defer_index_creation,
+    uses_per_bank_vector_indexes,
+    validate_extension,
+)
+from hindsight_api.engine.retain import bank_utils
+
+
+class RecordingConn:
+    def __init__(self):
+        self.statements = []
+
+    def execute(self, statement, *args, **kwargs):
+        self.statements.append(str(statement))
+
+
+def test_validate_extension_accepts_scann():
+    assert validate_extension("scann") == "scann"
+    assert validate_extension("ScaNN") == "scann"
+
+
+def test_pg_extension_name_maps_scann_to_alloydb_extension():
+    assert pg_extension_name("scann") == "alloydb_scann"
+
+
+def test_index_using_clause_scann_uses_cosine_auto_mode():
+    clause = index_using_clause("scann")
+
+    assert "USING scann (embedding cosine)" in clause
+    assert "mode = 'AUTO'" in clause
+
+
+def test_index_using_clause_pgvector_matches_existing_clause():
+    assert index_using_clause("pgvector") == "USING hnsw (embedding vector_cosine_ops)"
+
+
+def test_index_type_keyword_scann_round_trips_pg_indexes_indexdef():
+    keyword = index_type_keyword("scann")
+    indexdef = "CREATE INDEX idx ON memory_units USING scann (embedding cosine) WITH (mode='AUTO')"
+
+    assert keyword == "scann"
+    assert keyword in indexdef.lower()
+
+
+def test_bootstrap_extension_scann_installs_vector_before_alloydb_scann():
+    conn = RecordingConn()
+
+    bootstrap_extension(conn, "scann")
+
+    assert conn.statements == [
+        "CREATE EXTENSION IF NOT EXISTS vector",
+        "CREATE EXTENSION IF NOT EXISTS alloydb_scann CASCADE",
+    ]
+
+
+def test_scann_index_creation_defers_until_table_is_large_enough():
+    assert should_defer_index_creation("scann", 0)
+    assert should_defer_index_creation("scann", SCANN_MIN_ROWS_FOR_AUTO_INDEX - 1)
+    assert not should_defer_index_creation("scann", SCANN_MIN_ROWS_FOR_AUTO_INDEX)
+    assert not should_defer_index_creation("pgvector", 0)
+
+
+def test_scann_does_not_use_per_bank_partial_indexes():
+    assert not uses_per_bank_vector_indexes("scann")
+    assert uses_per_bank_vector_indexes("pgvector")
+    assert uses_per_bank_vector_indexes("pgvectorscale")
+    assert uses_per_bank_vector_indexes("vchord")
+
+
+def test_alembic_vector_migrations_freeze_vector_sql_locally():
+    migration_dir = Path(__file__).resolve().parent.parent / "hindsight_api/alembic/versions"
+    changed_migrations = [
+        "5a366d414dce_initial_schema.py",
+        "a4b5c6d7e8f9_fix_per_bank_vector_index_type.py",
+        "d5e6f7a8b9c0_add_bank_internal_id_and_per_bank_hnsw.py",
+        "n9i0j1k2l3m4_learnings_and_pinned_reflections.py",
+    ]
+
+    for migration in changed_migrations:
+        text = (migration_dir / migration).read_text()
+        assert "hindsight_api._vector_index" not in text
+
+
+class RecordingOps:
+    def __init__(self):
+        self.called = False
+
+    async def create_bank_vector_indexes(self, *args, **kwargs):
+        self.called = True
+
+
+class ScannConfig:
+    vector_extension = "scann"
+
+
+async def test_create_bank_vector_indexes_skips_scann(monkeypatch):
+    monkeypatch.setattr(bank_utils, "get_config", lambda: ScannConfig())
+    ops = RecordingOps()
+
+    await bank_utils.create_bank_vector_indexes(None, "bank", "00000000-0000-0000-0000-000000000000", ops=ops)
+
+    assert not ops.called

hindsight-api-slim/tests/test_worker.py

@@ -56,13 +56,15 @@ async def pool(backend):
 
 @pytest_asyncio.fixture
 async def clean_operations(pool):
-    """Clean up async_operations table before and after tests."""
-    # Clean before test - covers both 'test-worker-' and 'test_worker_recovery' patterns
-    await pool.execute(
-        "DELETE FROM async_operations WHERE bank_id LIKE 'test-worker-%' OR bank_id LIKE 'test_worker_%'"
-    )
+    """Clean up async_operations table before and after tests.
+
+    We must clean ALL pending operations (not just test-worker-* prefixed ones)
+    because WorkerPoller.claim_batch scans the entire schema for pending tasks.
+    Stale operations left by other tests (e.g. consolidation) cause spurious
+    failures when the poller picks them up unexpectedly.
+    """
+    await pool.execute("DELETE FROM async_operations WHERE status = 'pending'")
     yield
-    # Clean after test
     await pool.execute(
         "DELETE FROM async_operations WHERE bank_id LIKE 'test-worker-%' OR bank_id LIKE 'test_worker_%'"
     )
@@ -2392,6 +2394,75 @@ async def test_pending_breakdown_explains_unclaimable_rows(pool, backend, clean_
     assert buckets["consolidation"]["claimable"] >= 1
 
 
+class TestSummariseChildErrorMessages:
+    """Pure unit tests for the _summarise_child_error_messages helper.
+
+    The helper picks a representative error message for a parent whose
+    children failed. The integration tests above exercise the full path
+    through _mark_failed; these tests focus on the choice itself.
+    """
+
+    def _sib(self, status: str, error_message: str | None = None) -> dict:
+        return {"status": status, "error_message": error_message}
+
+    def test_all_failed_with_same_message_inherits_that_message(self):
+        from hindsight_api.worker.poller import _summarise_child_error_messages
+
+        siblings = [
+            self._sib("failed", "boom"),
+            self._sib("failed", "boom"),
+            self._sib("failed", "boom"),
+        ]
+        assert _summarise_child_error_messages(siblings) == "boom"
+
+    def test_mixed_failed_messages_picks_most_common(self):
+        from hindsight_api.worker.poller import _summarise_child_error_messages
+
+        siblings = [
+            self._sib("failed", "common cause"),
+            self._sib("failed", "common cause"),
+            self._sib("failed", "rare cause"),
+        ]
+        assert _summarise_child_error_messages(siblings) == "common cause"
+
+    def test_completed_siblings_ignored(self):
+        from hindsight_api.worker.poller import _summarise_child_error_messages
+
+        siblings = [
+            self._sib("completed", None),
+            self._sib("completed", None),
+            self._sib("failed", "the one real failure"),
+        ]
+        assert _summarise_child_error_messages(siblings) == "the one real failure"
+
+    def test_no_failed_siblings_falls_back_to_generic(self):
+        from hindsight_api.worker.poller import _summarise_child_error_messages
+
+        siblings = [
+            self._sib("completed", None),
+            self._sib("completed", None),
+        ]
+        assert _summarise_child_error_messages(siblings) == "One or more sub-batches failed"
+
+    def test_failed_siblings_with_no_error_message_falls_back_to_generic(self):
+        from hindsight_api.worker.poller import _summarise_child_error_messages
+
+        siblings = [
+            self._sib("failed", None),
+            self._sib("failed", ""),
+        ]
+        assert _summarise_child_error_messages(siblings) == "One or more sub-batches failed"
+
+    def test_whitespace_only_messages_treated_as_empty(self):
+        from hindsight_api.worker.poller import _summarise_child_error_messages
+
+        siblings = [
+            self._sib("failed", "   "),
+            self._sib("failed", "actual error"),
+        ]
+        assert _summarise_child_error_messages(siblings) == "actual error"
+
+
 class TestMarkFailedParentPropagation:
     """Tests for _mark_failed parent propagation in WorkerPoller.
 
@@ -2471,10 +2542,21 @@ class TestMarkFailedParentPropagation:
         assert "DB constraint violation" in child2_row["error_message"]
 
         # parent must now be failed (all siblings done, at least one failed)
-        parent_row = await pool.fetchrow("SELECT status FROM async_operations WHERE operation_id = $1", parent_id)
+        parent_row = await pool.fetchrow(
+            "SELECT status, error_message FROM async_operations WHERE operation_id = $1",
+            parent_id,
+        )
         assert parent_row["status"] == "failed", (
             f"Parent should be 'failed' when last sibling fails, got '{parent_row['status']}'"
         )
+        # Parent error_message must propagate the child's actual error reason,
+        # not the legacy generic "One or more sub-batches failed". Without this,
+        # downstream filters that classify failures by error_message lose all
+        # signal once a batch has children.
+        assert "DB constraint violation" in (parent_row["error_message"] or ""), (
+            f"Parent error_message should inherit child's reason, "
+            f"got: {parent_row['error_message']!r}"
+        )
 
     @pytest.mark.asyncio
     async def test_mark_failed_finalises_parent_when_last_sibling_is_sole_child(self, pool, backend, clean_operations):
@@ -2794,6 +2876,71 @@ class TestClaimBatchRotation:
         finally:
             await pool.execute("DELETE FROM async_operations WHERE operation_id = $1", op_id)
 
+    @pytest.mark.asyncio
+    async def test_scan_uses_optional_routine_when_installed(self, pool, backend, clean_operations):
+        """When ``public.schemas_with_pending_work()`` exists in pg_proc,
+        ``_scan_active_schemas`` invokes it instead of running per-schema
+        EXISTS queries from Python. Confirms the OptionalRoutines probe
+        path is wired correctly end-to-end.
+
+        The routine body installed here is a minimal stand-in that
+        satisfies the contract documented on
+        ``optional_routines.SCHEMAS_WITH_PENDING_WORK`` — Hindsight does
+        not own the canonical implementation.
+        """
+        from hindsight_api.engine.db.postgresql import PostgresConnection
+        from hindsight_api.worker import WorkerPoller
+
+        # Minimal contract-satisfying implementation: returns the empty
+        # set. Enough to prove the poller follows the server-side path.
+        await pool.execute(
+            "CREATE OR REPLACE FUNCTION public.schemas_with_pending_work() "
+            "RETURNS SETOF text AS $$ BEGIN RETURN; END $$ LANGUAGE plpgsql STABLE"
+        )
+        try:
+            poller = WorkerPoller(
+                backend=backend,
+                worker_id="test-routine",
+                executor=lambda x: None,
+            )
+
+            captured_fetch: list[str] = []
+            captured_fetchval: list[str] = []
+            original_fetch = PostgresConnection.fetch
+            original_fetchval = PostgresConnection.fetchval
+
+            async def spy_fetch(self, query, *args, timeout=None):
+                captured_fetch.append(query)
+                return await original_fetch(self, query, *args, timeout=timeout)
+
+            async def spy_fetchval(self, query, *args, column=0, timeout=None):
+                captured_fetchval.append(query)
+                return await original_fetchval(self, query, *args, column=column, timeout=timeout)
+
+            PostgresConnection.fetch = spy_fetch  # type: ignore[method-assign]
+            PostgresConnection.fetchval = spy_fetchval  # type: ignore[method-assign]
+            try:
+                await poller._scan_active_schemas([None])
+            finally:
+                PostgresConnection.fetch = original_fetch  # type: ignore[method-assign]
+                PostgresConnection.fetchval = original_fetchval  # type: ignore[method-assign]
+
+            # Routine was probed (one pg_proc lookup) and then invoked.
+            assert any("pg_proc" in q for q in captured_fetchval), (
+                f"Expected a pg_proc existence probe; fetchval queries: {captured_fetchval}"
+            )
+            assert any("schemas_with_pending_work" in q for q in captured_fetch), (
+                f"Expected schemas_with_pending_work() to be invoked; fetch queries: {captured_fetch}"
+            )
+            # Fallback per-schema EXISTS path must NOT have run.
+            assert not any("async_operations" in q and "EXISTS" in q for q in captured_fetchval), (
+                f"Fallback EXISTS path should be skipped when routine is installed; fetchval queries: {captured_fetchval}"
+            )
+            # Probe result is cached so the next scan skips the pg_proc lookup.
+            assert poller._optional_routines._cache.get("schemas_with_pending_work") is True
+        finally:
+            await pool.execute("DROP FUNCTION IF EXISTS public.schemas_with_pending_work()")
+
     @pytest.mark.asyncio
     async def test_claim_batch_only_queries_active_schemas(self, pool, backend, clean_operations):
         """claim_batch uses _scan_active_schemas to pre-filter, then

hindsight-api-slim/tests/test_worker_main.py

@@ -0,0 +1,35 @@
+"""Tests for hindsight_api.worker.main entry-point helpers."""
+
+import asyncio
+import signal
+from unittest.mock import MagicMock
+
+from hindsight_api.worker.main import _install_shutdown_signal_handlers
+
+
+def test_install_shutdown_signal_handlers_unix_path():
+    """On platforms where asyncio supports signal handlers (Unix), both
+    SIGINT and SIGTERM are registered and the helper reports success."""
+    loop = MagicMock(spec=asyncio.AbstractEventLoop)
+    handler = MagicMock()
+
+    installed = _install_shutdown_signal_handlers(loop, handler)
+
+    assert installed is True
+    loop.add_signal_handler.assert_any_call(signal.SIGINT, handler)
+    loop.add_signal_handler.assert_any_call(signal.SIGTERM, handler)
+    assert loop.add_signal_handler.call_count == 2
+
+
+def test_install_shutdown_signal_handlers_windows_path():
+    """On Windows, asyncio's ProactorEventLoop raises NotImplementedError
+    from add_signal_handler. The helper must swallow it and report failure
+    so the worker keeps running with default Python signal behavior
+    (regression test for issue #1411)."""
+    loop = MagicMock(spec=asyncio.AbstractEventLoop)
+    loop.add_signal_handler.side_effect = NotImplementedError
+    handler = MagicMock()
+
+    installed = _install_shutdown_signal_handlers(loop, handler)
+
+    assert installed is False

hindsight-api/pyproject.toml

@@ -4,12 +4,12 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hindsight-api"
-version = "0.5.6"
+version = "0.6.2"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "hindsight-api-slim[all]>=0.4.17",
+    "hindsight-api-slim[all]==0.6.2",
 ]
 
 [tool.uv.sources]

hindsight-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hindsight-cli"
-version = "0.5.6"
+version = "0.6.2"
 edition = "2021"
 authors = ["Hindsight Team"]
 description = "A beautiful CLI for Hindsight - semantic memory system"

hindsight-cli/src/api.rs

@@ -269,6 +269,7 @@ impl ApiClient {
         bank_id: &str,
         files: Vec<(String, Vec<u8>)>,
         context: Option<String>,
+        strategy: Option<String>,
         verbose: bool,
     ) -> Result<FileRetainResult> {
         self.runtime.block_on(async {
@@ -284,6 +285,9 @@ impl ApiClient {
                     if let Some(ctx) = &context {
                         meta["context"] = serde_json::Value::String(ctx.clone());
                     }
+                    if let Some(strat) = &strategy {
+                        meta["strategy"] = serde_json::Value::String(strat.clone());
+                    }
                     // Use filename stem as document_id for deduplication
                     if let Some(stem) = std::path::Path::new(name)
                         .file_stem()
@@ -1258,8 +1262,8 @@ impl ApiClient {
 
 // Re-export types from the generated client for use in commands
 pub use types::{
-    BankProfileResponse, MemoryItem, RecallRequest, RecallResponse, RecallResult, ReflectRequest,
-    ReflectResponse, RetainRequest,
+    BankProfileResponse, MemoryItem, MemoryItemTimestamp, RecallRequest, RecallResponse,
+    RecallResult, ReflectRequest, ReflectResponse, RetainRequest,
 };
 
 #[cfg(test)]

hindsight-cli/src/commands/memory.rs

@@ -3,7 +3,9 @@ use std::fs;
 use std::path::PathBuf;
 use walkdir::WalkDir;
 
-use crate::api::{ApiClient, MemoryItem, RecallRequest, ReflectRequest, RetainRequest};
+use crate::api::{
+    ApiClient, MemoryItem, MemoryItemTimestamp, RecallRequest, ReflectRequest, RetainRequest,
+};
 use crate::config;
 use crate::output::{self, OutputFormat};
 use crate::ui;
@@ -438,6 +440,7 @@ pub fn retain(
     content: String,
     doc_id: Option<String>,
     context: Option<String>,
+    timestamp: Option<String>,
     r#async: bool,
     document_tags: Option<Vec<String>>,
     verbose: bool,
@@ -451,11 +454,20 @@ pub fn retain(
         None
     };
 
+    // MemoryItem.timestamp is a progenitor anyOf enum; round-trip through JSON to pick the matching variant.
+    let timestamp = match timestamp {
+        Some(s) => Some(
+            serde_json::from_value::<MemoryItemTimestamp>(serde_json::Value::String(s.clone()))
+                .with_context(|| format!("invalid --timestamp value: {:?}", s))?,
+        ),
+        None => None,
+    };
+
     let item = MemoryItem {
         content: content.clone(),
         context,
         metadata: None,
-        timestamp: None,
+        timestamp,
         document_id: Some(doc_id.clone()),
         entities: None,
         tags: None,
@@ -498,6 +510,7 @@ pub fn retain(
     }
 }
 
+#[allow(clippy::too_many_arguments)]
 pub fn retain_files(
     client: &ApiClient,
     agent_id: &str,
@@ -505,6 +518,7 @@ pub fn retain_files(
     recursive: bool,
     context: Option<String>,
     r#async: bool,
+    strategy: Option<String>,
     verbose: bool,
     output_format: OutputFormat,
 ) -> Result<()> {
@@ -567,7 +581,8 @@ pub fn retain_files(
             pb.inc(1);
         }
 
-        let result = client.file_retain(agent_id, file_data, context.clone(), verbose)?;
+        let result =
+            client.file_retain(agent_id, file_data, context.clone(), strategy.clone(), verbose)?;
         all_operation_ids.extend(result.operation_ids);
     }
 

hindsight-cli/src/main.rs

@@ -591,6 +591,12 @@ enum MemoryCommands {
         #[arg(short = 'c', long)]
         context: Option<String>,
 
+        /// When the content occurred (ISO 8601 datetime, e.g. 2024-01-15T10:30:00Z
+        /// or 2024-01-15). Pass "unset" to store without a timestamp.
+        /// Omit to default to now.
+        #[arg(short = 't', long)]
+        timestamp: Option<String>,
+
         /// Queue for background processing
         #[arg(long)]
         r#async: bool,
@@ -619,6 +625,10 @@ enum MemoryCommands {
         /// Queue for background processing
         #[arg(long)]
         r#async: bool,
+
+        /// Named retain strategy to use for these files (overrides the bank's default strategy)
+        #[arg(short = 's', long)]
+        strategy: Option<String>,
     },
 
     /// Delete a memory unit
@@ -1430,6 +1440,7 @@ fn run() -> Result<()> {
                 content,
                 doc_id,
                 context,
+                timestamp,
                 r#async,
                 document_tags,
             } => commands::memory::retain(
@@ -1438,6 +1449,7 @@ fn run() -> Result<()> {
                 content,
                 doc_id,
                 context,
+                timestamp,
                 r#async,
                 document_tags,
                 verbose,
@@ -1449,6 +1461,7 @@ fn run() -> Result<()> {
                 recursive,
                 context,
                 r#async,
+                strategy,
             } => commands::memory::retain_files(
                 &client,
                 &bank_id,
@@ -1456,6 +1469,7 @@ fn run() -> Result<()> {
                 recursive,
                 context,
                 r#async,
+                strategy,
                 verbose,
                 output_format,
             ),

hindsight-cli/tests/integration_test.rs

@@ -91,6 +91,25 @@ fn test_ui_command_with_config() {
     std::fs::remove_dir_all(&temp_dir).ok();
 }
 
+#[test]
+fn test_memory_retain_exposes_timestamp_flag() {
+    // Regression: `hindsight memory retain` historically had no way to set the
+    // memory's event date even though the SDKs do. The flag must appear in
+    // --help so users (and docs) can discover it.
+    let output = Command::new("cargo")
+        .args(["run", "--", "memory", "retain", "--help"])
+        .output()
+        .expect("Failed to execute command");
+
+    assert!(output.status.success(), "retain --help failed");
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    assert!(
+        stdout.contains("--timestamp") && stdout.contains("-t"),
+        "expected --timestamp/-t flag in retain --help, got: {}",
+        stdout
+    );
+}
+
 #[test]
 fn test_configure_command() {
     // Test that configure command creates/updates config

hindsight-clients/go/api/openapi.yaml

@@ -7,7 +7,7 @@ info:
     name: Apache 2.0
     url: https://www.apache.org/licenses/LICENSE-2.0.html
   title: Hindsight HTTP API
-  version: 0.5.6
+  version: 0.6.2
 servers:
 - url: /
 paths:
@@ -3666,7 +3666,7 @@ components:
       - updates
       title: BankConfigUpdate
     BankListItem:
-      description: Bank list item with profile summary.
+      description: Bank list item with profile summary and stats.
       properties:
         bank_id:
           title: Bank Id
@@ -3685,6 +3685,13 @@ components:
         updated_at:
           nullable: true
           type: string
+        fact_count:
+          default: 0
+          title: Fact Count
+          type: integer
+        last_document_at:
+          nullable: true
+          type: string
       required:
       - bank_id
       - disposition
@@ -3699,6 +3706,8 @@ components:
             empathy: 3
             literalism: 3
             skepticism: 3
+          fact_count: 156
+          last_document_at: 2024-01-16T14:20:00Z
           mission: I am a software engineer helping my team ship quality code
           name: Alice
           updated_at: 2024-01-16T14:20:00Z

hindsight-clients/go/api_audit.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.6
+API version: 0.6.2
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_bank_templates.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.6
+API version: 0.6.2
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_banks.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.6
+API version: 0.6.2
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_directives.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.6
+API version: 0.6.2
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_documents.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.6
+API version: 0.6.2
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_entities.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.6
+API version: 0.6.2
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.