Add Python client get_version helper

Adds HindsightClient.get_version()/aget_version() convenience wrappers for the existing /version endpoint, re-exports VersionResponse for typed callers, and tests both paths against a mocked MonitoringApi. Python parity for #2252 (TypeScript getVersion). Fixes #2248.
Add TypeScript client version helper (#2252 )
2026-06-17 11:23:02 +02:00 · 2026-06-17 11:19:35 +02:00 · 2026-06-16 18:34:11 -07:00 · 2026-06-16 16:00:03 -04:00 · 2026-06-16 15:56:53 -04:00 · 2026-06-16 15:53:21 -04:00
682 changed files with 43248 additions and 2808 deletions
@@ -47,6 +47,13 @@ HINDSIGHT_API_LLM_BASE_URL=https://api.openai.com/v1
 HINDSIGHT_API_HOST=0.0.0.0
 HINDSIGHT_API_PORT=8888
 HINDSIGHT_API_LOG_LEVEL=info
+# Optional retain chunking override for structured logs/transcripts.
+# Unset uses HINDSIGHT_API_RETAIN_CHUNK_SIZE as the structured-chunk limit.
+# HINDSIGHT_API_RETAIN_STRUCTURED_CHUNK_SIZE=
+
+# Dry-run extraction preview endpoint (POST /memories/dry-run-extract). Enabled by default; it makes
+# a real LLM call but stores nothing. Set to false to remove the endpoint (returns 404).
+# HINDSIGHT_API_ENABLE_DRY_RUN_EXTRACT=true

 # Base Path / Reverse Proxy Support (Optional)
 # Set these when deploying behind a reverse proxy with path-based routing
@@ -59,6 +66,7 @@ HINDSIGHT_API_LOG_LEVEL=info
 # HINDSIGHT_API_READ_DATABASE_URL=  # Optional read-replica URL. When set, recall queries (semantic, BM25, graph, temporal) flow through a separate pool against this URL, offloading the primary. Typically points to a read-only endpoint (CNPG's <cluster>-ro service or Aurora reader endpoint).
 # HINDSIGHT_API_MIGRATION_DATABASE_URL=  # Direct PostgreSQL URL for migrations (bypasses PgBouncer). Falls back to DATABASE_URL.
 # HINDSIGHT_API_DATABASE_SCHEMA=public  # PostgreSQL schema name (default: public)
+# HINDSIGHT_API_MIGRATION_CONCURRENCY=1  # Tenant schemas to migrate concurrently (PG only, each in its own process; per-schema work stays sequential). Each worker has ~1-2s startup cost + uses ~3 DB connections, so it only pays off with many schemas (tens+) or slow migrations; keep concurrency*3 <= spare max_connections. Default: 1 (sequential).

 # Vector Extension (Optional - uses pgvector by default)
 # Options: "pgvector" (default), "vchord", "pgvectorscale" (DiskANN)
@@ -32,10 +32,13 @@ jobs:
      integration-tests: ${{ steps.filter.outputs.integration-tests }}
      integrations-openclaw: ${{ steps.filter.outputs.integrations-openclaw }}
      integrations-ai-sdk: ${{ steps.filter.outputs.integrations-ai-sdk }}
+      integrations-agent-framework: ${{ steps.filter.outputs.integrations-agent-framework }}
+      integrations-composio: ${{ steps.filter.outputs.integrations-composio }}
      integrations-chat: ${{ steps.filter.outputs.integrations-chat }}
      integrations-claude-code: ${{ steps.filter.outputs.integrations-claude-code }}
      integrations-cline: ${{ steps.filter.outputs.integrations-cline }}
      integrations-codex: ${{ steps.filter.outputs.integrations-codex }}
+      integrations-continue: ${{ steps.filter.outputs.integrations-continue }}
      integrations-cursor-cli: ${{ steps.filter.outputs.integrations-cursor-cli }}
      integrations-crewai: ${{ steps.filter.outputs.integrations-crewai }}
      integrations-litellm: ${{ steps.filter.outputs.integrations-litellm }}
@@ -48,6 +51,7 @@ jobs:
      integrations-opencode: ${{ steps.filter.outputs.integrations-opencode }}
      integrations-cursor: ${{ steps.filter.outputs.integrations-cursor }}
      integrations-n8n: ${{ steps.filter.outputs.integrations-n8n }}
+      integrations-zapier: ${{ steps.filter.outputs.integrations-zapier }}
      integrations-cloudflare-oauth-proxy: ${{ steps.filter.outputs.integrations-cloudflare-oauth-proxy }}
      integrations-superagent: ${{ steps.filter.outputs.integrations-superagent }}
      integrations-lockfiles: ${{ steps.filter.outputs.integrations-lockfiles }}
@@ -124,6 +128,10 @@ jobs:
            - 'hindsight-integrations/openclaw/**'
          integrations-ai-sdk:
            - 'hindsight-integrations/ai-sdk/**'
+          integrations-agent-framework:
+            - 'hindsight-integrations/agent-framework/**'
+          integrations-composio:
+            - 'hindsight-integrations/composio/**'
          integrations-chat:
            - 'hindsight-integrations/chat/**'
          integrations-claude-code:
@@ -132,6 +140,8 @@ jobs:
            - 'hindsight-integrations/cline/**'
          integrations-codex:
            - 'hindsight-integrations/codex/**'
+          integrations-continue:
+            - 'hindsight-integrations/continue/**'
          integrations-cursor-cli:
            - 'hindsight-integrations/cursor-cli/**'
          integrations-crewai:
@@ -158,6 +168,8 @@ jobs:
            - 'hindsight-integrations/cursor/**'
          integrations-n8n:
            - 'hindsight-integrations/n8n/**'
+          integrations-zapier:
+            - 'hindsight-integrations/zapier/**'
          integrations-cloudflare-oauth-proxy:
            - 'hindsight-integrations/cloudflare-oauth-proxy/**'
          integrations-superagent:
@@ -727,6 +739,37 @@ jobs:
      working-directory: ./hindsight-integrations/n8n
      run: npm run build

+  test-zapier-integration:
+    needs: [detect-changes]
+    if: >-
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-zapier == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v6
+      with:
+        node-version: '22'
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/zapier
+      run: npm install --no-fund --no-audit
+
+    - name: Validate app definition
+      working-directory: ./hindsight-integrations/zapier
+      run: npm run validate
+
+    - name: Run tests
+      working-directory: ./hindsight-integrations/zapier
+      run: npm test
+
  test-hindsight-agent-sdk:
    needs: [detect-changes]
    if: >-
@@ -3017,6 +3060,88 @@ jobs:
      # (requires_real_llm) needs a live Hindsight server and runs separately.
      run: uv run pytest tests -v -m "not requires_real_llm"

+  test-composio-integration:
+    needs: [detect-changes]
+    if: >-
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-composio == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version-file: ".python-version"
+
+    - name: Build composio integration
+      working-directory: ./hindsight-integrations/composio
+      run: uv build
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/composio
+      run: uv sync --frozen
+
+    - name: Lint
+      working-directory: ./hindsight-integrations/composio
+      run: uv run ruff check .
+
+    - name: Run tests
+      working-directory: ./hindsight-integrations/composio
+      # PR CI runs only the deterministic bucket; the real-LLM E2E bucket
+      # (requires_real_llm) needs a live Hindsight server and runs separately.
+      run: uv run pytest tests -v -m "not requires_real_llm"
+
+  test-continue-integration:
+    needs: [detect-changes]
+    if: >-
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-continue == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version-file: ".python-version"
+
+    - name: Build continue integration
+      working-directory: ./hindsight-integrations/continue
+      run: uv build
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/continue
+      run: uv sync --frozen
+
+    - name: Run tests
+      working-directory: ./hindsight-integrations/continue
+      # PR CI runs only the deterministic bucket; the real-LLM E2E bucket
+      # (requires_real_llm) needs a live Hindsight server and runs separately.
+      run: uv run pytest tests -v -m "not requires_real_llm"
+
  test-smolagents-integration:
    needs: [detect-changes]
    if: >-
@@ -3155,6 +3280,49 @@ jobs:
      working-directory: ./hindsight-integrations/obsidian
      run: npm test

+  test-agent-framework-integration:
+    needs: [detect-changes]
+    if: >-
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-agent-framework == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version-file: ".python-version"
+
+    - name: Build agent-framework integration
+      working-directory: ./hindsight-integrations/agent-framework
+      run: uv build
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/agent-framework
+      run: uv sync --frozen
+
+    - name: Lint
+      working-directory: ./hindsight-integrations/agent-framework
+      run: uv run ruff check .
+
+    - name: Run tests
+      working-directory: ./hindsight-integrations/agent-framework
+      # PR CI runs only the deterministic bucket; the real-LLM E2E bucket
+      # (requires_real_llm) needs a live Hindsight server and runs separately.
+      run: uv run pytest tests -v -m "not requires_real_llm"
+
  test-crewai-integration:
    needs: [detect-changes]
    if: >-
@@ -3645,6 +3813,49 @@ jobs:
        echo "=== API Server Logs ==="
        cat /tmp/slim-api-server.log 2>/dev/null || true

+  verify-embed-control-center-bundle:
+    # The control center UI (Preact + Tailwind) is built with Vite and its static
+    # output is committed (served as-is by the embed's Python http.server, no Node
+    # at runtime). We can't byte-diff the committed bundle against a fresh build —
+    # Vite's content-hashed asset filenames aren't reproducible across the CI
+    # runner's OS/arch vs the committer's. So instead verify: (1) the committed
+    # bundle is a real, wired Vite build (index.html references JS/CSS that exist),
+    # and (2) the source still builds cleanly.
+    needs: [detect-changes]
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.embed == 'true' ||
+      needs.detect-changes.outputs.ci == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Setup Node
+      uses: actions/setup-node@v6
+      with:
+        node-version: '22'
+
+    # Check the committed bundle BEFORE building (the build overwrites static/).
+    - name: Verify the committed bundle is wired
+      working-directory: ./hindsight-embed/hindsight_embed/control_center
+      run: |
+        test -f static/index.html || { echo "::error::static/index.html missing — run 'npm run build' in control_center/ui and commit static/"; exit 1; }
+        js=$(grep -oE 'assets/[A-Za-z0-9_.-]+\.js' static/index.html | head -1)
+        css=$(grep -oE 'assets/[A-Za-z0-9_.-]+\.css' static/index.html | head -1)
+        { [ -n "$js" ] && [ -f "static/$js" ]; } || { echo "::error::index.html does not reference a committed JS bundle — rebuild the UI and commit static/"; exit 1; }
+        { [ -n "$css" ] && [ -f "static/$css" ]; } || { echo "::error::index.html does not reference a committed CSS bundle — rebuild the UI and commit static/"; exit 1; }
+        echo "committed bundle is wired ✓"
+
+    - name: Verify the source builds cleanly
+      working-directory: ./hindsight-embed/hindsight_embed/control_center/ui
+      run: |
+        npm ci
+        npm run build
+        echo "control center UI builds ✓"
+
  test-embed:
    needs: [detect-changes]
    if: >-
@@ -3860,7 +4071,8 @@ jobs:
        target="$RUNNER_TEMP/install-test"
        PYTHONPATH="$target" python -c "
        from hindsight_embed.daemon_embed_manager import DaemonEmbedManager
-        cmd = DaemonEmbedManager()._find_api_command()
+        # api_version is only used for the uvx fallback; the binary branch ignores it.
+        cmd = DaemonEmbedManager()._find_api_command('0.0.0')
        print('Resolved command:', cmd)
        assert len(cmd) == 1 and cmd[0].endswith('hindsight-api.exe'), (
            f'Expected sibling hindsight-api.exe, got {cmd!r}. '
@@ -4221,6 +4433,60 @@ jobs:
          fi
        done

+  # Dead-code detection beyond what ruff's F401/F841 catch (those are already
+  # BLOCKING via the ruff config + the verify-generated-files job).
+  #
+  # - knip (control plane): BLOCKING on unused files / dependencies / unlisted
+  #   dependencies. These are unambiguous — an orphaned file or a dead
+  #   package.json entry — so they fail the build.
+  # - vulture (Python) + knip unused *exports*: ADVISORY only. vulture's
+  #   function/argument heuristics false-positive on FastAPI/SQLAlchemy/Pydantic
+  #   patterns, and the control plane intentionally keeps an unused shadcn/ui
+  #   component surface, so these are surfaced in the step summary, not gated.
+  check-unused-code:
+    needs: [detect-changes]
+    if: >-
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.core == 'true' ||
+      needs.detect-changes.outputs.control-plane == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v6
+      with:
+        node-version: '20'
+        cache: 'npm'
+        cache-dependency-path: package-lock.json
+
+    - name: Install Control Plane dependencies
+      run: npm install --workspace=hindsight-control-plane
+
+    - name: knip — unused files / dependencies (blocking)
+      working-directory: hindsight-control-plane
+      run: npx --yes knip@5 --no-progress --include files,dependencies,unlisted
+
+    - name: Advisory scan — vulture + knip exports
+      continue-on-error: true
+      run: |
+        {
+          echo '## Dead-code scan (advisory)'
+          echo ''
+          echo '```'
+          ./scripts/hooks/check-unused.sh 2>&1 | sed 's/\x1b\[[0-9;]*m//g'
+          echo '```'
+        } | tee -a "$GITHUB_STEP_SUMMARY"
+
  verify-generated-files:
    runs-on: ubuntu-latest
    timeout-minutes: 30
@@ -4438,10 +4704,12 @@ jobs:
      - test-integration
      - test-ag2-integration
      - test-autogen-integration
+      - test-continue-integration
      - test-smolagents-integration
      - test-dify-integration
      - test-flowise-integration
      - test-obsidian-integration
+      - test-agent-framework-integration
      - test-crewai-integration
      - test-langgraph-integration
      - test-superagent-integration
@@ -4454,6 +4722,7 @@ jobs:
      - test-pip-slim
      - test-embed
      - test-embed-windows
+      - verify-embed-control-center-bundle
      - test-hindsight-all
      - test-hindsight-agent-sdk
      - test-claude-agent-sdk-integration
@@ -216,6 +216,18 @@ migration file dispatches through `run_for_dialect`, which calls either
 ./scripts/hooks/lint.sh
 ```

+Dead-code detection runs in CI (the `check-unused-code` job) at two levels:
+- **Blocking:** unused imports (ruff `F401`) and variables (`F841`) — `lint.sh` auto-removes
+  them and `verify-generated-files` fails on any leftover diff; and **knip** for orphaned
+  control-plane files / unused (or unlisted) `package.json` dependencies.
+- **Advisory:** whole unused Python functions (vulture) and unused control-plane *exports*
+  (the shadcn/ui surface is kept on purpose) — surfaced, not gated.
+
+Run both locally with:
+```bash
+./scripts/hooks/check-unused.sh
+```
+
 **After completing any implementation work, run `/code-review`** to verify your changes against project standards (missing tests, dead code, type safety, etc.). Fix any "must fix" issues before considering the task done.

 **MANDATORY: Run `/code-review` before pushing code or creating a pull request.** Do not push or create a PR until all "must fix" issues are resolved.
@@ -315,7 +327,10 @@ Fields must be categorized as either **hierarchical** (can be overridden per-ten
   ```

 2. **main.py** (`hindsight-api-slim/hindsight_api/main.py`):
-   - Add field to the manual `HindsightConfig()` constructor call (search for "CLI override")
+   - No change is needed for ordinary environment-backed config fields. The CLI starts from `_get_raw_config()`,
+     so new `HindsightConfig` fields are carried through automatically.
+   - If the new field should be overridable by a CLI flag, add the argparse option in `_parse_cli_args()` and include
+     that field in the `dataclasses.replace(config, ...)` call near the "CLI override" comment.

 3. **Use hierarchical config in MemoryEngine**:
   ```python
@@ -361,7 +376,7 @@ Fields must be categorized as either **hierarchical** (can be overridden per-ten

 ```bash
 cp .env.example .env
-# Edit .env with LLM API key
+# Edit .env with the LLM provider/model and credentials for your setup

 # Python deps
 uv sync --directory hindsight-api-slim/
@@ -370,10 +385,10 @@ uv sync --directory hindsight-api-slim/
 npm install
 ```

-Required env vars:
+Common LLM settings:
 - `HINDSIGHT_API_LLM_PROVIDER`: openai, anthropic, gemini, groq, minimax, ollama, lmstudio
- `HINDSIGHT_API_LLM_API_KEY`: Your API key
- `HINDSIGHT_API_LLM_MODEL`: Model name (e.g., gpt-4o-mini, claude-sonnet-4-20250514)
+- `HINDSIGHT_API_LLM_API_KEY`: API key for providers that require one
+- `HINDSIGHT_API_LLM_MODEL`: Model name (defaults are provider-specific)

 Optional (uses local models by default):
 - `HINDSIGHT_API_EMBEDDINGS_PROVIDER`: local (default) or tei
@@ -7,7 +7,6 @@
 [![CI](https://github.com/vectorize-io/hindsight/actions/workflows/release.yml/badge.svg)](https://github.com/vectorize-io/hindsight/actions/workflows/release.yml)
 [![Slack Community](https://img.shields.io/badge/Slack-Join%20Community-4A154B?logo=slack)](https://join.slack.com/t/hindsight-space/shared_invite/zt-3nhbm4w29-LeSJ5Ixi6j8PdiYOCPlOgg)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
-[![gitcgr](https://gitcgr.com/badge/vectorize-io/hindsight.svg)](https://gitcgr.com/vectorize-io/hindsight)
 ![PyPI - Downloads](https://img.shields.io/pypi/dm/hindsight-api?label=PyPI)
 ![NPM Downloads](https://img.shields.io/npm/dm/%40vectorize-io%2Fhindsight-client?logoColor=orange&label=NPM&color=blue&link=https%3A%2F%2Fwww.npmjs.com%2Fpackage%2F%40vectorize-io%2Fhindsight-client)
 <br/>
@@ -311,7 +310,7 @@ client.reflect(bank_id="my-bank", query="What should I know about Alice?")
 | **macOS** (Intel / x86_64) | ✅ | ⚠️ | ✅ |
 | **Windows** (x86_64) | ✅ | ✅ | ✅ |

-⚠️ Intel Macs: use `hindsight-all-slim` — see the [installation guide](https://docs.hindsight.vectorize.io/docs/developer/installation#supported-platforms) for details.
+⚠️ Intel Macs: use `hindsight-all-slim` — see the [installation guide](https://hindsight.vectorize.io/developer/installation#supported-platforms) for details.

 ---

@@ -77,7 +77,6 @@
            "npm:@radix-ui/react-dropdown-menu@^2.1.16",
            "npm:@radix-ui/react-label@^2.1.8",
            "npm:@radix-ui/react-popover@^1.1.15",
-            "npm:@radix-ui/react-radio-group@^1.3.8",
            "npm:@radix-ui/react-select@^2.2.6",
            "npm:@radix-ui/react-slider@^1.3.6",
            "npm:@radix-ui/react-slot@^1.2.4",
@@ -1,6 +1,6 @@
 name: hindsight
 # Docker Compose file for Hindsight with PostgreSQL and vectorchord
-# docker compose -f docker/docker-compose/docker-compose.yaml down && sleep 2 && docker compose -f docker/docker-compose/docker-compose.yaml up -d
+# docker compose -f docker/docker-compose/vchord/docker-compose.yaml down && sleep 2 && docker compose -f docker/docker-compose/vchord/docker-compose.yaml up -d
 # Make sure to set the required environment variables before running:
 # - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
 # - Configure LLM provider variables as needed (see below in the hindsight service)
@@ -2,8 +2,8 @@ apiVersion: v2
 name: hindsight
 description: Hindsight helm chart
 type: application
-version: 0.8.1
-appVersion: "0.8.1"
+version: 0.8.2
+appVersion: "0.8.2"
 keywords:
  - ai
  - memory
@@ -66,13 +66,13 @@ helm install hindsight ./helm/hindsight -n hindsight --create-namespace -f value

 | Parameter | Description | Default |
 |-----------|-------------|---------|
-| `version` | Default image tag for all components | `0.1.0` |
+| `version` | Default image tag for all components | Chart `appVersion` |
 | `api.enabled` | Enable the API component | `true` |
-| `api.image.repository` | API image repository | `hindsight/api` |
+| `api.image.repository` | API image repository | `ghcr.io/vectorize-io/hindsight-api` |
 | `api.image.tag` | API image tag (defaults to `version`) | - |
 | `api.service.port` | API service port | `8888` |
 | `controlPlane.enabled` | Enable the control plane | `true` |
-| `controlPlane.image.repository` | Control plane image repository | `hindsight/control-plane` |
+| `controlPlane.image.repository` | Control plane image repository | `ghcr.io/vectorize-io/hindsight-control-plane` |
 | `controlPlane.image.tag` | Control plane image tag (defaults to `version`) | - |
 | `controlPlane.service.port` | Control plane service port | `3000` |
 | `postgresql.enabled` | Deploy PostgreSQL as subchart | `true` |
@@ -13,9 +13,6 @@
 #   - Any other env vars you want to inject
 # existingSecret: "my-hindsight-secret"

-# Global settings
-replicaCount: 1
-
 # Image settings for api
 api:
  enabled: true
@@ -1,6 +1,6 @@
 {
  "name": "@vectorize-io/hindsight-all",
-  "version": "0.8.1",
+  "version": "0.8.2",
  "description": "Node.js programmatic lifecycle manager for Hindsight — embeds a local hindsight daemon in a Node application. Pair with @vectorize-io/hindsight-client for memory operations.",
  "main": "dist/index.js",
  "types": "dist/index.d.ts",
@@ -4,12 +4,12 @@ build-backend = "setuptools.build_meta"

 [project]
 name = "hindsight-all-slim"
-version = "0.8.1"
+version = "0.8.2"
 description = "Hindsight: Agent Memory That Works Like Human Memory - Slim All-in-One Bundle"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "hindsight-api-slim==0.8.1",
+    "hindsight-api-slim==0.8.2",
    "hindsight-client>=0.0.7",
    "hindsight-embed>=0.1.0",
 ]
@@ -4,12 +4,12 @@ build-backend = "hatchling.build"

 [project]
 name = "hindsight-all"
-version = "0.8.1"
+version = "0.8.2"
 description = "Hindsight: Agent Memory That Works Like Human Memory - All-in-One Bundle"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "hindsight-api-slim[all]==0.8.1",
+    "hindsight-api-slim[all]==0.8.2",
    "hindsight-client>=0.0.7",
    "hindsight-embed>=0.1.0",
 ]
@@ -21,7 +21,7 @@ hindsight-embed = { workspace = true }

 [project.optional-dependencies]
 local-llm = [
-    "hindsight-api-slim[local-llm]==0.8.1",
+    "hindsight-api-slim[local-llm]==0.8.2",
 ]
 test = [
    "pytest>=7.0.0",
@@ -53,4 +53,4 @@ __all__ = [
    "RemoteTEICrossEncoder",
    "LLMConfig",
 ]
-__version__ = "0.8.1"
+__version__ = "0.8.2"
@@ -258,12 +258,7 @@ async def _run_migration(
    embedding_dimension: int | None = None,
 ) -> list[str]:
    """Resolve database URL and run migrations for one schema or all discovered schemas."""
-    from ..migrations import (
-        ensure_embedding_dimension,
-        ensure_text_search_extension,
-        ensure_vector_extension,
-        run_migrations,
-    )
+    from ..migrations import run_migrations_for_schemas

    is_pg0, instance_name, _ = parse_pg0_url(db_url)
    if is_pg0:
@@ -284,32 +279,21 @@ async def _run_migration(
        # Preserve order while removing duplicates.
        schemas = list(dict.fromkeys(schemas))

-    for schema in schemas:
-        run_migrations(resolved_url, schema=schema, migration_database_url=config.migration_database_url)
-
-    if embedding_dimension is not None:
-        for schema in schemas:
-            ensure_embedding_dimension(
-                resolved_url,
-                embedding_dimension,
-                schema=schema,
-                vector_extension=config.vector_extension,
-            )
-
-    for schema in schemas:
-        ensure_vector_extension(
-            resolved_url,
-            vector_extension=config.vector_extension,
-            schema=schema,
-        )
-
-    for schema in schemas:
-        ensure_text_search_extension(
-            resolved_url,
-            text_search_extension=config.text_search_extension,
-            pg_search_tokenizer=config.text_search_extension_pg_search_tokenizer,
-            schema=schema,
-        )
+    # Migrate up to `migration_concurrency` schemas at once (each in its own
+    # process); within a schema the work stays sequential. Run off the event
+    # loop so the process pool's blocking joins don't stall it.
+    await asyncio.to_thread(
+        run_migrations_for_schemas,
+        resolved_url,
+        schemas,
+        concurrency=config.migration_concurrency,
+        migration_database_url=config.migration_database_url,
+        embedding_dimension=embedding_dimension,
+        vector_extension=config.vector_extension,
+        text_search_extension=config.text_search_extension,
+        pg_search_tokenizer=config.text_search_extension_pg_search_tokenizer,
+        ensure_extensions=True,
+    )

    return schemas

@@ -0,0 +1,105 @@
+"""Add a composite index on memory_links(bank_id, link_type) (PostgreSQL).
+
+``bank_id`` was added to ``memory_links`` in ``c5d6e7f8a9b0`` precisely so that
+bank-scoped reads (e.g. the stats endpoint) could filter on the link table
+directly instead of joining ``memory_units`` — that JOIN took 18+ seconds on
+banks with millions of links. The column landed without an index, so every
+``bank_id = $1`` predicate still falls back to a sequential scan over the whole
+table.
+
+This adds the missing btree. It is composite on ``(bank_id, link_type)`` rather
+than ``bank_id`` alone because the hot query is the stats endpoint's
+``SELECT link_type, COUNT(*) ... WHERE bank_id = $1 GROUP BY link_type``: a
+``(bank_id, link_type)`` index serves that filter, grouping and count as an
+index-only scan, never touching the heap, whereas a ``bank_id``-only index would
+still have to read every matching row to recover ``link_type``. ``link_type`` is
+low-cardinality (only ``temporal``/``semantic``/``caused_by`` are written —
+entity edges were dropped in ``e9b2c7d1f3a4``), so the trailing column adds
+little to the index size while removing the heap fetch.
+
+The Oracle baseline (``o1a2b3c4d5e6``) already creates ``idx_ml_bank_id`` on
+``memory_links(bank_id)``; that single-column index already covers Oracle's
+bank-scoped filter, so the Oracle slot here is intentionally absent and only the
+PostgreSQL dialect gets the composite index.
+
+``memory_links`` can hold tens of millions of rows, so the index is built
+CONCURRENTLY to avoid taking a write lock on the table. CONCURRENTLY cannot run
+inside a transaction block, so the statement runs in an ``autocommit_block()``;
+``IF NOT EXISTS`` keeps it idempotent across retries and re-migrated tenant
+schemas. A CONCURRENTLY build interrupted partway (lock conflict, disk
+pressure, signal) leaves the index behind as *invalid*; ``IF NOT EXISTS`` would
+then skip over it forever, so the upgrade first drops any invalid leftover of
+this name before (re)creating it.
+
+Revision ID: 2071c7518f88
+Revises: a1d3f5b7c9e2
+Create Date: 2026-06-16
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+from sqlalchemy import text
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "2071c7518f88"
+down_revision: str | Sequence[str] | None = "a1d3f5b7c9e2"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_INDEX_NAME = "idx_memory_links_bank_id_link_type"
+
+
+def _get_schema_prefix() -> str:
+    """Schema-qualifier for raw SQL on PG (multi-tenant search_path)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def _pg_upgrade() -> None:
+    bind = op.get_bind()
+    # `or None` collapses an unset option and an explicit empty string into NULL
+    # so the COALESCE below falls back to current_schema() in both cases.
+    target_schema = context.config.get_main_option("target_schema") or None
+    schema = _get_schema_prefix()
+
+    # CREATE INDEX CONCURRENTLY cannot run inside a transaction block; the
+    # autocommit_block runs each statement outside Alembic's migration
+    # transaction.
+    with op.get_context().autocommit_block():
+        # A CONCURRENTLY build that errored on a previous run leaves an INVALID
+        # index of this name behind. `CREATE INDEX ... IF NOT EXISTS` would see
+        # that relation and skip, so bank_id queries would keep seq-scanning.
+        # Drop only the invalid leftover — never a healthy index — so the retry
+        # actually rebuilds a usable one.
+        leftover_invalid = bind.execute(
+            text(
+                "SELECT NOT i.indisvalid "
+                "FROM pg_class c "
+                "JOIN pg_index i ON c.oid = i.indexrelid "
+                "JOIN pg_namespace n ON c.relnamespace = n.oid "
+                "WHERE c.relname = :index_name "
+                "  AND n.nspname = COALESCE(:target_schema, current_schema())"
+            ),
+            {"index_name": _INDEX_NAME, "target_schema": target_schema},
+        ).scalar()
+        if leftover_invalid:
+            op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}{_INDEX_NAME}")
+
+        # IF NOT EXISTS keeps the create idempotent across retries and schemas.
+        op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {_INDEX_NAME} ON {schema}memory_links(bank_id, link_type)")
+
+
+def _pg_downgrade() -> None:
+    schema = _get_schema_prefix()
+    with op.get_context().autocommit_block():
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}{_INDEX_NAME}")
+
+
+def upgrade() -> None:
+    run_for_dialect(pg=_pg_upgrade)
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade)
@@ -0,0 +1,85 @@
+"""Repair: widen the remaining live ``bank_id`` columns from VARCHAR(64) to TEXT on PostgreSQL.
+
+Follow-up to ``c3e5a7b9d1f4`` (issue #2106), which widened the two *history*
+tables (``observation_history``, ``mental_model_history``) to ``TEXT`` after the
+narrow ``VARCHAR(64)`` declaration bricked startup. The same VARCHAR(64) / TEXT
+inconsistency still affects the live tables that store a user-supplied
+``bank_id``:
+
+* ``directives``    -- created VARCHAR(64) in ``p1k2l3m4n5o6``
+* ``mental_models`` -- VARCHAR(64) (origin ``pinned_reflections`` in
+                       ``n9i0j1k2l3m4``; recreated in ``h3c4d5e6f7g8``)
+
+``mental_model_versions`` is intentionally *not* widened here: it is created in
+``j5e6f7g8h9i0`` but dropped (``DROP TABLE ... CASCADE``) in ``o0j1k2l3m4n5`` and
+never recreated on the upgrade path, so it does not exist at head. Issuing
+``ALTER TABLE mental_model_versions ...`` would raise ``UndefinedTable`` and --
+because migrations run inside the lifespan-startup transaction -- roll the whole
+migration back, bricking the API. (It is unrelated to the live
+``mental_model_history`` table widened by ``c3e5a7b9d1f4``.)
+
+``banks.bank_id`` is ``TEXT`` (unbounded), so a deployment can create a bank
+whose id exceeds 64 chars -- the 78-char hierarchical org-unit shape reported in
+issue #2106 -- and the bank insert succeeds. The next write that propagates that
+id (``create_directive``, ``create_mental_model`` / consolidation, or
+mental-model versioning) then aborts with::
+
+    psycopg2.errors.StringDataRightTruncation: value too long for type
+    character varying(64)
+
+i.e. a 500 on core write endpoints, instead of the startup brick that
+``c3e5a7b9d1f4`` already repaired.
+
+``ALTER COLUMN ... TYPE TEXT`` is a no-op on a column that is already ``TEXT``,
+so every upgrade path converges on ``TEXT``. These tables are per-tenant (they
+live in each tenant schema, not ``public``), so this runs for every migrated
+schema via the search-path-aware prefix -- the same mechanism as
+``c3e5a7b9d1f4``.
+
+PostgreSQL only: these tables are created by PostgreSQL-only migrations
+(``run_for_dialect(pg=...)``); on Oracle they are absent or already
+``VARCHAR2(256)`` (consistent, never truncates), so the Oracle slot is
+intentionally absent -- mirroring ``c3e5a7b9d1f4``.
+
+Revision ID: a1d3f5b7c9e2
+Revises: c3e5a7b9d1f4
+Create Date: 2026-06-13
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "a1d3f5b7c9e2"
+down_revision: str | Sequence[str] | None = "c3e5a7b9d1f4"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Schema-qualifier for raw SQL on PG (multi-tenant search_path)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def _pg_upgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}directives ALTER COLUMN bank_id TYPE TEXT")
+    op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN bank_id TYPE TEXT")
+
+
+def _pg_downgrade() -> None:
+    # No-op: narrowing back to VARCHAR(64) could truncate real data and would
+    # re-introduce the bug this migration repairs. The column types are owned by
+    # the migrations that created the tables.
+    pass
+
+
+def upgrade() -> None:
+    run_for_dialect(pg=_pg_upgrade)
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade)
@@ -6,8 +6,11 @@ place. If a row is in ``memory_units`` it is live; if it is in
 ``invalidated_memory_units`` it has been retired. Recall/consolidation/graph
 queries never need a state predicate — the rows simply aren't there.

-The archive mirrors ``memory_units`` column-for-column (so a row round-trips
-losslessly on revert) plus:
+The archive mirrors ``memory_units`` column-for-column — except ``embedding``,
+which it never keeps: the archive is cold storage, never a recall surface, and
+revert recomputes the embedding from the unit's text/dates/entities. Keeping no
+archive vector also means a later embedding-model switch (which re-dimensions
+``memory_units``) can't trip a dimension mismatch on the move (#2209). Plus:
 - ``invalidation_reason``  optional free text recorded on invalidate
 - ``invalidated_at``       when it was retired
 - ``entity_ids``           snapshot of the unit's entity associations, so revert
@@ -49,13 +52,18 @@ def _pg_upgrade() -> None:
    # Add edited_at to the live table FIRST so the archive's LIKE clone below
    # inherits it (keeps the two tables column-for-column identical for round-trip).
    op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS edited_at TIMESTAMPTZ")
-    # LIKE ... INCLUDING DEFAULTS clones every memory_units column (incl. the
-    # embedding vector and edited_at) so an invalidated row can move back verbatim.
-    # We deliberately omit indexes/constraints — the archive is cold storage, not a
-    # recall surface; only the lookups below need indexing.
+    # LIKE ... INCLUDING DEFAULTS clones every memory_units column (incl.
+    # edited_at) so an invalidated row can move back verbatim. We deliberately
+    # omit indexes/constraints — the archive is cold storage, not a recall
+    # surface; only the lookups below need indexing.
    op.execute(
        f"CREATE TABLE IF NOT EXISTS {schema}invalidated_memory_units (LIKE {schema}memory_units INCLUDING DEFAULTS)"
    )
+    # ...then drop the inherited embedding: the archive never stores one (revert
+    # recomputes it), so it isn't created here only to be dropped again later by
+    # d4f6a8c2e1b3. That migration still runs as a no-op (DROP ... IF EXISTS) on
+    # fresh DBs and does the real drop on DBs created before this column was removed.
+    op.execute(f"ALTER TABLE {schema}invalidated_memory_units DROP COLUMN IF EXISTS embedding")
    op.execute(
        f"ALTER TABLE {schema}invalidated_memory_units "
        f"ADD COLUMN IF NOT EXISTS invalidation_reason TEXT, "
@@ -0,0 +1,93 @@
+"""Drop the embedding column from the curation archive (invalidated_memory_units).
+
+The archive is cold storage, never a recall surface, so it has no business
+keeping an embedding. Earlier curation code copied the live row's embedding into
+``invalidated_memory_units`` on invalidate; the engine now leaves it out on
+invalidate and recomputes it on revert, so the column is dead weight.
+
+Dropping it makes "the archive holds no embedding" a schema-enforced invariant
+rather than a convention the move queries have to honour, and removes a latent
+failure mode (#2209): after an embedding-model switch the live tables are
+re-dimensioned but the archive was not, so a stale old-dimension embedding in
+the archive tripped a vector-dimension mismatch on the INSERT … SELECT
+round-trip. With no column at all, there is nothing to mismatch.
+
+The creation sites no longer add the column (the PG ``LIKE`` clone in
+c9a1b2d3e4f5 drops it; the Oracle baseline omits it), so on a fresh database
+this migration is a no-op (DROP ... IF EXISTS / Oracle ORA-00904 swallow). It
+does the real work on databases created before the column was removed there.
+
+DROP COLUMN is a metadata-only operation on both PostgreSQL and Oracle 23ai (no
+table rewrite), so it is cheap even across many tenant schemas. The downgrade
+re-adds an unconstrained vector column (any dimension) — empty, since the
+embeddings are intentionally discarded.
+
+Revision ID: d4f6a8c2e1b3
+Revises: a1d3f5b7c9e2
+Create Date: 2026-06-15
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "d4f6a8c2e1b3"
+down_revision: str | Sequence[str] | None = "a1d3f5b7c9e2"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _pg_schema_prefix() -> str:
+    """Schema-qualifier for raw SQL on PG (multi-tenant search_path)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def _pg_upgrade() -> None:
+    schema = _pg_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}invalidated_memory_units DROP COLUMN IF EXISTS embedding")
+
+
+def _pg_downgrade() -> None:
+    schema = _pg_schema_prefix()
+    # Unconstrained `vector` (no dimension) so the re-added column accepts any
+    # model's embeddings; it comes back empty regardless.
+    op.execute(f"ALTER TABLE {schema}invalidated_memory_units ADD COLUMN IF NOT EXISTS embedding vector")
+
+
+def _oracle_upgrade() -> None:
+    # Oracle has no `DROP COLUMN IF EXISTS`; swallow ORA-00904 (column does not
+    # exist) so the migration is idempotent and safe on a fresh schema whose
+    # baseline already omits the column.
+    op.execute(
+        """
+        BEGIN
+            EXECUTE IMMEDIATE 'ALTER TABLE invalidated_memory_units DROP COLUMN embedding';
+        EXCEPTION WHEN OTHERS THEN
+            IF SQLCODE != -904 THEN RAISE; END IF;
+        END;
+        """
+    )
+
+
+def _oracle_downgrade() -> None:
+    # Swallow ORA-01430 (column already exists) for idempotency.
+    op.execute(
+        """
+        BEGIN
+            EXECUTE IMMEDIATE 'ALTER TABLE invalidated_memory_units ADD (embedding VECTOR)';
+        EXCEPTION WHEN OTHERS THEN
+            IF SQLCODE != -1430 THEN RAISE; END IF;
+        END;
+        """
+    )
+
+
+def upgrade() -> None:
+    run_for_dialect(pg=_pg_upgrade, oracle=_oracle_upgrade)
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade, oracle=_oracle_downgrade)
@@ -0,0 +1,39 @@
+"""Merge two divergent migration heads.
+
+``d4f6a8c2e1b3`` (drop the curation-archive embedding column) and
+``2071c7518f88`` (add the memory_links(bank_id, link_type) index) were authored
+in parallel off the same parent (``a1d3f5b7c9e2``) and merged independently,
+leaving the DAG with two heads. This is a no-op merge that re-unifies them so
+``alembic upgrade head`` is unambiguous again (enforced by
+``tests/test_alembic_dag.py::test_single_head``).
+
+Revision ID: e1f2a3b4c5d6
+Revises: d4f6a8c2e1b3, 2071c7518f88
+Create Date: 2026-06-16
+"""
+
+from collections.abc import Sequence
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "e1f2a3b4c5d6"
+down_revision: str | Sequence[str] | None = ("d4f6a8c2e1b3", "2071c7518f88")
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _pg_upgrade() -> None:
+    # Pure DAG merge — both parents already applied their schema changes.
+    pass
+
+
+def _pg_downgrade() -> None:
+    pass
+
+
+def upgrade() -> None:
+    run_for_dialect(pg=_pg_upgrade)
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade)
@@ -142,6 +142,9 @@ _TABLES: tuple[str, ...] = (
    # Cold archive for curation: invalidated facts are MOVED here out of
    # memory_units so the recall hot-path never sees them. Mirrors memory_units
    # plus invalidation bookkeeping and an entity-id snapshot for lossless revert.
+    # No `embedding` column: the archive is cold storage and revert recomputes the
+    # embedding, so there is no archive vector to fall out of sync with the live
+    # model's dimension on a model switch (#2209).
    """
    CREATE TABLE IF NOT EXISTS invalidated_memory_units (
        id                RAW(16)        NOT NULL,
@@ -149,7 +152,6 @@ _TABLES: tuple[str, ...] = (
        document_id       VARCHAR2(512),
        chunk_id          VARCHAR2(512),
        text              CLOB           NOT NULL,
-        embedding         VECTOR(384, FLOAT32),
        context           CLOB,
        event_date        TIMESTAMP WITH TIME ZONE NOT NULL,
        occurred_start    TIMESTAMP WITH TIME ZONE,
@@ -16,9 +16,7 @@ retention parameters, retrieval settings, etc.) in Python field name format.

 from collections.abc import Sequence

-import sqlalchemy as sa
 from alembic import context, op
-from sqlalchemy.dialects.postgresql import JSONB

 from hindsight_api.alembic._dialect import run_for_dialect

@@ -12,7 +12,7 @@ import re
 import uuid
 from collections.abc import Awaitable
 from contextlib import asynccontextmanager
-from datetime import datetime, timezone
+from datetime import datetime
 from typing import Any, Literal, TypeVar

 from fastapi import Depends, FastAPI, File, Form, Header, HTTPException, Query, Request, UploadFile
@@ -44,13 +44,78 @@ def _parse_metadata(metadata: Any) -> dict[str, Any]:
    return {}


-from typing import Callable
+from collections.abc import Iterable
+from types import UnionType
+from typing import Callable, Union, get_args, get_origin

+from fastapi.routing import APIRoute
 from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator

 from hindsight_api import MemoryEngine


+def _annotation_is_nullable(annotation: Any) -> bool:
+    """True if the annotation is a Union that includes None (i.e. ``X | None``)."""
+    if get_origin(annotation) in (Union, UnionType):
+        return any(arg is type(None) for arg in get_args(annotation))
+    return False
+
+
+def _iter_models(annotation: Any) -> Iterable[type[BaseModel]]:
+    """Yield every Pydantic model referenced by an annotation, recursing through generics."""
+    if isinstance(annotation, type) and issubclass(annotation, BaseModel):
+        yield annotation
+        return
+    for arg in get_args(annotation):
+        yield from _iter_models(arg)
+
+
+def _model_has_required_nullable(model: type[BaseModel], seen: set[type[BaseModel]]) -> bool:
+    """True if the model (or any nested model) declares a required *and* nullable field.
+
+    Such a field is in the OpenAPI ``required`` set but may serialize to null, so dropping
+    it (via ``exclude_none``) would omit a key that strict generated clients expect to be
+    present. Routes whose response model contains one of these must keep emitting nulls to
+    stay wire-compatible with already-generated clients.
+    """
+    if model in seen:
+        return False
+    seen.add(model)
+    for field in model.model_fields.values():
+        annotation = field.annotation
+        if field.is_required() and _annotation_is_nullable(annotation):
+            return True
+        for nested in _iter_models(annotation):
+            if _model_has_required_nullable(nested, seen):
+                return True
+    return False
+
+
+def _response_model_has_required_nullable(response_model: Any) -> bool:
+    seen: set[type[BaseModel]] = set()
+    return any(_model_has_required_nullable(model, seen) for model in _iter_models(response_model))
+
+
+class ExcludeNoneRoute(APIRoute):
+    """Route class that drops null fields from responses, preserving wire compatibility.
+
+    ``response_model_exclude_none`` is enabled automatically for every route whose response
+    model has no required-and-nullable field. Routes that *do* have such a field (where an
+    omitted key would break strict clients) are left untouched and keep emitting nulls.
+    An explicit ``response_model_exclude_none`` passed to the route decorator is respected.
+    """
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        response_model = kwargs.get("response_model")
+        if (
+            not kwargs.get("response_model_exclude_none")
+            and response_model is not None
+            and not _response_model_has_required_nullable(response_model)
+        ):
+            kwargs["response_model_exclude_none"] = True
+        super().__init__(*args, **kwargs)
+
+
 def FieldWithDefault(default_factory: Callable, **kwargs) -> Any:
    """
    Field wrapper that ensures default_factory values appear in OpenAPI schema.
@@ -83,9 +148,14 @@ def FieldWithDefault(default_factory: Callable, **kwargs) -> Any:


 from hindsight_api.config import get_config
-from hindsight_api.engine.memory_engine import Budget, _current_schema, _get_tiktoken_encoding, fq_table
+from hindsight_api.engine.memory_engine import Budget, _current_schema, _get_tiktoken_encoding
 from hindsight_api.engine.providers.none_llm import LLMNotAvailableError
-from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES, MemoryFact, TokenUsage
+from hindsight_api.engine.response_models import (
+    VALID_RECALL_FACT_TYPES,
+    DryRunExtractionResult,
+    MemoryFact,
+    TokenUsage,
+)
 from hindsight_api.engine.search.tags import TagGroup, TagsMatch
 from hindsight_api.extensions import HttpExtension, OperationValidationError, load_extension
 from hindsight_api.metrics import create_metrics_collector, get_metrics_collector, initialize_metrics
@@ -265,7 +335,7 @@ class RecallResult(BaseModel):

    id: str
    text: str
-    type: str | None = None  # fact type: world, experience, opinion, observation
+    type: str | None = None  # fact type: world, experience, observation
    entities: list[str] | None = None  # Entity names mentioned in this fact
    context: str | None = None
    occurred_start: str | None = None  # ISO format date when the event started
@@ -545,13 +615,16 @@ class MemoryItem(BaseModel):
            return [v]
        return v

-    observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = Field(
+    observation_scopes: Literal["per_tag", "combined", "all_combinations", "shared"] | list[list[str]] | None = Field(
        default=None,
        title="ObservationScopes",
        description=(
            "How to scope observations during consolidation. "
            "'per_tag' runs one consolidation pass per individual tag, creating separate observations for each tag. "
            "'combined' (default) runs a single pass with all tags together. "
+            "'shared' runs a single pass over one global, untagged scope, so memories consolidate together "
+            "regardless of their tags — useful for deduplicating across volatile per-call provenance tags "
+            "(e.g. per-session ids) while keeping those tags on the source facts. "
            "A list of tag lists runs one pass per inner list, giving full control over which combinations to use."
        ),
    )
@@ -852,7 +925,7 @@ class ReflectFact(BaseModel):
    text: str = Field(
        description="Fact text. When type='observation', this contains markdown-formatted consolidated knowledge"
    )
-    type: str | None = None  # fact type: world, experience, opinion, observation
+    type: str | None = None  # fact type: world, experience, observation
    context: str | None = None
    occurred_start: str | None = None
    occurred_end: str | None = None
@@ -1149,7 +1222,14 @@ class CreateBankRequest(BaseModel):
    )
    retain_chunk_size: int | None = Field(
        default=None,
-        description="Maximum token size for each content chunk during retain.",
+        description="Target maximum characters for each content chunk during retain.",
+    )
+    retain_structured_chunk_size: int | None = Field(
+        default=None,
+        description=(
+            "Maximum characters for a single JSONL line or conversation turn to keep whole during retain. "
+            "Defaults to retain_chunk_size when unset."
+        ),
    )
    enable_observations: bool | None = Field(
        default=None,
@@ -1189,6 +1269,7 @@ class CreateBankRequest(BaseModel):
            "retain_extraction_mode",
            "retain_custom_instructions",
            "retain_chunk_size",
+            "retain_structured_chunk_size",
            "enable_observations",
            "observations_mission",
        ):
@@ -1280,6 +1361,33 @@ class GraphDataResponse(BaseModel):
    limit: int


+class ObservationScope(BaseModel):
+    """A distinct observation scope: an exact tag set plus its observation count."""
+
+    tags: list[str] = Field(
+        description="The exact tag set defining this scope (normalized order). Empty list is the global/untagged scope."
+    )
+    count: int = Field(description="Number of observations that live under this scope")
+
+
+class ObservationScopesResponse(BaseModel):
+    """Response model for the observation scopes enumeration endpoint."""
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "scopes": [
+                    {"tags": ["user:alice"], "count": 12},
+                    {"tags": ["user:alice", "project:apollo"], "count": 4},
+                    {"tags": [], "count": 2},
+                ]
+            }
+        }
+    )
+
+    scopes: list[ObservationScope] = Field(description="Distinct observation scopes, most populous first")
+
+
 class ListMemoryUnitsResponse(BaseModel):
    """Response model for list memory units endpoint."""

@@ -1309,6 +1417,32 @@ class ListMemoryUnitsResponse(BaseModel):
    offset: int


+class DryRunExtractRequest(BaseModel):
+    """Request to run fact extraction ONLY (no resolution/links/embeddings/persistence).
+
+    Every field below the content/context/date is a prompt-affecting override applied just for this
+    call — used to preview what a candidate retain mission (or any extraction setting) would extract,
+    without changing the bank. Unset (null) fields fall back to the bank's resolved config.
+    """
+
+    content: str = Field(description="Text to extract facts from (e.g. a document or a single chunk).")
+    context: str = Field(default="", description="Optional context about the content.")
+    # Named `timestamp` to match the retain item payload (retain maps timestamp -> event_date internally).
+    timestamp: datetime | None = Field(
+        default=None, description="Reference timestamp for resolving relative times (ISO 8601)."
+    )
+    agent_name: str | None = Field(default=None, description="Narrator override (memory owner) primed in the prompt.")
+    # --- prompt-affecting config overrides (null = use the bank's value) ---
+    retain_mission: str | None = None
+    retain_extraction_mode: str | None = None
+    retain_custom_instructions: str | None = None
+    retain_extract_causal_links: bool | None = None
+    retain_chunk_size: int | None = None
+    entity_labels: list | None = None
+    entities_allow_free_form: bool | None = None
+    llm_output_language: str | None = None
+
+
 class ListDocumentsResponse(BaseModel):
    """Response model for list documents endpoint."""

@@ -1405,6 +1539,12 @@ class DocumentResponse(BaseModel):
    tags: list[str] = FieldWithDefault(list, description="Tags associated with this document")
    document_metadata: dict[str, Any] | None = Field(default=None, description="Document metadata")
    retain_params: dict[str, Any] | None = Field(default=None, description="Parameters used during retain")
+    observation_scopes: str | list[list[str]] | None = Field(
+        default=None,
+        description="The observation_scopes spec configured at retain time (e.g. 'all_combinations', "
+        "'per_tag', or explicit tag-set lists), captured into retain_params. None when none was set "
+        "(default 'combined' scoping) or for documents retained before this was captured.",
+    )


 class UpdateDocumentRequest(BaseModel):
@@ -1994,7 +2134,14 @@ class BankTemplateConfig(BaseModel):
    retain_custom_instructions: str | None = Field(
        default=None, description="Custom extraction prompt (when mode='custom')"
    )
-    retain_chunk_size: int | None = Field(default=None, description="Max token size for each content chunk")
+    retain_chunk_size: int | None = Field(default=None, description="Target max characters for each content chunk")
+    retain_structured_chunk_size: int | None = Field(
+        default=None,
+        description=(
+            "Max characters for a single JSONL line or conversation turn to keep whole; "
+            "defaults to retain_chunk_size when unset"
+        ),
+    )
    enable_observations: bool | None = Field(default=None, description="Toggle observation consolidation")
    observations_mission: str | None = Field(default=None, description="Controls what gets synthesised")
    disposition_skepticism: int | None = Field(default=None, ge=1, le=5, description="Skepticism trait (1-5)")
@@ -2030,6 +2177,16 @@ class BankTemplateConfig(BaseModel):
    max_observations_per_scope: int | None = Field(
        default=None, description="Max observations to retain per consolidation scope"
    )
+    observation_scope_limits: list[dict[str, Any]] | None = Field(
+        default=None,
+        description=(
+            "Per-scope overrides of max_observations_per_scope: "
+            '[{"scope": ["run_*", "shared"], "limit": 1}]. Each scope is a list of '
+            "fnmatch tag-globs; a consolidation scope matches under exact cover "
+            "(every tag matched by a glob and every glob matched by a tag). The first "
+            "matching rule wins; unmatched scopes fall back to max_observations_per_scope."
+        ),
+    )
    reflect_source_facts_max_tokens: int | None = Field(
        default=None, description="Max tokens of source facts per reflect call"
    )
@@ -2627,7 +2784,7 @@ class CreateWebhookRequest(BaseModel):
    secret: str | None = Field(default=None, description="HMAC-SHA256 signing secret (optional)")
    event_types: list[str] = Field(
        default=["consolidation.completed"],
-        description="List of event types to deliver. Currently supported: 'consolidation.completed'",
+        description="List of event types to deliver. Supported: 'retain.completed', 'consolidation.completed', 'memory_defense.triggered'.",
    )
    enabled: bool = Field(default=True, description="Whether this webhook is active")
    http_config: WebhookHttpConfig = Field(
@@ -2737,7 +2894,6 @@ def _make_audited_http(audit_logger_getter: Callable[[], AuditLogger | None]):
    from datetime import datetime as _dt
    from datetime import timezone as _tz
    from functools import wraps
-    from typing import Callable as _Callable

    def audited(action: str, *, request_param: str | None = "request"):
        """Decorator that wraps an HTTP handler with audit logging.
@@ -2968,6 +3124,10 @@ def create_app(
        root_path=config.base_path,
    )

+    # Drop null fields from responses (omit `"x": null`) for routes where it's wire-safe.
+    # Must be set before any route is registered so @app.<method> decorators pick it up.
+    app.router.route_class = ExcludeNoneRoute
+
    # IMPORTANT: Set memory on app.state immediately, don't wait for lifespan
    # This is required for mounted sub-applications where lifespan may not fire
    app.state.memory = memory
@@ -3081,8 +3241,6 @@ def create_app(
        # Replace UUIDs and numeric IDs with placeholders
        import re

-        from starlette.requests import Request
-
        path = request.url.path
        # Replace UUIDs
        path = re.sub(r"/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", "/{id}", path)
@@ -3283,7 +3441,7 @@ def _register_routes(app: FastAPI):
        "/v1/default/banks/{bank_id}/graph",
        response_model=GraphDataResponse,
        summary="Get memory graph data",
-        description="Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion).",
+        description="Retrieve graph data for visualization, optionally filtered by type (world/experience/observation).",
        operation_id="get_graph",
        tags=["Memory"],
    )
@@ -3350,7 +3508,7 @@ def _register_routes(app: FastAPI):

        Args:
            bank_id: Memory Bank ID (from path)
-            type: Filter by fact type (world, experience, opinion)
+            type: Filter by fact type (world, experience, observation)
            q: Search query for full-text search (searches text and context)
            consolidation_state: Filter by consolidation state for source memories
                (world/experience). One of 'failed', 'pending', or 'done'.
@@ -3383,6 +3541,75 @@ def _register_routes(app: FastAPI):
            logger.error(f"Error in /v1/default/banks/{bank_id}/memories/list: {error_detail}")
            raise HTTPException(status_code=500, detail=str(e))

+    async def _require_dry_run_enabled() -> None:
+        """Feature-flag gate for dry-run extraction.
+
+        Declared as a dependency BEFORE ``precheck_for("dry_run_extract")`` so a
+        disabled route returns 404 regardless of tenant/billing state — FastAPI
+        resolves path-operation dependencies in signature order, so this runs
+        first and preserves the original "disabled → 404" contract.
+        """
+        if not get_config().enable_dry_run_extract:
+            raise HTTPException(
+                status_code=404,
+                detail="Dry-run extraction is disabled. Set HINDSIGHT_API_ENABLE_DRY_RUN_EXTRACT=true to re-enable.",
+            )
+
+    @app.post(
+        "/v1/default/banks/{bank_id}/memories/dry-run-extract",
+        response_model=DryRunExtractionResult,
+        summary="Dry-run fact extraction (preview, no persistence)",
+        description=(
+            "Preview what the retain step would extract from text WITHOUT changing the bank — no "
+            "entity resolution, links, embeddings, or persistence. Returns the candidate facts and "
+            "the LLM token usage. Every prompt-affecting setting (retain mission, extraction mode, "
+            "chunk size, …) is overridable in the body to A/B a candidate config against the bank's "
+            "current one. This is a read-only tool: nothing is stored."
+        ),
+        operation_id="dry_run_extract_memories",
+        tags=["Memory"],
+    )
+    async def api_dry_run_extract(
+        bank_id: str,
+        body: DryRunExtractRequest,
+        request_context: RequestContext = Depends(get_request_context),
+        _enabled: None = Depends(_require_dry_run_enabled),
+        _precheck: None = Depends(precheck_for("dry_run_extract")),
+    ):
+        try:
+            override_fields = (
+                "retain_mission",
+                "retain_extraction_mode",
+                "retain_custom_instructions",
+                "retain_extract_causal_links",
+                "retain_chunk_size",
+                "entity_labels",
+                "entities_allow_free_form",
+                "llm_output_language",
+            )
+            overrides = {f: getattr(body, f) for f in override_fields if getattr(body, f) is not None}
+            return await app.state.memory.extract_dry_run(
+                bank_id,
+                body.content,
+                context=body.context or "",
+                event_date=body.timestamp,
+                overrides=overrides,
+                agent_name=body.agent_name,
+                request_context=request_context,
+            )
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in /v1/default/banks/{bank_id}/memories/dry-run-extract: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
    @app.get(
        "/v1/default/banks/{bank_id}/memories/{memory_id}",
        summary="Get memory unit",
@@ -3700,11 +3927,11 @@ def _register_routes(app: FastAPI):
        "/v1/default/banks/{bank_id}/reflect",
        response_model=ReflectResponse,
        summary="Reflect and generate answer",
-        description="Reflect and formulate an answer using bank identity, world facts, and opinions.\n\n"
+        description="Reflect and formulate an answer using bank identity, world facts, observations, and mental models.\n\n"
        "This endpoint:\n"
        "1. Retrieves experience (conversations and events)\n"
        "2. Retrieves world facts relevant to the query\n"
-        "3. Retrieves existing opinions (bank's perspectives)\n"
+        "3. Retrieves observations and mental models (bank's synthesized perspectives)\n"
        "4. Uses LLM to formulate a contextual answer\n"
        "5. Returns plain text answer and the facts used",
        operation_id="reflect",
@@ -5774,6 +6001,35 @@ def _register_routes(app: FastAPI):
            logger.error(f"Error in DELETE /v1/default/banks/{bank_id}/observations: {error_detail}")
            raise HTTPException(status_code=500, detail=str(e))

+    @app.get(
+        "/v1/default/banks/{bank_id}/observations/scopes",
+        response_model=ObservationScopesResponse,
+        summary="List observation scopes",
+        description=(
+            "Enumerate the distinct scopes across a bank's observations. Each observation lives "
+            "under a scope: the exact set of tags it was consolidated with. Returns every distinct "
+            "scope (tag order normalized) with the number of observations in it; the empty tag list "
+            "is the global/untagged scope. Use a returned scope with the graph endpoint "
+            "(tags=<scope> & tags_match=exact) to filter observations to exactly that scope."
+        ),
+        operation_id="list_observation_scopes",
+        tags=["Memory"],
+    )
+    async def api_list_observation_scopes(bank_id: str, request_context: RequestContext = Depends(get_request_context)):
+        """List the distinct observation scopes (exact tag sets) for a bank."""
+        try:
+            return await app.state.memory.list_observation_scopes(bank_id, request_context=request_context)
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in GET /v1/default/banks/{bank_id}/observations/scopes: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
    @app.post(
        "/v1/default/banks/{bank_id}/consolidation/recover",
        response_model=RecoverConsolidationResponse,
@@ -6585,7 +6841,6 @@ def _register_routes(app: FastAPI):
                _validate_parsers(_resolve_parser(request_data.parser), "request-level parser")

            # Prepare file items and calculate total batch size
-            import io

            file_items = []
            total_batch_size = 0
@@ -6664,14 +6919,14 @@ def _register_routes(app: FastAPI):
        "/v1/default/banks/{bank_id}/memories",
        response_model=DeleteResponse,
        summary="Clear memory bank memories",
-        description="Delete memory units for a memory bank. Optionally filter by type (world, experience, opinion) to delete only specific types. This is a destructive operation that cannot be undone. The bank profile (disposition and background) will be preserved.",
+        description="Delete memory units for a memory bank. Optionally filter by type (world, experience, observation) to delete only specific types. This is a destructive operation that cannot be undone. The bank profile (disposition and background) will be preserved.",
        operation_id="clear_bank_memories",
        tags=["Memory"],
    )
    @audited("clear_memories", request_param=None)
    async def api_clear_bank_memories(
        bank_id: str,
-        type: str | None = Query(None, description="Optional fact type filter (world, experience, opinion)"),
+        type: str | None = Query(None, description="Optional fact type filter (world, experience, observation)"),
        request_context: RequestContext = Depends(get_request_context),
    ):
        """Clear memories for a memory bank, optionally filtered by type."""
@@ -356,6 +356,7 @@ ENV_MCP_ENABLED_TOOLS = "HINDSIGHT_API_MCP_ENABLED_TOOLS"
 ENV_MCP_STATELESS = "HINDSIGHT_API_MCP_STATELESS"
 ENV_ENABLE_BANK_CONFIG_API = "HINDSIGHT_API_ENABLE_BANK_CONFIG_API"
 ENV_ENABLE_BANK_LLM_HEALTH = "HINDSIGHT_API_ENABLE_BANK_LLM_HEALTH"
+ENV_ENABLE_DRY_RUN_EXTRACT = "HINDSIGHT_API_ENABLE_DRY_RUN_EXTRACT"
 ENV_DEFAULT_BANK_TEMPLATE = "HINDSIGHT_API_DEFAULT_BANK_TEMPLATE"
 ENV_GRAPH_RETRIEVER = "HINDSIGHT_API_GRAPH_RETRIEVER"
 ENV_RECALL_MAX_CONCURRENT = "HINDSIGHT_API_RECALL_MAX_CONCURRENT"
@@ -396,6 +397,7 @@ ENV_LLM_PROMPT_CACHE_ENABLED = "HINDSIGHT_API_LLM_PROMPT_CACHE_ENABLED"
 # Retain settings
 ENV_RETAIN_MAX_COMPLETION_TOKENS = "HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS"
 ENV_RETAIN_CHUNK_SIZE = "HINDSIGHT_API_RETAIN_CHUNK_SIZE"
+ENV_RETAIN_STRUCTURED_CHUNK_SIZE = "HINDSIGHT_API_RETAIN_STRUCTURED_CHUNK_SIZE"
 ENV_RETAIN_EXTRACT_CAUSAL_LINKS = "HINDSIGHT_API_RETAIN_EXTRACT_CAUSAL_LINKS"
 ENV_RETAIN_EXTRACTION_MODE = "HINDSIGHT_API_RETAIN_EXTRACTION_MODE"
 ENV_RETAIN_MISSION = "HINDSIGHT_API_RETAIN_MISSION"
@@ -453,6 +455,7 @@ ENV_CONSOLIDATION_RECALL_BUDGET = "HINDSIGHT_API_CONSOLIDATION_RECALL_BUDGET"
 ENV_CONSOLIDATION_MAX_ATTEMPTS = "HINDSIGHT_API_CONSOLIDATION_MAX_ATTEMPTS"
 ENV_OBSERVATIONS_MISSION = "HINDSIGHT_API_OBSERVATIONS_MISSION"
 ENV_MAX_OBSERVATIONS_PER_SCOPE = "HINDSIGHT_API_MAX_OBSERVATIONS_PER_SCOPE"
+ENV_OBSERVATION_SCOPE_LIMITS = "HINDSIGHT_API_OBSERVATION_SCOPE_LIMITS"
 ENV_ENABLE_OBSERVATION_HISTORY = "HINDSIGHT_API_ENABLE_OBSERVATION_HISTORY"
 ENV_OBSERVATION_HISTORY_MAX_ENTRIES = "HINDSIGHT_API_OBSERVATION_HISTORY_MAX_ENTRIES"
 ENV_ENABLE_MENTAL_MODEL_HISTORY = "HINDSIGHT_API_ENABLE_MENTAL_MODEL_HISTORY"
@@ -478,6 +481,7 @@ ENV_LAZY_RERANKER = "HINDSIGHT_API_LAZY_RERANKER"

 # Database migrations
 ENV_RUN_MIGRATIONS_ON_STARTUP = "HINDSIGHT_API_RUN_MIGRATIONS_ON_STARTUP"
+ENV_MIGRATION_CONCURRENCY = "HINDSIGHT_API_MIGRATION_CONCURRENCY"

 # Database connection pool
 ENV_DB_POOL_MIN_SIZE = "HINDSIGHT_API_DB_POOL_MIN_SIZE"
@@ -801,6 +805,10 @@ DEFAULT_MCP_ENABLED = True
 DEFAULT_MCP_ENABLED_TOOLS: list[str] | None = None  # None = all tools enabled
 DEFAULT_MCP_STATELESS = False  # False = stateful (supports SSE/GET); True = stateless (POST-only)
 DEFAULT_ENABLE_BANK_CONFIG_API = True
+# Dry-run extraction is a preview tool that makes a real LLM call but stores nothing. Enabled by
+# default; set HINDSIGHT_API_ENABLE_DRY_RUN_EXTRACT=false to remove the endpoint (e.g. to cap
+# provider cost/abuse on untrusted deployments).
+DEFAULT_ENABLE_DRY_RUN_EXTRACT = True
 # The per-bank LLM connectivity probe makes a real provider call, so it's OFF by
 # default (cost/abuse concerns) and must be explicitly enabled to expose the endpoint.
 DEFAULT_ENABLE_BANK_LLM_HEALTH = False
@@ -893,9 +901,16 @@ DEFAULT_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION = (
 )
 DEFAULT_OBSERVATIONS_MISSION = None  # Declarative spec of what observations are for this bank
 DEFAULT_MAX_OBSERVATIONS_PER_SCOPE = -1  # Max observations per tag scope (-1 = unlimited)
+# Per-scope overrides of the cap above: list of {"scope": [tag-globs], "limit": int}.
+# First rule whose pattern exact-covers a scope's tags wins; else the default above.
+DEFAULT_OBSERVATION_SCOPE_LIMITS: list | None = None

 # Database migrations
 DEFAULT_RUN_MIGRATIONS_ON_STARTUP = True
+# Number of tenant schemas to migrate concurrently. Each schema runs in its own
+# process (Alembic's command.upgrade() is not thread-safe); within a schema the
+# work is always sequential. 1 = fully sequential (the safe default).
+DEFAULT_MIGRATION_CONCURRENCY = 1

 # Database connection pool
 DEFAULT_DB_POOL_MIN_SIZE = 5
@@ -1073,6 +1088,63 @@ def _parse_optional_positive_int(name: str, raw: str | None) -> int | None:
    return _parse_positive_int(name, raw, 1)


+def _validate_retain_chunking_int(name: str, value: Any) -> int:
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise ValueError(f"{name} must be an integer, got {value!r}")
+    if value < 1:
+        raise ValueError(f"{name} must be >= 1, got {value}")
+    return value
+
+
+def validate_retain_chunking_config(
+    retain_chunk_size: Any,
+    retain_structured_chunk_size: Any,
+    *,
+    retain_chunk_size_name: str = "retain_chunk_size",
+    retain_structured_chunk_size_name: str = "retain_structured_chunk_size",
+) -> None:
+    """Validate retain chunking size fields.
+
+    Defaults emit field-style names ("retain_chunk_size") so API/PATCH callers
+    don't have to override them. The startup validator (HindsightConfig.validate)
+    overrides to env-style names ("HINDSIGHT_API_RETAIN_CHUNK_SIZE") for env
+    misconfig errors.
+    """
+    _validate_retain_chunking_int(retain_chunk_size_name, retain_chunk_size)
+    if retain_structured_chunk_size is None:
+        return
+    _validate_retain_chunking_int(
+        retain_structured_chunk_size_name,
+        retain_structured_chunk_size,
+    )
+
+
+def validate_retain_completion_token_budget(
+    *,
+    llm_provider: str,
+    retain_max_completion_tokens: int,
+    retain_chunk_size: int,
+    retain_llm_model: str | None = None,
+    llm_model: str | None = None,
+    retain_llm_provider: str | None = None,
+    retain_max_completion_tokens_name: str = "retain_max_completion_tokens",
+    retain_chunk_size_name: str = "retain_chunk_size",
+) -> None:
+    """Validate that retain LLM output capacity exceeds the configured chunk size."""
+    if llm_provider == "none" or retain_max_completion_tokens > retain_chunk_size:
+        return
+    raise ValueError(
+        f"Invalid configuration: {retain_max_completion_tokens_name} "
+        f"({retain_max_completion_tokens}) must be greater than "
+        f"{retain_chunk_size_name} ({retain_chunk_size}). "
+        f"\n\nYou have two options to fix this:"
+        f"\n  1. Increase {retain_max_completion_tokens_name} to a value > {retain_chunk_size}"
+        f"\n  2. Use a model that supports at least {retain_max_completion_tokens} output tokens"
+        f"\n     (current model: {retain_llm_model or llm_model}, "
+        f"provider: {retain_llm_provider or llm_provider})"
+    )
+
+
 def _parse_optional_choice(name: str, raw: str | None, allowed: frozenset[str]) -> str | None:
    """Parse an optional string env var constrained to a small allowlist."""
    if raw is None or raw == "":
@@ -1408,6 +1480,7 @@ class HindsightConfig:
    mcp_stateless: bool  # True = stateless HTTP (POST-only); False = stateful (supports GET/SSE)
    enable_bank_config_api: bool
    enable_bank_llm_health: bool
+    enable_dry_run_extract: bool
    # Default bank template (static, server-level only). When set, the manifest is applied
    # to every newly-created bank, overriding the env/config defaults for any fields it sets.
    default_bank_template: dict | None
@@ -1426,6 +1499,7 @@ class HindsightConfig:
    # Retain settings
    retain_max_completion_tokens: int
    retain_chunk_size: int
+    retain_structured_chunk_size: int | None
    retain_extract_causal_links: bool
    retain_extraction_mode: str
    retain_mission: str | None
@@ -1484,6 +1558,10 @@ class HindsightConfig:
    consolidation_max_attempts: int
    observations_mission: str | None
    max_observations_per_scope: int
+    # Per-scope observation caps overriding max_observations_per_scope.
+    # Raw JSON shape: [{"scope": ["run_*", "shared"], "limit": 1}, ...]
+    # (validated/applied in engine.consolidation.consolidator._effective_scope_limit)
+    observation_scope_limits: list | None

    # Entity labels (controlled vocabulary of key:value classification labels extracted at retain time)
    # List of label group dicts: [{key, description, type, optional, values: [{value, description}]}]
@@ -1530,6 +1608,7 @@ class HindsightConfig:

    # Database migrations
    run_migrations_on_startup: bool
+    migration_concurrency: int

    # Database connection pool
    db_pool_min_size: int
@@ -1649,6 +1728,7 @@ class HindsightConfig:
        "mcp_enabled_tools",
        # Retention settings (behavioral)
        "retain_chunk_size",
+        "retain_structured_chunk_size",
        "retain_extraction_mode",
        "retain_mission",
        "retain_custom_instructions",
@@ -1668,6 +1748,7 @@ class HindsightConfig:
        "consolidation_source_facts_max_tokens_per_observation",
        "observations_mission",
        "max_observations_per_scope",
+        "observation_scope_limits",
        # Reflect settings
        "reflect_mission",
        "reflect_source_facts_max_tokens",
@@ -1807,20 +1888,23 @@ class HindsightConfig:
                "disabling observations/consolidation. Reflect will return HTTP 400."
            )

-        # RETAIN_MAX_COMPLETION_TOKENS must be greater than RETAIN_CHUNK_SIZE
-        # to ensure the LLM has enough output capacity to extract facts from chunks
-        # (not applicable when provider is "none" since no LLM calls are made)
-        if self.llm_provider != "none" and self.retain_max_completion_tokens <= self.retain_chunk_size:
-            raise ValueError(
-                f"Invalid configuration: HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS "
-                f"({self.retain_max_completion_tokens}) must be greater than "
-                f"HINDSIGHT_API_RETAIN_CHUNK_SIZE ({self.retain_chunk_size}). "
-                f"\n\nYou have two options to fix this:"
-                f"\n  1. Increase HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS to a value > {self.retain_chunk_size}"
-                f"\n  2. Use a model that supports at least {self.retain_max_completion_tokens} output tokens"
-                f"\n     (current model: {self.retain_llm_model or self.llm_model}, "
-                f"provider: {self.retain_llm_provider or self.llm_provider})"
-            )
+        validate_retain_chunking_config(
+            self.retain_chunk_size,
+            self.retain_structured_chunk_size,
+            retain_chunk_size_name="HINDSIGHT_API_RETAIN_CHUNK_SIZE",
+            retain_structured_chunk_size_name="HINDSIGHT_API_RETAIN_STRUCTURED_CHUNK_SIZE",
+        )
+
+        validate_retain_completion_token_budget(
+            llm_provider=self.llm_provider,
+            retain_max_completion_tokens=self.retain_max_completion_tokens,
+            retain_chunk_size=self.retain_chunk_size,
+            retain_llm_model=self.retain_llm_model,
+            llm_model=self.llm_model,
+            retain_llm_provider=self.retain_llm_provider,
+            retain_max_completion_tokens_name="HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS",
+            retain_chunk_size_name="HINDSIGHT_API_RETAIN_CHUNK_SIZE",
+        )

        # Warn if local ML dependencies are missing when configured.
        # Don't hard-fail here — the actual ImportError fires at model init time
@@ -2265,6 +2349,8 @@ class HindsightConfig:
            == "true",
            enable_bank_config_api=os.getenv(ENV_ENABLE_BANK_CONFIG_API, str(DEFAULT_ENABLE_BANK_CONFIG_API)).lower()
            == "true",
+            enable_dry_run_extract=os.getenv(ENV_ENABLE_DRY_RUN_EXTRACT, str(DEFAULT_ENABLE_DRY_RUN_EXTRACT)).lower()
+            == "true",
            default_bank_template=_parse_default_bank_template(os.getenv(ENV_DEFAULT_BANK_TEMPLATE)),
            # Recall
            graph_retriever=os.getenv(ENV_GRAPH_RETRIEVER, DEFAULT_GRAPH_RETRIEVER),
@@ -2294,6 +2380,10 @@ class HindsightConfig:
                os.getenv(ENV_RETAIN_MAX_COMPLETION_TOKENS, str(DEFAULT_RETAIN_MAX_COMPLETION_TOKENS))
            ),
            retain_chunk_size=int(os.getenv(ENV_RETAIN_CHUNK_SIZE, str(DEFAULT_RETAIN_CHUNK_SIZE))),
+            retain_structured_chunk_size=_parse_optional_positive_int(
+                ENV_RETAIN_STRUCTURED_CHUNK_SIZE,
+                os.getenv(ENV_RETAIN_STRUCTURED_CHUNK_SIZE),
+            ),
            retain_extract_causal_links=os.getenv(
                ENV_RETAIN_EXTRACT_CAUSAL_LINKS, str(DEFAULT_RETAIN_EXTRACT_CAUSAL_LINKS)
            ).lower()
@@ -2433,11 +2523,14 @@ class HindsightConfig:
            max_observations_per_scope=int(
                os.getenv(ENV_MAX_OBSERVATIONS_PER_SCOPE, str(DEFAULT_MAX_OBSERVATIONS_PER_SCOPE))
            ),
+            observation_scope_limits=json.loads(os.getenv(ENV_OBSERVATION_SCOPE_LIMITS, "null"))
+            or DEFAULT_OBSERVATION_SCOPE_LIMITS,
            entity_labels=None,
            entities_allow_free_form=True,
            memory_defense=None,
            # Database migrations
            run_migrations_on_startup=os.getenv(ENV_RUN_MIGRATIONS_ON_STARTUP, "true").lower() == "true",
+            migration_concurrency=int(os.getenv(ENV_MIGRATION_CONCURRENCY, str(DEFAULT_MIGRATION_CONCURRENCY))),
            # Database connection pool
            db_pool_min_size=int(os.getenv(ENV_DB_POOL_MIN_SIZE, str(DEFAULT_DB_POOL_MIN_SIZE))),
            db_pool_max_size=int(os.getenv(ENV_DB_POOL_MAX_SIZE, str(DEFAULT_DB_POOL_MAX_SIZE))),
@@ -18,6 +18,8 @@ from hindsight_api.config import (
    HindsightConfig,
    _get_raw_config,
    normalize_config_dict,
+    validate_retain_chunking_config,
+    validate_retain_completion_token_budget,
 )
 from hindsight_api.engine.memory_engine import fq_table
 from hindsight_api.extensions.tenant import TenantExtension
@@ -29,6 +31,35 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)


+def _validate_retain_strategy_chunking(base_config: HindsightConfig, strategies: Any) -> None:
+    """Validate retain strategy chunking with the same semantics as apply_strategy()."""
+    if not isinstance(strategies, dict):
+        return
+    configurable = HindsightConfig.get_configurable_fields()
+    for strategy_name, overrides in strategies.items():
+        if not isinstance(overrides, dict):
+            raise ValueError(f"Invalid retain strategy {strategy_name!r}: must be an object")
+        filtered = {k: v for k, v in overrides.items() if k in configurable}
+        if not filtered:
+            continue
+        try:
+            resolved = replace(base_config, **filtered)
+            validate_retain_chunking_config(
+                resolved.retain_chunk_size,
+                resolved.retain_structured_chunk_size,
+            )
+            validate_retain_completion_token_budget(
+                llm_provider=resolved.llm_provider,
+                retain_max_completion_tokens=resolved.retain_max_completion_tokens,
+                retain_chunk_size=resolved.retain_chunk_size,
+                retain_llm_model=resolved.retain_llm_model,
+                llm_model=resolved.llm_model,
+                retain_llm_provider=resolved.retain_llm_provider,
+            )
+        except ValueError as e:
+            raise ValueError(f"Invalid retain strategy {strategy_name!r}: {e}") from e
+
+
 class ConfigResolver:
    """Resolves hierarchical configuration with tenant/bank overrides."""

@@ -46,6 +77,26 @@ class ConfigResolver:
        self._configurable_fields = HindsightConfig.get_configurable_fields()
        self._credential_fields = HindsightConfig.get_credential_fields()

+    async def _resolve_parent_config_dict(self, bank_id: str, context: RequestContext | None = None) -> dict[str, Any]:
+        """Resolve global + tenant config before bank-level overrides."""
+        config_dict = asdict(self._global_config)
+
+        if self.tenant_extension and context:
+            try:
+                tenant_overrides = await self.tenant_extension.get_tenant_config(context)
+                if tenant_overrides:
+                    # Normalize keys and filter to configurable fields only
+                    normalized_tenant = normalize_config_dict(tenant_overrides)
+                    configurable_tenant = {k: v for k, v in normalized_tenant.items() if k in self._configurable_fields}
+                    config_dict.update(configurable_tenant)
+                    logger.debug(
+                        f"Applied tenant config overrides for bank {bank_id}: {list(configurable_tenant.keys())}"
+                    )
+            except Exception as e:
+                logger.warning(f"Failed to load tenant config for bank {bank_id}: {e}")
+
+        return config_dict
+
    async def resolve_full_config(self, bank_id: str, context: RequestContext | None = None) -> HindsightConfig:
        """
        Resolve full HindsightConfig for a bank with hierarchical overrides applied.
@@ -65,23 +116,7 @@ class ConfigResolver:
        Returns:
            Complete HindsightConfig with hierarchical overrides applied
        """
-        # Start with global config (all fields)
-        config_dict = asdict(self._global_config)
-
-        # Load tenant config overrides (if tenant extension available)
-        if self.tenant_extension and context:
-            try:
-                tenant_overrides = await self.tenant_extension.get_tenant_config(context)
-                if tenant_overrides:
-                    # Normalize keys and filter to configurable fields only
-                    normalized_tenant = normalize_config_dict(tenant_overrides)
-                    configurable_tenant = {k: v for k, v in normalized_tenant.items() if k in self._configurable_fields}
-                    config_dict.update(configurable_tenant)
-                    logger.debug(
-                        f"Applied tenant config overrides for bank {bank_id}: {list(configurable_tenant.keys())}"
-                    )
-            except Exception as e:
-                logger.warning(f"Failed to load tenant config for bank {bank_id}: {e}")
+        config_dict = await self._resolve_parent_config_dict(bank_id, context)

        # Load bank config overrides
        bank_overrides = await self._load_bank_config(bank_id)
@@ -92,6 +127,10 @@ class ConfigResolver:
        # Return full config object (dataclass doesn't have __init__ that accepts kwargs, so we update the object)
        # Create a new config instance by copying the global config and updating fields
        resolved_config = HindsightConfig(**config_dict)
+        validate_retain_chunking_config(
+            resolved_config.retain_chunk_size,
+            resolved_config.retain_structured_chunk_size,
+        )
        return resolved_config

    async def get_bank_config(self, bank_id: str, context: RequestContext | None = None) -> dict[str, Any]:
@@ -266,6 +305,29 @@ class ConfigResolver:
        # Validate recall budget fields
        _validate_recall_budget_updates(normalized_updates)

+        chunking_fields_updated = (
+            "retain_chunk_size" in normalized_updates
+            or "retain_structured_chunk_size" in normalized_updates
+            or "retain_strategies" in normalized_updates
+        )
+        if chunking_fields_updated:
+            config_dict = await self._resolve_parent_config_dict(bank_id, context)
+            active_bank_overrides = await self._load_bank_config(bank_id)
+            for key, value in normalized_updates.items():
+                if key not in self._configurable_fields:
+                    continue
+                if value is None:
+                    active_bank_overrides.pop(key, None)
+                else:
+                    active_bank_overrides[key] = value
+            config_dict.update(active_bank_overrides)
+            base_config = HindsightConfig(**config_dict)
+            validate_retain_chunking_config(
+                base_config.retain_chunk_size,
+                base_config.retain_structured_chunk_size,
+            )
+            _validate_retain_strategy_chunking(base_config, base_config.retain_strategies)
+
        # Persist the override. Banks are created lazily (on first retain), so a
        # PATCH that precedes any ingestion would otherwise UPDATE zero rows and
        # silently no-op while returning 200. Ensure the bank row exists first
@@ -364,7 +426,8 @@ def apply_strategy(config: HindsightConfig, strategy_name: str) -> HindsightConf
    A strategy is a named set of hierarchical field overrides stored in
    config.retain_strategies. Any field in _HIERARCHICAL_FIELDS can be
    overridden, including retain_extraction_mode, retain_chunk_size,
-    entity_labels, entities_allow_free_form, etc.
+    retain_structured_chunk_size, entity_labels,
+    entities_allow_free_form, etc.

    Unknown strategy names log a warning and return config unchanged.
    Unknown or non-hierarchical fields in the strategy are silently ignored.
@@ -386,4 +449,17 @@ def apply_strategy(config: HindsightConfig, strategy_name: str) -> HindsightConf
        return config

    logger.debug(f"Applying retain strategy '{strategy_name}': {list(filtered.keys())}")
-    return replace(config, **filtered)
+    resolved = replace(config, **filtered)
+    validate_retain_chunking_config(
+        resolved.retain_chunk_size,
+        resolved.retain_structured_chunk_size,
+    )
+    validate_retain_completion_token_budget(
+        llm_provider=resolved.llm_provider,
+        retain_max_completion_tokens=resolved.retain_max_completion_tokens,
+        retain_chunk_size=resolved.retain_chunk_size,
+        retain_llm_model=resolved.retain_llm_model,
+        llm_model=resolved.llm_model,
+        retain_llm_provider=resolved.retain_llm_provider,
+    )
+    return resolved
@@ -24,6 +24,7 @@ from collections import defaultdict
 from contextlib import AsyncExitStack
 from dataclasses import asdict, dataclass, field
 from datetime import datetime, timezone
+from fnmatch import fnmatchcase
 from itertools import combinations
 from typing import TYPE_CHECKING, Any, Literal

@@ -334,7 +335,15 @@ def _resolve_obs_tags_list(memory: dict[str, Any]) -> list[list[str]] | None:

    Returns ``None`` for the default ``combined``-mode single pass (caller uses
    the memory's own tags). Returns a list[list[str]] when the memory requested
-    multi-pass scoping (``per_tag``, ``all_combinations``, or an explicit list).
+    multi-pass scoping (``per_tag``, ``all_combinations``, ``shared``, or an
+    explicit list).
+
+    ``shared`` resolves to ``[[]]`` — a single pass over the empty (untagged)
+    scope. The created observation carries no tags and recall/dedup match it with
+    ``tags_match="any"``, so every memory consolidates into one shared observation
+    regardless of its own tags. Use it to deduplicate across volatile per-call
+    provenance tags (e.g. per-session ids) without dropping those tags from the
+    source facts.
    """
    parsed = _parse_observation_scopes(memory)
    tags = list(memory.get("tags") or [])
@@ -345,6 +354,8 @@ def _resolve_obs_tags_list(memory: dict[str, Any]) -> list[list[str]] | None:
        if not tags:
            return None
        return [list(c) for r in range(1, len(tags) + 1) for c in combinations(tags, r)]
+    if parsed == "shared":
+        return [[]]
    if parsed == "combined" or parsed is None:
        return None
    return parsed  # explicit list[list[str]]
@@ -361,6 +372,7 @@ def _resolve_write_scopes(memory: dict[str, Any]) -> list[frozenset[str]]:
    - ``combined`` / ``None``    -> ``[frozenset(memory.tags)]``
    - ``per_tag``                -> ``[frozenset({t}) for t in memory.tags]``
    - ``all_combinations``       -> one frozenset per nonempty subset of tags
+    - ``shared``                 -> ``[frozenset()]`` (the single untagged scope)
    - explicit ``list[list[str]]`` -> one frozenset per declared scope

    Empty-tag memories collapse to a single ``frozenset()`` in all modes so they
@@ -375,6 +387,8 @@ def _resolve_write_scopes(memory: dict[str, Any]) -> list[frozenset[str]]:
        if not tags:
            return [frozenset()]
        return [frozenset(c) for r in range(1, len(tags) + 1) for c in combinations(tags, r)]
+    if parsed == "shared":
+        return [frozenset()]
    if parsed == "combined" or parsed is None:
        return [frozenset(tags)]
    return [frozenset(s) for s in parsed]  # explicit list[list[str]]
@@ -523,6 +537,86 @@ async def _count_observations_for_scope(
    )


+@dataclass(frozen=True)
+class _ScopeLimitRule:
+    """One ``observation_scope_limits`` rule: a scope pattern -> an observation cap.
+
+    ``globs`` is a tuple of fnmatch tag-globs describing one consolidation scope.
+    A concrete scope (the set of ``fact_tags`` for a consolidation pass) matches
+    under *exact cover*: every tag is matched by some glob AND every glob matches
+    some tag. So ``["shared"]`` matches the scope ``{shared}`` but not
+    ``{run_1, shared}``, and ``["run_*", "shared"]`` matches ``{run_1, shared}``
+    but not ``{shared}``.
+
+    ``limit`` is the cap applied to matching scopes (-1 = unlimited, 0 = no new
+    observations, >0 = hard cap), mirroring ``max_observations_per_scope``.
+    """
+
+    globs: tuple[str, ...]
+    limit: int
+
+
+def _parse_scope_limit_rules(raw: Any) -> list[_ScopeLimitRule]:
+    """Parse the raw ``observation_scope_limits`` config into ordered rules.
+
+    The config round-trips as JSON through env and the bank-config API, so this
+    is defensive: malformed entries are skipped rather than raising, and list
+    order is preserved (first match wins in :func:`_effective_scope_limit`).
+    """
+    if not isinstance(raw, list):
+        return []
+    rules: list[_ScopeLimitRule] = []
+    for entry in raw:
+        if not isinstance(entry, dict):
+            continue
+        scope = entry.get("scope")
+        limit = entry.get("limit")
+        if not isinstance(scope, list) or not scope:
+            continue
+        if not all(isinstance(g, str) and g for g in scope):
+            continue
+        # bool is an int subclass — reject True/False masquerading as a limit.
+        if not isinstance(limit, int) or isinstance(limit, bool):
+            continue
+        rules.append(_ScopeLimitRule(globs=tuple(scope), limit=limit))
+    return rules
+
+
+def _scope_matches_globs(globs: tuple[str, ...], tags: list[str]) -> bool:
+    """Exact-cover match between a scope pattern and a concrete tag set.
+
+    True iff every tag is covered by at least one glob AND every glob covers at
+    least one tag (no uncovered tags, no vacuous globs). Untagged scopes never
+    match, so a scope limit never applies to untagged observations (consistent
+    with the ``and fact_tags`` guard at the call site). Matching is
+    case-sensitive (``fnmatchcase``) for deterministic cross-platform behaviour.
+    """
+    tagset = set(tags)
+    if not tagset:
+        return False
+    if not all(any(fnmatchcase(t, g) for g in globs) for t in tagset):
+        return False
+    if not all(any(fnmatchcase(t, g) for t in tagset) for g in globs):
+        return False
+    return True
+
+
+def _effective_scope_limit(config: Any, fact_tags: list[str]) -> int:
+    """Resolve the observation cap for one concrete consolidation scope.
+
+    The first rule in ``observation_scope_limits`` whose pattern exact-covers
+    ``fact_tags`` wins; otherwise falls back to the bank-wide
+    ``max_observations_per_scope``. Wildcards live only here, matched against the
+    already-resolved concrete tags — the SQL count stays exact and indexed.
+    """
+    if config is None:
+        return -1
+    for rule in _parse_scope_limit_rules(getattr(config, "observation_scope_limits", None)):
+        if _scope_matches_globs(rule.globs, fact_tags):
+            return rule.limit
+    return config.max_observations_per_scope
+
+
 def _build_response_model(max_creates: int | None = None) -> type[_ConsolidationBatchResponse]:
    """Build a response model, optionally constraining max creates via JSON schema."""
    if max_creates is None or max_creates < 0:
@@ -1403,11 +1497,15 @@ async def _process_memory_batch(
        # All memories in the batch share the same tag set (enforced by batching)
        fact_tags = memories[0].get("tags") or [] if memories else []

-    # 2b. Compute remaining observation slots for this scope (if limit configured)
-    max_obs = config.max_observations_per_scope if config is not None else -1
+    # 2b. Compute remaining observation slots for this scope (if limit configured).
+    # The cap is resolved per-scope: an observation_scope_limits rule may override
+    # the bank-wide max_observations_per_scope for scopes matching its tag pattern.
+    max_obs = _effective_scope_limit(config, fact_tags)
    remaining_observation_slots: int | None = None
-    if max_obs > 0 and fact_tags:
-        current_count = await _count_observations_for_scope(conn, bank_id, fact_tags)
+    if max_obs >= 0 and fact_tags:
+        # max_obs == 0 means "no new observations": there are no slots regardless
+        # of the current count, so skip the count query for that case.
+        current_count = await _count_observations_for_scope(conn, bank_id, fact_tags) if max_obs > 0 else 0
        remaining_observation_slots = max(max_obs - current_count, 0)
        if remaining_observation_slots == 0:
            logger.info(
@@ -2045,7 +2143,7 @@ async def _consolidate_batch_with_llm(

    # Build capacity note for the prompt when observation limit is configured
    observation_capacity_note: str | None = None
-    if remaining_observation_slots is not None and max_observations_per_scope > 0:
+    if remaining_observation_slots is not None and max_observations_per_scope >= 0:
        if remaining_observation_slots == 0:
            observation_capacity_note = (
                f"OBSERVATION LIMIT REACHED ({max_observations_per_scope}/{max_observations_per_scope}). "
@@ -27,34 +27,21 @@ from ..config import (
    DEFAULT_RERANKER_LITELLM_MODEL,
    DEFAULT_RERANKER_LITELLM_SDK_MODEL,
    DEFAULT_RERANKER_LOCAL_BATCH_SIZE,
-    DEFAULT_RERANKER_LOCAL_FORCE_CPU,
-    DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT,
    DEFAULT_RERANKER_LOCAL_MODEL,
-    DEFAULT_RERANKER_LOCAL_TRUST_REMOTE_CODE,
-    DEFAULT_RERANKER_PROVIDER,
    DEFAULT_RERANKER_SILICONFLOW_BASE_URL,
    DEFAULT_RERANKER_SILICONFLOW_MODEL,
    DEFAULT_RERANKER_TEI_BATCH_SIZE,
-    DEFAULT_RERANKER_TEI_HTTP_TIMEOUT,
    DEFAULT_RERANKER_TEI_MAX_CONCURRENT,
    DEFAULT_RERANKER_ZEROENTROPY_MODEL,
    DEFAULT_ZEROENTROPY_BASE_URL,
    ENV_RERANKER_ALIBABA_API_KEY,
    ENV_RERANKER_COHERE_API_KEY,
-    ENV_RERANKER_COHERE_MODEL,
    ENV_RERANKER_FLASHRANK_CACHE_DIR,
    ENV_RERANKER_FLASHRANK_CPU_MEM_ARENA,
    ENV_RERANKER_FLASHRANK_MODEL,
    ENV_RERANKER_GOOGLE_PROJECT_ID,
-    ENV_RERANKER_LOCAL_FORCE_CPU,
-    ENV_RERANKER_LOCAL_MAX_CONCURRENT,
-    ENV_RERANKER_LOCAL_MODEL,
-    ENV_RERANKER_LOCAL_TRUST_REMOTE_CODE,
    ENV_RERANKER_PROVIDER,
    ENV_RERANKER_SILICONFLOW_API_KEY,
-    ENV_RERANKER_TEI_BATCH_SIZE,
-    ENV_RERANKER_TEI_HTTP_TIMEOUT,
-    ENV_RERANKER_TEI_MAX_CONCURRENT,
    ENV_RERANKER_TEI_URL,
    ENV_RERANKER_ZEROENTROPY_API_KEY,
 )
@@ -303,7 +290,6 @@ class LocalSTCrossEncoder(CrossEncoderModel):
        - bucket_batching: sort pairs by token length to reduce padding waste (36-54% speedup)
        - batch_size: explicit batch size for predict() calls (MPS optimal: 32)
        """
-        import numpy as np

        try:
            if self.bucket_batching and len(pairs) > 1:
@@ -19,7 +19,6 @@ and mirrors Django's ``DatabaseOperations`` architecture.
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import Any
-from uuid import UUID

 from .base import DatabaseConnection
 from .result import ResultRow
@@ -8,8 +8,6 @@ columns can't appear in GROUP BY).
 import json
 import uuid as uuid_mod
 from datetime import UTC, datetime
-from typing import Any
-from uuid import UUID

 from .base import DatabaseConnection
 from .ops import DataAccessOps, TagListingParts
@@ -4,11 +4,6 @@ Uses unnest(), LATERAL, DISTINCT ON, and native array operations for
 efficient batch operations.
 """

-import json
-from datetime import UTC, datetime
-from typing import Any
-from uuid import UUID
-
 from .base import DatabaseConnection
 from .ops import DataAccessOps, TagListingParts
 from .result import ResultRow
@@ -620,7 +615,6 @@ class PostgreSQLOps(DataAccessOps):
        per_entity_limit: int,
    ) -> tuple[list[ResultRow], list[ResultRow], list[ResultRow]]:
        # v0.5.6 array ops: unnest, &&, COUNT(DISTINCT) on source_memory_ids.
-        from ..schema import fq_table

        entity_rows = await conn.fetch(
            f"""
@@ -106,6 +106,15 @@ SCHEMAS_WITH_PENDING_WORK = OptionalRoutine(
        deployment.
      * Should be cheap and idempotent — called every poll cycle (~30s).

+    The poller trusts the result wholesale: any schema the routine does
+    not return is treated as having no work this cycle. It does NOT
+    second-guess omissions with a per-schema scan — that would re-run the
+    exact queries this routine exists to avoid. Consequently the routine
+    is *only* appropriate for multi-tenant deployments. Single-schema
+    (default/public only) installs should NOT create it: the per-schema
+    fallback below is a single cheap EXISTS check that covers ``public``
+    correctly and cannot starve.
+
    Fallback when the routine is absent: per-schema ``EXISTS`` queries
    from Python (~4ms per schema). The server-side path is a single-
    round-trip optimisation worth ~200ms in deployments with thousands
@@ -26,11 +26,8 @@ from ..config import (
    DEFAULT_EMBEDDINGS_GEMINI_MODEL,
    DEFAULT_EMBEDDINGS_LITELLM_MODEL,
    DEFAULT_EMBEDDINGS_LITELLM_SDK_MODEL,
-    DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU,
    DEFAULT_EMBEDDINGS_LOCAL_MODEL,
-    DEFAULT_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE,
    DEFAULT_EMBEDDINGS_OPENAI_MODEL,
-    DEFAULT_EMBEDDINGS_PROVIDER,
    DEFAULT_EMBEDDINGS_ZEROENTROPY_BATCH_SIZE,
    DEFAULT_EMBEDDINGS_ZEROENTROPY_DIMENSIONS,
    DEFAULT_EMBEDDINGS_ZEROENTROPY_ENCODING_FORMAT,
@@ -40,13 +37,6 @@ from ..config import (
    DEFAULT_ZEROENTROPY_BASE_URL,
    ENV_EMBEDDINGS_COHERE_API_KEY,
    ENV_EMBEDDINGS_GEMINI_API_KEY,
-    ENV_EMBEDDINGS_LOCAL_FORCE_CPU,
-    ENV_EMBEDDINGS_LOCAL_MODEL,
-    ENV_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE,
-    ENV_EMBEDDINGS_ONNX_DIMENSIONS,
-    ENV_EMBEDDINGS_ONNX_MODEL_ID,
-    ENV_EMBEDDINGS_ONNX_MODEL_PATH,
-    ENV_EMBEDDINGS_ONNX_TOKENIZER_NAME_OR_PATH,
    ENV_EMBEDDINGS_OPENAI_API_KEY,
    ENV_EMBEDDINGS_OPENAI_BASE_URL,
    ENV_EMBEDDINGS_OPENAI_MODEL,
@@ -834,14 +834,12 @@ class EntityResolver:

            best_candidate = None
            best_score = 0.0
-            best_name_similarity = 0.0

            nearby_entity_set = {e["text"].lower() for e in nearby_entities if e["text"] != entity_text}

            for row in candidates:
                candidate_id = row["id"]
                canonical_name = row["canonical_name"]
-                metadata = row["metadata"]
                last_seen = row["last_seen"]
                score = 0.0

@@ -888,7 +886,6 @@ class EntityResolver:
                if score > best_score:
                    best_score = score
                    best_candidate = candidate_id
-                    best_name_similarity = name_similarity

            # Threshold for considering it the same entity
            threshold = 0.6
@@ -8,7 +8,7 @@ enabling support for multiple LLM backends (OpenAI, Anthropic, Gemini, Codex, et
 from abc import ABC, abstractmethod
 from typing import Any

-from .response_models import LLMToolCallResult, TokenUsage
+from .response_models import LLMToolCallResult


 class LLMInterface(ABC):
@@ -11,14 +11,10 @@ import time
 import uuid
 from contextlib import AsyncExitStack
 from pathlib import Path
-from typing import Any
-
-import httpx
-from openai import APIConnectionError, APIStatusError, AsyncOpenAI, LengthFinishReasonError
+from typing import TYPE_CHECKING, Any

 # Vertex AI imports (conditional - for LLMProvider to pass credentials to GeminiLLM)
 try:
-    import google.auth
    from google.oauth2 import service_account

    VERTEXAI_AVAILABLE = True
@@ -27,16 +23,14 @@ except ImportError:

 from ..config import (
    DEFAULT_LLM_MAX_CONCURRENT,
-    DEFAULT_LLM_TIMEOUT,
    ENV_CONSOLIDATION_LLM_MAX_CONCURRENT,
-    ENV_LLM_GROQ_SERVICE_TIER,
    ENV_LLM_MAX_CONCURRENT,
-    ENV_LLM_TIMEOUT,
    ENV_REFLECT_LLM_MAX_CONCURRENT,
    ENV_RETAIN_LLM_MAX_CONCURRENT,
 )
-from ..metrics import get_metrics_collector
-from .response_models import TokenUsage
+
+if TYPE_CHECKING:
+    from .response_models import LLMToolCallResult

 # Seed applied to every Groq request for deterministic behavior.
 DEFAULT_LLM_SEED = 4242
@@ -287,7 +281,6 @@ def create_llm_provider(
    Returns:
        LLMInterface implementation for the specified provider.
    """
-    from .llm_interface import LLMInterface
    from .providers import (
        AnthropicLLM,
        ClaudeCodeLLM,
@@ -15,6 +15,7 @@ import functools
 import inspect
 import json
 import logging
+import sys
 import time
 import uuid
 from collections.abc import Awaitable, Callable
@@ -36,8 +37,6 @@ from ..config import (
    HindsightConfig,
    get_config,
 )
-from ..db_url import to_libpq_url
-from ..metrics import get_metrics_collector
 from ..tracing import create_operation_span
 from ..utils import mask_network_location
 from ..worker.exceptions import DeferOperation, RetryTaskAt
@@ -58,10 +57,7 @@ from .llm_trace import (
 from .operation_metadata import (
    BatchRetainChildMetadata,
    BatchRetainParentMetadata,
-    ConsolidationMetadata,
-    RefreshMentalModelMetadata,
    RetainExtractionErrors,
-    RetainMetadata,
    RetainOutcomeAggregate,
    RetainOutcomeMetadata,
 )
@@ -334,16 +330,12 @@ def validate_sql_schema(sql: str) -> None:
                        )


-import asyncpg
-import numpy as np
-from pydantic import BaseModel, Field
-
 from .cross_encoder import CrossEncoderModel
 from .embeddings import Embeddings, create_embeddings_from_env
 from .interface import MemoryEngineInterface

 if TYPE_CHECKING:
-    from hindsight_api.extensions import OperationValidatorExtension, TenantExtension
+    from hindsight_api.extensions import OperationValidatorExtension, TenantExtension, ValidationResult
    from hindsight_api.models import RequestContext

    from .audit import AuditLogListResponse, AuditLogStatsResponse
@@ -352,21 +344,18 @@ if TYPE_CHECKING:

 from enum import Enum

-from ..metrics import get_metrics_collector
 from ..pg0 import EmbeddedPostgres, parse_pg0_url
 from .entity_resolver import EntityResolver
 from .llm_wrapper import LLMConfig, requires_api_key, sanitize_llm_output, sanitize_text
 from .query_analyzer import QueryAnalyzer
 from .reflect import run_reflect_agent
-from .reflect.prompts import DELTA_SYSTEM_PROMPT, build_delta_prompt
 from .reflect.tools import tool_expand, tool_recall, tool_search_mental_models, tool_search_observations
 from .response_models import (
    VALID_RECALL_FACT_TYPES,
-    EntityObservation,
+    DryRunExtractionResult,
    EntityState,
    LLMCallTrace,
    MemoryFact,
-    ObservationRef,
    ReflectResult,
    TokenUsage,
    ToolCallTrace,
@@ -374,7 +363,6 @@ from .response_models import (
 from .response_models import RecallResult as RecallResultModel
 from .retain import bank_utils, embedding_utils
 from .retain.types import RetainContentDict
-from .search import think_utils
 from .search.reranking import CrossEncoderReranker, apply_combined_scoring
 from .search.tags import TagGroup, TagsMatch, build_tag_groups_where_clause, build_tags_where_clause
 from .search.types import ScoredResult
@@ -437,6 +425,12 @@ class _SubBatchSplit:
    document_body_overrides: list[str | None] = field(default_factory=list)


+@dataclass(frozen=True)
+class _RetainChunkingConfig:
+    chunk_size: int
+    structured_chunk_size: int | None
+
+
 def _split_contents_into_sub_batches(
    contents: list[RetainContentDict],
    tokens_per_batch: int,
@@ -1161,7 +1155,7 @@ class MemoryEngine(MemoryEngineInterface):
        if self._operation_validator is None:
            return None

-        from hindsight_api.extensions import OperationValidationError, ValidationResult
+        from hindsight_api.extensions import OperationValidationError

        result = await validation_coro
        if not result.allowed:
@@ -2581,6 +2575,28 @@ class MemoryEngine(MemoryEngineInterface):
                f"first-time model download legitimately needs more time."
            ) from e

+        # Normalize torch's process-global default dtype back to float32 after the
+        # concurrent local model loads. transformers' dtype context manager (entered
+        # by SentenceTransformer / CrossEncoder / from_pretrained) does a
+        # NON-thread-safe save/restore of the global default dtype: when an fp16 and
+        # an fp32 model load in parallel above, an unlucky interleave can leave the
+        # default stuck at float16, after which every encode() emits NaN vectors that
+        # pgvector rejects ("NaN not allowed in vector") on MPS, or raises
+        # "c10::Half != float" on CPU — non-deterministically across restarts. By the
+        # time gather() returns, all load threads have joined, so resetting the
+        # default here is race-free, keeps the loads fully parallel, and converges on
+        # the float32 inference state a healthy boot already reaches. torch is only
+        # imported (in sys.modules) if a local provider actually loaded a model.
+        # See https://github.com/vectorize-io/hindsight/issues/2162.
+        torch_mod = sys.modules.get("torch")
+        if torch_mod is not None and torch_mod.get_default_dtype() != torch_mod.float32:
+            logger.warning(
+                "torch default dtype was left at %s after concurrent model init; "
+                "restoring float32 to avoid NaN embedding vectors (issue #2162).",
+                torch_mod.get_default_dtype(),
+            )
+            torch_mod.set_default_dtype(torch_mod.float32)
+
        # Run database migrations if enabled
        if self._run_migrations:
            if not self.db_url:
@@ -2596,51 +2612,40 @@ class MemoryEngine(MemoryEngineInterface):
            tenants = await self._tenant_extension.list_tenants()
            if tenants:
                logger.info(f"Running migrations on {len(tenants)} schema(s)...")
-                for tenant in tenants:
-                    schema = tenant.schema
-                    if schema:
-                        schema = self._backend.normalize_schema(schema)
-                        self._backend.run_migrations(self.db_url, schema=schema)
+                if self._database_backend_type == "postgresql":
+                    # PG: fan out across schemas (up to migration_concurrency, each
+                    # in its own process) and fold the PG-specific post-migration
+                    # extension/dimension sync into the same per-schema unit. Run
+                    # off the event loop so the process pool's blocking joins don't
+                    # stall it.
+                    from ..migrations import run_migrations_for_schemas
+
+                    schemas = [tenant.schema for tenant in tenants if tenant.schema]
+                    await asyncio.to_thread(
+                        run_migrations_for_schemas,
+                        self.db_url,
+                        schemas,
+                        concurrency=config.migration_concurrency,
+                        migration_database_url=config.migration_database_url,
+                        embedding_dimension=self.embeddings.dimension,
+                        vector_extension=config.vector_extension,
+                        text_search_extension=config.text_search_extension,
+                        pg_search_tokenizer=config.text_search_extension_pg_search_tokenizer,
+                        ensure_extensions=self._backend.supports_bm25,
+                    )
+                else:
+                    # Oracle and other backends: Alembic's non-thread-safe globals
+                    # and the absence of per-schema extension steps make parallelism
+                    # unnecessary; run sequentially via the backend's own runner.
+                    # normalize_schema() maps PG's "public" default to None (the
+                    # connecting user's schema) on Oracle.
+                    for tenant in tenants:
+                        if tenant.schema:
+                            self._backend.run_migrations(
+                                self.db_url, schema=self._backend.normalize_schema(tenant.schema)
+                            )
                logger.info("Schema migrations completed")

-            # PG-specific post-migration steps: ensure vector/text search extensions
-            # and embedding dimensions match configuration. These are no-ops for
-            # non-PG backends since they use different indexing strategies.
-            if self._backend.supports_bm25:
-                from ..migrations import (
-                    ensure_embedding_dimension,
-                    ensure_text_search_extension,
-                    ensure_vector_extension,
-                )
-
-                if tenants:
-                    for tenant in tenants:
-                        schema = tenant.schema
-                        if schema:
-                            ensure_embedding_dimension(
-                                self.db_url,
-                                self.embeddings.dimension,
-                                schema=schema,
-                                vector_extension=config.vector_extension,
-                            )
-
-                    for tenant in tenants:
-                        schema = tenant.schema
-                        if schema:
-                            ensure_vector_extension(
-                                self.db_url, vector_extension=config.vector_extension, schema=schema
-                            )
-
-                    for tenant in tenants:
-                        schema = tenant.schema
-                        if schema:
-                            ensure_text_search_extension(
-                                self.db_url,
-                                text_search_extension=config.text_search_extension,
-                                pg_search_tokenizer=config.text_search_extension_pg_search_tokenizer,
-                                schema=schema,
-                            )
-
        logger.info(f"Connecting to database at {mask_network_location(self.db_url)}")

        # Create SQL dialect via abstraction layer
@@ -3115,7 +3120,7 @@ class MemoryEngine(MemoryEngineInterface):
            )
            result = await self._validate_operation(self._operation_validator.validate_retain(ctx))
            if result and result.contents is not None:
-                contents = result.contents
+                contents = cast(list[RetainContentDict], result.contents)

        # Engine-owned copy: the orchestrator clears per-item "content" strings
        # after building the document's combined text (memory pressure
@@ -3225,7 +3230,7 @@ class MemoryEngine(MemoryEngineInterface):
            # with, so the offsets match the chunk_index values it assigns.
            from .retain import fact_extraction, fact_storage

-            sub_chunk_size = await self._resolve_retain_chunk_size(bank_id, request_context, strategy)
+            chunking_config = await self._resolve_retain_chunking_config(bank_id, request_context, strategy)
            chunk_offsets: dict[str, int] = {}

            # In update_mode="append", retain_batch prepends the existing document
@@ -3248,7 +3253,11 @@ class MemoryEngine(MemoryEngineInterface):
                    existing_text = await fact_storage.get_document_content(conn, bank_id, append_doc_id)
                if existing_text:
                    append_prepend_chunks[append_doc_id] = len(
-                        fact_extraction.chunk_text(existing_text, sub_chunk_size)
+                        fact_extraction.chunk_text(
+                            existing_text,
+                            chunking_config.chunk_size,
+                            structured_chunk_size=chunking_config.structured_chunk_size,
+                        )
                    )

            for i, (sub_batch, sub_origins) in enumerate(zip(sub_batches, origin_indices), 1):
@@ -3302,7 +3311,13 @@ class MemoryEngine(MemoryEngineInterface):
                # document continues the sequence.
                if sub_doc_id:
                    sub_chunk_count = sum(
-                        len(fact_extraction.chunk_text(item.get("content", "") or "", sub_chunk_size))
+                        len(
+                            fact_extraction.chunk_text(
+                                item.get("content", "") or "",
+                                chunking_config.chunk_size,
+                                structured_chunk_size=chunking_config.structured_chunk_size,
+                            )
+                        )
                        for item in sub_batch
                    )
                    # retain_batch only prepends the existing body on the global
@@ -3413,13 +3428,13 @@ class MemoryEngine(MemoryEngineInterface):
        except Exception as e:
            logger.warning(f"Failed to submit graph maintenance task for bank {bank_id}: {e}")

-    async def _resolve_retain_chunk_size(
+    async def _resolve_retain_chunking_config(
        self,
        bank_id: str,
        request_context: "RequestContext",
        strategy: str | None,
-    ) -> int:
-        """Resolve the effective ``retain_chunk_size`` for a bank.
+    ) -> _RetainChunkingConfig:
+        """Resolve the effective retain chunking settings for a bank.

        Mirrors the bank-config + strategy resolution that
        ``_retain_batch_async_internal`` applies before handing config to the
@@ -3433,7 +3448,10 @@ class MemoryEngine(MemoryEngineInterface):
        effective_strategy = strategy or resolved_config.retain_default_strategy
        if effective_strategy:
            resolved_config = apply_strategy(resolved_config, effective_strategy)
-        return getattr(resolved_config, "retain_chunk_size", 3000)
+        return _RetainChunkingConfig(
+            chunk_size=getattr(resolved_config, "retain_chunk_size", 3000),
+            structured_chunk_size=getattr(resolved_config, "retain_structured_chunk_size", None),
+        )

    async def _retain_batch_async_internal(
        self,
@@ -3476,7 +3494,7 @@ class MemoryEngine(MemoryEngineInterface):
        # Use the new modular orchestrator
        from .retain import orchestrator

-        backend = await self._get_backend()
+        await self._get_backend()

        # Resolve bank-specific config for this operation
        resolved_config = await self._config_resolver.resolve_full_config(bank_id, request_context)
@@ -3612,7 +3630,7 @@ class MemoryEngine(MemoryEngineInterface):
        parse_archive(archive_bytes)

        await self._authenticate_tenant(request_context)
-        backend = await self._get_backend()
+        await self._get_backend()
        # Ensure the bank (and its per-bank vector indexes) exist before inserts.
        # Import has no single write transaction to join — the archive is written
        # by a worker later — so the bank is created on its own connection.
@@ -4178,9 +4196,6 @@ class MemoryEngine(MemoryEngineInterface):
                retrieve_all_fact_types_parallel,
            )

-            # Track each retrieval start time
-            retrieval_start = time.time()
-
            retrieval_span = tracer_otel.start_span("hindsight.recall_retrieval")
            retrieval_span.set_attribute("hindsight.bank_id", bank_id)
            retrieval_span.set_attribute("hindsight.fact_types", ",".join(fact_type))
@@ -4291,10 +4306,7 @@ class MemoryEngine(MemoryEngineInterface):
                        f"graph {pre_cap_counts[2]}->{len(graph_results)}"
                    )

-            retrieval_duration = time.time() - retrieval_start
-
            step_duration = time.time() - step_start
-            total_retrievals = len(fact_type) * (4 if temporal_results else 3)
            # Format per-method timings
            timing_parts = [
                f"semantic={len(semantic_results)}({aggregated_timings['semantic']:.3f}s)",
@@ -5173,6 +5185,12 @@ class MemoryEngine(MemoryEngineInterface):
            # document_metadata is sourced from retain_params.metadata
            document_metadata = retain_params_parsed.get("metadata") if retain_params_parsed else None

+            # observation_scopes is captured into retain_params at retain time
+            # (see _build_retain_params); surface it as a top-level field so the
+            # UI can show which scoping was requested. Only present for documents
+            # retained after this was added.
+            observation_scopes = retain_params_parsed.get("observation_scopes") if retain_params_parsed else None
+
            return {
                "id": doc["id"],
                "bank_id": doc["bank_id"],
@@ -5189,6 +5207,7 @@ class MemoryEngine(MemoryEngineInterface):
                "tags": list(doc["tags"]) if doc["tags"] else [],
                "document_metadata": document_metadata or None,
                "retain_params": retain_params_parsed or None,
+                "observation_scopes": observation_scopes or None,
            }

    async def delete_document(
@@ -5738,6 +5757,46 @@ class MemoryEngine(MemoryEngineInterface):

                return {"deleted_count": count or 0}

+    async def list_observation_scopes(
+        self,
+        bank_id: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """List the distinct scopes across a bank's observations.
+
+        Every consolidated observation lives under a "scope": the exact set of
+        tags it was consolidated with. This enumerates each distinct scope (tag
+        order normalized so ``[a, b]`` and ``[b, a]`` collapse) together with the
+        number of observations in it. The empty list ``[]`` is the "global" scope
+        of untagged observations. Results are ordered most-populous first.
+
+        Returns:
+            Dict with ``scopes``: list of ``{"tags": list[str], "count": int}``.
+        """
+        await self._authenticate_tenant(request_context)
+        if self._operation_validator:
+            from hindsight_api.extensions import BankReadContext
+
+            ctx = BankReadContext(bank_id=bank_id, operation="list_observation_scopes", request_context=request_context)
+            await self._validate_operation(self._operation_validator.validate_bank_read(ctx))
+        backend = await self._get_backend()
+        async with acquire_with_retry(backend) as conn:
+            rows = await conn.fetch(
+                f"""
+                SELECT scope, COUNT(*) AS count
+                FROM (
+                    SELECT COALESCE(ARRAY(SELECT unnest(tags) ORDER BY 1), '{{}}'::text[]) AS scope
+                    FROM {fq_table("memory_units")}
+                    WHERE bank_id = $1 AND fact_type = 'observation'
+                ) s
+                GROUP BY scope
+                ORDER BY count DESC, scope
+                """,
+                bank_id,
+            )
+            return {"scopes": [{"tags": list(r["scope"]), "count": r["count"]} for r in rows]}
+
    async def retry_failed_consolidation(
        self,
        bank_id: str,
@@ -5933,9 +5992,10 @@ class MemoryEngine(MemoryEngineInterface):
          not ``memory_links``, so there is nothing to relink directly).
        - **Invalidate** (``state='invalidated'``): move the row to the archive
          (cascade-pruning its links/entity associations and re-deriving dependent
-          observations). The embedding + an entity-id snapshot travel with it.
+          observations). The archive is cold storage, so the embedding is dropped
+          (only an entity-id snapshot travels with it).
        - **Revert** (``state='valid'``): move the row back, restore its entity
-          associations, and re-consolidate.
+          associations, recompute its embedding, and re-consolidate.

        Only ``world``/``experience`` facts can be curated — observations are
        derived and regenerate from their sources. Returns the updated memory
@@ -6027,6 +6087,13 @@ class MemoryEngine(MemoryEngineInterface):
                    )

                collist = await self._memory_unit_columns(conn)
+                # The archive is cold storage, never a recall surface, so the schema gives it
+                # no `embedding` column at all (dropped in d4f6a8c2e1b3). The move in/out is
+                # therefore over every memory_units column EXCEPT embedding; on revert the
+                # embedding is recomputed from the unit's text/dates/entities below. This makes
+                # a model switch (which re-dimensions memory_units) structurally unable to trip
+                # a vector-dimension mismatch on the INSERT … SELECT round-trip (#2209).
+                arch_cols = ", ".join(c for c in (s.strip() for s in collist.split(",")) if c != '"embedding"')

                # --- Edit fields (live rows only): text / context / dates / fact_type / entities ---
                doing_edit = any(
@@ -6118,8 +6185,8 @@ class MemoryEngine(MemoryEngineInterface):
                    # Capture relink victims BEFORE the row (and its links) disappear.
                    await enqueue_relink_victims(conn, bank_id, [memory_id], ops=backend.ops)
                    await conn.execute(
-                        f"INSERT INTO {arch} ({collist}, invalidation_reason, invalidated_at, entity_ids) "
-                        f"SELECT {collist}, $2, now(), $3::uuid[] FROM {mu} WHERE id = $1 AND bank_id = $4",
+                        f"INSERT INTO {arch} ({arch_cols}, invalidation_reason, invalidated_at, entity_ids) "
+                        f"SELECT {arch_cols}, $2, now(), $3::uuid[] FROM {mu} WHERE id = $1 AND bank_id = $4",
                        str(memory_uuid),
                        reason,
                        entity_ids,
@@ -6145,8 +6212,11 @@ class MemoryEngine(MemoryEngineInterface):
                    arch_row = await conn.fetchrow(
                        f"SELECT entity_ids FROM {arch} WHERE id = $1 AND bank_id = $2", str(memory_uuid), bank_id
                    )
+                    # The archive has no embedding column (see arch_cols above), so the live
+                    # row's embedding defaults to NULL on the way back and is recomputed below
+                    # once entities are restored.
                    await conn.execute(
-                        f"INSERT INTO {mu} ({collist}) SELECT {collist} FROM {arch} WHERE id = $1 AND bank_id = $2",
+                        f"INSERT INTO {mu} ({arch_cols}) SELECT {arch_cols} FROM {arch} WHERE id = $1 AND bank_id = $2",
                        str(memory_uuid),
                        bank_id,
                    )
@@ -6169,6 +6239,35 @@ class MemoryEngine(MemoryEngineInterface):
                            arch_row["entity_ids"],
                            bank_id,
                        )
+                    # Recompute the embedding (the archive doesn't keep one) so the reverted
+                    # unit is searchable again, using the now-current model's dimension and the
+                    # restored entity set — mirroring how an edit re-embeds.
+                    reverted = await conn.fetchrow(
+                        f"SELECT text, occurred_start, occurred_end, mentioned_at FROM {mu} "
+                        f"WHERE id = $1 AND bank_id = $2",
+                        str(memory_uuid),
+                        bank_id,
+                    )
+                    if reverted:
+                        ent_rows = await conn.fetch(
+                            f"SELECT e.canonical_name FROM {ue} ue JOIN {ent} e ON ue.entity_id = e.id "
+                            f"WHERE ue.unit_id = $1",
+                            str(memory_uuid),
+                        )
+                        new_emb = await self._reembed_memory_text(
+                            text=reverted["text"],
+                            occurred_start=reverted["occurred_start"],
+                            occurred_end=reverted["occurred_end"],
+                            mentioned_at=reverted["mentioned_at"],
+                            entities=[r["canonical_name"] for r in ent_rows],
+                        )
+                        if new_emb is not None:
+                            await conn.execute(
+                                f"UPDATE {mu} SET embedding = $3::vector WHERE id = $1 AND bank_id = $2",
+                                str(memory_uuid),
+                                bank_id,
+                                new_emb,
+                            )
                    await conn.execute(f"DELETE FROM {arch} WHERE id = $1 AND bank_id = $2", str(memory_uuid), bank_id)
                    need_consolidation = True
                    need_graph = True
@@ -6317,6 +6416,10 @@ class MemoryEngine(MemoryEngineInterface):
                    query_conditions.append(tag_clause.removeprefix("AND "))
                    param_count += 1
                    query_params.append(tags)
+            elif tags_match == "exact":
+                # Exact match with no tags is the "global" scope: rows that carry no
+                # tags at all. (Other match modes treat empty tags as "no filter".)
+                query_conditions.append("(tags IS NULL OR tags = '{}')")

            where_clause = "WHERE " + " AND ".join(query_conditions) if query_conditions else ""

@@ -6357,9 +6460,10 @@ class MemoryEngine(MemoryEngineInterface):
                    source_memory_ids.extend(unit["source_memory_ids"])
            source_memory_ids = list(set(source_memory_ids))  # Deduplicate

-            # Fetch non-entity links where BOTH endpoints are in the visible set (or
-            # source memories). Entity edges are derived below from unit_entities so
-            # we don't materialize them in memory_links anymore.
+            # Fetch links where BOTH endpoints are in the visible set (or source
+            # memories). Entity edges are derived below from unit_entities so we
+            # don't materialize them in memory_links anymore (dropped in migration
+            # e9b2c7d1f3a4) — no link_type filter is needed.
            # Cap at 10k edges — the UI can't usefully render more, and uncapped queries
            # on highly-connected graphs (e.g. 1000 nodes with 500k+ edges) are too slow.
            max_edges = 10000
@@ -6373,8 +6477,7 @@ class MemoryEngine(MemoryEngineInterface):
                           ml.weight,
                           NULL::text AS entity_name
                    FROM {fq_table("memory_links")} ml
-                    WHERE ml.link_type <> 'entity'
-                      AND ml.from_unit_id = ANY($1::uuid[])
+                    WHERE ml.from_unit_id = ANY($1::uuid[])
                      AND ml.to_unit_id = ANY($1::uuid[])
                    ORDER BY ml.weight DESC NULLS LAST
                    LIMIT $2
@@ -6651,6 +6754,84 @@ class MemoryEngine(MemoryEngineInterface):

        return {"nodes": nodes, "edges": edges, "table_rows": table_rows, "total_units": total_count, "limit": limit}

+    # Prompt-affecting settings overridable per dry-run extraction call.
+    _EXTRACTION_OVERRIDE_FIELDS = frozenset(
+        {
+            "retain_mission",
+            "retain_extraction_mode",
+            "retain_custom_instructions",
+            "retain_extract_causal_links",
+            "retain_chunk_size",
+            "entity_labels",
+            "entities_allow_free_form",
+            "llm_output_language",
+        }
+    )
+
+    async def extract_dry_run(
+        self,
+        bank_id: str,
+        content: str,
+        *,
+        context: str = "",
+        event_date: "datetime | None" = None,
+        overrides: dict | None = None,
+        agent_name: str | None = None,
+        request_context: "RequestContext",
+    ) -> "DryRunExtractionResult":
+        """Run fact extraction ONLY — no entity resolution, links, embeddings, or persistence.
+
+        Returns candidate facts (a subset of the ``list_memory_units`` item shape) plus the LLM token
+        usage, so callers can diff a mission's extraction output against stored memories without
+        mutating the bank. Every prompt-affecting setting is overridable per call via ``overrides``
+        (e.g. to test a candidate retain mission); ``agent_name`` overrides the narrator.
+        Side-effect-free and idempotent.
+        """
+        from .response_models import ExtractedFact
+        from .retain import bank_utils, fact_extraction
+
+        # Resolve the tenant schema before touching any bank-scoped data (config, bank profile).
+        await self._authenticate_tenant(request_context)
+        resolved_config = await self._config_resolver.resolve_full_config(bank_id, request_context)
+        if self._llm_config.provider == "none":
+            resolved_config.retain_extraction_mode = "chunks"
+
+        for key, value in (overrides or {}).items():
+            if key not in self._EXTRACTION_OVERRIDE_FIELDS:
+                raise ValueError(
+                    f"Unsupported extraction override '{key}'. Allowed: {sorted(self._EXTRACTION_OVERRIDE_FIELDS)}"
+                )
+            setattr(resolved_config, key, value)
+
+        backend = await self._get_backend()
+        # Narrator primes the "Narrator:" line in the prompt — resolve it the same way retain does.
+        if agent_name is None:
+            profile = await bank_utils.get_bank_profile(backend, bank_id)
+            profile_name = profile["name"] if profile else bank_id
+            agent_name = None if profile_name == bank_id else profile_name
+
+        retain_llm = self._retain_llm_config.with_config(resolved_config, bank_id=bank_id, operation="retain")
+        facts, _chunks, usage = await fact_extraction.extract_facts_from_text(
+            text=content,
+            event_date=event_date,
+            llm_config=retain_llm,
+            agent_name=agent_name or "",
+            config=resolved_config,
+            context=context,
+        )
+
+        extracted = [
+            ExtractedFact(
+                text=fact.fact,
+                fact_type=fact.fact_type,
+                occurred_start=fact.occurred_start,
+                occurred_end=fact.occurred_end,
+                entities=[e.text for e in (fact.entities or []) if getattr(e, "text", None)],
+            )
+            for fact in facts
+        ]
+        return DryRunExtractionResult(facts=extracted, usage=usage)
+
    async def list_memory_units(
        self,
        bank_id: str,
@@ -8081,7 +8262,7 @@ class MemoryEngine(MemoryEngineInterface):
                bank_id=bank_id, operation="update_bank_disposition", request_context=request_context
            )
            await self._validate_operation(self._operation_validator.validate_bank_write(ctx))
-        backend = await self._get_backend()
+        await self._get_backend()
        await bank_utils.update_bank_disposition(self._backend, bank_id, disposition)

    async def set_bank_mission(
@@ -8108,7 +8289,7 @@ class MemoryEngine(MemoryEngineInterface):

            ctx = BankWriteContext(bank_id=bank_id, operation="set_bank_mission", request_context=request_context)
            await self._validate_operation(self._operation_validator.validate_bank_write(ctx))
-        backend = await self._get_backend()
+        await self._get_backend()
        await bank_utils.set_bank_mission(self._backend, bank_id, mission)
        return {"bank_id": bank_id, "mission": mission}

@@ -8137,7 +8318,7 @@ class MemoryEngine(MemoryEngineInterface):

            ctx = BankWriteContext(bank_id=bank_id, operation="merge_bank_mission", request_context=request_context)
            await self._validate_operation(self._operation_validator.validate_bank_write(ctx))
-        backend = await self._get_backend()
+        await self._get_backend()
        return await bank_utils.merge_bank_mission(self._backend, self._reflect_llm_config, bank_id, new_info)

    async def list_banks(
@@ -8155,7 +8336,7 @@ class MemoryEngine(MemoryEngineInterface):
            List of dicts with bank_id, name, disposition, mission, created_at, updated_at
        """
        await self._authenticate_tenant(request_context)
-        backend = await self._get_backend()
+        await self._get_backend()
        banks = await bank_utils.list_banks(self._backend)
        if self._operation_validator:
            from hindsight_api.extensions import BankListContext
@@ -8195,12 +8376,15 @@ class MemoryEngine(MemoryEngineInterface):
        """
        Reflect and formulate an answer using an agentic loop with tools.

-        The reflect agent iteratively uses tools to:
+        The reflect agent iteratively uses read-only tools to:
        1. lookup: Get mental models (synthesized knowledge)
        2. recall: Search facts (semantic + temporal retrieval)
-        3. learn: Create/update mental models with new insights
+        3. search observations: Retrieve prior observations
        4. expand: Get chunk/document context for memories

+        Reflect is read-only: it synthesizes an answer from the bank's stored
+        memories and persists nothing.
+
        The agent starts with empty context and must call tools to gather
        information. On the last iteration, tools are removed to force a
        final text response.
@@ -9086,11 +9270,16 @@ class MemoryEngine(MemoryEngineInterface):
            # per-fact-type slice, and it tolerates empty maps (the section
            # prints with no rows). Response keys are kept populated below for
            # schema stability so existing SDK deserializers don't break.
+            # No link_type filter: entity edges are no longer stored in
+            # memory_links (dropped in migration e9b2c7d1f3a4 — derived on demand
+            # from unit_entities), so only temporal/semantic/caused_by rows exist
+            # here. Omitting the predicate lets the (bank_id, link_type) index
+            # serve this bank-scoped GROUP BY as an index-only scan.
            non_entity_link_rows = await conn.fetch(
                f"""
                SELECT link_type, COUNT(*) as count
                FROM {fq_table("memory_links")}
-                WHERE bank_id = $1 AND link_type <> 'entity'
+                WHERE bank_id = $1
                GROUP BY link_type
                """,
                bank_id,
@@ -9432,87 +9621,6 @@ class MemoryEngine(MemoryEngineInterface):
            "observations": [],
        }

-    def _parse_observations(self, observations_raw: list):
-        """Parse raw observation dicts into typed Observation models.
-
-        Returns list of Observation models with computed trend/evidence_span/evidence_count.
-        """
-        from .reflect.observations import Observation, ObservationEvidence
-
-        observations: list[Observation] = []
-        for obs in observations_raw:
-            if not isinstance(obs, dict):
-                continue
-
-            try:
-                parsed = Observation(
-                    title=obs.get("title", ""),
-                    content=obs.get("content", ""),
-                    evidence=[
-                        ObservationEvidence(
-                            memory_id=ev.get("memory_id", ""),
-                            quote=ev.get("quote", ""),
-                            relevance=ev.get("relevance", ""),
-                            timestamp=ev.get("timestamp"),
-                        )
-                        for ev in obs.get("evidence", [])
-                        if isinstance(ev, dict)
-                    ],
-                    created_at=obs.get("created_at"),
-                )
-                observations.append(parsed)
-            except Exception as e:
-                logger.warning(f"Failed to parse observation: {e}")
-                continue
-
-        return observations
-
-    async def _count_memories_since(
-        self,
-        bank_id: str,
-        since_timestamp: str | None,
-        backend=None,
-    ) -> int:
-        """
-        Count memories created after a given timestamp.
-
-        Args:
-            bank_id: Bank identifier
-            since_timestamp: ISO timestamp string. If None, returns total count.
-            backend: Optional database backend (uses default if not provided)
-
-        Returns:
-            Number of memories created since the timestamp
-        """
-        if backend is None:
-            backend = await self._get_backend()
-
-        async with acquire_with_retry(backend) as conn:
-            if since_timestamp:
-                # Parse the timestamp
-                from datetime import datetime
-
-                try:
-                    ts = datetime.fromisoformat(since_timestamp.replace("Z", "+00:00"))
-                except ValueError:
-                    # Invalid timestamp, return total count
-                    ts = None
-
-                if ts:
-                    count = await conn.fetchval(
-                        f"SELECT COUNT(*) FROM {fq_table('memory_units')} WHERE bank_id = $1 AND created_at > $2",
-                        bank_id,
-                        ts,
-                    )
-                    return count or 0
-
-            # No timestamp or invalid, return total count
-            count = await conn.fetchval(
-                f"SELECT COUNT(*) FROM {fq_table('memory_units')} WHERE bank_id = $1",
-                bank_id,
-            )
-            return count or 0
-
    async def _delete_stale_observations_for_memories(
        self,
        conn,
@@ -9529,149 +9637,6 @@ class MemoryEngine(MemoryEngineInterface):

        return await delete_stale_observations_for_memories(conn, bank_id, fact_ids, ops=self._backend.ops)

-    # =========================================================================
-    # MENTAL MODELS (CONSOLIDATED) - Read-only access to auto-consolidated mental models
-    # =========================================================================
-
-    async def list_mental_models_consolidated(
-        self,
-        bank_id: str,
-        *,
-        tags: list[str] | None = None,
-        tags_match: str = "any",
-        limit: int = 100,
-        offset: int = 0,
-        request_context: "RequestContext",
-    ) -> list[dict[str, Any]]:
-        """List auto-consolidated observations for a bank.
-
-        Observations are stored in memory_units with fact_type='observation'.
-        They are automatically created and updated by the consolidation engine.
-
-        Args:
-            bank_id: Bank identifier
-            tags: Optional tags to filter by
-            tags_match: How to match tags - 'any', 'all', or 'exact'
-            limit: Maximum number of results
-            offset: Offset for pagination
-            request_context: Request context for authentication
-
-        Returns:
-            List of observation dicts
-        """
-        await self._authenticate_tenant(request_context)
-        backend = await self._get_backend()
-
-        async with acquire_with_retry(backend) as conn:
-            # Build tag filter
-            tag_filter = ""
-            params: list[Any] = [bank_id, limit, offset]
-            if tags:
-                if tags_match == "all":
-                    tag_filter = " AND tags @> $4::varchar[]"
-                elif tags_match == "exact":
-                    tag_filter = " AND tags = $4::varchar[]"
-                else:  # any
-                    tag_filter = " AND tags && $4::varchar[]"
-                params.append(tags)
-
-            rows = await conn.fetch(
-                f"""
-                SELECT id, bank_id, text, proof_count, tags, source_memory_ids, created_at, updated_at
-                FROM {fq_table("memory_units")}
-                WHERE bank_id = $1 AND fact_type = 'observation' {tag_filter}
-                ORDER BY updated_at DESC NULLS LAST
-                LIMIT $2 OFFSET $3
-                """,
-                *params,
-            )
-
-            return [self._row_to_observation_consolidated(row) for row in rows]
-
-    async def get_observation_consolidated(
-        self,
-        bank_id: str,
-        observation_id: str,
-        *,
-        include_source_memories: bool = True,
-        request_context: "RequestContext",
-    ) -> dict[str, Any] | None:
-        """Get a single observation by ID.
-
-        Args:
-            bank_id: Bank identifier
-            observation_id: Observation ID
-            include_source_memories: Whether to include full source memory details
-            request_context: Request context for authentication
-
-        Returns:
-            Observation dict or None if not found
-        """
-        await self._authenticate_tenant(request_context)
-        backend = await self._get_backend()
-
-        async with acquire_with_retry(backend) as conn:
-            row = await conn.fetchrow(
-                f"""
-                SELECT id, bank_id, text, proof_count, tags, source_memory_ids, created_at, updated_at
-                FROM {fq_table("memory_units")}
-                WHERE bank_id = $1 AND id = $2 AND fact_type = 'observation'
-                """,
-                bank_id,
-                observation_id,
-            )
-
-            if not row:
-                return None
-
-            result = self._row_to_observation_consolidated(row)
-
-            # Fetch source memories if requested and source_memory_ids exist
-            if include_source_memories and result.get("source_memory_ids"):
-                source_ids = [uuid.UUID(sid) if isinstance(sid, str) else sid for sid in result["source_memory_ids"]]
-                source_rows = await conn.fetch(
-                    f"""
-                    SELECT id, text, fact_type, context, occurred_start, mentioned_at
-                    FROM {fq_table("memory_units")}
-                    WHERE id = ANY($1::uuid[])
-                    ORDER BY mentioned_at DESC NULLS LAST
-                    """,
-                    source_ids,
-                )
-                result["source_memories"] = [
-                    {
-                        "id": str(r["id"]),
-                        "text": r["text"],
-                        "type": r["fact_type"],
-                        "context": r["context"],
-                        "occurred_start": r["occurred_start"].isoformat() if r["occurred_start"] else None,
-                        "mentioned_at": r["mentioned_at"].isoformat() if r["mentioned_at"] else None,
-                    }
-                    for r in source_rows
-                ]
-
-            return result
-
-    def _row_to_observation_consolidated(self, row: Any) -> dict[str, Any]:
-        """Convert a database row to an observation dict."""
-        # Convert source_memory_ids to strings
-        source_memory_ids = row.get("source_memory_ids") or []
-        source_memory_ids = [str(sid) for sid in source_memory_ids]
-
-        return {
-            "id": str(row["id"]),
-            "bank_id": row["bank_id"],
-            "text": row["text"],
-            "proof_count": row["proof_count"] or 1,
-            # Deprecated inline field — full history via GET .../{id}/history.
-            "history": [],
-            "tags": row["tags"] or [],
-            "source_memory_ids": source_memory_ids,
-            "source_memories": [],  # Populated separately when fetching full details
-            "created_at": row["created_at"].isoformat() if row["created_at"] else None,
-            "updated_at": row["updated_at"].isoformat() if row["updated_at"] else None,
-        }
-
    # =========================================================================
    # MENTAL MODELS CRUD
    # =========================================================================
@@ -10130,6 +10095,13 @@ class MemoryEngine(MemoryEngineInterface):
                        )
                based_on_serialized_payload[fact_type] = serialized_facts

+            # Facts from this reflect only — for the structured-delta LLM prompt.
+            # Accumulated based_on below is audit/grounding; re-sending all historical
+            # facts each refresh blows past provider input limits (e.g. Z.ai 1261).
+            delta_supporting_facts: list[dict[str, Any]] = []
+            for _facts in based_on_serialized_payload.values():
+                delta_supporting_facts.extend(_facts)
+
            # In delta mode, based_on must accumulate: the mental model is
            # grounded on ALL facts ever used, not just the latest delta's new
            # ones. Merge previous based_on with current, deduplicating by id.
@@ -10156,8 +10128,8 @@ class MemoryEngine(MemoryEngineInterface):
            # drift is structurally impossible. Falls back to the full candidate
            # markdown if either the structuring or the LLM op call fails.
            from .reflect.delta_ops import (
-                DeltaOperationList,
                apply_operations,
+                parse_delta_operation_list,
            )
            from .reflect.prompts import (
                STRUCTURED_DELTA_SYSTEM_PROMPT,
@@ -10192,9 +10164,7 @@ class MemoryEngine(MemoryEngineInterface):
                    current_doc = None

                if current_doc is not None:
-                    supporting_facts: list[dict[str, Any]] = []
-                    for _ftype, facts in based_on_serialized_payload.items():
-                        supporting_facts.extend(facts)
+                    supporting_facts = delta_supporting_facts

                    # No new facts since last refresh — skip the delta LLM call
                    # and preserve existing content unchanged.
@@ -10222,7 +10192,7 @@ class MemoryEngine(MemoryEngineInterface):
                    doc_max_tokens = mental_model.get("max_tokens") or 2048
                    delta_max_tokens = max(2048, int(doc_max_tokens * 1.5))
                    user_prompt = build_structured_delta_prompt(
-                        current_document_json=current_doc.model_dump_json(indent=2),
+                        current_document_json=current_doc.model_dump_json(),
                        candidate_markdown=reflect_result.text,
                        supporting_facts=supporting_facts,
                        source_query=current_source_query,
@@ -10243,19 +10213,7 @@ class MemoryEngine(MemoryEngineInterface):
                            temperature=0.0,
                            scope="mental_model_delta_ops",
                        )
-                        op_list: DeltaOperationList
-                        if isinstance(raw, DeltaOperationList):
-                            op_list = raw
-                        elif isinstance(raw, dict):
-                            op_list = DeltaOperationList.model_validate(raw)
-                        else:
-                            text = (raw or "").strip()
-                            # Strip optional fenced code block.
-                            if text.startswith("```"):
-                                text = text.split("\n", 1)[1] if "\n" in text else ""
-                                if text.endswith("```"):
-                                    text = text[:-3].rstrip()
-                            op_list = DeltaOperationList.model_validate_json(text)
+                        op_list = parse_delta_operation_list(raw)
                        outcome = apply_operations(current_doc, op_list.operations)
                        final_structured = outcome.document
                        final_content = render_document(outcome.document)
@@ -11139,10 +11097,6 @@ class MemoryEngine(MemoryEngineInterface):
            # Parent operations have their status updated when all children complete/fail
            operation_list = []
            for row in operations:
-                # Map DB status to API status (pending includes processing)
-                db_status = row["status"]
-                api_status = "pending" if db_status in ("pending", "processing") else db_status
-
                result_metadata = conn.parse_json(row["result_metadata"]) or {}

                next_retry_at = row["next_retry_at"]
@@ -11237,7 +11191,6 @@ class MemoryEngine(MemoryEngineInterface):
                    child_statuses = []
                    all_done = True
                    any_failed = False
-                    all_completed = True

                    for child_row in child_rows:
                        raw_crm = child_row["result_metadata"]
@@ -11258,9 +11211,6 @@ class MemoryEngine(MemoryEngineInterface):
                            all_done = False
                        if child_status == "failed":
                            any_failed = True
-                        if child_status != "completed":
-                            all_completed = False
-
                    # Self-healing: if parent status is out of sync with children, update it
                    if all_done and api_status == "pending":
                        correct_status = "failed" if any_failed else "completed"
@@ -14,7 +14,7 @@ import logging
 import time
 from typing import Any

-from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError
+from hindsight_api.engine.llm_interface import LLMInterface
 from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
 from hindsight_api.metrics import get_metrics_collector

@@ -15,7 +15,7 @@ from typing import Any

 from pydantic import ValidationError

-from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError
+from hindsight_api.engine.llm_interface import LLMInterface
 from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
 from hindsight_api.metrics import get_metrics_collector

@@ -24,7 +24,7 @@ from typing import Any

 import httpx

-from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError
+from hindsight_api.engine.llm_interface import LLMInterface
 from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
 from hindsight_api.metrics import get_metrics_collector

@@ -397,7 +397,6 @@ class CodexLLM(LLMInterface):
        }

        url = f"{self.base_url}/codex/responses"
-        last_exception = None

        # Manual attempt tracking instead of ``for attempt in range(...)`` so
        # that the reactive-refresh path can retry once without consuming a
@@ -428,7 +427,6 @@ class CodexLLM(LLMInterface):
                        if attempt < max_retries:
                            backoff = min(initial_backoff * (2**attempt), max_backoff)
                            await asyncio.sleep(backoff)
-                            last_exception = e
                            attempt += 1
                            continue
                        raise
@@ -490,7 +488,6 @@ class CodexLLM(LLMInterface):
                return result

            except httpx.HTTPStatusError as e:
-                last_exception = e
                status_code = e.response.status_code

                # Auth error: try one OAuth refresh + retry before giving up.
@@ -549,7 +546,6 @@ class CodexLLM(LLMInterface):
                    raise

            except httpx.RequestError as e:
-                last_exception = e
                if attempt < max_retries:
                    backoff = min(initial_backoff * (2**attempt), max_backoff)
                    logger.warning(f"Codex connection error (attempt {attempt + 1}/{max_retries + 1}): {e}")
@@ -564,10 +560,6 @@ class CodexLLM(LLMInterface):
                logger.error(f"Unexpected Codex error: {type(e).__name__}: {e}")
                raise

-        if last_exception:
-            raise last_exception
-        raise RuntimeError("Codex call failed after all retries")
-
    async def _parse_sse_stream(self, response: httpx.Response) -> str:
        """
        Parse Server-Sent Events (SSE) stream from Codex API.
@@ -11,7 +11,6 @@ import base64
 import io
 import json
 import logging
-import os
 import time
 from contextvars import ContextVar
 from typing import Any
@@ -20,7 +19,7 @@ from google import genai
 from google.genai import errors as genai_errors
 from google.genai import types as genai_types

-from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError
+from hindsight_api.engine.llm_interface import LLMInterface
 from hindsight_api.engine.llm_wrapper import parse_llm_json
 from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
 from hindsight_api.metrics import get_metrics_collector
@@ -36,7 +35,6 @@ _safety_settings_ctx: ContextVar[list | None] = ContextVar("gemini_safety_settin

 # Vertex AI imports (optional)
 try:
-    import google.auth
    from google.oauth2 import service_account

    VERTEXAI_AVAILABLE = True
@@ -6,12 +6,17 @@ structured information like temporal constraints.
 """

 import logging
-import re
 from abc import ABC, abstractmethod
 from datetime import datetime, timedelta

 from pydantic import BaseModel, Field

+from hindsight_api.engine.temporal_periods import (
+    NO_TEMPORAL_CONSTRAINT,
+    extract_period,
+    is_embedded_cjk_dateparser_match,
+)
+
 logger = logging.getLogger(__name__)


@@ -123,9 +128,12 @@ class DateparserQueryAnalyzer(QueryAnalyzer):

        # Check for period expressions first (these need special handling)
        query_lower = query.lower()
-        period_result = self._extract_period(query_lower, reference_date)
-        if period_result is not None:
-            return QueryAnalysis(temporal_constraint=period_result)
+        period_result = extract_period(query_lower, reference_date)
+        if period_result is NO_TEMPORAL_CONSTRAINT:
+            return QueryAnalysis(temporal_constraint=None)
+        if isinstance(period_result, tuple):
+            start_date, end_date = period_result
+            return QueryAnalysis(temporal_constraint=TemporalConstraint(start_date=start_date, end_date=end_date))

        # Lazy load dateparser (only imports on first call, then cached)
        self.load()
@@ -158,7 +166,12 @@ class DateparserQueryAnalyzer(QueryAnalyzer):

        # Filter out false positives (common words parsed as dates)
        false_positives = {"do", "may", "march", "will", "can", "sat", "sun", "mon", "tue", "wed", "thu", "fri"}
-        valid_results = [(text, date) for text, date in results if text.lower() not in false_positives or len(text) > 3]
+        valid_results = [
+            (text, date)
+            for text, date in results
+            if (text.lower() not in false_positives or len(text) > 3)
+            and not is_embedded_cjk_dateparser_match(query, text)
+        ]

        if not valid_results:
            return QueryAnalysis(temporal_constraint=None)
@@ -172,127 +185,6 @@ class DateparserQueryAnalyzer(QueryAnalyzer):

        return QueryAnalysis(temporal_constraint=TemporalConstraint(start_date=start_date, end_date=end_date))

-    def _extract_period(self, query: str, reference_date: datetime) -> TemporalConstraint | None:
-        """
-        Extract period-based temporal expressions (week, month, year, weekend).
-
-        These need special handling as they represent date ranges, not single dates.
-        Supports multiple languages.
-        """
-
-        def constraint(start: datetime, end: datetime) -> TemporalConstraint:
-            return TemporalConstraint(
-                start_date=start.replace(hour=0, minute=0, second=0, microsecond=0),
-                end_date=end.replace(hour=23, minute=59, second=59, microsecond=999999),
-            )
-
-        # Yesterday patterns (English, Spanish, Italian, French, German)
-        if re.search(r"\b(yesterday|ayer|ieri|hier|gestern)\b", query, re.IGNORECASE):
-            d = reference_date - timedelta(days=1)
-            return constraint(d, d)
-
-        # Today patterns
-        if re.search(r"\b(today|hoy|oggi|aujourd\'?hui|heute)\b", query, re.IGNORECASE):
-            return constraint(reference_date, reference_date)
-
-        # "a couple of days ago" / "a few days ago" patterns
-        # These are imprecise so we create a range
-        if re.search(r"\b(a\s+)?couple\s+(of\s+)?days?\s+ago\b", query, re.IGNORECASE):
-            # "a couple of days" = approximately 2 days, give range of 1-3 days
-            return constraint(reference_date - timedelta(days=3), reference_date - timedelta(days=1))
-
-        if re.search(r"\b(a\s+)?few\s+days?\s+ago\b", query, re.IGNORECASE):
-            # "a few days" = approximately 3-4 days, give range of 2-5 days
-            return constraint(reference_date - timedelta(days=5), reference_date - timedelta(days=2))
-
-        # "a couple of weeks ago" / "a few weeks ago" patterns
-        if re.search(r"\b(a\s+)?couple\s+(of\s+)?weeks?\s+ago\b", query, re.IGNORECASE):
-            # "a couple of weeks" = approximately 2 weeks, give range of 1-3 weeks
-            return constraint(reference_date - timedelta(weeks=3), reference_date - timedelta(weeks=1))
-
-        if re.search(r"\b(a\s+)?few\s+weeks?\s+ago\b", query, re.IGNORECASE):
-            # "a few weeks" = approximately 3-4 weeks, give range of 2-5 weeks
-            return constraint(reference_date - timedelta(weeks=5), reference_date - timedelta(weeks=2))
-
-        # "a couple of months ago" / "a few months ago" patterns
-        if re.search(r"\b(a\s+)?couple\s+(of\s+)?months?\s+ago\b", query, re.IGNORECASE):
-            # "a couple of months" = approximately 2 months, give range of 1-3 months
-            return constraint(reference_date - timedelta(days=90), reference_date - timedelta(days=30))
-
-        if re.search(r"\b(a\s+)?few\s+months?\s+ago\b", query, re.IGNORECASE):
-            # "a few months" = approximately 3-4 months, give range of 2-5 months
-            return constraint(reference_date - timedelta(days=150), reference_date - timedelta(days=60))
-
-        # Last week patterns (English, Spanish, Italian, French, German)
-        if re.search(
-            r"\b(last\s+week|la\s+semana\s+pasada|la\s+settimana\s+scorsa|la\s+semaine\s+derni[eè]re|letzte\s+woche)\b",
-            query,
-            re.IGNORECASE,
-        ):
-            start = reference_date - timedelta(days=reference_date.weekday() + 7)
-            return constraint(start, start + timedelta(days=6))
-
-        # Last month patterns
-        if re.search(
-            r"\b(last\s+month|el\s+mes\s+pasado|il\s+mese\s+scorso|le\s+mois\s+dernier|letzten?\s+monat)\b",
-            query,
-            re.IGNORECASE,
-        ):
-            first = reference_date.replace(day=1)
-            end = first - timedelta(days=1)
-            start = end.replace(day=1)
-            return constraint(start, end)
-
-        # Last year patterns
-        if re.search(
-            r"\b(last\s+year|el\s+a[ñn]o\s+pasado|l\'anno\s+scorso|l\'ann[ée]e\s+derni[eè]re|letztes?\s+jahr)\b",
-            query,
-            re.IGNORECASE,
-        ):
-            year = reference_date.year - 1
-            return constraint(datetime(year, 1, 1), datetime(year, 12, 31))
-
-        # Last weekend patterns
-        if re.search(
-            r"\b(last\s+weekend|el\s+fin\s+de\s+semana\s+pasado|lo\s+scorso\s+fine\s+settimana|le\s+week-?end\s+dernier|letztes?\s+wochenende)\b",
-            query,
-            re.IGNORECASE,
-        ):
-            days_since_sat = (reference_date.weekday() + 2) % 7
-            if days_since_sat == 0:
-                days_since_sat = 7
-            sat = reference_date - timedelta(days=days_since_sat)
-            return constraint(sat, sat + timedelta(days=1))
-
-        # Month + Year patterns (e.g., "June 2024", "junio 2024", "giugno 2024")
-        month_patterns = {
-            "january|enero|gennaio|janvier|januar": 1,
-            "february|febrero|febbraio|f[ée]vrier|februar": 2,
-            "march|marzo|mars|m[äa]rz": 3,
-            "april|abril|aprile|avril": 4,
-            "may|mayo|maggio|mai": 5,
-            "june|junio|giugno|juin|juni": 6,
-            "july|julio|luglio|juillet|juli": 7,
-            "august|agosto|ao[uû]t": 8,
-            "september|septiembre|settembre|septembre": 9,
-            "october|octubre|ottobre|octobre|oktober": 10,
-            "november|noviembre|novembre": 11,
-            "december|diciembre|dicembre|d[ée]cembre|dezember": 12,
-        }
-
-        for pattern, month_num in month_patterns.items():
-            match = re.search(rf"\b({pattern})\s+(\d{{4}})\b", query, re.IGNORECASE)
-            if match:
-                year = int(match.group(2))
-                start = datetime(year, month_num, 1)
-                if month_num == 12:
-                    end = datetime(year, 12, 31)
-                else:
-                    end = datetime(year, month_num + 1, 1) - timedelta(days=1)
-                return constraint(start, end)
-
-        return None
-

 class TransformerQueryAnalyzer(QueryAnalyzer):
    """
@@ -26,10 +26,13 @@ or stay the same per refresh, never get worse.

 from __future__ import annotations

+import json
 import logging
 from typing import Annotated, Any, Literal, Union

-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
+
+from hindsight_api.engine.llm_wrapper import parse_llm_json

 from .structured_doc import (
    Block,
@@ -144,6 +147,27 @@ Operation = Annotated[
    Field(discriminator="op"),
 ]

+_OPERATION_ADAPTER: TypeAdapter[Operation] = TypeAdapter(Operation)
+
+
+def _validate_operations_list(raw_ops: Any) -> tuple[list[Operation], list[dict[str, Any]]]:
+    """Validate each operation independently; drop invalid ops instead of failing the batch."""
+    if not isinstance(raw_ops, list):
+        raise TypeError(f"operations must be a list, got {type(raw_ops)!r}")
+    valid: list[Operation] = []
+    skipped: list[dict[str, Any]] = []
+    for i, item in enumerate(raw_ops):
+        try:
+            valid.append(_OPERATION_ADAPTER.validate_python(item))
+        except ValidationError as exc:
+            skipped.append({"index": i, "op": item, "error": exc.errors(include_url=False)})
+            logger.warning(
+                "[STRUCTURED_DELTA] skipping invalid operation at index %s: %s",
+                i,
+                exc.errors(include_url=False),
+            )
+    return valid, skipped
+

 class DeltaOperationList(BaseModel):
    """Container for the operations produced by an LLM delta call."""
@@ -152,6 +176,104 @@ class DeltaOperationList(BaseModel):
    operations: list[Operation] = Field(default_factory=list)


+class DeltaAllOpsInvalidError(ValueError):
+    """Raised when the model emitted operations but none survived validation.
+
+    Distinct from an empty ``operations`` array (a legitimate no-op): here every
+    op was malformed, so returning zero valid ops would make the caller apply
+    nothing and silently drop this refresh's new facts. Raising instead lets the
+    caller fall back to a full rewrite, which still integrates the new facts.
+    """
+
+
+def _finalize_operations(valid: list[Operation], skipped: list[dict[str, Any]]) -> DeltaOperationList:
+    """Build the result, but refuse a wholesale validation failure as a silent no-op."""
+    if skipped and not valid:
+        raise DeltaAllOpsInvalidError(f"all {len(skipped)} delta operation(s) failed validation")
+    return DeltaOperationList(operations=valid)
+
+
+def _extract_balanced_json_object(text: str) -> str | None:
+    """Return the first top-level ``{...}`` slice, ignoring trailing junk."""
+    start = text.find("{")
+    if start < 0:
+        return None
+    depth = 0
+    in_string = False
+    escape = False
+    for i in range(start, len(text)):
+        ch = text[i]
+        if in_string:
+            if escape:
+                escape = False
+            elif ch == "\\":
+                escape = True
+            elif ch == '"':
+                in_string = False
+            continue
+        if ch == '"':
+            in_string = True
+        elif ch == "{":
+            depth += 1
+        elif ch == "}":
+            depth -= 1
+            if depth == 0:
+                return text[start : i + 1]
+    return None
+
+
+def parse_delta_operation_list(raw: Any) -> DeltaOperationList:
+    """Parse structured-delta LLM output into a validated operation list."""
+    if isinstance(raw, DeltaOperationList):
+        return raw
+    if isinstance(raw, dict):
+        ops_raw = raw.get("operations", [])
+        valid, skipped = _validate_operations_list(ops_raw)
+        if skipped:
+            logger.info(
+                "[STRUCTURED_DELTA] parsed %s op(s), skipped %s invalid op(s) from dict payload",
+                len(valid),
+                len(skipped),
+            )
+        return _finalize_operations(valid, skipped)
+
+    text = (raw or "").strip()
+    if not text:
+        return DeltaOperationList()
+
+    candidates: list[str] = [text]
+    extracted = _extract_balanced_json_object(text)
+    if extracted and extracted != text:
+        candidates.append(extracted)
+
+    last_error: Exception | None = None
+    for candidate in candidates:
+        try:
+            payload = parse_llm_json(candidate)
+        except json.JSONDecodeError as exc:
+            last_error = exc
+            continue
+        if not isinstance(payload, dict) or "operations" not in payload:
+            last_error = ValueError("delta payload must be an object with an operations array")
+            continue
+        try:
+            valid, skipped = _validate_operations_list(payload["operations"])
+        except TypeError as exc:
+            last_error = exc
+            continue
+        if skipped:
+            logger.info(
+                "[STRUCTURED_DELTA] parsed %s op(s), skipped %s invalid op(s)",
+                len(valid),
+                len(skipped),
+            )
+        return _finalize_operations(valid, skipped)
+
+    if last_error is not None:
+        raise last_error
+    return DeltaOperationList()
+
+
 # Application ---------------------------------------------------------------


@@ -734,7 +734,65 @@ Examples
  ``{"operations": [{"op": "replace_block", "section_id": "overview",
  "index": 0, "block": {"type": "paragraph", "text": "Updated summary."}}]}``
 - Remove an obsolete block →
-  ``{"operations": [{"op": "remove_block", "section_id": "status", "index": 2}]}``"""
+  ``{"operations": [{"op": "remove_block", "section_id": "status", "index": 2}]}``
+
+JSON STRING RULES (critical)
+- Every ``text`` and ``items`` string must be valid JSON: escape ``"`` as ``\\"``,
+  backslashes as ``\\\\``, and newlines as ``\\n``. Do not use raw backticks inside
+  strings unless needed; prefer plain quotes for file paths.
+- ``replace_block``, ``insert_block``, and ``remove_block`` MUST include ``index`` (0-based block position in that section). Use ``replace_section_blocks`` only when replacing every block in a section.
+
+- Do not append extra ``]`` or ``}`` after the closing ``}`` of the root object."""
+
+_STRUCTURED_DELTA_DEFAULT_MAX_INPUT_TOKENS = 24_000
+
+
+def _truncate_cl100k(text: str, max_tokens: int) -> str:
+    """Truncate text to at most max_tokens using cl100k_base."""
+    if max_tokens <= 0:
+        return ""
+    from .tokenization import count_cl100k_tokens
+
+    if count_cl100k_tokens(text) <= max_tokens:
+        return text
+    enc = __import__("tiktoken").get_encoding("cl100k_base")
+    return enc.decode(enc.encode(text)[:max_tokens])
+
+
+def _fit_structured_delta_prompt_parts(
+    *,
+    source_query: str,
+    current_document_json: str,
+    candidate_markdown: str,
+    facts_block: str,
+    budget_hint: str,
+    task_footer: str,
+    max_input_tokens: int,
+) -> tuple[str, str, str, bool]:
+    """Shrink large prompt sections to fit within max_input_tokens (cl100k estimate)."""
+    from .tokenization import count_cl100k_tokens
+
+    fixed = (
+        f"## Topic\n{source_query}\n\n"
+        f"## CURRENT DOCUMENT (apply ops to this; reference section ids as listed)\n"
+        f"```json\n\n```\n\n"
+        f"## NEW INFORMATION SYNTHESIS (context for how new facts relate to the topic)\n"
+        f"```markdown\n\n```\n\n"
+        f"## SUPPORTING FACTS (new since last refresh — integrate these)\n"
+        f"{budget_hint}\n\n"
+        f"{task_footer}"
+    )
+    facts_header = "## SUPPORTING FACTS (new since last refresh — integrate these)\n"
+    facts_prefix_tokens = count_cl100k_tokens(facts_header)
+    reserved_facts = min(4096, max(512, max_input_tokens // 8))
+    doc_budget = max(1024, (max_input_tokens - count_cl100k_tokens(fixed) - reserved_facts) * 55 // 100)
+    cand_budget = max(512, (max_input_tokens - count_cl100k_tokens(fixed) - reserved_facts) * 30 // 100)
+    facts_budget = max(256, reserved_facts - facts_prefix_tokens)
+    doc_json = _truncate_cl100k(current_document_json, doc_budget)
+    candidate = _truncate_cl100k(candidate_markdown, cand_budget)
+    facts_body = _truncate_cl100k(facts_block, facts_budget)
+    truncated = doc_json != current_document_json or candidate != candidate_markdown or facts_body != facts_block
+    return doc_json, candidate, facts_body, truncated


 def build_structured_delta_prompt(
@@ -744,6 +802,7 @@ def build_structured_delta_prompt(
    supporting_facts: list[dict[str, Any]],
    source_query: str,
    max_output_tokens: int | None = None,
+    max_input_tokens: int | None = None,
 ) -> str:
    """Build the user prompt for a structured-delta mental model refresh.

@@ -774,19 +833,39 @@ def build_structured_delta_prompt(
            "block-level ops) so the response always parses as valid JSON."
        )

-    return (
-        f"## Topic\n{source_query}\n\n"
-        f"## CURRENT DOCUMENT (apply ops to this; reference section ids as listed)\n"
-        f"```json\n{current_document_json}\n```\n\n"
-        f"## NEW INFORMATION SYNTHESIS (context for how new facts relate to the topic)\n"
-        f"```markdown\n{candidate_markdown}\n```\n\n"
-        f"## SUPPORTING FACTS (new since last refresh — integrate these)\n{facts_block}"
-        f"{budget_hint}\n\n"
+    task_footer = (
        "## Task\n"
        "Output a JSON object matching the operations schema. Integrate the new "
        "supporting facts into CURRENT DOCUMENT. Add, update, or remove content "
        "as needed. Preserve unchanged sections and blocks by not mentioning them."
    )
+    input_cap = max_input_tokens if max_input_tokens is not None else _STRUCTURED_DELTA_DEFAULT_MAX_INPUT_TOKENS
+    doc_json, candidate, facts_body, input_truncated = _fit_structured_delta_prompt_parts(
+        source_query=source_query,
+        current_document_json=current_document_json,
+        candidate_markdown=candidate_markdown,
+        facts_block=facts_block,
+        budget_hint=budget_hint,
+        task_footer=task_footer,
+        max_input_tokens=input_cap,
+    )
+    truncation_note = ""
+    if input_truncated:
+        truncation_note = (
+            "\n\n*Note: Document, synthesis, or facts were truncated to fit the model "
+            "context window. Prefer minimal, high-leverage operations.*"
+        )
+
+    return (
+        f"## Topic\n{source_query}\n\n"
+        f"## CURRENT DOCUMENT (apply ops to this; reference section ids as listed)\n"
+        f"```json\n{doc_json}\n```\n\n"
+        f"## NEW INFORMATION SYNTHESIS (context for how new facts relate to the topic)\n"
+        f"```markdown\n{candidate}\n```\n\n"
+        f"## SUPPORTING FACTS (new since last refresh — integrate these)\n{facts_body}"
+        f"{budget_hint}{truncation_note}\n\n"
+        f"{task_footer}"
+    )


 DELTA_SYSTEM_PROMPT = """You are performing a surgical delta update to an existing mental model document.
@@ -105,6 +105,34 @@ class TokenUsage(BaseModel):
        )


+class ExtractedFact(BaseModel):
+    """A single candidate fact produced by dry-run extraction (no resolution/links/persistence).
+
+    A deliberate subset of the persisted memory-unit shape — only the fields a fresh extraction
+    yields. Storage/consolidation/curation fields (id, document_id, chunk_id, proof_count, state, …)
+    are omitted because nothing is stored. Entities are raw, unresolved names.
+    """
+
+    text: str = Field(description="The extracted fact text.")
+    fact_type: str = Field(description="Perspective classification: 'world' or 'experience'.")
+    occurred_start: str | None = Field(default=None, description="ISO timestamp the fact's event started, if dated.")
+    occurred_end: str | None = Field(default=None, description="ISO timestamp the fact's event ended, if dated.")
+    entities: list[str] = Field(
+        default_factory=list, description="Raw (unresolved) entity names mentioned in the fact."
+    )
+
+
+class DryRunExtractionResult(BaseModel):
+    """Result of dry-run fact extraction: candidate facts plus aggregated LLM token usage."""
+
+    facts: list[ExtractedFact] = Field(
+        default_factory=list, description="Candidate facts the retain step would extract."
+    )
+    usage: TokenUsage = Field(
+        default_factory=TokenUsage, description="Aggregated token usage across the extraction LLM calls."
+    )
+
+
 class DispositionTraits(BaseModel):
    """
    Disposition traits for a memory bank.
@@ -4,7 +4,6 @@ bank profile utilities for disposition and mission management.

 import json
 import logging
-import re
 import uuid
 from dataclasses import dataclass
 from typing import TypedDict
@@ -14,7 +14,6 @@ from typing import Any, Literal, cast

 from pydantic import BaseModel, ConfigDict, Field, create_model, field_validator

-from ...config import get_config
 from ..llm_wrapper import LLMConfig, OutputTooLongError, sanitize_llm_output
 from ..operation_metadata import RetainExtractionErrors
 from ..response_models import TokenUsage
@@ -421,19 +420,14 @@ _RECURSIVE_TEXT_SEPARATORS = [
    "",  # Characters (last resort)
 ]

-# A single structured unit (a JSONL line or a conversation turn) is kept whole
-# even when it overflows the budget — but only up to this multiple. Beyond it,
-# the unit is split as text rather than handed to the LLM wildly over budget
-# (the extractor has no second re-chunk pass; an oversized chunk just errors).
-_CHUNK_OVERFLOW_FACTOR = 1.5
-

 def _split_oversized_unit(text: str, max_chars: int) -> list[str]:
    """Sentence-aware split of a single unit that overflowed the budget.

    Used when one JSONL line / conversation turn is so large it can't be kept
-    whole within ``_CHUNK_OVERFLOW_FACTOR``. The resulting fragments are no
-    longer valid JSON, but the fact extractor treats every chunk as plain text.
+    whole within the configured structured-chunk limit. The resulting fragments
+    are no longer valid JSON, but the fact extractor treats every chunk as plain
+    text.
    """
    from langchain_text_splitters import RecursiveCharacterTextSplitter

@@ -447,18 +441,21 @@ def _split_oversized_unit(text: str, max_chars: int) -> list[str]:
    return splitter.split_text(text)


-def chunk_text(text: str, max_chars: int) -> list[str]:
+def chunk_text(text: str, max_chars: int, structured_chunk_size: int | None = None) -> list[str]:
    """
    Split text into chunks, preserving conversation structure when possible.

    For JSON conversation arrays (user/assistant turns) and JSONL (newline-delimited
    JSON objects), splits at turn/line boundaries so no object is split across chunks.
-    A single turn/line that overflows is kept whole up to ``_CHUNK_OVERFLOW_FACTOR``×
-    the budget, then split as text. For plain text, uses sentence-aware splitting.
+    A single turn/line that overflows ``max_chars`` is kept whole only up to
+    ``structured_chunk_size``. When unset, that limit defaults to ``max_chars``.
+    For plain text, uses sentence-aware splitting.

    Args:
        text: Input text to chunk (plain text, JSON conversation, or JSONL)
-        max_chars: Maximum characters per chunk (default 120k ≈ 30k tokens)
+        max_chars: Target maximum characters per chunk
+        structured_chunk_size: Maximum characters for a single JSONL line or
+            conversation turn to keep whole. Defaults to ``max_chars``.

    Returns:
        List of text chunks, roughly under max_chars
@@ -467,17 +464,19 @@ def chunk_text(text: str, max_chars: int) -> list[str]:
    if len(text) <= max_chars:
        return [text]

+    structured_limit = structured_chunk_size if structured_chunk_size is not None else max_chars
+
    # Try to parse as JSON conversation array
    try:
        parsed = json.loads(text)
        if isinstance(parsed, list) and all(isinstance(turn, dict) for turn in parsed):
            # This looks like a conversation - chunk at turn boundaries
-            return _chunk_conversation(parsed, max_chars)
+            return _chunk_conversation(parsed, max_chars, structured_limit)
    except (json.JSONDecodeError, ValueError):
        pass

    # Try to parse as JSONL (newline-delimited JSON objects, e.g. session logs)
-    jsonl_chunks = _chunk_jsonl(text, max_chars)
+    jsonl_chunks = _chunk_jsonl(text, max_chars, structured_limit)
    if jsonl_chunks is not None:
        return jsonl_chunks

@@ -485,20 +484,19 @@ def chunk_text(text: str, max_chars: int) -> list[str]:
    return _split_oversized_unit(text, max_chars)


-def _chunk_conversation(turns: list[dict], max_chars: int) -> list[str]:
+def _chunk_conversation(turns: list[dict], max_chars: int, structured_limit: int) -> list[str]:
    """
    Chunk a conversation array at turn boundaries, preserving complete turns.

    Args:
        turns: List of conversation turn dicts (with 'role' and 'content' keys)
        max_chars: Maximum characters per chunk
+        structured_limit: Maximum characters for a single turn to keep whole

    Returns:
        List of JSON-serialized chunks, each containing complete turns
    """

-    overflow_limit = int(max_chars * _CHUNK_OVERFLOW_FACTOR)
-
    chunks = []
    current_chunk = []
    current_size = 2  # Account for "[]"
@@ -513,13 +511,14 @@ def _chunk_conversation(turns: list[dict], max_chars: int) -> list[str]:
    for turn in turns:
        # Estimate size of this turn when serialized (with comma separator)
        turn_json = json.dumps(turn, ensure_ascii=False)
-        turn_size = len(turn_json) + 1  # +1 for comma
+        turn_unit_size = len(turn_json)
+        turn_size = turn_unit_size + 1  # +1 for comma

        # A turn too large to keep whole even alone: flush, then split it as
        # text so no chunk runs far over budget (the extractor won't re-chunk).
-        if turn_size > overflow_limit:
+        if turn_unit_size > structured_limit:
            _flush()
-            chunks.extend(_split_oversized_unit(turn_json, max_chars))
+            chunks.extend(_split_oversized_unit(turn_json, structured_limit))
            continue

        # If adding this turn would exceed limit and we have turns, save current chunk
@@ -536,18 +535,20 @@ def _chunk_conversation(turns: list[dict], max_chars: int) -> list[str]:
    return chunks if chunks else [json.dumps(turns, ensure_ascii=False)]


-def _chunk_jsonl(text: str, max_chars: int) -> list[str] | None:
+def _chunk_jsonl(text: str, max_chars: int, structured_limit: int) -> list[str] | None:
    """Chunk newline-delimited JSON (JSONL) at line boundaries.

    Detects JSONL — two or more non-empty lines, each a complete JSON object —
    and packs whole lines into chunks so no line is split across chunks (multiple
-    short lines may share a chunk). A line that overflows is kept whole up to
-    ``_CHUNK_OVERFLOW_FACTOR``× the budget, then split as text. Returns ``None``
-    if the input is not JSONL, so the caller falls back to plain-text splitting.
+    short lines may share a chunk). A line that overflows ``max_chars`` is kept
+    whole only up to ``structured_limit``. Returns ``None`` if the input is not
+    JSONL, so the caller falls back to plain-text splitting.

    Args:
        text: Input text to inspect/chunk.
        max_chars: Maximum characters per chunk.
+        structured_limit: Maximum characters for a single JSONL line to
+            keep whole.

    Returns:
        List of JSONL chunks (lines joined by newline), or ``None`` if not JSONL.
@@ -564,8 +565,6 @@ def _chunk_jsonl(text: str, max_chars: int) -> list[str] | None:
        if not isinstance(obj, dict):
            return None

-    overflow_limit = int(max_chars * _CHUNK_OVERFLOW_FACTOR)
-
    chunks: list[str] = []
    current_chunk: list[str] = []
    current_size = 0
@@ -578,17 +577,18 @@ def _chunk_jsonl(text: str, max_chars: int) -> list[str] | None:
            current_size = 0

    for line in lines:
+        line_unit_size = len(line)
        line_size = len(line) + 1  # +1 for the joining newline

        # A line too large to keep whole even alone: flush, then split it as
        # text so no chunk runs far over budget (the extractor won't re-chunk).
-        if line_size > overflow_limit:
+        if line_unit_size > structured_limit:
            _flush()
-            chunks.extend(_split_oversized_unit(line, max_chars))
+            chunks.extend(_split_oversized_unit(line, structured_limit))
            continue

        # If adding this line would exceed the limit and we have lines, flush.
-        # A line up to overflow_limit is kept whole (a small, bounded overflow).
+        # A line up to structured_limit is kept whole (a bounded overflow).
        if current_size + line_size > max_chars and current_chunk:
            _flush()

@@ -1739,7 +1739,11 @@ async def extract_facts_from_text(
        - chunks: List of tuples (chunk_text, fact_count) for each chunk
        - usage: Aggregated token usage across all LLM calls
    """
-    chunks = chunk_text(text, max_chars=config.retain_chunk_size)
+    chunks = chunk_text(
+        text,
+        max_chars=config.retain_chunk_size,
+        structured_chunk_size=config.retain_structured_chunk_size,
+    )

    # Log chunk count before starting LLM requests
    total_chars = sum(len(c) for c in chunks)
@@ -1881,8 +1885,7 @@ async def extract_facts_from_contents_batch_api(

    logger.info(f"Using Batch API for fact extraction ({len(contents)} contents)")

-    # Check config for extraction mode and causal link extraction (used throughout)
-    extraction_mode = config.retain_extraction_mode
+    # Check config for causal link extraction (used throughout)
    extract_causal_links = config.retain_extract_causal_links

    # Check if provider supports batch API
@@ -1923,7 +1926,11 @@ async def extract_facts_from_contents_batch_api(
    prompt, response_schema = _build_extraction_prompt_and_schema(config)

    for content_index, item in enumerate(contents):
-        chunks = chunk_text(item.content, max_chars=config.retain_chunk_size)
+        chunks = chunk_text(
+            item.content,
+            max_chars=config.retain_chunk_size,
+            structured_chunk_size=config.retain_structured_chunk_size,
+        )

        for chunk_index_in_content, chunk in enumerate(chunks):
            all_chunks_info.append((chunk, content_index, chunk_index_in_content, item.event_date, item.context))
@@ -2344,7 +2351,11 @@ def _extract_facts_chunks(
    global_chunk_idx = 0

    for content_index, content in enumerate(contents):
-        chunks = chunk_text(content.content, config.retain_chunk_size)
+        chunks = chunk_text(
+            content.content,
+            config.retain_chunk_size,
+            structured_chunk_size=config.retain_structured_chunk_size,
+        )
        for chunk in chunks:
            chunks_metadata.append(
                ChunkMetadata(
@@ -800,8 +800,6 @@ async def create_causal_links_batch(
    try:
        import time as time_mod

-        create_start = time_mod.time()
-
        # Build links list
        links = []
        for fact_idx, causal_relations in enumerate(causal_relations_per_fact):
@@ -23,7 +23,6 @@ from ...extensions.memory_defense import (
    parse_policy,
 )
 from ...worker.stage import set_stage
-from ..db.base import DatabaseBackend
 from ..db_utils import acquire_with_retry
 from ..memory_engine import count_tokens, fq_table
 from . import bank_utils
@@ -90,7 +89,29 @@ async def _fire_memory_defense_webhook(
    if webhook_manager is None:
        return
    try:
-        from ...webhooks import MemoryDefenseEventData, WebhookEvent, WebhookEventType
+        from ...webhooks import (
+            MemoryDefenseEventData,
+            MemoryDefenseHit,
+            WebhookEvent,
+            WebhookEventType,
+        )
+
+        # Translate per-match raw dicts on the decision into MemoryDefenseHit
+        # entries on the wire. The decision's hits list is already fingerprinted
+        # by apply_redaction (the raw value never lands in hits, by contract),
+        # so this is purely a shape conversion. None when no per-hit data is
+        # available so receivers can distinguish "no preview info" from
+        # "scanned, nothing matched" (the latter wouldn't be a webhook delivery
+        # in the first place).
+        decision_hits = getattr(decision, "hits", None) or []
+        hits: list[MemoryDefenseHit] | None = [
+            MemoryDefenseHit(
+                detector=str(h.get("detector") or ""),
+                preview=str(h.get("preview") or ""),
+            )
+            for h in decision_hits
+            if h.get("detector") and h.get("preview")
+        ] or None

        event = WebhookEvent(
            event=WebhookEventType.MEMORY_DEFENSE_TRIGGERED,
@@ -104,6 +125,17 @@ async def _fire_memory_defense_webhook(
                document_id=document_id,
                matched_types=decision.matched_types or None,
                message=decision.message or None,
+                hits=hits,
+                # Optional SIEM-enrichment fields populated by downstream
+                # extensions (e.g. hindsight-cloud's _CloudDefenseDecision
+                # subclass). Read via getattr so OSS doesn't need to know
+                # about extension subclasses. Combined with the manager's
+                # exclude_none serialization, missing values stay absent
+                # from the wire entirely rather than appearing as null.
+                severity=getattr(decision, "severity", None),
+                api_key_name=getattr(decision, "api_key_name", None),
+                memory_unit_id=getattr(decision, "memory_unit_id", None),
+                receipt_uri=getattr(decision, "receipt_uri", None),
            ),
        )
        await webhook_manager.fire_event_with_conn(event, conn, schema=schema)
@@ -274,6 +306,8 @@ def _build_retain_params(contents_dicts, document_tags=None, doc_contents=None):
            )
        if first_item.get("metadata"):
            retain_params["metadata"] = first_item["metadata"]
+        if first_item.get("observation_scopes") is not None:
+            retain_params["observation_scopes"] = first_item["observation_scopes"]

    return retain_params, merged_tags

@@ -864,10 +898,15 @@ async def retain_batch(
    # retain code paths.
    chunk_batch_size = getattr(config, "retain_chunk_batch_size", 100)
    chunk_size = getattr(config, "retain_chunk_size", 3000)
+    structured_chunk_size = getattr(config, "retain_structured_chunk_size", None)
    all_pre_chunks: list[str] = []
    chunk_to_content: list[int] = []  # maps chunk index -> index into contents
    for content_idx, content in enumerate(contents):
-        content_chunks = fact_extraction.chunk_text(content.content, chunk_size)
+        content_chunks = fact_extraction.chunk_text(
+            content.content,
+            chunk_size,
+            structured_chunk_size=structured_chunk_size,
+        )
        all_pre_chunks.extend(content_chunks)
        chunk_to_content.extend([content_idx] * len(content_chunks))

@@ -2247,9 +2286,14 @@ def _chunk_contents_for_delta(contents: list[RetainContent], config) -> dict[int
    """
    result = {}
    global_chunk_idx = 0
+    chunk_size = getattr(config, "retain_chunk_size", 3000)
+    structured_chunk_size = getattr(config, "retain_structured_chunk_size", None)
    for content in contents:
-        chunk_size = getattr(config, "retain_chunk_size", 3000)
-        chunks = fact_extraction.chunk_text(content.content, chunk_size)
+        chunks = fact_extraction.chunk_text(
+            content.content,
+            chunk_size,
+            structured_chunk_size=structured_chunk_size,
+        )
        for chunk_text in chunks:
            result[global_chunk_idx] = chunk_text
            global_chunk_idx += 1
@@ -24,7 +24,9 @@ class RetainContentDict(TypedDict, total=False):
        tags: Visibility scope tags for this content item (optional)
        observation_scopes: How to scope observations for consolidation (optional).
            "per_tag" runs one pass per individual tag; "combined" (default) runs a
-            single pass with all tags; a list[list[str]] specifies exact passes.
+            single pass with all tags; "shared" runs a single pass over one global,
+            untagged scope so memories consolidate together regardless of tags;
+            a list[list[str]] specifies exact passes.
        update_mode: How to handle existing documents with the same document_id (optional).
            "replace" (default) deletes old data and reprocesses. "append" concatenates
            new content to the existing document and reprocesses.
@@ -38,7 +40,7 @@ class RetainContentDict(TypedDict, total=False):
    entities: list[dict[str, str]]  # [{"text": "...", "type": "..."}]
    tags: list[str]  # Visibility scope tags
    observation_scopes: (
-        Literal["per_tag", "combined", "all_combinations"] | list[list[str]]
+        Literal["per_tag", "combined", "all_combinations", "shared"] | list[list[str]]
    )  # Observation scopes for consolidation
    update_mode: Literal["replace", "append"]

@@ -57,7 +59,7 @@ class RetainContent:
    metadata: dict[str, str] = field(default_factory=dict)
    entities: list[dict[str, str]] = field(default_factory=list)  # User-provided entities
    tags: list[str] = field(default_factory=list)  # Visibility scope tags
-    observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = (
+    observation_scopes: Literal["per_tag", "combined", "all_combinations", "shared"] | list[list[str]] | None = (
        None  # Observation scopes
    )

@@ -124,7 +126,7 @@ class ExtractedFact:
    mentioned_at: datetime | None = None
    metadata: dict[str, str] = field(default_factory=dict)
    tags: list[str] = field(default_factory=list)  # Visibility scope tags
-    observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = (
+    observation_scopes: Literal["per_tag", "combined", "all_combinations", "shared"] | list[list[str]] | None = (
        None  # Observation scopes
    )

@@ -176,7 +178,7 @@ class ProcessedFact:
    tags: list[str] = field(default_factory=list)

    # Observation scopes for consolidation
-    observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = None
+    observation_scopes: Literal["per_tag", "combined", "all_combinations", "shared"] | list[list[str]] | None = None

    @property
    def is_duplicate(self) -> bool:
@@ -2,8 +2,6 @@
 Helper functions for hybrid search (semantic + BM25 + graph).
 """

-from typing import Any
-
 from .types import MergedCandidate, RetrievalResult


@@ -156,39 +154,3 @@ def interleave_fusion(result_lists: list[list[RetrievalResult]]) -> list[MergedC
        )
        for pos, doc_id in enumerate(ordered_ids)
    ]
-
-
-def normalize_scores_on_deltas(results: list[dict[str, Any]], score_keys: list[str]) -> list[dict[str, Any]]:
-    """
-    Normalize scores based on deltas (min-max normalization within result set).
-
-    This ensures all scores are in [0, 1] range based on the spread in THIS result set.
-
-    Args:
-        results: List of result dicts
-        score_keys: Keys to normalize (e.g., ["recency", "frequency"])
-
-    Returns:
-        Results with normalized scores added as "{key}_normalized"
-    """
-    for key in score_keys:
-        values = [r.get(key, 0.0) for r in results if key in r]
-
-        if not values:
-            continue
-
-        min_val = min(values)
-        max_val = max(values)
-        delta = max_val - min_val
-
-        if delta > 0:
-            for r in results:
-                if key in r:
-                    r[f"{key}_normalized"] = (r[key] - min_val) / delta
-        else:
-            # All values are the same, set to 0.5
-            for r in results:
-                if key in r:
-                    r[f"{key}_normalized"] = 0.5
-
-    return results
@@ -99,9 +99,15 @@ def apply_combined_scoring(

    for sr in scored_results:
        # Recency: linear decay over 365 days → [0.1, 1.0]; neutral 0.5 if no date.
+        # Use the unit's effective time (occurred_start, then mentioned_at, then
+        # occurred_end) — the same COALESCE order as retrieval._coalesce_date — so a
+        # memory that carries only a mentioned_at / occurred_end (e.g. conversation
+        # facts or ongoing states that intentionally lack occurred_start) still gets
+        # correct recency ordering instead of a flat neutral 0.5.
        sr.recency = 0.5
-        if sr.retrieval.occurred_start:
-            occurred = sr.retrieval.occurred_start
+        effective = sr.retrieval.occurred_start or sr.retrieval.mentioned_at or sr.retrieval.occurred_end
+        if effective:
+            occurred = effective
            if occurred.tzinfo is None:
                occurred = occurred.replace(tzinfo=UTC)
            days_ago = (now - occurred).total_seconds() / 86400
@@ -13,7 +13,7 @@ import logging
 import re
 from dataclasses import dataclass, field
 from datetime import UTC, datetime
-from typing import Any, Optional
+from typing import TYPE_CHECKING, Any, Optional

 from ...config import get_config
 from ..db_utils import acquire_with_retry
@@ -24,6 +24,9 @@ from .link_expansion_retrieval import LinkExpansionRetriever
 from .tags import TagGroup, TagsMatch, build_tag_groups_where_clause, build_tags_where_clause_simple
 from .types import GraphRetrievalTimings, RetrievalResult

+if TYPE_CHECKING:
+    from ..query_analyzer import QueryAnalyzer
+
 logger = logging.getLogger(__name__)


@@ -2,14 +2,18 @@
 Tags filtering utilities for retrieval.

 Provides SQL building functions for filtering memories by tags.
-Supports four matching modes via TagsMatch enum:
+Supports five matching modes via TagsMatch enum:
 - "any": OR matching, includes untagged memories (default, backward compatible)
 - "all": AND matching, includes untagged memories
 - "any_strict": OR matching, excludes untagged memories
 - "all_strict": AND matching, excludes untagged memories
+- "exact": set-equality matching, excludes untagged memories

 OR matching (any/any_strict): Memory matches if ANY of its tags overlap with request tags
 AND matching (all/all_strict): Memory matches if ALL request tags are present in its tags
+EXACT matching: Memory matches only if its tag set EQUALS the request tag set (order-
+    independent). Used for observation "scope" filtering, where each observation lives
+    under exactly one scope (its full tag set) and "scope [a]" must not match "[a, b]".
 """

 from __future__ import annotations
@@ -18,7 +22,7 @@ from typing import Annotated, Literal

 from pydantic import BaseModel, ConfigDict, Field

-TagsMatch = Literal["any", "all", "any_strict", "all_strict"]
+TagsMatch = Literal["any", "all", "any_strict", "all_strict", "exact"]


 def _parse_tags_match(match: TagsMatch) -> tuple[str, bool]:
@@ -38,6 +42,10 @@ def _parse_tags_match(match: TagsMatch) -> tuple[str, bool]:
        return "&&", False
    elif match == "all_strict":
        return "@>", False
+    elif match == "exact":
+        # Set equality is handled by the callers via `@> AND <@`; the operator
+        # here is unused. Untagged rows never equal a non-empty scope.
+        return "@>", False
    else:
        # Default to "any" behavior
        return "&&", True
@@ -78,6 +86,13 @@ def build_tags_where_clause(
        return "", [], param_offset

    column = f"{table_alias}tags" if table_alias else "tags"
+
+    if match == "exact":
+        # Set equality (order-independent): superset AND subset. Untagged rows
+        # (empty array) never satisfy `@>` of a non-empty scope, so they're excluded.
+        clause = f"AND ({column} @> ${param_offset} AND {column} <@ ${param_offset})"
+        return clause, [tags], param_offset + 1
+
    operator, include_untagged = _parse_tags_match(match)

    if include_untagged:
@@ -115,6 +130,12 @@ def build_tags_where_clause_simple(
        return ""

    column = f"{table_alias}tags" if table_alias else "tags"
+
+    if match == "exact":
+        # Set equality (order-independent): superset AND subset. Untagged rows
+        # (empty array) never satisfy `@>` of a non-empty scope, so they're excluded.
+        return f"AND ({column} @> ${param_num} AND {column} <@ ${param_num})"
+
    operator, include_untagged = _parse_tags_match(match)

    if include_untagged:
@@ -164,7 +185,11 @@ def filter_results_by_tags(
            # else: skip untagged
        else:
            result_tags_set = set(result_tags)
-            if is_any_match:
+            if match == "exact":
+                # Set equality: tag set must match the scope exactly
+                if result_tags_set == tags_set:
+                    filtered.append(result)
+            elif is_any_match:
                # Any overlap
                if result_tags_set & tags_set:
                    filtered.append(result)
@@ -241,6 +266,9 @@ def _build_group_clause(
    """
    if isinstance(group, TagGroupLeaf):
        column = f"{table_alias}tags" if table_alias else "tags"
+        if group.match == "exact":
+            clause = f"({column} @> ${param_offset} AND {column} <@ ${param_offset})"
+            return clause, [group.tags], param_offset + 1
        operator, include_untagged = _parse_tags_match(group.match)
        if include_untagged:
            clause = f"({column} IS NULL OR {column} = '{{}}' OR {column} {operator} ${param_offset})"
@@ -349,6 +377,8 @@ def _match_group(result: object, group: TagGroup) -> bool:
            return include_untagged
        else:
            result_tags_set = set(result_tags)
+            if group.match == "exact":
+                return result_tags_set == tags_set
            if is_any_match:
                return bool(result_tags_set & tags_set)
            else:
@@ -2,7 +2,7 @@

 import logging
 import os
-from datetime import datetime, timedelta, timezone
+from datetime import timedelta, timezone

 import obstore as obs
 from obstore.store import GCSStore
@@ -0,0 +1,155 @@
+"""Explicit period extraction helpers for DateparserQueryAnalyzer.
+
+This module keeps the public period-extraction API and the non-Chinese period
+rules. Chinese rules live in chinese_temporal_periods.py because that rule set is
+substantially larger and has different boundary behavior from whitespace-based
+languages.
+"""
+
+import calendar
+import re
+import unicodedata
+from datetime import datetime, timedelta
+
+DateRange = tuple[datetime, datetime]
+
+
+class NoTemporalConstraintSentinel:
+    pass
+
+
+NO_TEMPORAL_CONSTRAINT = NoTemporalConstraintSentinel()
+
+__all__ = [
+    "NO_TEMPORAL_CONSTRAINT",
+    "extract_period",
+    "is_embedded_cjk_dateparser_match",
+]
+
+
+def _is_cjk_character(char: str) -> bool:
+    return "\u4e00" <= char <= "\u9fff"
+
+
+def is_embedded_cjk_dateparser_match(query: str, matched_text: str) -> bool:
+    from hindsight_api.engine.chinese_temporal_periods import (
+        is_embedded_cjk_dateparser_match as chinese_is_embedded_cjk_dateparser_match,
+    )
+
+    return chinese_is_embedded_cjk_dateparser_match(query, matched_text)
+
+
+def _constraint(start: datetime, end: datetime) -> DateRange:
+    return (
+        start.replace(hour=0, minute=0, second=0, microsecond=0),
+        end.replace(hour=23, minute=59, second=59, microsecond=999999),
+    )
+
+
+def _month_end(year: int, month: int) -> datetime:
+    return datetime(year, month, calendar.monthrange(year, month)[1])
+
+
+def _extract_non_chinese_period(query: str, reference_date: datetime) -> DateRange | None:
+    if re.search(r"\b(yesterday|ayer|ieri|hier|gestern)\b", query, re.IGNORECASE):
+        d = reference_date - timedelta(days=1)
+        return _constraint(d, d)
+
+    if re.search(r"\b(today|hoy|oggi|aujourd\'?hui|heute)\b", query, re.IGNORECASE):
+        return _constraint(reference_date, reference_date)
+
+    if re.search(r"\b(a\s+)?couple\s+(of\s+)?days?\s+ago\b", query, re.IGNORECASE):
+        return _constraint(reference_date - timedelta(days=3), reference_date - timedelta(days=1))
+
+    if re.search(r"\b(a\s+)?few\s+days?\s+ago\b", query, re.IGNORECASE):
+        return _constraint(reference_date - timedelta(days=5), reference_date - timedelta(days=2))
+
+    if re.search(r"\b(a\s+)?couple\s+(of\s+)?weeks?\s+ago\b", query, re.IGNORECASE):
+        return _constraint(reference_date - timedelta(weeks=3), reference_date - timedelta(weeks=1))
+
+    if re.search(r"\b(a\s+)?few\s+weeks?\s+ago\b", query, re.IGNORECASE):
+        return _constraint(reference_date - timedelta(weeks=5), reference_date - timedelta(weeks=2))
+
+    if re.search(r"\b(a\s+)?couple\s+(of\s+)?months?\s+ago\b", query, re.IGNORECASE):
+        return _constraint(reference_date - timedelta(days=90), reference_date - timedelta(days=30))
+
+    if re.search(r"\b(a\s+)?few\s+months?\s+ago\b", query, re.IGNORECASE):
+        return _constraint(reference_date - timedelta(days=150), reference_date - timedelta(days=60))
+
+    if re.search(
+        r"\b(last\s+week|la\s+semana\s+pasada|la\s+settimana\s+scorsa|la\s+semaine\s+derni[eè]re|letzte\s+woche)\b",
+        query,
+        re.IGNORECASE,
+    ):
+        start = reference_date - timedelta(days=reference_date.weekday() + 7)
+        return _constraint(start, start + timedelta(days=6))
+
+    if re.search(
+        r"\b(last\s+month|el\s+mes\s+pasado|il\s+mese\s+scorso|le\s+mois\s+dernier|letzten?\s+monat)\b",
+        query,
+        re.IGNORECASE,
+    ):
+        first = reference_date.replace(day=1)
+        end = first - timedelta(days=1)
+        start = end.replace(day=1)
+        return _constraint(start, end)
+
+    if re.search(
+        r"\b(last\s+year|el\s+a[ñn]o\s+pasado|l\'anno\s+scorso|l\'ann[ée]e\s+derni[eè]re|letztes?\s+jahr)\b",
+        query,
+        re.IGNORECASE,
+    ):
+        year = reference_date.year - 1
+        return _constraint(datetime(year, 1, 1), datetime(year, 12, 31))
+
+    if re.search(
+        r"\b(last\s+weekend|el\s+fin\s+de\s+semana\s+pasado|lo\s+scorso\s+fine\s+settimana|le\s+week-?end\s+dernier|letztes?\s+wochenende)\b",
+        query,
+        re.IGNORECASE,
+    ):
+        days_since_sat = (reference_date.weekday() + 2) % 7
+        if days_since_sat == 0:
+            days_since_sat = 7
+        sat = reference_date - timedelta(days=days_since_sat)
+        return _constraint(sat, sat + timedelta(days=1))
+
+    month_patterns = {
+        "january|enero|gennaio|janvier|januar": 1,
+        "february|febrero|febbraio|f[ée]vrier|februar": 2,
+        "march|marzo|mars|m[äa]rz": 3,
+        "april|abril|aprile|avril": 4,
+        "may|mayo|maggio|mai": 5,
+        "june|junio|giugno|juin|juni": 6,
+        "july|julio|luglio|juillet|juli": 7,
+        "august|agosto|ao[uû]t": 8,
+        "september|septiembre|settembre|septembre": 9,
+        "october|octubre|ottobre|octobre|oktober": 10,
+        "november|noviembre|novembre": 11,
+        "december|diciembre|dicembre|d[ée]cembre|dezember": 12,
+    }
+    for pattern, month_num in month_patterns.items():
+        match = re.search(rf"\b({pattern})\s+(\d{{4}})\b", query, re.IGNORECASE)
+        if match:
+            year = int(match.group(2))
+            start = datetime(year, month_num, 1)
+            return _constraint(start, _month_end(year, month_num))
+
+    return None
+
+
+def extract_period(query: str, reference_date: datetime) -> DateRange | NoTemporalConstraintSentinel | None:
+    """Extract explicit period-based temporal expressions.
+
+    Non-Chinese rules are kept here. Chinese rules are delegated to
+    chinese_temporal_periods.py and are skipped entirely for non-CJK queries.
+    """
+    query = unicodedata.normalize("NFKC", query)
+
+    if any(_is_cjk_character(char) for char in query):
+        from hindsight_api.engine.chinese_temporal_periods import extract_chinese_period
+
+        chinese_result = extract_chinese_period(query, reference_date)
+        if chinese_result is not None:
+            return chinese_result
+
+    return _extract_non_chinese_period(query, reference_date)
@@ -22,7 +22,7 @@ from pydantic import BaseModel, Field
 # Bump when the archive layout changes in a backward-incompatible way.
 SCHEMA_VERSION = 1

-ObservationScopes = Literal["per_tag", "combined", "all_combinations"] | list[list[str]]
+ObservationScopes = Literal["per_tag", "combined", "all_combinations", "shared"] | list[list[str]]


 class TransferCausalRelation(BaseModel):
@@ -52,4 +52,5 @@ class MemoryDefenseRegexExtension(MemoryDefenseExtension):
            message=f"Sensitive data pattern matched: {', '.join(result.matched_types)}",
            redacted_content=result.content if rule.action is DefenseAction.REDACT else None,
            matched_types=result.matched_types,
+            hits=result.hits,
        )
@@ -29,9 +29,14 @@ class DefenseAction(str, Enum):

 _VALID_ACTIONS = {a.value for a in DefenseAction}

-# Detector identifiers valid as ``policy.rules[*].on``. The OSS extension only
-# screens for sensitive data (secrets/PII), so that's the only accepted value.
-_VALID_DETECTORS = {"sensitive_data"}
+# ``policy.rules[*].on`` names a detector. The OSS extension only screens for
+# ``sensitive_data``; any other name is a silent no-op here and is dispatched
+# by whichever extension is loaded (e.g. hindsight-cloud screens cloud-only
+# detectors). The parser therefore does NOT validate ``on`` against a fixed
+# list — pinning the OSS roster to cloud's would force an OSS bump for every
+# new cloud detector just to avoid 422-ing a write it never interprets. We
+# only require ``on`` to be a non-empty string; entitlement and dispatch are
+# the loaded extension's ``screen()`` job.


@dataclass(frozen=True)
@@ -53,19 +58,58 @@ class DefenseDecision:
    message: str = ""
    redacted_content: str | None = None
    matched_types: list[str] = field(default_factory=list)
+    # Per-match fingerprinted previews. Each entry is
+    # ``{"detector": <pattern label>, "preview": <fingerprinted value>}``.
+    # The preview is *never* the raw value — see :func:`_fingerprint_value`.
+    # OSS populates this from ``apply_redaction``; downstream extensions
+    # populate it from their own detectors. Optional: empty when the
+    # match path didn't capture per-hit values.
+    hits: list[dict] = field(default_factory=list)


@dataclass
 class RedactionResult:
    content: str
    matched_types: list[str]
+    # Same shape as ``DefenseDecision.hits`` — one entry per matched value
+    # (so a single content with two GitHub tokens produces two entries).
+    hits: list[dict] = field(default_factory=list)
+
+
+def _fingerprint_value(value: str) -> str:
+    """Return a redaction-identifiable preview of a matched value.
+
+    The preview keeps the prefix and a short suffix so a SIEM operator can
+    correlate against their credential inventory (the prefix names the
+    provider; the suffix disambiguates specific instances) without the raw
+    secret crossing the wire. Length-aware so short values don't accidentally
+    leak material:
+
+    - Length < 6:  redact entirely (return a fixed-length mask). Catches
+      noise like a single ``-----BEGIN...`` marker line.
+    - Length 6-15: keep the first 2 + last 2 around an ellipsis.
+    - Length > 15: keep the first 4 + last 4 around an ellipsis.
+
+    Examples::
+
+        _fingerprint_value("ghp_AAAA...AAAA" + "A" * 36)  -> "ghp_...AAAA"
+        _fingerprint_value("AKIA" + "B" * 16)              -> "AKIA...BBBB"
+        _fingerprint_value("123-45-6789")                  -> "12...89"
+        _fingerprint_value("abc")                          -> "[redacted]"
+    """
+    n = len(value)
+    if n < 6:
+        return "[redacted]"
+    if n <= 15:
+        return f"{value[:2]}...{value[-2:]}"
+    return f"{value[:4]}...{value[-4:]}"


 def parse_policy(raw: dict | None) -> DefensePolicy:
    """Parse a raw bank-config dict into a frozen DefensePolicy.

-    Raises ValueError for unknown detectors or actions; the HTTP layer
-    converts those into a 422 response.
+    Raises ValueError for a missing/empty ``on`` or an unknown action; the
+    HTTP layer converts those into a 422 response.
    """
    if raw is None:
        return DefensePolicy()
@@ -73,8 +117,8 @@ def parse_policy(raw: dict | None) -> DefensePolicy:
    rules: list[PolicyRule] = []
    for item in raw.get("rules", []) or []:
        on_raw = item.get("on")
-        if on_raw not in _VALID_DETECTORS:
-            raise ValueError(f"invalid on {on_raw!r}; must be one of {sorted(_VALID_DETECTORS)}")
+        if not isinstance(on_raw, str) or not on_raw:
+            raise ValueError(f"invalid on {on_raw!r}; must be a non-empty string")
        action_raw = item.get("action")
        if action_raw not in _VALID_ACTIONS:
            raise ValueError(f"invalid action {action_raw!r}; must be one of {sorted(_VALID_ACTIONS)}")
@@ -166,16 +210,42 @@ _COMPILED_REDACTIONS: list[tuple[str, re.Pattern]] = [
 def apply_redaction(content: str) -> RedactionResult:
    """Scrub known secret/PII patterns from content with [REDACTED:type] markers.

-    Returns the (possibly unchanged) content alongside the list of pattern
-    labels that matched (empty when nothing matched).
+    Returns the (possibly unchanged) content alongside:
+      - ``matched_types``: pattern labels that matched (deduplicated, in
+        first-occurrence order). Empty when nothing matched.
+      - ``hits``: per-match fingerprinted previews — one entry per matched
+        substring (so two GitHub tokens in the same content produce two
+        entries). Each entry is ``{"detector": label, "preview": fingerprint}``
+        where ``preview`` is a length-aware redaction of the original value.
+        The raw secret never appears in ``hits``.
+
+    The two-pass shape (find matches first, then substitute) lets us capture
+    raw values for fingerprinting before they're replaced by ``[REDACTED:type]``
+    markers. A single-pass approach would lose the originals.
    """
    matched: list[str] = []
+    hits: list[dict] = []
    for label, pattern in _COMPILED_REDACTIONS:
-        new_content = pattern.sub(f"[REDACTED:{label}]", content)
-        if new_content != content:
+        raw_hits = pattern.findall(content)
+        if not raw_hits:
+            continue
+        if label not in matched:
            matched.append(label)
-            content = new_content
-    return RedactionResult(content=content, matched_types=matched)
+        for raw in raw_hits:
+            # findall returns either a string or a tuple of capture groups
+            # depending on the pattern. The redaction-pattern catalog uses a
+            # mix; coerce to the matched substring as best we can.
+            if isinstance(raw, tuple):
+                # Pick the longest non-empty group as the canonical match.
+                non_empty = [g for g in raw if g]
+                raw_str = max(non_empty, key=len) if non_empty else ""
+            else:
+                raw_str = raw
+            if not raw_str:
+                continue
+            hits.append({"detector": label, "preview": _fingerprint_value(raw_str)})
+        content = pattern.sub(f"[REDACTED:{label}]", content)
+    return RedactionResult(content=content, matched_types=matched, hits=hits)


 class MemoryDefenseExtension(Extension, ABC):
@@ -3,7 +3,7 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING

 from hindsight_api.extensions.base import Extension

@@ -2321,7 +2321,7 @@ def _register_update_memory(mcp: FastMCP, memory: MemoryEngine, config: MCPTools

    if config.include_bank_id_param:

-        @mcp.tool()
+        @mcp.tool(description=_EDIT_DOC)
        async def update_memory(
            memory_id: str,
            text: str | None = None,
@@ -2332,7 +2332,7 @@ def _register_update_memory(mcp: FastMCP, memory: MemoryEngine, config: MCPTools
            entities: list[str] | None = None,
            bank_id: str | None = None,
        ) -> str:
-            f"""{_EDIT_DOC}
+            """
            Args:
                memory_id: The ID of the memory unit to edit.
                bank_id: Optional bank (defaults to session bank). Use for cross-bank operations.
@@ -2367,7 +2367,7 @@ def _register_update_memory(mcp: FastMCP, memory: MemoryEngine, config: MCPTools

    else:

-        @mcp.tool()
+        @mcp.tool(description=_EDIT_DOC)
        async def update_memory(
            memory_id: str,
            text: str | None = None,
@@ -2377,7 +2377,7 @@ def _register_update_memory(mcp: FastMCP, memory: MemoryEngine, config: MCPTools
            fact_type: str | None = None,
            entities: list[str] | None = None,
        ) -> dict:
-            f"""{_EDIT_DOC}
+            """
            Args:
                memory_id: The ID of the memory unit to edit.
            """
@@ -2426,14 +2426,14 @@ def _register_invalidate_memory(mcp: FastMCP, memory: MemoryEngine, config: MCPT

    if config.include_bank_id_param:

-        @mcp.tool()
+        @mcp.tool(description=_INVALIDATE_DOC)
        async def invalidate_memory(
            memory_id: str,
            reason: str | None = None,
            restore: bool = False,
            bank_id: str | None = None,
        ) -> str:
-            f"""{_INVALIDATE_DOC}
+            """
            Args:
                memory_id: The ID of the memory unit to retire (or restore).
                reason: Optional free-text reason recorded when invalidating.
@@ -2466,13 +2466,13 @@ def _register_invalidate_memory(mcp: FastMCP, memory: MemoryEngine, config: MCPT

    else:

-        @mcp.tool()
+        @mcp.tool(description=_INVALIDATE_DOC)
        async def invalidate_memory(
            memory_id: str,
            reason: str | None = None,
            restore: bool = False,
        ) -> dict:
-            f"""{_INVALIDATE_DOC}
+            """
            Args:
                memory_id: The ID of the memory unit to retire (or restore).
                reason: Optional free-text reason recorded when invalidating.
@@ -3191,7 +3191,8 @@ def _register_update_bank(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsCo
                    - retain_mission: Steers what gets extracted during retain().
                    - retain_extraction_mode: 'concise' (default), 'verbose', or 'custom'.
                    - retain_custom_instructions: Custom extraction prompt (active when mode is 'custom').
-                    - retain_chunk_size: Maximum token size for each content chunk.
+                    - retain_chunk_size: Target maximum characters for each content chunk.
+                    - retain_structured_chunk_size: Maximum characters for a single JSONL line or conversation turn to keep whole.
                    - retain_chunk_batch_size: Number of chunks to process in parallel.
                    - enable_observations: Toggle observation consolidation after retain().
                    - observations_mission: Controls observation synthesis rules.
@@ -3250,7 +3251,8 @@ def _register_update_bank(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsCo
                    - retain_mission: Steers what gets extracted during retain().
                    - retain_extraction_mode: 'concise' (default), 'verbose', or 'custom'.
                    - retain_custom_instructions: Custom extraction prompt (active when mode is 'custom').
-                    - retain_chunk_size: Maximum token size for each content chunk.
+                    - retain_chunk_size: Target maximum characters for each content chunk.
+                    - retain_structured_chunk_size: Maximum characters for a single JSONL line or conversation turn to keep whole.
                    - retain_chunk_batch_size: Number of chunks to process in parallel.
                    - enable_observations: Toggle observation consolidation after retain().
                    - observations_mission: Controls observation synthesis rules.
@@ -39,6 +39,32 @@ def _get_tenant() -> str:
    return get_current_schema()


+def _is_client_cancellation(exc: BaseException) -> bool:
+    """Whether *exc* is a client-disconnect cancellation rather than a failure.
+
+    An abandoned recall/reflect raises OperationCancelledError (issue #2122);
+    the HTTP layer re-raises it as ``HTTPException(499) from exc`` (see
+    api/http.py run_cancellable_on_disconnect). The exception itself, or any
+    link in its ``__cause__`` chain, being an OperationCancelledError marks it
+    as a cancellation. Matching on the cause chain rather than a bare status
+    code avoids misclassifying an unrelated 499 as a cancellation. Per the
+    engine contract a cancellation is "not a failure to retry or report"
+    (cancellation.OperationCancelledError), so it must not be counted against
+    ``hindsight.operation.total``.
+    """
+    # Imported lazily to avoid import-time coupling (cf. _get_tenant above).
+    from hindsight_api.cancellation import OperationCancelledError
+
+    cause: BaseException | None = exc
+    seen: set[int] = set()  # guard against a cyclic __cause__ chain
+    while cause is not None and id(cause) not in seen:
+        if isinstance(cause, OperationCancelledError):
+            return True
+        seen.add(id(cause))
+        cause = cause.__cause__
+    return False
+
+
 # Custom bucket boundaries for operation duration (in seconds)
 # Fine granularity in 0-30s range where most operations complete
 DURATION_BUCKETS = (0.1, 0.25, 0.5, 0.75, 1.0, 2.0, 3.0, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, 120.0)
@@ -373,20 +399,31 @@ class MetricsCollector(MetricsCollectorBase):
            attributes["max_tokens"] = str(max_tokens)

        success = True
+        cancelled = False
        try:
            yield
-        except Exception:
-            success = False
+        except Exception as exc:
+            # A client disconnect cancels the operation cooperatively (#2122),
+            # raised as OperationCancelledError and re-raised by the HTTP layer
+            # as HTTPException(499) from it. An abandoned request is neither a
+            # success nor a failure, so it is excluded from the metric entirely
+            # rather than inflating either the failure or the success rate on
+            # hindsight.operation.total.
+            if _is_client_cancellation(exc):
+                cancelled = True
+            else:
+                success = False
            raise
        finally:
-            duration = time.time() - start_time
-            attributes["success"] = str(success).lower()
+            if not cancelled:
+                duration = time.time() - start_time
+                attributes["success"] = str(success).lower()

-            # Record duration
-            self.operation_duration.record(duration, attributes)
+                # Record duration
+                self.operation_duration.record(duration, attributes)

-            # Record operation count
-            self.operation_total.add(1, attributes)
+                # Record operation count
+                self.operation_total.add(1, attributes)

    def record_llm_call(
        self,
@@ -27,6 +27,7 @@ from alembic.config import Config
 from alembic.script.revision import ResolutionError
 from alembic.util.exc import CommandError
 from sqlalchemy import Connection, create_engine, text
+from sqlalchemy.pool import NullPool

 from ._pg_search import normalize_pg_search_tokenizer, pg_search_bm25_columns
 from ._vector_index import (
@@ -247,7 +248,14 @@ def run_migrations(
        #   2. After acquiring the lock, COMMIT the transaction on the advisory-lock
        #      connection itself before running migrations.  pg_advisory_lock is
        #      session-level, so the lock survives the COMMIT.
-        engine = create_engine(migration_url)
+        # NullPool: do not retain the connection in a pool after the migration.
+        # Each schema migration opens a few short-lived engines (here plus the
+        # ensure_* steps); with the default QueuePool those connections linger
+        # until GC, and running many schemas in parallel (migration_concurrency)
+        # multiplies that footprint and exhausts max_connections — observed as
+        # "FATAL: sorry, too many clients already" sweeping 20k schemas at
+        # concurrency 12. NullPool closes the connection on return.
+        engine = create_engine(migration_url, poolclass=NullPool)
        with engine.connect() as conn:
            logger.debug(f"Acquiring migration advisory lock for schema '{schema_name}' (id={lock_id})...")
            while True:
@@ -400,7 +408,7 @@ def check_migration_status(
            return None, None

        # Get current revision from database
-        engine = create_engine(to_libpq_url(database_url))
+        engine = create_engine(to_libpq_url(database_url), poolclass=NullPool)
        with engine.connect() as connection:
            context = MigrationContext.configure(connection)
            current_rev = context.get_current_revision()
@@ -573,7 +581,7 @@ def ensure_embedding_dimension(
    """
    schema_name = schema or "public"

-    engine = create_engine(to_libpq_url(database_url))
+    engine = create_engine(to_libpq_url(database_url), poolclass=NullPool)
    with engine.connect() as conn:
        # Check if memory_units table exists (proxy for schema being initialized)
        table_exists = conn.execute(
@@ -596,6 +604,10 @@ def ensure_embedding_dimension(

        _migrate_table_embedding_dimension(conn, schema_name, "memory_units", required_dimension, vector_ext)
        _migrate_table_embedding_dimension(conn, schema_name, "mental_models", required_dimension, vector_ext)
+        # NOTE: invalidated_memory_units is deliberately omitted. The curation archive has no
+        # embedding column at all (dropped in migration d4f6a8c2e1b3) — invalidate stores no
+        # embedding and revert recomputes one — so there is no archive vector to re-dimension
+        # and a model switch can't trip a dimension mismatch there (#2209).


 def ensure_vector_extension(
@@ -622,7 +634,7 @@ def ensure_vector_extension(
    """
    schema_name = schema or "public"

-    engine = create_engine(to_libpq_url(database_url))
+    engine = create_engine(to_libpq_url(database_url), poolclass=NullPool)
    with engine.connect() as conn:
        # Detect which vector extension should be used
        target_ext = _detect_vector_extension(conn, vector_extension)
@@ -836,7 +848,7 @@ def ensure_text_search_extension(
    schema_name = schema or "public"
    pg_search_tokenizer = normalize_pg_search_tokenizer(pg_search_tokenizer)

-    engine = create_engine(to_libpq_url(database_url))
+    engine = create_engine(to_libpq_url(database_url), poolclass=NullPool)
    with engine.connect() as conn:
        # Tables with search_vector columns to check
        tables_to_check = [
@@ -1129,3 +1141,129 @@ def ensure_text_search_extension(

        conn.commit()
        logger.info(f"Successfully migrated text search to {text_search_extension}")
+
+
+def _migrate_one_schema_pg(
+    database_url: str,
+    schema: str,
+    *,
+    migration_database_url: str | None,
+    embedding_dimension: int | None,
+    vector_extension: str,
+    text_search_extension: str,
+    pg_search_tokenizer: str | None,
+    ensure_extensions: bool,
+) -> str:
+    """Run migrations + post-migration extension setup for a SINGLE PG schema.
+
+    Module-level (not a closure) so it is picklable and can run inside a
+    ``ProcessPoolExecutor`` worker. The steps run strictly in order — this is
+    the per-tenant sequential unit; parallelism happens only *across* schemas.
+    Returns the schema name on success; raises on the first failing step so the
+    caller can attribute the failure back to this schema.
+    """
+    run_migrations(database_url, schema=schema, migration_database_url=migration_database_url)
+    if embedding_dimension is not None:
+        ensure_embedding_dimension(
+            database_url,
+            embedding_dimension,
+            schema=schema,
+            vector_extension=vector_extension,
+        )
+    if ensure_extensions:
+        ensure_vector_extension(database_url, vector_extension=vector_extension, schema=schema)
+        ensure_text_search_extension(
+            database_url,
+            text_search_extension=text_search_extension,
+            schema=schema,
+            pg_search_tokenizer=pg_search_tokenizer,
+        )
+    return schema
+
+
+def _make_migration_executor(max_workers: int):
+    """Build the executor that runs per-schema migrations in parallel.
+
+    Each schema must run in its OWN process — Alembic's ``command.upgrade()``
+    uses non-thread-safe module globals (serialized in-process by
+    ``_alembic_lock``), so a thread pool would not actually run two upgrades at
+    once. ``spawn`` gives every worker a clean interpreter on all platforms,
+    avoiding the fork-of-a-multithreaded-process deadlock hazard (the API server
+    holds threads/pools when migrations run on startup).
+
+    Factored out so tests can substitute an in-process executor.
+    """
+    import multiprocessing
+    from concurrent.futures import ProcessPoolExecutor
+
+    return ProcessPoolExecutor(max_workers=max_workers, mp_context=multiprocessing.get_context("spawn"))
+
+
+def run_migrations_for_schemas(
+    database_url: str,
+    schemas: list[str],
+    *,
+    concurrency: int = 1,
+    migration_database_url: str | None = None,
+    embedding_dimension: int | None = None,
+    vector_extension: str = "pgvector",
+    text_search_extension: str = "native",
+    pg_search_tokenizer: str | None = None,
+    ensure_extensions: bool = True,
+) -> None:
+    """Run PostgreSQL migrations for many schemas, up to ``concurrency`` at once.
+
+    Within a schema the work is always sequential (migrate → embedding dim →
+    vector ext → text-search ext). Across schemas, when ``concurrency > 1`` each
+    schema is migrated in its OWN process: Alembic's ``command.upgrade()`` relies
+    on non-thread-safe module-level globals (serialized in-process by
+    ``_alembic_lock``), so threads would gain nothing — separate interpreters
+    each get a clean Alembic context. Per-schema advisory locks
+    (``_get_schema_lock_id``) keep concurrent processes from colliding on the
+    same schema across replicas.
+
+    ``database_url`` must already be resolved (e.g. an embedded ``pg0`` instance
+    started in the parent) — workers receive it verbatim and only connect.
+
+    Failures are collected per schema and re-raised together so one bad tenant
+    does not hide the status of the others.
+    """
+    if not schemas:
+        return
+
+    worker_kwargs = dict(
+        migration_database_url=migration_database_url,
+        embedding_dimension=embedding_dimension,
+        vector_extension=vector_extension,
+        text_search_extension=text_search_extension,
+        pg_search_tokenizer=pg_search_tokenizer,
+        ensure_extensions=ensure_extensions,
+    )
+
+    effective = max(1, min(concurrency, len(schemas)))
+    if effective == 1:
+        # Inline, in-process — no subprocess overhead for the common single
+        # tenant / sequential case (and keeps embedded pg0 dev simple).
+        for schema in schemas:
+            _migrate_one_schema_pg(database_url, schema, **worker_kwargs)
+        return
+
+    logger.info("Migrating %d schema(s) with concurrency=%d", len(schemas), effective)
+    errors: dict[str, BaseException] = {}
+    with _make_migration_executor(effective) as executor:
+        futures = {
+            executor.submit(_migrate_one_schema_pg, database_url, schema, **worker_kwargs): schema for schema in schemas
+        }
+        for future in futures:
+            schema = futures[future]
+            try:
+                future.result()
+            except Exception as exc:  # noqa: BLE001 — aggregate per-schema, re-raise below
+                errors[schema] = exc
+                logger.error("Migration failed for schema '%s': %s", schema, exc)
+
+    if errors:
+        failed = ", ".join(sorted(errors))
+        raise RuntimeError(
+            f"Database migrations failed for {len(errors)} of {len(schemas)} schema(s): {failed}"
+        ) from next(iter(errors.values()))
@@ -4,6 +4,7 @@ from .manager import WebhookManager
 from .models import (
    ConsolidationEventData,
    MemoryDefenseEventData,
+    MemoryDefenseHit,
    RetainEventData,
    WebhookConfig,
    WebhookEvent,
@@ -17,5 +18,6 @@ __all__ = [
    "WebhookEventType",
    "ConsolidationEventData",
    "MemoryDefenseEventData",
+    "MemoryDefenseHit",
    "RetainEventData",
 ]
@@ -70,7 +70,10 @@ class WebhookManager:
        webhook_table = _fq_table("webhooks", schema)
        ops_table = _fq_table("async_operations", schema)
        now = datetime.now(timezone.utc)
-        payload_str = event.model_dump_json()
+        # Drop null fields so receivers don't see promised-but-unfilled keys.
+        # OSS leaves SIEM-enrichment fields (severity, api_key_name, etc.) None
+        # because it doesn't have the data; cloud populates them when it does.
+        payload_str = event.model_dump_json(exclude_none=True)

        try:
            async with self._backend.acquire() as conn:
@@ -150,7 +153,10 @@ class WebhookManager:
        webhook_table = _fq_table("webhooks", schema)
        ops_table = _fq_table("async_operations", schema)
        now = datetime.now(timezone.utc)
-        payload_str = event.model_dump_json()
+        # Drop null fields so receivers don't see promised-but-unfilled keys.
+        # OSS leaves SIEM-enrichment fields (severity, api_key_name, etc.) None
+        # because it doesn't have the data; cloud populates them when it does.
+        payload_str = event.model_dump_json(exclude_none=True)

        try:
            rows = await self._backend.ops.get_webhooks_for_dispatch(
@@ -24,14 +24,43 @@ class RetainEventData(BaseModel):
    tags: list[str] | None = None


+class MemoryDefenseHit(BaseModel):
+    """A single secret match inside a non-allow decision.
+
+    ``preview`` is a fingerprinted, redaction-identifiable rendering of the
+    matched value (e.g. ``ghp_AAAA...BBBB``) so SIEM operators can correlate
+    against their credential inventory WITHOUT the raw secret crossing the
+    network. Implementations must never put the raw value here.
+    """
+
+    detector: str  # the inner detector that matched (e.g. "GitHub Token")
+    preview: str  # fingerprinted value, never the raw secret
+
+
 class MemoryDefenseEventData(BaseModel):
-    """Payload for a memory_defense.triggered event (one item, one non-allow decision)."""
+    """Payload for a memory_defense.triggered event (one item, one non-allow decision).
+
+    The four base fields (``action``/``detector``/``document_id``/``message``)
+    plus ``matched_types`` are populated by every implementation including OSS's
+    built-in regex defense. The remaining fields are optional SIEM-enrichment
+    surfaces that downstream extensions (e.g. hindsight-cloud) populate when
+    they have richer per-decision context — severity classification, the API
+    key that submitted the retain, fingerprinted hit previews for SIEM
+    correlation, and pointers into the audit trail. OSS leaves them ``None``;
+    receivers should treat absence as "not provided" rather than "no match".
+    """

    action: str  # "redact" or "block"
    detector: str | None = None  # e.g. "sensitive_data"
    document_id: str | None = None
    matched_types: list[str] | None = None  # redaction pattern labels that fired
    message: str | None = None
+    # --- Optional SIEM enrichment (populated by extensions, not OSS) ---
+    severity: str | None = None  # "low" / "medium" / "high" / "critical"
+    api_key_name: str | None = None  # human-readable name of the submitting API key
+    hits: list[MemoryDefenseHit] | None = None  # per-match fingerprints for correlation
+    memory_unit_id: str | None = None  # drill-down pointer (when the decision was REDACT)
+    receipt_uri: str | None = None  # storage pointer for the audit trail entry


 class WebhookEvent(BaseModel):
@@ -16,7 +16,7 @@ import time
 import traceback
 from collections import Counter
 from collections.abc import Awaitable, Callable, Iterable
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any

 from ..engine.schema import fq_table_explicit as fq_table
@@ -226,38 +226,23 @@ class WorkerPoller:
        """
        async with self._backend.acquire() as conn:
            if await self._optional_routines.is_installed(conn, "schemas_with_pending_work"):
+                # The routine IS the authority on where work exists: every schema
+                # it returns is claimable, and every schema it does NOT return is
+                # treated as having nothing to do this cycle. That is the entire
+                # point of installing it — one round-trip replaces N per-schema
+                # EXISTS probes. We deliberately do NOT re-verify the omitted
+                # schemas with a per-schema scan: that re-runs the exact queries
+                # the routine exists to avoid, on every idle poll, silently
+                # negating the optimisation.
+                #
+                # Because the result is trusted wholesale, the routine is only
+                # appropriate for multi-tenant deployments. A single-schema
+                # (default/public only) install should NOT create it and instead
+                # falls through to the per-schema path below — a single cheap
+                # EXISTS check that cannot starve. See
+                # ``hindsight_api.engine.db.optional_routines``.
                rows = await conn.fetch("SELECT * FROM public.schemas_with_pending_work()")
-                routine_active = {self._normalize_poll_schema(r[0]) for r in rows}
-                known_schemas = set(schemas)
-                active = routine_active & known_schemas
-                unknown = routine_active - known_schemas
-                if unknown:
-                    logger.warning(
-                        "Optional PG routine public.schemas_with_pending_work() returned schema(s) "
-                        "not present in tenant discovery: %s",
-                        sorted(str(s) for s in unknown),
-                    )
-
-                # The optional routine returns PostgreSQL schema names, but the poller uses
-                # None for the default schema. Older operator-supplied implementations also
-                # commonly scan tenant_% only; when the default schema is in scope but absent
-                # from the routine result, verify via the fully-correct per-schema fallback so
-                # public single-tenant deployments cannot silently starve.
-                should_verify_with_fallback = (None in known_schemas and None not in active) or (
-                    bool(routine_active) and not active
-                )
-                if not should_verify_with_fallback:
-                    return active
-
-                fallback_active = await self._scan_active_schemas_by_exists(conn, schemas)
-                missed = fallback_active - active
-                if missed:
-                    logger.warning(
-                        "Optional PG routine public.schemas_with_pending_work() missed claimable schema(s) %s; "
-                        "using per-schema fallback for this poll",
-                        sorted(str(s) for s in missed),
-                    )
-                return fallback_active
+                return {self._normalize_poll_schema(r[0]) for r in rows}

            return await self._scan_active_schemas_by_exists(conn, schemas)

@@ -824,7 +809,6 @@ class WorkerPoller:
            recovered = 0
            for row in rows:
                operation_id = str(row["operation_id"])
-                task_payload = row["task_payload"]
                result_metadata = row["result_metadata"]

                # Parse metadata
@@ -838,12 +822,6 @@ class WorkerPoller:
                    f"Recovering batch operation: operation_id={operation_id}, batch_id={batch_id}, provider={batch_provider}"
                )

-                # Parse task_payload
-                if isinstance(task_payload, str):
-                    task_dict = json.loads(task_payload)
-                else:
-                    task_dict = task_payload
-
                # Mark operation as ready for re-processing
                # Reset to pending with task_payload intact so worker picks it up again
                async with self._backend.acquire() as conn:
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"

 [project]
 name = "hindsight-api-slim"
-version = "0.8.1"
+version = "0.8.2"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -200,12 +200,11 @@ select = [
    "W",      # pycodestyle warnings
    "F",      # Pyflakes
    "I",      # isort
+    "B021",   # flake8-bugbear: f-string used as docstring (leaves __doc__ None)
 ]
 ignore = [
    "E501",   # line too long (handled by formatter)
    "E402",   # module import not at top of file
-    "F401",   # unused import (too noisy during development)
-    "F841",   # unused variable (too noisy during development)
    "F811",   # redefined while unused
    "F821",   # undefined name (forward references in type hints)
 ]
@@ -16,6 +16,27 @@ from hindsight_api.engine.cross_encoder import LocalSTCrossEncoder
 from hindsight_api.engine.query_analyzer import DateparserQueryAnalyzer
 from hindsight_api.engine.task_backend import SyncTaskBackend
 from hindsight_api.pg0 import EmbeddedPostgres
+from hindsight_api.tracing import unregister_span_recorder
+
+
+async def _teardown_memory_engine(mem: MemoryEngine) -> None:
+    """Tear down a test MemoryEngine, guaranteeing its span recorder is unregistered.
+
+    LLM-trace recorders live in a process-global registry; ``MemoryEngine.close()`` is
+    the only thing that removes the engine's recorder from it. If close() is skipped
+    (pool already closing) or raises before that step, the recorder leaks and a later
+    test's LLM calls get recorded into the shared DB — the flaky
+    test_llm_trace::test_disabled_writes_no_rows (#2229). Unregister unconditionally;
+    it's a no-op when close() already did it.
+    """
+    try:
+        if mem._pool and not mem._pool._closing:
+            await mem.close()
+    except Exception:
+        pass
+    finally:
+        unregister_span_recorder(mem._llm_recorder)
+

 # Default pg0 instance configuration for tests
 DEFAULT_PG0_INSTANCE_NAME = "hindsight-test"
@@ -342,10 +363,7 @@ async def oracle_memory(oracle_db_url, embeddings, cross_encoder, query_analyzer
        )
        await mem.initialize()
        yield mem
-        try:
-            await mem.close()
-        except Exception:
-            pass
+        await _teardown_memory_engine(mem)
    finally:
        # Restore original env var and clear config cache
        if old_backend is None:
@@ -463,11 +481,7 @@ async def memory(pg0_db_url, embeddings, cross_encoder, query_analyzer):
    )
    await mem.initialize()
    yield mem
-    try:
-        if mem._pool and not mem._pool._closing:
-            await mem.close()
-    except Exception:
-        pass
+    await _teardown_memory_engine(mem)


@pytest_asyncio.fixture(scope="function")
@@ -496,11 +510,7 @@ async def memory_real_llm(pg0_db_url, embeddings, cross_encoder, query_analyzer)
    )
    await mem.initialize()
    yield mem
-    try:
-        if mem._pool and not mem._pool._closing:
-            await mem.close()
-    except Exception:
-        pass
+    await _teardown_memory_engine(mem)


@pytest_asyncio.fixture(scope="function")
@@ -527,11 +537,7 @@ async def memory_no_llm_verify(pg0_db_url, embeddings, cross_encoder, query_anal
    )
    await mem.initialize()
    yield mem
-    try:
-        if mem._pool and not mem._pool._closing:
-            await mem.close()
-    except Exception:
-        pass
+    await _teardown_memory_engine(mem)


@pytest_asyncio.fixture
@@ -82,7 +82,8 @@ async def test_bank_llm_not_configured(api_client, memory, monkeypatch):
        monkeypatch.setattr(cfg, "provider", "none")
    body = (await api_client.post("/v1/default/banks/llm-none/health/llm")).json()
    assert all(op["status"] == "not_configured" and op["ok"] is False for op in body["operations"])
-    assert all(op["latency_ms"] is None for op in body["operations"])
+    # latency_ms is null when not configured; responses omit null fields, so use .get().
+    assert all(op.get("latency_ms") is None for op in body["operations"])


@pytest.mark.asyncio
@@ -30,6 +30,7 @@ from hindsight_api.api.http import BankTemplateConfig
 # Each tuple is (field_name, applied_value). Values chosen to differ
 # visibly from defaults so round-trip bugs surface.
 NEW_FIELDS: list[tuple[str, object]] = [
+    ("retain_structured_chunk_size", 6000),
    ("retain_default_strategy", "strategy-a"),
    ("retain_strategies", {"strategy-a": {"mode": "concise", "max_tokens": 512}}),
    ("retain_chunk_batch_size", 7),
@@ -495,9 +495,10 @@ class TestExport:
        assert resp.status_code == 200
        data = resp.json()
        assert data["version"] == "1"
-        assert data["bank"] is None
-        assert data["mental_models"] is None
-        assert data["directives"] is None
+        # An empty bank has no overrides; these null fields are omitted from the response.
+        assert data.get("bank") is None
+        assert data.get("mental_models") is None
+        assert data.get("directives") is None

    @pytest.mark.asyncio
    async def test_export_after_import(self, api_client, bank_id):
@@ -12,11 +12,6 @@ import pytest

 from hindsight_api.engine.retain.fact_extraction import chunk_text

-# Mirror of fact_extraction._CHUNK_OVERFLOW_FACTOR — a unit is kept whole only
-# up to this multiple of the budget before being split as text.
-OVERFLOW_FACTOR = 1.5
-
-
 # ---------------------------------------------------------------------------
 # Plain text
 # ---------------------------------------------------------------------------
@@ -137,22 +132,70 @@ def test_chunk_jsonl_splits_at_line_boundaries():
    assert seen == [json.loads(line) for line in lines], "Lines must be preserved in order"


-def test_chunk_jsonl_small_overflow_kept_whole():
-    """A JSONL line that overflows by less than 1.5x is kept whole, not split."""
-    big = json.dumps({"c": "y" * 20})  # 29 chars; budget 25, cap 37 -> kept whole
+def test_chunk_jsonl_default_structured_unit_limit_matches_budget():
+    """A JSONL line over the budget is split when no larger structured-chunk cap is set."""
+    big = json.dumps({"c": "y" * 20})  # 29 chars; budget 25 -> split
    small = json.dumps({"c": "ok"})
    text = "\n".join([big, small])
-    assert 25 < len(big) <= int(25 * OVERFLOW_FACTOR)

    chunks = chunk_text(text, max_chars=25)

-    # The line overflows the budget but stays a single intact chunk of its own.
+    assert chunks == [
+        '{"c":',
+        '"yyyyyyyyyyyyyyyyyyyy"}',
+        small,
+    ]
+
+
+def test_chunk_jsonl_custom_structured_unit_limit_keeps_overflow_whole():
+    """A JSONL line over the budget is kept whole when the explicit cap allows it."""
+    big = json.dumps({"c": "y" * 20})  # 29 chars
+    small = json.dumps({"c": "ok"})
+    text = "\n".join([big, small])
+
+    chunks = chunk_text(text, max_chars=25, structured_chunk_size=len(big))
+
    assert chunks == [big, small]


+def test_chunk_structured_unit_limit_above_chunk_size_preserves_small_overflows():
+    """Structured units between max_chars and the structured cap remain intact."""
+    jsonl_line = json.dumps({"c": "y" * 20})  # 29 chars; over budget 25, within cap 29
+    conversation = json.dumps([{"c": "y" * 20}])
+
+    jsonl_chunks = chunk_text(
+        "\n".join([jsonl_line, json.dumps({"c": "ok"})]),
+        max_chars=25,
+        structured_chunk_size=29,
+    )
+    conversation_chunks = chunk_text(conversation, max_chars=25, structured_chunk_size=29)
+
+    assert jsonl_chunks[0] == jsonl_line
+    assert conversation_chunks == [conversation]
+
+
+def test_chunk_jsonl_structured_unit_limit_can_be_below_chunk_size():
+    """An oversized JSONL line is split by the structured cap, not the larger chunk budget."""
+    huge = json.dumps({"c": "y" * 40})  # 49 chars; over cap 20 but under budget 55
+    small = json.dumps({"c": "ok"})
+    text = "\n".join([huge, small])
+
+    chunks = chunk_text(text, max_chars=55, structured_chunk_size=20)
+
+    assert chunks == [
+        '{"c":',
+        '"yyyyyyyyyyyyyyyyyy',
+        "yyyyyyyyyyyyyyyyyyyy",
+        'yy"}',
+        small,
+    ]
+    for chunk in chunks:
+        assert len(chunk) <= 20
+
+
 def test_chunk_jsonl_huge_line_is_split():
-    """A JSONL line past the 1.5x overflow cap is split as text — exact fragments."""
-    huge = json.dumps({"c": "y" * 40})  # 50 chars; budget 20, cap 30 -> must split
+    """A JSONL line past the structured-chunk cap is split as text — exact fragments."""
+    huge = json.dumps({"c": "y" * 40})  # 49 chars; budget/cap 20 -> must split
    small = json.dumps({"c": "ok"})
    text = "\n".join([huge, small])

@@ -166,9 +209,9 @@ def test_chunk_jsonl_huge_line_is_split():
        'yy"}',
        '{"c": "ok"}',
    ]
-    # No fragment exceeds the overflow cap.
+    # No fragment exceeds the configured split budget.
    for chunk in chunks:
-        assert len(chunk) <= int(20 * OVERFLOW_FACTOR)
+        assert len(chunk) <= 20


 # ---------------------------------------------------------------------------
@@ -212,8 +255,40 @@ def test_chunk_conversation_splits_at_turn_boundaries():
    assert seen == turns


+def test_chunk_conversation_custom_structured_unit_limit_keeps_overflow_whole():
+    """A conversation turn over the budget is kept whole when the explicit cap allows it."""
+    turns = [{"c": "y" * 20}, {"c": "ok"}]
+    text = json.dumps(turns)
+    turn_size = len(json.dumps(turns[0]))
+
+    chunks = chunk_text(text, max_chars=25, structured_chunk_size=turn_size)
+
+    assert chunks == [
+        '[{"c": "yyyyyyyyyyyyyyyyyyyy"}]',
+        '[{"c": "ok"}]',
+    ]
+
+
+def test_chunk_conversation_structured_unit_limit_can_be_below_chunk_size():
+    """An oversized conversation turn is split by the structured cap, not the larger chunk budget."""
+    turns = [{"c": "y" * 40}, {"c": "ok"}]
+    text = json.dumps(turns)
+
+    chunks = chunk_text(text, max_chars=55, structured_chunk_size=20)
+
+    assert chunks == [
+        '{"c":',
+        '"yyyyyyyyyyyyyyyyyy',
+        "yyyyyyyyyyyyyyyyyyyy",
+        'yy"}',
+        '[{"c": "ok"}]',
+    ]
+    for chunk in chunks:
+        assert len(chunk) <= 20
+
+
 def test_chunk_conversation_huge_turn_is_split():
-    """A single turn past the 1.5x overflow cap is split as text — exact fragments."""
+    """A single turn past the structured-chunk cap is split as text — exact fragments."""
    turns = [{"c": "y" * 40}, {"c": "ok"}]
    text = json.dumps(turns)

@@ -228,7 +303,7 @@ def test_chunk_conversation_huge_turn_is_split():
        '[{"c": "ok"}]',
    ]
    for chunk in chunks:
-        assert len(chunk) <= int(20 * OVERFLOW_FACTOR)
+        assert len(chunk) <= 20


 # ---------------------------------------------------------------------------
@@ -21,12 +21,16 @@ def _make_result(
    ce_norm: float,
    occurred_start: datetime | None = None,
    temporal_proximity: float | None = None,
+    mentioned_at: datetime | None = None,
+    occurred_end: datetime | None = None,
 ) -> ScoredResult:
    retrieval = RetrievalResult(
        id="test",
        text="test",
        fact_type="world",
        occurred_start=occurred_start,
+        occurred_end=occurred_end,
+        mentioned_at=mentioned_at,
        temporal_proximity=temporal_proximity,
    )

@@ -141,13 +145,36 @@ class TestBoostFormula:
        apply_combined_scoring([l_relevant, l_recent], now=NOW)
        assert l_relevant.weight > l_recent.weight, "Low-CE model: relevance should still win"

-    def test_no_occurred_start_defaults_recency_neutral(self):
-        """Missing occurred_start → recency=0.5 → no boost/penalty."""
-        sr = _make_result(ce_norm=0.5, occurred_start=None)
+    def test_no_effective_time_defaults_recency_neutral(self):
+        """No effective time at all (occurred_start/mentioned_at/occurred_end) → recency=0.5."""
+        sr = _make_result(ce_norm=0.5)
        apply_combined_scoring([sr], now=NOW)
        assert sr.recency == 0.5
        assert abs(sr.weight - 0.5) < 1e-9

+    def test_mentioned_at_drives_recency_when_no_occurred_start(self):
+        """A memory with only mentioned_at must derive recency from it, not stay neutral."""
+        sr = _make_result(ce_norm=0.5, mentioned_at=NOW)
+        apply_combined_scoring([sr], now=NOW)
+        assert sr.recency == 1.0
+        assert sr.weight > 0.5
+
+    def test_occurred_end_is_last_recency_fallback(self):
+        """occurred_end feeds recency when neither occurred_start nor mentioned_at is set."""
+        old = NOW - timedelta(days=400)
+        sr = _make_result(ce_norm=0.5, occurred_end=old)
+        apply_combined_scoring([sr], now=NOW)
+        assert sr.recency == 0.1
+        assert sr.weight < 0.5
+
+    def test_occurred_start_takes_precedence_over_mentioned_at(self):
+        """occurred_start wins over mentioned_at (matches _coalesce_date COALESCE order)."""
+        recent = NOW - timedelta(days=10)
+        old = NOW - timedelta(days=400)
+        sr = _make_result(ce_norm=0.5, occurred_start=recent, mentioned_at=old)
+        apply_combined_scoring([sr], now=NOW)
+        assert sr.recency > 0.9
+
    def test_timezone_naive_occurred_start_handled(self):
        """Naive datetimes in occurred_start should not raise."""
        naive_date = datetime(2024, 1, 1)  # no tzinfo
@@ -20,6 +20,7 @@ def setup_test_env():
        "HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS",
        "HINDSIGHT_API_CONSOLIDATION_MAX_COMPLETION_TOKENS",
        "HINDSIGHT_API_RETAIN_CHUNK_SIZE",
+        "HINDSIGHT_API_RETAIN_STRUCTURED_CHUNK_SIZE",
        "HINDSIGHT_API_LLM_PROVIDER",
        "HINDSIGHT_API_LLM_MODEL",
        "HINDSIGHT_API_LLM_REASONING_EFFORT",
@@ -104,6 +105,54 @@ def test_valid_retain_config_succeeds():
    config = HindsightConfig.from_env()
    assert config.retain_max_completion_tokens == 64000
    assert config.retain_chunk_size == 3000
+    assert config.retain_structured_chunk_size is None
+
+
+def test_retain_structured_chunk_size_reads_from_env():
+    """Structured JSONL/conversation units can have an explicit character cap."""
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS"] = "64000"
+    os.environ["HINDSIGHT_API_RETAIN_CHUNK_SIZE"] = "3000"
+    os.environ["HINDSIGHT_API_RETAIN_STRUCTURED_CHUNK_SIZE"] = "9000"
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+
+    config = HindsightConfig.from_env()
+    assert config.retain_structured_chunk_size == 9000
+
+
+def test_retain_structured_chunk_size_can_be_less_than_chunk_size():
+    """Structured-chunk cap can be smaller than the retain chunk target."""
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS"] = "64000"
+    os.environ["HINDSIGHT_API_RETAIN_CHUNK_SIZE"] = "3000"
+    os.environ["HINDSIGHT_API_RETAIN_STRUCTURED_CHUNK_SIZE"] = "2000"
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+
+    config = HindsightConfig.from_env()
+    assert config.retain_chunk_size == 3000
+    assert config.retain_structured_chunk_size == 2000
+
+
+def test_retain_strategy_structured_chunk_size_validation():
+    """Retain strategies allow structured-chunk caps below chunk size."""
+    from hindsight_api.config import HindsightConfig
+    from hindsight_api.config_resolver import apply_strategy
+
+    os.environ["HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS"] = "64000"
+    os.environ["HINDSIGHT_API_RETAIN_CHUNK_SIZE"] = "3000"
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+
+    config = HindsightConfig.from_env()
+    config.retain_strategies = {
+        "jsonl": {
+            "retain_structured_chunk_size": 2000,
+        }
+    }
+
+    resolved = apply_strategy(config, "jsonl")
+    assert resolved.retain_structured_chunk_size == 2000


 def test_semantic_min_similarity_reads_from_env():
@@ -3114,6 +3114,59 @@ async def test_max_observations_per_scope_limits_creates(memory: MemoryEngine, r
        await memory.delete_bank(bank_id, request_context=request_context)


+@pytest.mark.asyncio
+async def test_max_observations_per_scope_zero_forbids_all_creates(memory: MemoryEngine, request_context):
+    """limit=0 means "no new observations": consolidation must create none.
+
+    Regression for the ``> 0`` call-site guards that excluded 0, leaving
+    ``remaining_observation_slots=None`` (unconstrained) so a limit of 0 behaved
+    like unlimited — the inverse of the documented ``0 = no new observations``.
+    """
+    bank_id = f"test-max-obs-zero-{uuid.uuid4().hex[:8]}"
+    await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
+
+    raw = _get_raw_config()
+    fake_config = type(raw)(
+        **{
+            **{f: getattr(raw, f) for f in raw.__dataclass_fields__},
+            "max_observations_per_scope": 0,
+        }
+    )
+
+    try:
+        original_global_config = memory._config_resolver._global_config
+        memory._config_resolver._global_config = fake_config
+        wrapper, mock_llm = _make_mock_llm_one_obs_per_fact()
+        original_llm = memory._consolidation_llm_config
+        memory._consolidation_llm_config = wrapper
+
+        try:
+            # Insert tagged memories; the mock LLM will try to create 1 obs per
+            # fact, but limit=0 must block every create.
+            async with memory._pool.acquire() as conn:
+                await _insert_memories_with_tags(
+                    conn,
+                    bank_id,
+                    ["Alice loves hiking.", "Bob swims daily.", "Charlie does yoga."],
+                    tags=["scope:test"],
+                )
+
+            for _ in range(3):
+                await run_consolidation_job(memory_engine=memory, bank_id=bank_id, request_context=request_context)
+
+            async with memory._pool.acquire() as conn:
+                count = await _count_observations_for_scope(conn, bank_id, ["scope:test"])
+                assert count == 0, f"Expected 0 observations (limit=0), got {count}"
+
+                consolidation_calls = [c for c in mock_llm.get_mock_calls() if c["scope"] == "consolidation"]
+                assert len(consolidation_calls) >= 1, "LLM should have been called at least once"
+        finally:
+            memory._config_resolver._global_config = original_global_config
+            memory._consolidation_llm_config = original_llm
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
@pytest.mark.asyncio
 async def test_max_observations_per_scope_allows_updates_at_capacity(memory: MemoryEngine, request_context):
    """At capacity, the LLM can still update existing observations."""
@@ -178,6 +178,42 @@ async def test_combined_mode_parallel_writes_to_memory_tag_set(memory: MemoryEng
        await memory.delete_bank(bank_id, request_context=request_context)


+@pytest.mark.asyncio
+async def test_shared_mode_parallel_writes_only_untagged_scope(memory: MemoryEngine, request_context):
+    """shared → every memory writes to the single untagged scope, ignoring its
+    own tags. Three memories with disjoint tags therefore all consolidate into
+    the same global scope (the per-session-tag dedup use case) instead of one
+    isolated observation per tag."""
+    bank_id = f"test-shared-{uuid.uuid4().hex[:8]}"
+    await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
+    try:
+        async with memory._pool.acquire() as conn:
+            await _insert_memory(conn, bank_id, "Alice likes tea", ["session:s1"], "shared")
+            await _insert_memory(conn, bank_id, "Bob bikes daily", ["session:s2"], "shared")
+            await _insert_memory(conn, bank_id, "Carol reads books", ["session:s3"], "shared")
+
+        wrapper, _ = _mock_llm_one_obs_per_fact()
+        original_llm = memory._consolidation_llm_config
+        memory._consolidation_llm_config = wrapper
+        try:
+            with (
+                _override_config(memory, consolidation_llm_parallelism=3, consolidation_llm_batch_size=1),
+                patch.object(memory, "submit_async_consolidation"),
+            ):
+                result = await run_consolidation_job(
+                    memory_engine=memory, bank_id=bank_id, request_context=request_context
+                )
+        finally:
+            memory._consolidation_llm_config = original_llm
+
+        assert result["status"] == "completed"
+        tag_sets = await _fetch_observation_tag_sets(memory, bank_id)
+        # Every observation lands at the untagged scope — none carries a session tag.
+        assert tag_sets and all(t == frozenset() for t in tag_sets), tag_sets
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
@pytest.mark.asyncio
 async def test_per_tag_mode_parallel_writes_one_observation_per_tag(memory: MemoryEngine, request_context):
    """per_tag with tags [a, b] → two observations, tagged [a] and [b] respectively.
@@ -92,6 +92,18 @@ class TestResolveWriteScopesAllCombinations:
        assert _resolve_write_scopes(memory) == [frozenset()]


+class TestResolveWriteScopesShared:
+    def test_collapses_to_single_untagged_scope_regardless_of_tags(self):
+        # "shared" ignores the memory's own tags and writes to one global scope,
+        # so every memory deduplicates against the same observation.
+        memory = {"tags": ["alice", "session"], "observation_scopes": _as_json_string("shared")}
+        assert _resolve_write_scopes(memory) == [frozenset()]
+
+    def test_empty_tags_also_untagged_scope(self):
+        memory = {"tags": [], "observation_scopes": _as_json_string("shared")}
+        assert _resolve_write_scopes(memory) == [frozenset()]
+
+
 class TestResolveWriteScopesExplicitList:
    def test_uses_declared_scopes_verbatim(self):
        memory = {
@@ -166,6 +178,12 @@ class TestResolveObsTagsList:
        memory = {"tags": ["a", "b"], "observation_scopes": json.dumps(spec)}
        assert _resolve_obs_tags_list(memory) == spec

+    def test_shared_returns_single_empty_scope(self):
+        # One pass over the empty (untagged) scope; the memory's own tags are
+        # ignored so cross-tag memories consolidate into one observation.
+        memory = {"tags": ["a", "b"], "observation_scopes": json.dumps("shared")}
+        assert _resolve_obs_tags_list(memory) == [[]]
+

 # ---------------------------------------------------------------------------
 # Agreement between obs_tags_list (dispatch) and write_scopes (locks)
@@ -185,6 +203,7 @@ class TestDispatchLockAgreement:
            {"tags": ["a", "b", "c"], "observation_scopes": json.dumps("combined")},
            {"tags": ["a", "b"], "observation_scopes": json.dumps("per_tag")},
            {"tags": ["a", "b", "c"], "observation_scopes": json.dumps("all_combinations")},
+            {"tags": ["a", "b"], "observation_scopes": json.dumps("shared")},
            {"tags": ["a", "b"], "observation_scopes": json.dumps([["a"], ["b"], ["a", "b"]])},
            {"tags": ["a"], "observation_scopes": json.dumps([["a"], ["x"]])},
            # Pre-parsed Python shape (defensive — covers callers that hand the
@@ -0,0 +1,110 @@
+"""Tests for structured-delta LLM JSON parsing."""
+
+from __future__ import annotations
+
+import pytest
+
+from hindsight_api.engine.reflect.delta_ops import (
+    AppendBlockOp,
+    DeltaAllOpsInvalidError,
+    DeltaOperationList,
+    parse_delta_operation_list,
+)
+from hindsight_api.engine.reflect.structured_doc import BulletListBlock
+
+
+def test_parse_delta_operation_list_trailing_brackets():
+    """glm-style output with extra ]} after the root object."""
+    raw = (
+        '{"operations":[{"op":"append_block","section_id":"members",'
+        '"block":{"type":"bullet_list","items":["knip ignore react-dom"]}}]}]}'
+    )
+    op_list = parse_delta_operation_list(raw)
+    assert len(op_list.operations) == 1
+    assert isinstance(op_list.operations[0], AppendBlockOp)
+
+
+def test_parse_delta_operation_list_backticks_in_path():
+    raw = (
+        '{"operations":[{"op":"append_block","section_id":"conventions",'
+        '"block":{"type":"bullet_list","items":["hindsight-control-plane/knip.json"]}}]}'
+    )
+    op_list = parse_delta_operation_list(raw)
+    assert len(op_list.operations) == 1
+    op = op_list.operations[0]
+    assert op.section_id == "conventions"
+    assert op.block.items == ["hindsight-control-plane/knip.json"]
+
+
+def test_parse_delta_operation_list_prose_prefix():
+    raw = (
+        'Here is the update:\n{"operations": [{"op": "append_block", '
+        '"section_id": "x", "block": {"type": "paragraph", "text": "ok"}}]}'
+        "\nDone."
+    )
+    op_list = parse_delta_operation_list(raw)
+    assert len(op_list.operations) == 1
+
+
+def test_parse_delta_operation_list_skips_invalid_op_keeps_valid():
+    """One bad replace_block (missing index) must not discard the whole batch."""
+    raw = (
+        '{"operations": ['
+        '{"op": "append_block", "section_id": "s", '
+        '"block": {"type": "paragraph", "text": "ok"}}, '
+        '{"op": "replace_block", "section_id": "s", '
+        '"block": {"type": "paragraph", "text": "missing index"}}, '
+        '{"op": "append_block", "section_id": "s", '
+        '"block": {"type": "paragraph", "text": "also ok"}}'
+        "]}"
+    )
+    op_list = parse_delta_operation_list(raw)
+    assert len(op_list.operations) == 2
+    assert all(isinstance(o, AppendBlockOp) for o in op_list.operations)
+
+
+def test_parse_delta_operation_list_empty():
+    assert parse_delta_operation_list("").operations == []
+
+
+def test_parse_delta_operation_list_empty_operations_is_noop():
+    """A genuine empty operations array is a valid no-op, not an error."""
+    assert parse_delta_operation_list('{"operations": []}').operations == []
+    assert parse_delta_operation_list({"operations": []}).operations == []
+
+
+def test_parse_delta_operation_list_all_invalid_raises():
+    """If the model emits ops but every one is malformed, raise so the caller
+    falls back to a full rewrite instead of applying zero ops — which would
+    silently drop this refresh's new facts."""
+    raw = (
+        '{"operations": ['
+        '{"op": "replace_block", "section_id": "s", '
+        '"block": {"type": "paragraph", "text": "missing index a"}}, '
+        '{"op": "replace_block", "section_id": "s", '
+        '"block": {"type": "paragraph", "text": "missing index b"}}'
+        "]}"
+    )
+    with pytest.raises(DeltaAllOpsInvalidError):
+        parse_delta_operation_list(raw)
+    # Same payload shape as a dict must behave identically.
+    with pytest.raises(DeltaAllOpsInvalidError):
+        parse_delta_operation_list(
+            {
+                "operations": [
+                    {"op": "replace_block", "section_id": "s", "block": {"type": "paragraph", "text": "no index"}},
+                ]
+            }
+        )
+
+
+def test_parse_delta_operation_list_pydantic_instance():
+    original = DeltaOperationList(
+        operations=[
+            AppendBlockOp(
+                section_id="s",
+                block=BulletListBlock(items=["a"]),
+            )
+        ]
+    )
+    assert parse_delta_operation_list(original) is original
@@ -206,6 +206,58 @@ async def test_document_without_metadata(memory, request_context):
        await memory.delete_bank(bank_id, request_context=request_context)


+@pytest.mark.asyncio
+async def test_document_observation_scopes_from_retain_params(memory, request_context):
+    """observation_scopes passed at retain time is captured into retain_params and surfaced by get_document."""
+    bank_id = f"test_doc_obs_scopes_{datetime.now(timezone.utc).timestamp()}"
+
+    try:
+        document_id = "doc-with-scopes"
+        await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {
+                    "content": "Alice and Bob are friends.",
+                    "tags": ["alice", "bob"],
+                    "observation_scopes": "all_combinations",
+                }
+            ],
+            document_id=document_id,
+            request_context=request_context,
+        )
+
+        doc = await memory.get_document(document_id, bank_id, request_context=request_context)
+        assert doc is not None
+        # Surfaced as a top-level field and persisted in retain_params.
+        assert doc["observation_scopes"] == "all_combinations"
+        assert doc["retain_params"]["observation_scopes"] == "all_combinations"
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_document_observation_scopes_none_when_unset(memory, request_context):
+    """get_document returns observation_scopes None when none was configured at retain time."""
+    bank_id = f"test_doc_no_scopes_{datetime.now(timezone.utc).timestamp()}"
+
+    try:
+        document_id = "doc-no-scopes"
+        await memory.retain_async(
+            bank_id=bank_id,
+            content="Bob works at Microsoft.",
+            document_id=document_id,
+            request_context=request_context,
+        )
+
+        doc = await memory.get_document(document_id, bank_id, request_context=request_context)
+        assert doc is not None
+        assert doc["observation_scopes"] is None
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
@pytest.mark.asyncio
@pytest.mark.hs_llm_core
 async def test_document_persisted_with_zero_facts(memory_real_llm, request_context):
@@ -0,0 +1,179 @@
+"""HTTP + engine tests for dry-run fact extraction.
+
+POST /memories/dry-run-extract runs extraction ONLY (no resolution/links/embeddings/persistence) and
+returns candidate facts (a subset of the memory-unit shape) plus LLM token usage. Uses the
+deterministic mock-LLM `memory` fixture, so extraction yields canned facts without a real provider.
+"""
+
+import os
+import uuid
+from unittest.mock import patch
+
+import httpx
+import pytest
+import pytest_asyncio
+
+from hindsight_api import RequestContext
+from hindsight_api.api import create_app
+from hindsight_api.config import clear_config_cache
+from hindsight_api.extensions import (
+    OperationValidatorExtension,
+    PrecheckContext,
+    ValidationResult,
+)
+
+# Dry-run facts are a subset of the memory-unit shape — only fields a fresh extraction produces
+# (no storage/consolidation/curation fields, since nothing is persisted).
+FACT_KEYS = {
+    "text",
+    "fact_type",
+    "occurred_start",
+    "occurred_end",
+    "entities",
+}
+
+
+@pytest_asyncio.fixture
+async def api_client(memory):
+    app = create_app(memory, initialize_memory=False)
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
+        yield client
+
+
+@pytest.mark.asyncio
+async def test_dry_run_extracts_without_persisting(api_client, memory):
+    bank_id = f"dryrun-{uuid.uuid4().hex[:8]}"
+    await memory.get_bank_profile(bank_id=bank_id, request_context=RequestContext())
+
+    before = await memory.list_memory_units(bank_id=bank_id, request_context=RequestContext())
+
+    resp = await api_client.post(
+        f"/v1/default/banks/{bank_id}/memories/dry-run-extract",
+        json={
+            "content": "Alice moved to Berlin in 2021 and works as a nurse.",
+            "retain_mission": "Capture where people live and their jobs.",
+            "retain_chunk_size": 4000,
+        },
+    )
+    assert resp.status_code == 200, resp.text
+    body = resp.json()
+    assert isinstance(body["facts"], list) and body["facts"], "expected candidate facts"
+
+    for fact in body["facts"]:
+        # A subset of the memory-unit shape — no persistence/curation fields leak in. Null fields
+        # are omitted from responses API-wide (#2204), so the optional date fields may be absent;
+        # assert no UNEXPECTED key appears and the always-present ones are there.
+        assert set(fact) <= FACT_KEYS, f"unexpected keys: {set(fact) - FACT_KEYS}"
+        assert {"text", "fact_type", "entities"} <= set(fact)
+        assert fact["fact_type"] in ("world", "experience")
+        assert isinstance(fact["entities"], list)  # raw extraction → array, not a joined string
+
+    # Token usage is reported alongside the facts.
+    assert set(body["usage"]) >= {"input_tokens", "output_tokens", "total_tokens"}
+
+    # No persistence: the bank's stored memory count is unchanged.
+    after = await memory.list_memory_units(bank_id=bank_id, request_context=RequestContext())
+    assert after["total"] == before["total"]
+
+
+@pytest.mark.asyncio
+async def test_dry_run_disabled_returns_404(api_client, memory):
+    """With HINDSIGHT_API_ENABLE_DRY_RUN_EXTRACT=false the endpoint is removed (returns 404)."""
+    bank_id = f"dryrun-{uuid.uuid4().hex[:8]}"
+    await memory.get_bank_profile(bank_id=bank_id, request_context=RequestContext())
+
+    try:
+        with patch.dict(os.environ, {"HINDSIGHT_API_ENABLE_DRY_RUN_EXTRACT": "false"}):
+            clear_config_cache()  # force get_config() to re-read the patched env
+            resp = await api_client.post(
+                f"/v1/default/banks/{bank_id}/memories/dry-run-extract",
+                json={"content": "Alice moved to Berlin in 2021."},
+            )
+        assert resp.status_code == 404, resp.text
+        assert "disabled" in resp.json()["detail"].lower()
+    finally:
+        clear_config_cache()  # env restored on with-exit; reset so later tests see the default
+
+
+@pytest.mark.asyncio
+async def test_dry_run_rejects_unknown_override(memory):
+    bank_id = f"dryrun-{uuid.uuid4().hex[:8]}"
+    await memory.get_bank_profile(bank_id=bank_id, request_context=RequestContext())
+    with pytest.raises(ValueError, match="Unsupported extraction override"):
+        await memory.extract_dry_run(
+            bank_id,
+            "some content",
+            overrides={"embeddings_provider": "evil"},
+            request_context=RequestContext(),
+        )
+
+
+class _DryRunRejectingValidator(OperationValidatorExtension):
+    """Operation validator that rejects the dry-run-extract precheck.
+
+    Models an extension that gates LLM-billable routes (revoked key / exhausted
+    balance / rate-limited tenant). It rejects only the ``dry_run_extract``
+    operation so the test asserts the dry-run route is actually wired to the
+    precheck, not that the validator rejects everything.
+    """
+
+    async def precheck(self, ctx: PrecheckContext) -> ValidationResult:
+        if ctx.operation == "dry_run_extract":
+            return ValidationResult.reject("dry-run extraction not allowed", status_code=402)
+        return ValidationResult.accept()
+
+    async def validate_retain(self, ctx) -> ValidationResult:
+        return ValidationResult.accept()
+
+    async def validate_recall(self, ctx) -> ValidationResult:
+        return ValidationResult.accept()
+
+    async def validate_reflect(self, ctx) -> ValidationResult:
+        return ValidationResult.accept()
+
+
+@pytest.mark.asyncio
+async def test_dry_run_honors_operation_precheck(api_client, memory):
+    """dry-run-extract makes a real LLM call, so it must run the same billing/quota/rate-limit
+    precheck the other LLM-billable POST routes (retain/recall/reflect/mental_model_*/files_retain)
+    already wire. A validator that rejects the operation must short-circuit the request before any
+    extraction runs — without the precheck dependency the route would proceed to a 200."""
+    bank_id = f"dryrun-{uuid.uuid4().hex[:8]}"
+    await memory.get_bank_profile(bank_id=bank_id, request_context=RequestContext())
+
+    previous = getattr(memory, "_operation_validator", None)
+    memory._operation_validator = _DryRunRejectingValidator({})
+    try:
+        resp = await api_client.post(
+            f"/v1/default/banks/{bank_id}/memories/dry-run-extract",
+            json={"content": "Alice moved to Berlin in 2021."},
+        )
+        assert resp.status_code == 402, resp.text
+        assert "not allowed" in resp.json()["detail"].lower()
+    finally:
+        memory._operation_validator = previous
+
+
+@pytest.mark.asyncio
+async def test_dry_run_disabled_returns_404_even_with_validator(api_client, memory):
+    """A disabled dry-run route must 404 before the billing/quota precheck runs, even with a
+    configured validator — the feature-flag gate is declared as a dependency before the precheck,
+    so it preserves the original "disabled → 404" contract instead of leaking a 402/401/429."""
+    bank_id = f"dryrun-{uuid.uuid4().hex[:8]}"
+    await memory.get_bank_profile(bank_id=bank_id, request_context=RequestContext())
+
+    previous = getattr(memory, "_operation_validator", None)
+    memory._operation_validator = _DryRunRejectingValidator({})
+    try:
+        with patch.dict(os.environ, {"HINDSIGHT_API_ENABLE_DRY_RUN_EXTRACT": "false"}):
+            clear_config_cache()  # force get_config() to re-read the patched env
+            resp = await api_client.post(
+                f"/v1/default/banks/{bank_id}/memories/dry-run-extract",
+                json={"content": "Alice moved to Berlin in 2021."},
+            )
+        assert resp.status_code == 404, resp.text
+        assert "disabled" in resp.json()["detail"].lower()
+    finally:
+        clear_config_cache()  # env restored on with-exit; reset so later tests see the default
+        memory._operation_validator = previous
@@ -269,3 +269,82 @@ async def test_graph_q_filter_empty_results(api_client, test_bank_id):
    assert response.status_code == 200
    data = response.json()
    assert data["table_rows"] == []
+
+
+async def _seed_scoped_observations(memory, bank_id, request_context):
+    """Seed observations under scopes [a], [b], [a,b] (x2) and the global scope."""
+    await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
+    rows = [
+        (uuid.uuid4(), "obs scope a", ["a"]),
+        (uuid.uuid4(), "obs scope b", ["b"]),
+        (uuid.uuid4(), "obs scope ab one", ["a", "b"]),
+        (uuid.uuid4(), "obs scope ab two", ["b", "a"]),  # same scope as above, different order
+        (uuid.uuid4(), "obs global", []),
+    ]
+    async with memory._pool.acquire() as conn:
+        for obs_id, text, tags in rows:
+            await conn.execute(
+                """
+                INSERT INTO memory_units (id, bank_id, text, fact_type, tags, proof_count)
+                VALUES ($1, $2, $3, 'observation', $4::text[], 1)
+                """,
+                obs_id,
+                bank_id,
+                text,
+                tags,
+            )
+    return rows
+
+
+@pytest.mark.asyncio
+async def test_observation_scopes_enumeration(memory, api_client, test_bank_id, request_context):
+    """The scopes endpoint enumerates distinct tag sets (order-normalized) with counts."""
+    await _seed_scoped_observations(memory, test_bank_id, request_context)
+
+    response = await api_client.get(f"/v1/default/banks/{test_bank_id}/observations/scopes")
+    assert response.status_code == 200
+    scopes = response.json()["scopes"]
+
+    # [a,b] and [b,a] collapse into one scope with count 2; global scope is [].
+    as_map = {tuple(s["tags"]): s["count"] for s in scopes}
+    assert as_map == {("a",): 1, ("b",): 1, ("a", "b"): 2, (): 1}
+    # Most populous scope is first.
+    assert scopes[0]["tags"] == ["a", "b"]
+
+
+@pytest.mark.asyncio
+async def test_graph_exact_scope_filter(memory, api_client, test_bank_id, request_context):
+    """tags_match=exact filters observations to exactly one scope, not supersets."""
+    await _seed_scoped_observations(memory, test_bank_id, request_context)
+
+    # Exact scope [a] returns only the [a] observation, NOT the [a,b] ones.
+    response = await api_client.get(
+        f"/v1/default/banks/{test_bank_id}/graph",
+        params={"type": "observation", "tags": ["a"], "tags_match": "exact"},
+    )
+    assert response.status_code == 200
+    texts = {row["text"] for row in response.json()["table_rows"]}
+    assert texts == {"obs scope a"}
+
+    # Exact scope [a,b] returns both [a,b] observations regardless of stored order.
+    response = await api_client.get(
+        f"/v1/default/banks/{test_bank_id}/graph",
+        params={"type": "observation", "tags": ["a", "b"], "tags_match": "exact"},
+    )
+    assert response.status_code == 200
+    texts = {row["text"] for row in response.json()["table_rows"]}
+    assert texts == {"obs scope ab one", "obs scope ab two"}
+
+
+@pytest.mark.asyncio
+async def test_graph_exact_global_scope_filter(memory, api_client, test_bank_id, request_context):
+    """tags_match=exact with no tags is the global scope: untagged observations only."""
+    await _seed_scoped_observations(memory, test_bank_id, request_context)
+
+    response = await api_client.get(
+        f"/v1/default/banks/{test_bank_id}/graph",
+        params={"type": "observation", "tags_match": "exact"},
+    )
+    assert response.status_code == 200
+    texts = {row["text"] for row in response.json()["table_rows"]}
+    assert texts == {"obs global"}
@@ -6,11 +6,9 @@ key normalization, API endpoints, validation, and caching.
 """

 import json
-import os

 import pytest

-from hindsight_api import MemoryEngine
 from hindsight_api.config import HindsightConfig, normalize_config_dict, normalize_config_key
 from hindsight_api.config_resolver import ConfigResolver
 from hindsight_api.extensions.tenant import TenantExtension
@@ -113,6 +111,7 @@ async def test_hierarchical_fields_categorization():
    assert "retain_mission" in configurable
    assert "retain_custom_instructions" in configurable
    assert "retain_chunk_size" in configurable
+    assert "retain_structured_chunk_size" in configurable
    assert "enable_observations" in configurable
    assert "consolidation_llm_batch_size" in configurable
    assert "consolidation_source_facts_max_tokens" in configurable
@@ -131,6 +130,7 @@ async def test_hierarchical_fields_categorization():
    assert "retain_default_strategy" in configurable
    assert "retain_strategies" in configurable
    assert "max_observations_per_scope" in configurable
+    assert "observation_scope_limits" in configurable
    assert "reflect_source_facts_max_tokens" in configurable
    assert "llm_gemini_safety_settings" in configurable
    assert "mcp_enabled_tools" in configurable
@@ -139,7 +139,7 @@ async def test_hierarchical_fields_categorization():
    assert "consolidation_llm_parallelism" in configurable

    # Verify count is correct
-    assert len(configurable) == 38
+    assert len(configurable) == 40

    # Verify credential fields (NEVER exposed)
    assert "llm_api_key" in credentials
@@ -250,6 +250,230 @@ async def test_bank_config_null_consolidation_overrides_use_server_defaults():
        assert field_name not in bank_overrides


+@pytest.mark.asyncio
+async def test_retain_chunking_null_overrides_use_server_defaults():
+    """JSON null retain chunking overrides should behave like Server Default."""
+    bank_id = "test-null-retain-chunking-config-bank"
+    resolver = ConfigResolver(backend=FakeBankConfigBackend())
+
+    await resolver.update_bank_config(
+        bank_id,
+        {
+            "retain_chunk_size": 5000,
+            "retain_structured_chunk_size": 7000,
+        },
+    )
+    config = await resolver.resolve_full_config(bank_id)
+    assert config.retain_chunk_size == 5000
+    assert config.retain_structured_chunk_size == 7000
+
+    await resolver.update_bank_config(
+        bank_id,
+        {
+            "retain_chunk_size": None,
+            "retain_structured_chunk_size": None,
+        },
+    )
+
+    resolved_config = await resolver.resolve_full_config(bank_id)
+    global_config = resolver._global_config
+    assert resolved_config.retain_chunk_size == global_config.retain_chunk_size
+    assert resolved_config.retain_structured_chunk_size == global_config.retain_structured_chunk_size
+
+
+@pytest.mark.asyncio
+async def test_retain_chunking_validation_uses_null_cleared_chunk_size():
+    """Chunking validation should apply JSON null tombstones before checking final values."""
+    bank_id = "test-null-retain-chunking-validation-bank"
+    resolver = ConfigResolver(backend=FakeBankConfigBackend())
+
+    await resolver.update_bank_config(
+        bank_id,
+        {
+            "retain_chunk_size": 5000,
+            "retain_structured_chunk_size": 7000,
+        },
+    )
+
+    await resolver.update_bank_config(
+        bank_id,
+        {
+            "retain_chunk_size": None,
+            "retain_structured_chunk_size": 4000,
+        },
+    )
+
+    resolved_config = await resolver.resolve_full_config(bank_id)
+    assert resolved_config.retain_chunk_size == resolver._global_config.retain_chunk_size
+    assert resolved_config.retain_structured_chunk_size == 4000
+
+
+@pytest.mark.asyncio
+async def test_existing_retain_strategy_structured_chunking_survives_chunk_size_changes():
+    """Top-level chunk size updates can exceed existing structured chunk caps."""
+    from hindsight_api.config_resolver import apply_strategy
+
+    bank_id = "test-existing-retain-strategy-chunking-bank"
+    resolver = ConfigResolver(backend=FakeBankConfigBackend())
+
+    await resolver.update_bank_config(
+        bank_id,
+        {
+            "retain_strategies": {
+                "jsonl": {
+                    "retain_structured_chunk_size": 4000,
+                },
+            },
+        },
+    )
+
+    await resolver.update_bank_config(bank_id, {"retain_chunk_size": 5000})
+
+    config = await resolver.resolve_full_config(bank_id)
+    strategy_config = apply_strategy(config, "jsonl")
+    assert strategy_config.retain_chunk_size == 5000
+    assert strategy_config.retain_structured_chunk_size == 4000
+
+
+@pytest.mark.asyncio
+async def test_retain_strategy_chunking_null_matches_apply_strategy_semantics():
+    """Strategy null values are direct overrides, not bank-config tombstones."""
+    from hindsight_api.config_resolver import apply_strategy
+
+    bank_id = "test-retain-strategy-null-chunking-bank"
+    resolver = ConfigResolver(backend=FakeBankConfigBackend())
+
+    await resolver.update_bank_config(
+        bank_id,
+        {
+            "retain_structured_chunk_size": 5000,
+            "retain_strategies": {
+                "large-turns": {
+                    "retain_chunk_size": 8000,
+                    "retain_structured_chunk_size": None,
+                },
+            },
+        },
+    )
+
+    resolved_config = await resolver.resolve_full_config(bank_id)
+    strategy_config = apply_strategy(resolved_config, "large-turns")
+    assert strategy_config.retain_chunk_size == 8000
+    assert strategy_config.retain_structured_chunk_size is None
+
+
+@pytest.mark.parametrize(
+    "updates",
+    [
+        {"retain_chunk_size": "5000"},
+        {"retain_chunk_size": 5000.5},
+        {"retain_chunk_size": True},
+        {"retain_structured_chunk_size": "5000"},
+        {"retain_structured_chunk_size": 5000.5},
+        {"retain_structured_chunk_size": False},
+    ],
+)
+@pytest.mark.asyncio
+async def test_retain_chunking_raw_patch_values_must_be_integers(updates):
+    """Raw config PATCH values should fail as 400-style ValueError, not TypeError."""
+    resolver = ConfigResolver(backend=FakeBankConfigBackend())
+
+    with pytest.raises(ValueError) as exc_info:
+        await resolver.update_bank_config("test-retain-chunking-malformed-patch-bank", updates)
+
+    error_message = str(exc_info.value)
+    assert "must be an integer" in error_message
+    assert "HINDSIGHT_API_" not in error_message
+
+
+@pytest.mark.asyncio
+async def test_retain_strategy_chunk_size_null_rejected_with_value_error():
+    """Strategy retain_chunk_size cannot be null because apply_strategy would use it directly."""
+    resolver = ConfigResolver(backend=FakeBankConfigBackend())
+
+    with pytest.raises(ValueError) as exc_info:
+        await resolver.update_bank_config(
+            "test-retain-strategy-null-chunk-size-bank",
+            {
+                "retain_strategies": {
+                    "bad": {
+                        "retain_chunk_size": None,
+                    },
+                },
+            },
+        )
+
+    error_message = str(exc_info.value)
+    assert "Invalid retain strategy 'bad'" in error_message
+    assert "retain_chunk_size must be an integer" in error_message
+
+
+@pytest.mark.asyncio
+async def test_retain_strategy_non_object_rejected_with_value_error():
+    """Strategy entries must be objects so apply_strategy cannot fail later."""
+    resolver = ConfigResolver(backend=FakeBankConfigBackend())
+
+    with pytest.raises(ValueError) as exc_info:
+        await resolver.update_bank_config(
+            "test-retain-strategy-non-object-bank",
+            {
+                "retain_strategies": {
+                    "bad": "not-a-dict",
+                },
+            },
+        )
+
+    assert "Invalid retain strategy 'bad': must be an object" in str(exc_info.value)
+
+
+@pytest.mark.asyncio
+async def test_retain_strategy_chunk_size_must_remain_below_max_completion_tokens():
+    """Strategy chunk-size overrides must preserve the existing retain output-token invariant."""
+    resolver = ConfigResolver(backend=FakeBankConfigBackend())
+
+    with pytest.raises(ValueError) as exc_info:
+        await resolver.update_bank_config(
+            "test-retain-strategy-max-completion-bank",
+            {
+                "retain_strategies": {
+                    "bad": {
+                        "retain_chunk_size": 64000,
+                    },
+                },
+            },
+        )
+
+    error_message = str(exc_info.value)
+    assert "Invalid retain strategy 'bad'" in error_message
+    assert "retain_max_completion_tokens" in error_message
+    assert "must be greater than retain_chunk_size" in error_message
+
+
+@pytest.mark.asyncio
+async def test_retain_strategy_structured_chunk_size_can_be_below_same_update_chunk_size():
+    """Strategy structured chunk size can be lower than the top-level chunk size."""
+    from hindsight_api.config_resolver import apply_strategy
+
+    resolver = ConfigResolver(backend=FakeBankConfigBackend())
+
+    await resolver.update_bank_config(
+        "test-retain-strategy-chunking-bank",
+        {
+            "retain_chunk_size": 5000,
+            "retain_strategies": {
+                "jsonl": {
+                    "retain_structured_chunk_size": 4000,
+                },
+            },
+        },
+    )
+
+    config = await resolver.resolve_full_config("test-retain-strategy-chunking-bank")
+    strategy_config = apply_strategy(config, "jsonl")
+    assert strategy_config.retain_chunk_size == 5000
+    assert strategy_config.retain_structured_chunk_size == 4000
+
+
@pytest.mark.asyncio
 async def test_config_validation_rejects_static_fields(memory, request_context):
    """Test that attempting to override static fields raises ValueError."""
@@ -501,7 +725,6 @@ async def test_config_get_bank_config_no_static_or_credential_fields_leak(memory
        # Get field categorizations
        configurable_fields = HindsightConfig.get_configurable_fields()
        credential_fields = HindsightConfig.get_credential_fields()
-        static_fields = HindsightConfig.get_static_fields()

        # SECURITY: Verify ONLY configurable fields are returned (NO static, NO credentials)
        for key in config.keys():
@@ -534,7 +757,12 @@ async def test_config_get_bank_config_no_static_or_credential_fields_leak(memory
            )

        # Verify we have the expected configurable fields (small set)
-        expected_configurable = ["retain_chunk_size", "retain_extraction_mode", "enable_observations"]
+        expected_configurable = [
+            "retain_chunk_size",
+            "retain_structured_chunk_size",
+            "retain_extraction_mode",
+            "enable_observations",
+        ]
        for field in expected_configurable:
            assert field in config, f"Expected configurable field '{field}' missing from config"

@@ -141,7 +141,8 @@ async def test_full_api_workflow(api_client, test_bank_id):
    reflect_result = response.json()
    assert "text" in reflect_result
    assert len(reflect_result["text"]) > 0
-    assert "based_on" in reflect_result
+    # based_on is only populated when facts are requested; it's null (and thus omitted) here.
+    assert reflect_result.get("based_on") is None

    # Verify the reflect endpoint returned a non-trivial response
    assert len(reflect_result["text"]) > 5, "Reflect should return a substantive response"
@@ -222,6 +222,41 @@ async def test_configured_provider_binds_bank_context(registered_recorder):
    assert current_trace_context() is None  # unwound after the call


+@pytest.mark.asyncio
+async def test_engine_teardown_unregisters_recorder_even_when_close_skipped():
+    """Regression for #2229.
+
+    Span recorders live in a process-global registry, and providers fan every call
+    out to ALL registered recorders. The engine fixtures must remove their recorder
+    on teardown even when ``close()`` is skipped (pool already closing/absent) or
+    raises before the unregister step — otherwise a leaked, still-enabled recorder
+    from an earlier test records a later test's LLM calls into the shared DB, which
+    is what made ``test_disabled_writes_no_rows`` flaky. The teardown helper must
+    leave the registry exactly as it found it.
+    """
+    from hindsight_api import tracing
+
+    from tests.conftest import _teardown_memory_engine
+
+    sentinel = object()
+    tracing.register_span_recorder(sentinel)
+    try:
+        assert sentinel in tracing.get_span_recorder()._recorders
+
+        # _pool=None makes the helper's gated close() a no-op, exercising the exact
+        # leak path; the finally must still unregister the recorder.
+        class _FakeEngine:
+            _pool = None
+            _llm_recorder = sentinel
+
+        await _teardown_memory_engine(_FakeEngine())
+        assert sentinel not in tracing.get_span_recorder()._recorders
+    finally:
+        # Belt-and-suspenders: don't leave the sentinel in the global registry if an
+        # assertion above fails (idempotent — the helper normally already removed it).
+        tracing.unregister_span_recorder(sentinel)
+
+
 # ── HTTP read API (integration) ───────────────────────────────────────────────


@@ -21,7 +21,7 @@ from hindsight_api.engine.cross_encoder import LocalSTCrossEncoder
 from hindsight_api.engine.query_analyzer import DateparserQueryAnalyzer
 from hindsight_api.engine.task_backend import SyncTaskBackend
 from hindsight_api.engine.retain.fact_extraction import FactExtractionResponse, ExtractedFact
-from hindsight_api.engine.llm_wrapper import TokenUsage
+from hindsight_api.engine.response_models import TokenUsage

 logger = logging.getLogger(__name__)

@@ -366,6 +366,114 @@ class TestMentalModelToolRegistration:
        assert "invalidate_memory" in tools
        assert len(tools) == 32

+    def test_all_tools_have_nonempty_descriptions(self):
+        """Every registered tool must expose a non-empty description.
+
+        Amazon Bedrock's Converse API rejects any toolSpec whose description is
+        an empty string, so a tool with no description breaks every Bedrock
+        request that includes it. This regressed once because update_memory and
+        invalidate_memory used an f-string as their "docstring"
+        (f\"\"\"{_DOC}...\"\"\"), which is an expression rather than a string
+        literal — so __doc__ was None and FastMCP emitted an empty description.
+        """
+        from fastmcp import FastMCP
+
+        memory = MagicMock()
+        # Mock all engine methods that tools reference
+        memory.retain_batch_async = AsyncMock()
+        memory.submit_async_retain = AsyncMock(return_value={"operation_id": "op"})
+        memory.recall_async = AsyncMock(return_value=MagicMock(results=[]))
+        memory.reflect_async = AsyncMock()
+        memory.list_banks = AsyncMock(return_value=[])
+        memory.get_bank_profile = AsyncMock(return_value={})
+        memory.update_bank = AsyncMock()
+        memory.list_mental_models = AsyncMock(return_value=[])
+        memory.get_mental_model = AsyncMock()
+        memory.create_mental_model = AsyncMock()
+        memory.submit_async_refresh_mental_model = AsyncMock()
+        memory.update_mental_model = AsyncMock()
+        memory.delete_mental_model = AsyncMock()
+        memory.list_directives = AsyncMock(return_value=[])
+        memory.create_directive = AsyncMock()
+        memory.delete_directive = AsyncMock()
+        memory.list_memory_units = AsyncMock(return_value={})
+        memory.get_memory_unit = AsyncMock()
+        memory.list_documents = AsyncMock(return_value={})
+        memory.get_document = AsyncMock()
+        memory.delete_document = AsyncMock()
+        memory.list_operations = AsyncMock(return_value={})
+        memory.get_operation_status = AsyncMock()
+        memory.cancel_operation = AsyncMock()
+        memory.list_tags = AsyncMock(return_value={})
+        memory.get_bank_stats = AsyncMock(return_value={})
+        memory.delete_bank = AsyncMock(return_value={})
+
+        # Cover both registration paths: multi-bank (include_bank_id_param=True)
+        # and single-bank (False), since each registers a distinct function.
+        for include_bank_id_param in (True, False):
+            mcp = FastMCP("test")
+            config = MCPToolsConfig(
+                bank_id_resolver=lambda: "bank",
+                include_bank_id_param=include_bank_id_param,
+                tools=None,  # Default - all tools
+            )
+            register_mcp_tools(mcp, memory, config)
+            tools = _tools(mcp)
+            missing = [name for name, tool in tools.items() if not (getattr(tool, "description", None) or "").strip()]
+            assert not missing, (
+                f"tools with empty descriptions (include_bank_id_param={include_bank_id_param}): {missing}"
+            )
+
+    def test_no_mcp_tool_definition_can_lack_a_description(self):
+        """Statically reject any @mcp.tool that would register without a description.
+
+        Complements test_all_tools_have_nonempty_descriptions: that test exercises
+        the *default* tool set at runtime, this one parses the source so it also
+        covers tools gated behind feature flags / non-default configs, and points
+        at the offending line directly. A tool must carry either a ``description=``
+        kwarg on the decorator or a real string-literal docstring. An f-string
+        ``docstring`` (f\"\"\"{_DOC}...\"\"\") is an expression, not a literal, so
+        __doc__ stays None and FastMCP emits an empty description — which Bedrock's
+        Converse API rejects, breaking every request that advertises the tool.
+        """
+        import ast
+        import pathlib
+
+        from hindsight_api import mcp_tools
+
+        source = pathlib.Path(mcp_tools.__file__).read_text()
+        tree = ast.parse(source)
+
+        def is_tool_decorator(dec: ast.expr) -> bool:
+            target = dec.func if isinstance(dec, ast.Call) else dec
+            return isinstance(target, ast.Attribute) and target.attr == "tool"
+
+        def has_valid_docstring(fn: ast.FunctionDef | ast.AsyncFunctionDef) -> bool:
+            first = fn.body[0] if fn.body else None
+            if not isinstance(first, ast.Expr):
+                return False
+            value = first.value
+            # ast.JoinedStr == f-string: __doc__ becomes None, not a docstring.
+            return isinstance(value, ast.Constant) and isinstance(value.value, str) and bool(value.value.strip())
+
+        offenders: list[str] = []
+        for node in ast.walk(tree):
+            if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                continue
+            tool_decorators = [d for d in node.decorator_list if is_tool_decorator(d)]
+            if not tool_decorators:
+                continue
+            dec = tool_decorators[0]
+            has_description_kwarg = isinstance(dec, ast.Call) and any(k.arg == "description" for k in dec.keywords)
+            if has_description_kwarg or has_valid_docstring(node):
+                continue
+            offenders.append(f"{node.name} (line {node.lineno})")
+
+        assert not offenders, (
+            "@mcp.tool definitions missing a description (need a description= kwarg or a "
+            f"plain-literal docstring, not an f-string): {offenders}"
+        )
+

@pytest.fixture
 def no_bank_mcp_server(mock_memory):
@@ -1572,6 +1680,7 @@ class TestUpdateBankVariants:
                "retain_extraction_mode": "custom",
                "retain_custom_instructions": "Extract only action items",
                "retain_chunk_size": 2000,
+                "retain_structured_chunk_size": 5000,
            }
        )
        config_call = mock_memory._config_resolver.update_bank_config.call_args
@@ -1585,6 +1694,7 @@ class TestUpdateBankVariants:
        assert updates["retain_extraction_mode"] == "custom"
        assert updates["retain_custom_instructions"] == "Extract only action items"
        assert updates["retain_chunk_size"] == 2000
+        assert updates["retain_structured_chunk_size"] == 5000

    async def test_update_bank_name_and_config_together(self, mock_memory):
        """name goes to engine, config_updates goes to config resolver."""
@@ -93,14 +93,23 @@ async def _in_live(conn, mem_id: uuid.UUID) -> bool:


 async def _archive_row(conn, mem_id: uuid.UUID) -> dict | None:
+    # No `embedding` column: the archive is cold storage and the schema drops it (#2209).
    row = await conn.fetchrow(
-        "SELECT text, embedding, invalidation_reason, invalidated_at, entity_ids "
-        "FROM invalidated_memory_units WHERE id = $1",
+        "SELECT text, invalidation_reason, invalidated_at, entity_ids FROM invalidated_memory_units WHERE id = $1",
        mem_id,
    )
    return dict(row) if row else None


+async def _archive_has_embedding_column(conn) -> bool:
+    return bool(
+        await conn.fetchval(
+            "SELECT 1 FROM information_schema.columns "
+            "WHERE table_name = 'invalidated_memory_units' AND column_name = 'embedding'"
+        )
+    )
+
+
 async def _link_count(conn, mem_id: uuid.UUID) -> int:
    return await conn.fetchval(
        "SELECT COUNT(*) FROM memory_links WHERE from_unit_id = $1 OR to_unit_id = $1",
@@ -165,7 +174,9 @@ class TestInvalidate:
            arch = await _archive_row(conn, m1)
            assert arch is not None, "row must be in the archive"
            assert arch["invalidation_reason"] == "decommissioned"
-            assert arch["embedding"] is not None, "embedding travels with the archived row"
+            assert not await _archive_has_embedding_column(conn), (
+                "archive is cold storage; the schema drops the embedding column (#2209)"
+            )
            assert await _link_count(conn, m1) == 0, "links cascade-pruned on move"
            assert str(obs_id) not in await _obs_ids(conn, bank_id), "derived observation removed"
            assert await _consolidated_at(conn, m2) is None, "surviving source reset for re-consolidation"
@@ -203,6 +214,8 @@ class TestInvalidate:
            assert await _archive_row(conn, m1) is None, "archive row removed on revert"
            assert await _consolidated_at(conn, m1) is None, "reverted memory re-consolidates"
            assert e1 in await _entity_ids_for(conn, m1), "entity associations restored on revert"
+            reverted_emb = await conn.fetchval("SELECT embedding FROM memory_units WHERE id = $1", m1)
+            assert reverted_emb is not None, "embedding recomputed on revert (archive keeps none)"

        await memory.delete_bank(bank_id, request_context=request_context)

@@ -19,6 +19,8 @@ from hindsight_api.extensions.loader import ExtensionLoadError, load_extension
 from hindsight_api.extensions.memory_defense import (
    DefenseAction,
    MemoryDefenseExtension,
+    _fingerprint_value,
+    apply_redaction,
    parse_policy,
 )

@@ -50,9 +52,36 @@ def test_parse_policy_rejects_invalid_action() -> None:
        parse_policy({"enabled": True, "rules": [{"on": "sensitive_data", "action": "lol"}]})


-def test_parse_policy_rejects_unknown_detector() -> None:
+@pytest.mark.parametrize("on", [None, "", 123])
+def test_parse_policy_rejects_empty_or_non_string_on(on: object) -> None:
    with pytest.raises(ValueError, match="invalid on"):
-        parse_policy({"enabled": True, "rules": [{"on": "nope", "action": "block"}]})
+        parse_policy({"enabled": True, "rules": [{"on": on, "action": "block"}]})
+
+
+@pytest.mark.parametrize(
+    "detector",
+    [
+        "sensitive_data",
+        "prompt_injection",
+        "size_anomaly",
+        "protected_keys",
+        "detect_secrets",
+        "base64_decode",
+        "llm_screen",
+        # An unknown future name passes too: the parser doesn't gate ``on``
+        # against a fixed roster.
+        "some_future_cloud_detector",
+    ],
+)
+def test_parse_policy_accepts_any_detector_name(detector: str) -> None:
+    """The parser accepts any non-empty detector name so cloud-shape policies
+    pass through the OSS PATCH layer unchanged. The OSS regex extension only
+    actually screens ``sensitive_data``; the rest are silent no-ops here and
+    are dispatched by downstream extensions (e.g. hindsight-cloud)."""
+    policy = parse_policy({"enabled": True, "rules": [{"on": detector, "action": "block"}]})
+    assert len(policy.rules) == 1
+    assert policy.rules[0].on == detector
+    assert policy.rules[0].action is DefenseAction.BLOCK


 def test_disabled_policy_is_inert() -> None:
@@ -65,6 +94,73 @@ def test_defense_action_string_round_trip() -> None:
    assert DefenseAction.BLOCK.value == "block"


+# ---------------------------------------------------------------------------
+# Fingerprinting (unit)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    "value,expected",
+    [
+        # Length > 15 → first-4 + ellipsis + last-4.
+        ("ghp_" + "A" * 36, "ghp_...AAAA"),
+        ("AKIA" + "B" * 16, "AKIA...BBBB"),
+        ("sk-ant-" + "Z" * 40, "sk-a...ZZZZ"),
+        # Length 6–15 → first-2 + ellipsis + last-2.
+        ("123-45-6789", "12...89"),
+        ("xoxb-12345", "xo...45"),
+        # Length < 6 → fully masked; we don't preview anything.
+        ("abcde", "[redacted]"),
+        ("", "[redacted]"),
+    ],
+)
+def test_fingerprint_value_shape(value: str, expected: str) -> None:
+    """_fingerprint_value never returns the raw value and uses length-aware
+    bracketing so short matches don't leak material."""
+    out = _fingerprint_value(value)
+    assert out == expected
+    if value:
+        assert value not in out, f"raw value leaked into fingerprint: {out!r}"
+
+
+def test_apply_redaction_hits_carry_fingerprinted_previews() -> None:
+    """apply_redaction returns per-match fingerprinted previews — one entry
+    per matched substring — with the raw secret nowhere present in the hits."""
+    s1 = "ghp_" + "A" * 36
+    s2 = "AKIA" + "B" * 16
+    s3 = "123-45-6789"
+    content = f"rotate {s1}, drop {s2}, also ssn {s3}"
+
+    result = apply_redaction(content)
+
+    # Same-shape labels still flow to matched_types (deduplicated).
+    assert set(result.matched_types) >= {"github_token", "aws_access_key", "ssn_us"}
+
+    # One hit per matched substring; raw secret never appears.
+    by_detector = {h["detector"]: h["preview"] for h in result.hits}
+    assert by_detector["github_token"] == "ghp_...AAAA"
+    assert by_detector["aws_access_key"] == "AKIA...BBBB"
+    assert by_detector["ssn_us"] == "12...89"
+    for h in result.hits:
+        assert s1 not in h["preview"]
+        assert s2 not in h["preview"]
+        assert s3 not in h["preview"]
+
+
+def test_apply_redaction_multiple_hits_per_pattern() -> None:
+    """Two matches of the same pattern produce two hits — receivers can count
+    occurrences, not just types."""
+    a = "ghp_" + "A" * 36
+    b = "ghp_" + "B" * 36
+    content = f"old {a} new {b}"
+    result = apply_redaction(content)
+
+    gh_hits = [h for h in result.hits if h["detector"] == "github_token"]
+    assert len(gh_hits) == 2
+    previews = {h["preview"] for h in gh_hits}
+    assert previews == {"ghp_...AAAA", "ghp_...BBBB"}
+
+
 # ---------------------------------------------------------------------------
 # Regex screening (unit)
 # ---------------------------------------------------------------------------
@@ -107,6 +203,14 @@ async def test_screen_redacts_secret(regex_defense, redact_policy) -> None:
    assert secret not in decision.redacted_content
    assert "[REDACTED:github_token]" in decision.redacted_content
    assert "github_token" in decision.matched_types
+    # The decision carries a per-match fingerprinted preview — never the raw
+    # value — so SIEM receivers can correlate without the secret crossing
+    # the wire.
+    assert decision.hits, "OSS should populate at least one hit"
+    hit = decision.hits[0]
+    assert hit["detector"] == "github_token"
+    assert hit["preview"] == "ghp_...AAAA"
+    assert secret not in hit["preview"]


@pytest.mark.asyncio
@@ -298,11 +402,27 @@ async def test_patch_rejects_invalid_action(api_client) -> None:


@pytest.mark.asyncio
-async def test_patch_rejects_unknown_detector(api_client) -> None:
+async def test_patch_accepts_cloud_only_detector(api_client) -> None:
+    # A cloud-only detector the OSS extension doesn't implement still persists
+    # through the PATCH layer (it's a silent no-op here, dispatched downstream).
    await api_client.put("/v1/default/banks/md-cfg-3", json={})
    r = await api_client.patch(
        "/v1/default/banks/md-cfg-3/config",
-        json={"updates": {"memory_defense": {"enabled": True, "rules": [{"on": "nope", "action": "redact"}]}}},
+        json={
+            "updates": {"memory_defense": {"enabled": True, "rules": [{"on": "prompt_injection", "action": "block"}]}}
+        },
+    )
+    assert r.status_code == 200, r.text
+    r2 = await api_client.get("/v1/default/banks/md-cfg-3/config")
+    assert r2.json()["config"]["memory_defense"]["rules"][0]["on"] == "prompt_injection"
+
+
+@pytest.mark.asyncio
+async def test_patch_rejects_empty_detector(api_client) -> None:
+    await api_client.put("/v1/default/banks/md-cfg-4", json={})
+    r = await api_client.patch(
+        "/v1/default/banks/md-cfg-4/config",
+        json={"updates": {"memory_defense": {"enabled": True, "rules": [{"on": "", "action": "redact"}]}}},
    )
    assert r.status_code == 422, r.text
    assert "on" in str(r.json()["detail"]).lower()
@@ -374,8 +494,13 @@ async def _memory_defense_webhook_events(memory, bank: str) -> list[dict]:
    deliveries queued for ``bank``. The webhook_delivery task_payload nests the
    serialized event under ``payload`` (a JSON string)."""
    async with memory._pool.acquire() as conn:
+        # Order most-recent-first so callers using ``events[0]`` always see
+        # the latest queued delivery — otherwise pollution from earlier test
+        # runs against the same bank surfaces stale payloads.
        rows = await conn.fetch(
-            "SELECT task_payload FROM async_operations WHERE operation_type = 'webhook_delivery' AND bank_id = $1",
+            "SELECT task_payload FROM async_operations "
+            "WHERE operation_type = 'webhook_delivery' AND bank_id = $1 "
+            "ORDER BY created_at DESC",
            bank,
        )
    events: list[dict] = []
@@ -419,6 +544,14 @@ async def test_retain_fires_webhook_on_redact(api_client, memory) -> None:
    assert data["detector"] == "sensitive_data"
    assert "github_token" in data["matched_types"]
    assert data["message"]
+    # The webhook payload carries a per-match fingerprinted preview — the raw
+    # secret never crosses the wire, but a SIEM can still correlate against
+    # its credential inventory using the leading provider prefix + trailing
+    # discriminator (e.g. `ghp_...AAAA`). Populated by OSS as of #2157.
+    hits = data.get("hits") or []
+    assert any(h.get("detector") == "github_token" and h.get("preview") == "ghp_...AAAA" for h in hits), hits
+    for h in hits:
+        assert secret not in (h.get("preview") or ""), "raw secret leaked into preview"


@pytest.mark.asyncio
@@ -342,7 +342,7 @@ class TestDeltaRefreshPlumbing:
        assert "obs-bob" in user_msg
        assert "Bob joined" in user_msg
        # The structured JSON of the current doc must include the section id "members".
-        assert '"id": "members"' in user_msg
+        assert '"members"' in user_msg

        # New content includes the new bullet.
        assert "Bob — junior engineer" in refreshed["content"]
@@ -357,6 +357,96 @@ class TestDeltaRefreshPlumbing:

        await memory.delete_bank(bank_id, request_context=request_context)

+    async def test_delta_prompt_sends_only_new_facts_not_accumulated_history(
+        self,
+        memory: MemoryEngine,
+        request_context: RequestContext,
+        patch_reflect,
+        patch_llm_call,
+    ):
+        """Regression: the delta prompt carries only THIS refresh's facts.
+
+        ``based_on`` accumulates across refreshes for grounding/audit, but the
+        structured-delta LLM call must receive only the facts produced by the
+        current reflect. Re-sending every historical fact each refresh grows the
+        prompt without bound and trips provider input limits (e.g. Z.ai 1261).
+        The accumulated set is still persisted in ``reflect_response.based_on``.
+        """
+        bank_id = f"test-delta-newfacts-{uuid.uuid4().hex[:8]}"
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        existing = "# Team\n\nAlice is the lead.\n\n## Members\n\n- Alice — lead\n"
+        mm = await memory.create_mental_model(
+            bank_id=bank_id,
+            name="Team Info",
+            source_query="Tell me about the team",
+            content=existing,
+            trigger={"mode": "delta"},
+            request_context=request_context,
+        )
+
+        # First refresh seeds prior based_on with an OLD fact (zero ops applied).
+        patch_reflect(
+            memory,
+            text="ignored — delta keeps existing",
+            facts=[
+                {
+                    "id": "obs-old-alice",
+                    "text": "Alice has been the team lead since 2019",
+                    "type": "observation",
+                    "context": None,
+                }
+            ],
+        )
+        patch_llm_call(memory, returns=[])
+        first = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+        first_based_on = (first.get("reflect_response") or {}).get("based_on") or {}
+        assert "obs-old-alice" in {f.get("id") for f in first_based_on.get("observation", [])}
+
+        # Second refresh brings only a NEW fact.
+        patch_reflect(
+            memory,
+            text="# Team\n\nAlice is the lead. Bob joined.",
+            facts=[
+                {
+                    "id": "obs-new-bob",
+                    "text": "Bob joined the team as junior engineer",
+                    "type": "observation",
+                    "context": None,
+                }
+            ],
+        )
+        ops = [
+            {
+                "op": "append_block",
+                "section_id": "members",
+                "block": {"type": "bullet_list", "items": ["Bob — junior engineer"]},
+            }
+        ]
+        llm_calls = patch_llm_call(memory, returns=ops)
+
+        refreshed = await memory.refresh_mental_model(
+            bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
+        )
+
+        assert len(llm_calls) == 1
+        user_msg = llm_calls[0]["messages"][1]["content"]
+        # The NEW fact is sent to the delta call...
+        assert "obs-new-bob" in user_msg
+        assert "Bob joined the team" in user_msg
+        # ...but the accumulated OLD fact must NOT be re-sent (the regression).
+        assert "obs-old-alice" not in user_msg
+        assert "Alice has been the team lead since 2019" not in user_msg
+
+        # based_on still ACCUMULATES both facts for grounding/audit.
+        based_on = (refreshed.get("reflect_response") or {}).get("based_on") or {}
+        obs_ids = {f.get("id") for f in based_on.get("observation", [])}
+        assert obs_ids == {"obs-new-bob", "obs-old-alice"}
+
+        await memory.delete_bank(bank_id, request_context=request_context)
+
    async def test_delta_zero_ops_keeps_existing_content_byte_identical(
        self,
        memory: MemoryEngine,
@@ -117,6 +117,54 @@ class TestMetricsCollector:
        attributes = call_args[0][1]
        assert attributes["success"] == "false"

+    def test_record_operation_cancellation_excluded_from_metric(self, collector):
+        """A client-disconnect cancellation is neither a success nor a failure.
+
+        Recall/reflect run the engine call inside record_operation; when the
+        client disconnects the engine raises OperationCancelledError (issue
+        #2122). That abandoned request must not be recorded on
+        hindsight.operation.total at all -- inflating neither the failure nor
+        the success rate -- even though the exception still propagates.
+        """
+        from hindsight_api.cancellation import OperationCancelledError
+
+        with pytest.raises(OperationCancelledError):
+            with collector.record_operation("recall", bank_id="test_bank", source="api"):
+                raise OperationCancelledError("client disconnected")
+
+        collector.operation_total.add.assert_not_called()
+        collector.operation_duration.record.assert_not_called()
+
+    def test_record_operation_http_499_from_cancellation_excluded(self, collector):
+        """run_cancellable_on_disconnect re-raises the cancellation as
+        ``HTTPException(499) from exc``; the cause chain marks it as a
+        cancellation, so it is excluded from the metric too."""
+        from fastapi import HTTPException
+
+        from hindsight_api.cancellation import OperationCancelledError
+
+        with pytest.raises(HTTPException):
+            with collector.record_operation("reflect", bank_id="test_bank", source="api"):
+                try:
+                    raise OperationCancelledError("client disconnected")
+                except OperationCancelledError as cancel:
+                    raise HTTPException(status_code=499, detail="client disconnected") from cancel
+
+        collector.operation_total.add.assert_not_called()
+        collector.operation_duration.record.assert_not_called()
+
+    def test_record_operation_unrelated_499_still_recorded_as_failure(self, collector):
+        """A 499 that is NOT caused by a cancellation (no OperationCancelledError
+        in the cause chain) is a real failure and must still be recorded."""
+        from fastapi import HTTPException
+
+        with pytest.raises(HTTPException):
+            with collector.record_operation("recall", bank_id="test_bank", source="api"):
+                raise HTTPException(status_code=499, detail="unrelated downstream error")
+
+        attributes = collector.operation_duration.record.call_args[0][1]
+        assert attributes["success"] == "false"
+
    def test_record_operation_with_budget(self, collector):
        """Test that budget is included in attributes when provided."""
        with collector.record_operation("recall", bank_id="test_bank", source="api", budget="mid"):
@@ -0,0 +1,123 @@
+"""Regression for the issue #2106 follow-up: the live ``bank_id`` columns must
+not truncate on PostgreSQL.
+
+``c3e5a7b9d1f4`` widened the *history* tables to ``TEXT``, but ``directives`` and
+``mental_models`` kept their original ``VARCHAR(64)`` ``bank_id`` while
+``banks.bank_id`` is ``TEXT``. A bank_id longer than 64 chars (the 78-char shape
+reported in #2106) can create the bank but then 500s with
+``StringDataRightTruncation`` on the next write to those tables.
+
+This test migrates a dedicated pg0 instance to head, asserts both columns are
+``TEXT``, then writes a >64-char bank_id through every widened table. Uses a
+dedicated pg0 instance (mirrors test_migration_history_long_bank_id) so the
+migrated schema is well defined.
+
+``mental_model_versions`` is deliberately excluded: it is dropped on the upgrade
+path (``o0j1k2l3m4n5``) and does not exist at head, so the migration must not
+touch it.
+"""
+
+import asyncio
+import uuid
+from pathlib import Path
+
+import pytest
+from alembic import command
+from alembic.config import Config
+from sqlalchemy import create_engine, text
+
+# Both tests in this module share one module-scoped pg0 on a fixed port (5568).
+# CI runs with `--dist loadgroup`, which, absent an xdist_group, may scatter the
+# two tests across workers that then each instantiate the module fixture and race
+# to provision the SAME instance — surfacing as flaky "Instance already running",
+# a pg_type UniqueViolation (concurrent CREATE EXTENSION), or "server closed the
+# connection". Pinning the module to a single worker serialises that provisioning.
+pytestmark = pytest.mark.xdist_group("migration-remaining-bankid-pg0")
+
+_SCRIPT_LOCATION = str(Path(__file__).parent.parent / "hindsight_api" / "alembic")
+
+_WIDEN_TABLES = ("directives", "mental_models")
+
+
+def _alembic_cfg(db_url: str) -> Config:
+    cfg = Config()
+    cfg.set_main_option("script_location", _SCRIPT_LOCATION)
+    cfg.set_main_option("sqlalchemy.url", db_url)
+    cfg.set_main_option("prepend_sys_path", ".")
+    cfg.set_main_option("path_separator", "os")
+    return cfg
+
+
+def _col_type(conn, table: str) -> str:
+    return conn.execute(
+        text("SELECT data_type FROM information_schema.columns WHERE table_name = :t AND column_name = 'bank_id'"),
+        {"t": table},
+    ).scalar()
+
+
+@pytest.fixture(scope="module")
+def head_db_url():
+    """pg0 instance migrated to head (includes the widen migration)."""
+    from hindsight_api.pg0 import EmbeddedPostgres
+
+    pg0 = EmbeddedPostgres(name="hindsight-remaining-bankid-test", port=5568)
+    loop = asyncio.new_event_loop()
+    try:
+        url = loop.run_until_complete(pg0.ensure_running())
+    finally:
+        loop.close()
+
+    command.upgrade(_alembic_cfg(url), "heads")
+    return url
+
+
+def test_remaining_bank_id_columns_are_text(head_db_url):
+    engine = create_engine(head_db_url)
+    try:
+        with engine.connect() as conn:
+            for table in _WIDEN_TABLES:
+                assert _col_type(conn, table) == "text", f"{table}.bank_id must be TEXT to match banks.bank_id"
+    finally:
+        engine.dispose()
+
+
+def test_long_bank_id_round_trips_through_widened_tables(head_db_url):
+    # bank_id matching the shape reported in the issue, well over the old 64-char
+    # cap. Unique suffixes keep the test idempotent against pg0 data dirs that
+    # persist across runs (otherwise a re-run collides on the banks PK).
+    long_bank = f"tenantA::ou_{uuid.uuid4().hex}::ou_{uuid.uuid4().hex}"
+    assert len(long_bank) > 64
+    mm_id = f"mm-{uuid.uuid4().hex}"  # explicit id keeps re-runs from colliding
+
+    engine = create_engine(head_db_url)
+    try:
+        with engine.connect() as conn:
+            conn.execute(text("INSERT INTO banks (bank_id) VALUES (:b)"), {"b": long_bank})
+            conn.execute(
+                text("INSERT INTO directives (bank_id, name, content) VALUES (:b, :n, :c)"),
+                {"b": long_bank, "n": "long-bank directive", "c": "rule body"},
+            )
+            conn.execute(
+                text(
+                    "INSERT INTO mental_models "
+                    "(id, bank_id, subtype, name, source_query, content) "
+                    "VALUES (:mid, :b, 'pinned', :n, :q, :c)"
+                ),
+                {
+                    "mid": mm_id,
+                    "b": long_bank,
+                    "n": "long-bank model",
+                    "q": "what does the user prefer",
+                    "c": "model body",
+                },
+            )
+            conn.commit()
+
+            for table in _WIDEN_TABLES:
+                got = conn.execute(
+                    text(f"SELECT bank_id FROM {table} WHERE bank_id = :b LIMIT 1"),
+                    {"b": long_bank},
+                ).scalar()
+                assert got == long_bank, f"bank_id was truncated on the widened {table} table"
+    finally:
+        engine.dispose()
@@ -0,0 +1,145 @@
+"""Orchestration tests for run_migrations_for_schemas (per-tenant parallelism).
+
+These cover the fan-out logic deterministically without a real database by
+stubbing the per-step migration functions. The real cross-process path is
+exercised by the standard migration/integration suites that run against pg0.
+"""
+
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+import pytest
+
+from hindsight_api import migrations
+
+
+@pytest.fixture
+def record_steps(monkeypatch):
+    """Replace the real migration steps with recorders; return the call log."""
+    calls: list[tuple[str, str]] = []
+    lock = threading.Lock()
+
+    def make(step):
+        def _step(database_url, *args, schema=None, **kwargs):
+            with lock:
+                calls.append((step, schema))
+
+        return _step
+
+    monkeypatch.setattr(migrations, "run_migrations", make("run_migrations"))
+    monkeypatch.setattr(migrations, "ensure_embedding_dimension", make("embedding_dimension"))
+    monkeypatch.setattr(migrations, "ensure_vector_extension", make("vector_extension"))
+    monkeypatch.setattr(migrations, "ensure_text_search_extension", make("text_search_extension"))
+    return calls
+
+
+def test_empty_schema_list_is_noop(record_steps):
+    migrations.run_migrations_for_schemas("postgresql://x/db", [])
+    assert record_steps == []
+
+
+def test_sequential_runs_all_steps_in_order_per_schema(record_steps):
+    migrations.run_migrations_for_schemas(
+        "postgresql://x/db",
+        ["a", "b"],
+        concurrency=1,
+        embedding_dimension=768,
+    )
+    # Each schema: migrate -> embedding dim -> vector ext -> text-search ext.
+    assert record_steps == [
+        ("run_migrations", "a"),
+        ("embedding_dimension", "a"),
+        ("vector_extension", "a"),
+        ("text_search_extension", "a"),
+        ("run_migrations", "b"),
+        ("embedding_dimension", "b"),
+        ("vector_extension", "b"),
+        ("text_search_extension", "b"),
+    ]
+
+
+def test_skips_embedding_dim_when_none_and_extensions_when_disabled(record_steps):
+    migrations.run_migrations_for_schemas(
+        "postgresql://x/db",
+        ["a"],
+        concurrency=1,
+        embedding_dimension=None,
+        ensure_extensions=False,
+    )
+    assert record_steps == [("run_migrations", "a")]
+
+
+def test_parallel_fans_out_across_schemas(monkeypatch):
+    """concurrency>1 runs distinct schemas at the same time (not serialized)."""
+    max_active = 0
+    active = 0
+    lock = threading.Lock()
+
+    def slow_migrate(database_url, *args, schema=None, **kwargs):
+        nonlocal max_active, active
+        with lock:
+            active += 1
+            max_active = max(max_active, active)
+        time.sleep(0.05)
+        with lock:
+            active -= 1
+
+    monkeypatch.setattr(migrations, "run_migrations", slow_migrate)
+    monkeypatch.setattr(migrations, "ensure_embedding_dimension", lambda *a, **k: None)
+    monkeypatch.setattr(migrations, "ensure_vector_extension", lambda *a, **k: None)
+    monkeypatch.setattr(migrations, "ensure_text_search_extension", lambda *a, **k: None)
+    # Run the parallel branch in-process so the monkeypatched steps are visible.
+    monkeypatch.setattr(
+        migrations,
+        "_make_migration_executor",
+        lambda max_workers: ThreadPoolExecutor(max_workers=max_workers),
+    )
+
+    migrations.run_migrations_for_schemas(
+        "postgresql://x/db",
+        ["a", "b", "c", "d"],
+        concurrency=3,
+    )
+    assert max_active == 3
+
+
+def test_parallel_aggregates_per_schema_failures(monkeypatch):
+    """One failing schema does not hide the others, and all are still attempted."""
+    attempted: list[str] = []
+    lock = threading.Lock()
+
+    def migrate(database_url, *args, schema=None, **kwargs):
+        with lock:
+            attempted.append(schema)
+        if schema in ("b", "d"):
+            raise RuntimeError(f"boom {schema}")
+
+    monkeypatch.setattr(migrations, "run_migrations", migrate)
+    monkeypatch.setattr(migrations, "ensure_embedding_dimension", lambda *a, **k: None)
+    monkeypatch.setattr(migrations, "ensure_vector_extension", lambda *a, **k: None)
+    monkeypatch.setattr(migrations, "ensure_text_search_extension", lambda *a, **k: None)
+    monkeypatch.setattr(
+        migrations,
+        "_make_migration_executor",
+        lambda max_workers: ThreadPoolExecutor(max_workers=max_workers),
+    )
+
+    with pytest.raises(RuntimeError) as exc_info:
+        migrations.run_migrations_for_schemas(
+            "postgresql://x/db",
+            ["a", "b", "c", "d"],
+            concurrency=2,
+        )
+
+    assert set(attempted) == {"a", "b", "c", "d"}
+    message = str(exc_info.value)
+    assert "b" in message and "d" in message
+    assert "2 of 4" in message
+
+
+def test_worker_is_picklable():
+    """ProcessPoolExecutor requires the worker to be importable/picklable."""
+    import pickle
+
+    pickle.loads(pickle.dumps(migrations._migrate_one_schema_pg))
@@ -0,0 +1,90 @@
+"""
+Startup must leave torch's global default dtype at float32 regardless of how the
+concurrent local model loads interleave.
+
+Covers issue #2162: transformers' dtype context manager (entered by
+SentenceTransformer / CrossEncoder / from_pretrained) does a NON-thread-safe
+save/restore of the *process-global* default dtype. When an fp16 embedding model
+and an fp32 reranker/query-analyzer load in parallel at startup, an unlucky
+interleave leaves the global default stuck at float16 — every later encode() then
+emits NaN vectors that pgvector rejects ("NaN not allowed in vector") on MPS, or
+raises "c10::Half != float" on CPU, non-deterministically across restarts.
+
+MemoryEngine.initialize() loads the models in parallel (for speed) and then, once
+the gather has joined every load thread, normalizes the global default dtype back
+to float32 — the inference state a healthy boot already converges to. This test
+simulates the poisoning by having a model load flip the default to float16, then
+asserts initialize() leaves it at float32.
+"""
+
+import pytest
+
+from hindsight_api import MemoryEngine
+from hindsight_api.engine.task_backend import SyncTaskBackend
+
+
+class _StopInit(Exception):
+    """Sentinel to abort initialize() right after the model-load gather."""
+
+
+class _PoisoningEmbeddings:
+    """Local embedding stub that mimics an fp16 load poisoning the global dtype."""
+
+    provider_name = "local"
+
+    async def initialize(self) -> None:
+        import torch
+
+        # Reproduce the symptom of transformers' racy dtype restore: the global
+        # default is left at float16 after the (parallel) load.
+        torch.set_default_dtype(torch.float16)
+
+
+class _NoopCrossEncoder:
+    provider_name = "local"
+
+    async def initialize(self) -> None:
+        return None
+
+
+class _NoopQueryAnalyzer:
+    def load(self) -> None:
+        return None
+
+
+@pytest.mark.asyncio
+async def test_global_default_dtype_restored_to_float32_after_init():
+    """A load that leaves the torch default at float16 is normalized back to float32."""
+    import torch
+
+    original = torch.get_default_dtype()
+    try:
+        engine = MemoryEngine(
+            # Non-pg0 URL so start_pg0() is a no-op and __init__ never connects.
+            db_url="postgresql://u:p@localhost:5999/db",
+            memory_llm_provider="none",
+            memory_llm_api_key=None,
+            memory_llm_model="none",
+            embeddings=_PoisoningEmbeddings(),
+            cross_encoder=_NoopCrossEncoder(),
+            query_analyzer=_NoopQueryAnalyzer(),
+            run_migrations=False,
+            skip_llm_verification=True,
+            lazy_reranker=False,  # load the cross-encoder eagerly, in the gather
+            task_backend=SyncTaskBackend(),
+        )
+
+        # Abort right after the post-gather dtype restore, before any real DB work.
+        async def _stop(*args, **kwargs):
+            raise _StopInit
+
+        engine._backend.initialize = _stop  # type: ignore[method-assign]
+
+        with pytest.raises(_StopInit):
+            await engine.initialize()
+
+        # The embedding load poisoned the default to float16; initialize() must
+        # have normalized it back so later encode() can't emit NaN vectors.
+        assert torch.get_default_dtype() == torch.float32
+    finally:
+        torch.set_default_dtype(original)
@@ -0,0 +1,156 @@
+"""Unit tests for per-scope observation-limit resolution.
+
+These cover the three pure helpers behind the ``observation_scope_limits``
+config field, which lets a bank cap observations differently per consolidation
+scope (e.g. one tag's scope unlimited, while scopes that also carry a wildcard
+tag are capped):
+
+- ``_scope_matches_globs``  — exact-cover match between a glob pattern and a
+  concrete tag set (the crux: ``{a}`` and ``{run_1, a}`` must resolve to
+  *different* rules even though both contain ``a``).
+- ``_parse_scope_limit_rules`` — defensive parsing of the raw JSON config.
+- ``_effective_scope_limit`` — first-match-wins resolution with fallback to the
+  bank-wide ``max_observations_per_scope``.
+
+All deterministic — direct asserts, no LLM.
+"""
+
+from types import SimpleNamespace
+
+import pytest
+
+from hindsight_api.engine.consolidation.consolidator import (
+    _effective_scope_limit,
+    _parse_scope_limit_rules,
+    _scope_matches_globs,
+    _ScopeLimitRule,
+)
+
+
+def _config(scope_limits, default=50):
+    """A minimal stand-in for the resolved HindsightConfig fields we read."""
+    return SimpleNamespace(
+        observation_scope_limits=scope_limits,
+        max_observations_per_scope=default,
+    )
+
+
+# ---------------------------------------------------------------------------
+# _scope_matches_globs — exact cover (every tag covered, every glob used)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    "globs,tags,expected",
+    [
+        # Literal single-tag scope: matches only the exact set.
+        (("shared",), ["shared"], True),
+        (("shared",), ["run_1", "shared"], False),  # run_1 uncovered
+        (("shared",), [], False),  # untagged never matches
+        # Wildcard + literal combined scope.
+        (("run_*", "shared"), ["run_1", "shared"], True),
+        (("run_*", "shared"), ["shared"], False),  # run_* glob is vacuous
+        (("run_*", "shared"), ["run_1"], False),  # shared glob is vacuous
+        (("run_*", "shared"), ["run_1", "shared", "extra"], False),  # extra uncovered
+        # One glob may cover several tags (still exact cover).
+        (("run_*", "shared"), ["run_1", "run_2", "shared"], True),
+        # Catch-all single glob matches any tagged scope but not the untagged one.
+        (("*",), ["anything"], True),
+        (("*",), ["a", "b"], True),
+        (("*",), [], False),
+        # Matching is case-sensitive.
+        (("SHARED",), ["shared"], False),
+    ],
+)
+def test_scope_matches_globs_exact_cover(globs, tags, expected):
+    assert _scope_matches_globs(globs, tags) is expected
+
+
+def test_scope_matches_globs_is_order_independent():
+    # Tags are a set; pattern order must not change the verdict.
+    assert _scope_matches_globs(("run_*", "shared"), ["shared", "run_9"]) is True
+    assert _scope_matches_globs(("shared", "run_*"), ["run_9", "shared"]) is True
+
+
+# ---------------------------------------------------------------------------
+# _parse_scope_limit_rules — defensive, order-preserving
+# ---------------------------------------------------------------------------
+
+
+def test_parse_rules_happy_path_preserves_order():
+    raw = [
+        {"scope": ["shared"], "limit": -1},
+        {"scope": ["run_*", "shared"], "limit": 1},
+    ]
+    rules = _parse_scope_limit_rules(raw)
+    assert rules == [
+        _ScopeLimitRule(globs=("shared",), limit=-1),
+        _ScopeLimitRule(globs=("run_*", "shared"), limit=1),
+    ]
+
+
+@pytest.mark.parametrize("raw", [None, "not-a-list", 42, {}, {"scope": ["shared"], "limit": 1}])
+def test_parse_rules_non_list_yields_empty(raw):
+    assert _parse_scope_limit_rules(raw) == []
+
+
+@pytest.mark.parametrize(
+    "entry",
+    [
+        "string-entry",  # not a dict
+        {"limit": 1},  # missing scope
+        {"scope": ["a"]},  # missing limit
+        {"scope": [], "limit": 1},  # empty scope
+        {"scope": "a", "limit": 1},  # scope not a list
+        {"scope": ["a", 7], "limit": 1},  # non-str glob
+        {"scope": ["a", ""], "limit": 1},  # empty glob string
+        {"scope": ["a"], "limit": "1"},  # limit not an int
+        {"scope": ["a"], "limit": True},  # bool masquerading as int
+    ],
+)
+def test_parse_rules_skips_malformed_entries(entry):
+    # Malformed entries are dropped; a following valid entry still parses.
+    raw = [entry, {"scope": ["ok"], "limit": 3}]
+    assert _parse_scope_limit_rules(raw) == [_ScopeLimitRule(globs=("ok",), limit=3)]
+
+
+# ---------------------------------------------------------------------------
+# _effective_scope_limit — first match wins, else bank default
+# ---------------------------------------------------------------------------
+
+
+def test_effective_limit_literal_vs_wildcard_scope():
+    """Literal scope unlimited, wildcard+literal scope capped, everything else default."""
+    config = _config(
+        [
+            {"scope": ["shared"], "limit": -1},
+            {"scope": ["run_*", "shared"], "limit": 1},
+        ],
+        default=50,
+    )
+    assert _effective_scope_limit(config, ["shared"]) == -1
+    assert _effective_scope_limit(config, ["run_42", "shared"]) == 1
+    assert _effective_scope_limit(config, ["some_other_tag"]) == 50  # fallback
+    assert _effective_scope_limit(config, []) == 50  # untagged → fallback (no rule matches)
+
+
+def test_effective_limit_first_match_wins():
+    # A broad catch-all placed first shadows a more specific later rule.
+    config = _config(
+        [
+            {"scope": ["*"], "limit": 5},
+            {"scope": ["shared"], "limit": -1},
+        ],
+        default=50,
+    )
+    assert _effective_scope_limit(config, ["shared"]) == 5
+
+
+def test_effective_limit_falls_back_when_no_rules():
+    assert _effective_scope_limit(_config(None, default=7), ["shared"]) == 7
+    assert _effective_scope_limit(_config([], default=7), ["shared"]) == 7
+
+
+def test_effective_limit_none_config_is_unlimited():
+    # Mirrors the old `config is None` branch at the call site.
+    assert _effective_scope_limit(None, ["shared"]) == -1
@@ -141,7 +141,11 @@ async def test_progress_surfaced_via_get_and_list(api_client, memory: MemoryEngi

@pytest.mark.asyncio
 async def test_progress_absent_returns_null(api_client, memory: MemoryEngine):
-    """Operations that never reached a checkpoint expose progress=null (shape unchanged)."""
+    """Operations that never reached a checkpoint expose no progress value.
+
+    The field is omitted from the JSON when null (responses drop null fields), so
+    ``.get("progress")`` is None whether the key is absent or explicitly null.
+    """
    bank_id = f"op_progress_none_{uuid.uuid4().hex[:8]}"
    pool = memory._pool
    await _ensure_bank(pool, bank_id)
@@ -149,11 +153,11 @@ async def test_progress_absent_returns_null(api_client, memory: MemoryEngine):

    get_resp = await api_client.get(f"/v1/default/banks/{bank_id}/operations/{op_id}")
    assert get_resp.status_code == 200
-    assert get_resp.json()["progress"] is None
+    assert get_resp.json().get("progress") is None

    list_resp = await api_client.get(f"/v1/default/banks/{bank_id}/operations")
    op = next(o for o in list_resp.json()["operations"] if o["id"] == op_id)
-    assert op["progress"] is None
+    assert op.get("progress") is None


@pytest.mark.asyncio
@@ -0,0 +1,46 @@
+"""Prompt-length 400s follow the normal APIStatusError retry path."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from openai import APIStatusError
+
+from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
+
+
+def _llm() -> OpenAICompatibleLLM:
+    return OpenAICompatibleLLM(
+        provider="zai",
+        model="glm-5-turbo",
+        api_key="test",
+        base_url="https://example.com/v1",
+    )
+
+
+def _length_error() -> APIStatusError:
+    response = MagicMock()
+    response.status_code = 400
+    response.text = '{"code": "1261", "message": "Prompt exceeds max length"}'
+    return APIStatusError(
+        "bad",
+        response=response,
+        body={"code": "1261", "message": "Prompt exceeds max length"},
+    )
+
+
+@pytest.mark.asyncio
+async def test_prompt_length_400_is_retried():
+    llm = _llm()
+    create = AsyncMock(side_effect=_length_error())
+    llm._client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create)))
+
+    with patch("hindsight_api.engine.providers.openai_compatible_llm.asyncio.sleep", new_callable=AsyncMock):
+        with pytest.raises(APIStatusError):
+            await llm.call(
+                messages=[{"role": "user", "content": "x"}],
+                scope="mental_model_delta_ops",
+                max_retries=2,
+            )
+
+    assert create.await_count == 3
@@ -2,9 +2,9 @@
 Test query analyzer for temporal extraction.
 """

-import pytest
 from datetime import datetime
-from hindsight_api.engine.query_analyzer import DateparserQueryAnalyzer, QueryAnalysis
+
+import pytest


 def test_query_analyzer_june_2024(query_analyzer):
@@ -267,6 +267,452 @@ def test_query_analyzer_few_days_ago(query_analyzer):
    assert analysis.temporal_constraint.end_date.day == 13


+@pytest.mark.parametrize(
+    ("query", "start", "end"),
+    [
+        ("今天做了什么", datetime(2025, 1, 15), datetime(2025, 1, 15)),
+        ("本日记录", datetime(2025, 1, 15), datetime(2025, 1, 15)),
+        ("今天清晨的记录", datetime(2025, 1, 15), datetime(2025, 1, 15)),
+        ("今天能做什么", datetime(2025, 1, 15), datetime(2025, 1, 15)),
+        ("今天下雨了吗", datetime(2025, 1, 15), datetime(2025, 1, 15)),
+        ("昨天做了什么", datetime(2025, 1, 14), datetime(2025, 1, 14)),
+        ("昨天还说过什么", datetime(2025, 1, 14), datetime(2025, 1, 14)),
+        ("昨天紀錄", datetime(2025, 1, 14), datetime(2025, 1, 14)),
+        ("昨天傍晚发生了什么", datetime(2025, 1, 14), datetime(2025, 1, 14)),
+        ("昨天說了什麼", datetime(2025, 1, 14), datetime(2025, 1, 14)),
+        ("昨天帮我做了什么", datetime(2025, 1, 14), datetime(2025, 1, 14)),
+        ("这周有哪些会议", datetime(2025, 1, 13), datetime(2025, 1, 19)),
+        ("這週有哪些會議", datetime(2025, 1, 13), datetime(2025, 1, 19)),
+        ("這週以內的記錄", datetime(2025, 1, 13), datetime(2025, 1, 19)),
+        ("本周有哪些会议", datetime(2025, 1, 13), datetime(2025, 1, 19)),
+        ("这个月的费用", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("這個月的費用", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("這個月期間的費用", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("这一个月的费用", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("本月的费用", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("本月中了奖", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("本月经费", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("本月資料", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("本月報告", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("本月工资", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("本月收入多少", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("月初的事", datetime(2025, 1, 1), datetime(2025, 1, 10)),
+        ("上旬的记录", datetime(2025, 1, 1), datetime(2025, 1, 10)),
+        ("中旬的记录", datetime(2025, 1, 11), datetime(2025, 1, 20)),
+        ("月底的安排", datetime(2025, 1, 21), datetime(2025, 1, 31)),
+        ("月尾的安排", datetime(2025, 1, 21), datetime(2025, 1, 31)),
+        ("今年讨论过什么", datetime(2025, 1, 1), datetime(2025, 12, 31)),
+        ("今年初的计划", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("当日安排", datetime(2025, 1, 15), datetime(2025, 1, 15)),
+        ("当天安排", datetime(2025, 1, 15), datetime(2025, 1, 15)),
+        ("當天記錄", datetime(2025, 1, 15), datetime(2025, 1, 15)),
+        ("当年计划", datetime(2025, 1, 1), datetime(2025, 12, 31)),
+        ("年初的计划", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("年底的计划", datetime(2025, 12, 1), datetime(2025, 12, 31)),
+        ("年尾计划", datetime(2025, 12, 1), datetime(2025, 12, 31)),
+        ("下周有哪些会议", datetime(2025, 1, 20), datetime(2025, 1, 26)),
+        ("下周拜访客户", datetime(2025, 1, 20), datetime(2025, 1, 26)),
+        ("下周再安排", datetime(2025, 1, 20), datetime(2025, 1, 26)),
+        ("下一个星期有哪些会议", datetime(2025, 1, 20), datetime(2025, 1, 26)),
+        ("上周一的会议", datetime(2025, 1, 6), datetime(2025, 1, 6)),
+        ("上周星期一的会议", datetime(2025, 1, 6), datetime(2025, 1, 6)),
+        ("上星期天去了哪里", datetime(2025, 1, 12), datetime(2025, 1, 12)),
+        ("这周五聊了什么", datetime(2025, 1, 17), datetime(2025, 1, 17)),
+        ("下周三的安排", datetime(2025, 1, 22), datetime(2025, 1, 22)),
+        ("下周星期三的安排", datetime(2025, 1, 22), datetime(2025, 1, 22)),
+        ("周一的会议", datetime(2025, 1, 13), datetime(2025, 1, 13)),
+        ("星期天去哪", datetime(2025, 1, 19), datetime(2025, 1, 19)),
+        ("礼拜五安排", datetime(2025, 1, 17), datetime(2025, 1, 17)),
+        ("下周末去哪", datetime(2025, 1, 25), datetime(2025, 1, 26)),
+        ("下一个周末去哪", datetime(2025, 1, 25), datetime(2025, 1, 26)),
+        ("下下周有哪些会议", datetime(2025, 1, 27), datetime(2025, 2, 2)),
+        ("下下周末去哪", datetime(2025, 2, 1), datetime(2025, 2, 2)),
+        ("大下周有哪些会议", datetime(2025, 1, 27), datetime(2025, 2, 2)),
+        ("下个月的费用", datetime(2025, 2, 1), datetime(2025, 2, 28)),
+        ("下下个月的费用", datetime(2025, 3, 1), datetime(2025, 3, 31)),
+        ("大下个月的费用", datetime(2025, 3, 1), datetime(2025, 3, 31)),
+        ("下一个月的费用", datetime(2025, 2, 1), datetime(2025, 2, 28)),
+        ("明年讨论什么", datetime(2026, 1, 1), datetime(2026, 12, 31)),
+        ("下一个年度计划", datetime(2026, 1, 1), datetime(2026, 12, 31)),
+        ("后年讨论什么", datetime(2027, 1, 1), datetime(2027, 12, 31)),
+        ("大后年计划", datetime(2028, 1, 1), datetime(2028, 12, 31)),
+        ("周末去哪", datetime(2025, 1, 18), datetime(2025, 1, 19)),
+        ("这周末去哪", datetime(2025, 1, 18), datetime(2025, 1, 19)),
+        ("本周末去哪", datetime(2025, 1, 18), datetime(2025, 1, 19)),
+        ("上周有哪些会议", datetime(2025, 1, 6), datetime(2025, 1, 12)),
+        ("上周代码改动", datetime(2025, 1, 6), datetime(2025, 1, 12)),
+        ("上周又改了什么", datetime(2025, 1, 6), datetime(2025, 1, 12)),
+        ("上周部署了什么", datetime(2025, 1, 6), datetime(2025, 1, 12)),
+        ("上周转账记录", datetime(2025, 1, 6), datetime(2025, 1, 12)),
+        ("上週開會說了什麼", datetime(2025, 1, 6), datetime(2025, 1, 12)),
+        ("上週紀錄", datetime(2025, 1, 6), datetime(2025, 1, 12)),
+        ("當週記錄", datetime(2025, 1, 13), datetime(2025, 1, 19)),
+        ("上一个星期有哪些会议", datetime(2025, 1, 6), datetime(2025, 1, 12)),
+        ("前一周有哪些会议", datetime(2025, 1, 6), datetime(2025, 1, 12)),
+        ("前一个星期有哪些会议", datetime(2025, 1, 6), datetime(2025, 1, 12)),
+        ("上上周有哪些会议", datetime(2024, 12, 30), datetime(2025, 1, 5)),
+        ("上上个星期有哪些会议", datetime(2024, 12, 30), datetime(2025, 1, 5)),
+        ("大上周有哪些会议", datetime(2024, 12, 30), datetime(2025, 1, 5)),
+        ("上週有哪些會議", datetime(2025, 1, 6), datetime(2025, 1, 12)),
+        ("上个月的费用", datetime(2024, 12, 1), datetime(2024, 12, 31)),
+        ("上个月3号的事", datetime(2024, 12, 3), datetime(2024, 12, 3)),
+        ("本月5日的记录", datetime(2025, 1, 5), datetime(2025, 1, 5)),
+        ("下个月10号安排", datetime(2025, 2, 10), datetime(2025, 2, 10)),
+        ("當月計劃", datetime(2025, 1, 1), datetime(2025, 1, 31)),
+        ("上月底的事", datetime(2024, 12, 21), datetime(2024, 12, 31)),
+        ("这个月初的事", datetime(2025, 1, 1), datetime(2025, 1, 10)),
+        ("上一个月的费用", datetime(2024, 12, 1), datetime(2024, 12, 31)),
+        ("前一个月的费用", datetime(2024, 12, 1), datetime(2024, 12, 31)),
+        ("上上个月的费用", datetime(2024, 11, 1), datetime(2024, 11, 30)),
+        ("上個月的費用", datetime(2024, 12, 1), datetime(2024, 12, 31)),
+        ("去年讨论过什么", datetime(2024, 1, 1), datetime(2024, 12, 31)),
+        ("今年曾经做过什么", datetime(2025, 1, 1), datetime(2025, 12, 31)),
+        ("去年申请了什么", datetime(2024, 1, 1), datetime(2024, 12, 31)),
+        ("去年總結", datetime(2024, 1, 1), datetime(2024, 12, 31)),
+        ("去年报销记录", datetime(2024, 1, 1), datetime(2024, 12, 31)),
+        ("去年底的事", datetime(2024, 12, 1), datetime(2024, 12, 31)),
+        ("去年年末的事", datetime(2024, 12, 1), datetime(2024, 12, 31)),
+        ("本年度计划", datetime(2025, 1, 1), datetime(2025, 12, 31)),
+        ("上一年度计划", datetime(2024, 1, 1), datetime(2024, 12, 31)),
+        ("前一年讨论过什么", datetime(2024, 1, 1), datetime(2024, 12, 31)),
+        ("前年讨论过什么", datetime(2023, 1, 1), datetime(2023, 12, 31)),
+        ("大前年讨论过什么", datetime(2022, 1, 1), datetime(2022, 12, 31)),
+        ("本季度计划", datetime(2025, 1, 1), datetime(2025, 3, 31)),
+        ("今年第一季计划", datetime(2025, 1, 1), datetime(2025, 3, 31)),
+        ("这一个季度计划", datetime(2025, 1, 1), datetime(2025, 3, 31)),
+        ("上季度计划", datetime(2024, 10, 1), datetime(2024, 12, 31)),
+        ("上一季计划", datetime(2024, 10, 1), datetime(2024, 12, 31)),
+        ("上一个季度计划", datetime(2024, 10, 1), datetime(2024, 12, 31)),
+        ("上一季度计划", datetime(2024, 10, 1), datetime(2024, 12, 31)),
+        ("上上季度计划", datetime(2024, 7, 1), datetime(2024, 9, 30)),
+        ("下季度计划", datetime(2025, 4, 1), datetime(2025, 6, 30)),
+        ("下一季计划", datetime(2025, 4, 1), datetime(2025, 6, 30)),
+        ("下一个季度计划", datetime(2025, 4, 1), datetime(2025, 6, 30)),
+        ("下下季度计划", datetime(2025, 7, 1), datetime(2025, 9, 30)),
+        ("第一季度计划", datetime(2025, 1, 1), datetime(2025, 3, 31)),
+        ("去年第四季度计划", datetime(2024, 10, 1), datetime(2024, 12, 31)),
+        ("上一年度第二季度计划", datetime(2024, 4, 1), datetime(2024, 6, 30)),
+        ("下一年度第三季度计划", datetime(2026, 7, 1), datetime(2026, 9, 30)),
+        ("2024年第二季度计划", datetime(2024, 4, 1), datetime(2024, 6, 30)),
+        ("2024年第二季计划", datetime(2024, 4, 1), datetime(2024, 6, 30)),
+        ("二零二四年第三季度计划", datetime(2024, 7, 1), datetime(2024, 9, 30)),
+        ("2024年上半年计划", datetime(2024, 1, 1), datetime(2024, 6, 30)),
+        ("2024年下半年计划", datetime(2024, 7, 1), datetime(2024, 12, 31)),
+        ("二零二四年上半年计划", datetime(2024, 1, 1), datetime(2024, 6, 30)),
+        ("去年上半年计划", datetime(2024, 1, 1), datetime(2024, 6, 30)),
+        ("去年下半年计划", datetime(2024, 7, 1), datetime(2024, 12, 31)),
+        ("今年下半年计划", datetime(2025, 7, 1), datetime(2025, 12, 31)),
+        ("明年上半年计划", datetime(2026, 1, 1), datetime(2026, 6, 30)),
+        ("上半年计划", datetime(2025, 1, 1), datetime(2025, 6, 30)),
+        ("下半年计划", datetime(2025, 7, 1), datetime(2025, 12, 31)),
+        ("今年六月中旬的活动", datetime(2025, 6, 11), datetime(2025, 6, 20)),
+        ("2024年6月下旬的活动", datetime(2024, 6, 21), datetime(2024, 6, 30)),
+        ("2024年6月底的活动", datetime(2024, 6, 21), datetime(2024, 6, 30)),
+        ("六月初的活动", datetime(2024, 6, 1), datetime(2024, 6, 10)),
+        ("6月底的活动", datetime(2024, 6, 21), datetime(2024, 6, 30)),
+        ("前年六月的活动", datetime(2023, 6, 1), datetime(2023, 6, 30)),
+        ("去年六月的活动", datetime(2024, 6, 1), datetime(2024, 6, 30)),
+        ("今年六月的活动", datetime(2025, 6, 1), datetime(2025, 6, 30)),
+        ("明年六月的活动", datetime(2026, 6, 1), datetime(2026, 6, 30)),
+        ("后年六月的活动", datetime(2027, 6, 1), datetime(2027, 6, 30)),
+        ("上周末去了哪里", datetime(2025, 1, 11), datetime(2025, 1, 12)),
+        ("上一个周末去了哪里", datetime(2025, 1, 11), datetime(2025, 1, 12)),
+        ("上上周末去了哪里", datetime(2025, 1, 4), datetime(2025, 1, 5)),
+        ("上星期末去了哪里", datetime(2025, 1, 11), datetime(2025, 1, 12)),
+        ("上礼拜末去了哪里", datetime(2025, 1, 11), datetime(2025, 1, 12)),
+        ("上週末去了哪裡", datetime(2025, 1, 11), datetime(2025, 1, 12)),
+        ("上年讨论过什么", datetime(2024, 1, 1), datetime(2024, 12, 31)),
+        ("2024年6月的活动", datetime(2024, 6, 1), datetime(2024, 6, 30)),
+        ("2024年6月份的活动", datetime(2024, 6, 1), datetime(2024, 6, 30)),
+        ("２０２４年６月的活动", datetime(2024, 6, 1), datetime(2024, 6, 30)),
+        ("2024年０６月的活动", datetime(2024, 6, 1), datetime(2024, 6, 30)),
+        ("2024年六月的活动", datetime(2024, 6, 1), datetime(2024, 6, 30)),
+        ("六月中了奖", datetime(2024, 6, 1), datetime(2024, 6, 30)),
+        ("二零二四年六月的活动", datetime(2024, 6, 1), datetime(2024, 6, 30)),
+        ("二Ｏ二四年六月的活动", datetime(2024, 6, 1), datetime(2024, 6, 30)),
+        ("二○二四年六月的活动", datetime(2024, 6, 1), datetime(2024, 6, 30)),
+        ("二〇二四年十一月的活动", datetime(2024, 11, 1), datetime(2024, 11, 30)),
+        ("2024年六月五日的活动", datetime(2024, 6, 5), datetime(2024, 6, 5)),
+        ("2024年6月廿一日的活动", datetime(2024, 6, 21), datetime(2024, 6, 21)),
+        ("二零二四年十二月卅一日的活动", datetime(2024, 12, 31), datetime(2024, 12, 31)),
+        ("今年6月5日的活动", datetime(2025, 6, 5), datetime(2025, 6, 5)),
+        ("六月五日的活动", datetime(2024, 6, 5), datetime(2024, 6, 5)),
+        ("2024年6月5號的活动", datetime(2024, 6, 5), datetime(2024, 6, 5)),
+        ("去年今天做了什么", datetime(2024, 1, 15), datetime(2024, 1, 15)),
+        ("明年今日安排", datetime(2026, 1, 15), datetime(2026, 1, 15)),
+        ("去年本日做了什么", datetime(2024, 1, 15), datetime(2024, 1, 15)),
+        ("明年今晚安排", datetime(2026, 1, 15), datetime(2026, 1, 15)),
+        ("去年昨晚吃了什么", datetime(2024, 1, 14), datetime(2024, 1, 14)),
+    ],
+)
+def test_query_analyzer_chinese_periods(query_analyzer, query, start, end):
+    """Test deterministic Chinese period extraction."""
+    reference_date = datetime(2025, 1, 15, 12, 0, 0)
+
+    analysis = query_analyzer.analyze(query, reference_date)
+
+    assert analysis.temporal_constraint is not None
+    assert analysis.temporal_constraint.start_date.date() == start.date()
+    assert analysis.temporal_constraint.end_date.date() == end.date()
+
+
+@pytest.mark.parametrize(
+    ("query", "expected"),
+    [
+        ("两天前提到的菜是什么", datetime(2025, 1, 13)),
+        ("明天要做什么", datetime(2025, 1, 16)),
+        ("明天才开会", datetime(2025, 1, 16)),
+        ("明天開會提醒我", datetime(2025, 1, 16)),
+        ("明天半夜提醒我", datetime(2025, 1, 16)),
+        ("明日要做什么", datetime(2025, 1, 16)),
+        ("后天要做什么", datetime(2025, 1, 17)),
+        ("大后天要做什么", datetime(2025, 1, 18)),
+        ("大後天要做什么", datetime(2025, 1, 18)),
+        ("大大后天要做什么", datetime(2025, 1, 19)),
+        ("前天提到的菜是什么", datetime(2025, 1, 13)),
+        ("大前天提到的菜是什么", datetime(2025, 1, 12)),
+        ("大大前天提到的菜是什么", datetime(2025, 1, 11)),
+        ("三天前提到的菜是什么", datetime(2025, 1, 12)),
+        ("三日前的记录", datetime(2025, 1, 12)),
+        ("十天前提到的菜是什么", datetime(2025, 1, 5)),
+        ("十二天前提到的菜是什么", datetime(2025, 1, 3)),
+        ("一百天前提到的菜是什么", datetime(2024, 10, 7)),
+        ("两周前讨论了这个", datetime(2025, 1, 1)),
+        ("一周前讨论了这个", datetime(2025, 1, 8)),
+        ("一个星期前讨论了这个", datetime(2025, 1, 8)),
+        ("兩週前討論了這個", datetime(2025, 1, 1)),
+        ("两个月前的计划", datetime(2024, 11, 15)),
+        ("俩月前的计划", datetime(2024, 11, 15)),
+        ("倆月前的計畫", datetime(2024, 11, 15)),
+        ("一个月前的计划", datetime(2024, 12, 15)),
+        ("三个月前的计划", datetime(2024, 10, 15)),
+        ("二十二个月前的计划", datetime(2023, 3, 15)),
+        ("两年前的计划", datetime(2023, 1, 15)),
+        ("三天后提醒我", datetime(2025, 1, 18)),
+        ("三天之后提醒我", datetime(2025, 1, 18)),
+        ("一个月以后提醒我", datetime(2025, 2, 15)),
+        ("两年后提醒我", datetime(2027, 1, 15)),
+        ("半个月后提醒我", datetime(2025, 1, 30)),
+        ("一年半后提醒我", datetime(2026, 7, 15)),
+        ("两年半以后提醒我", datetime(2027, 7, 15)),
+        ("昨晚吃了什么", datetime(2025, 1, 14)),
+        ("今晚安排", datetime(2025, 1, 15)),
+        ("明早安排", datetime(2025, 1, 16)),
+        ("半个月前的计划", datetime(2024, 12, 31)),
+        ("一个半月前的计划", datetime(2024, 11, 30)),
+        ("半年前的计划", datetime(2024, 7, 15)),
+        ("一年半前的计划", datetime(2023, 7, 15)),
+        ("两年半前的计划", datetime(2022, 7, 15)),
+        ("兩個月前的計畫", datetime(2024, 11, 15)),
+    ],
+)
+def test_query_analyzer_chinese_exact_relative_periods(query_analyzer, query, expected):
+    """Test Chinese exact relative time expressions are not treated as fuzzy couple ranges."""
+    reference_date = datetime(2025, 1, 15, 12, 0, 0)
+
+    analysis = query_analyzer.analyze(query, reference_date)
+
+    assert analysis.temporal_constraint is not None
+    assert analysis.temporal_constraint.start_date.date() == expected.date()
+    assert analysis.temporal_constraint.end_date.date() == expected.date()
+
+
+@pytest.mark.parametrize(
+    ("query", "start", "end"),
+    [
+        ("前两天提到的菜是什么", datetime(2025, 1, 12), datetime(2025, 1, 14)),
+        ("三两天前提到的菜是什么", datetime(2025, 1, 12), datetime(2025, 1, 14)),
+        ("几天前我做了什么", datetime(2025, 1, 10), datetime(2025, 1, 13)),
+        ("前幾天我做了什麼", datetime(2025, 1, 10), datetime(2025, 1, 13)),
+        ("一两周前讨论了这个", datetime(2024, 12, 25), datetime(2025, 1, 8)),
+        ("两三周前讨论了这个", datetime(2024, 12, 25), datetime(2025, 1, 8)),
+        ("几周前讨论了这个", datetime(2024, 12, 11), datetime(2025, 1, 1)),
+        ("几个星期前讨论了这个", datetime(2024, 12, 11), datetime(2025, 1, 1)),
+        ("几个礼拜前讨论了这个", datetime(2024, 12, 11), datetime(2025, 1, 1)),
+        ("一两个月前的计划", datetime(2024, 10, 17), datetime(2024, 12, 16)),
+        ("两三个月前的计划", datetime(2024, 10, 17), datetime(2024, 12, 16)),
+        ("几个月前的计划", datetime(2024, 8, 18), datetime(2024, 11, 16)),
+        ("一两年前的计划", datetime(2022, 1, 15), datetime(2024, 1, 15)),
+        ("三四天前的记录", datetime(2025, 1, 11), datetime(2025, 1, 12)),
+        ("四五周前讨论了什么", datetime(2024, 12, 11), datetime(2024, 12, 18)),
+        ("數天前的記錄", datetime(2025, 1, 10), datetime(2025, 1, 13)),
+    ],
+)
+def test_query_analyzer_chinese_fuzzy_periods(query_analyzer, query, start, end):
+    """Test Chinese fuzzy relative period extraction mirrors English ranges."""
+    reference_date = datetime(2025, 1, 15, 12, 0, 0)
+
+    analysis = query_analyzer.analyze(query, reference_date)
+
+    assert analysis.temporal_constraint is not None
+    assert analysis.temporal_constraint.start_date.date() == start.date()
+    assert analysis.temporal_constraint.end_date.date() == end.date()
+
+
+@pytest.mark.parametrize(
+    ("query", "start", "end"),
+    [
+        ("过去一周的记录", datetime(2025, 1, 8), datetime(2025, 1, 15)),
+        ("過去一週的記錄", datetime(2025, 1, 8), datetime(2025, 1, 15)),
+        ("過去一週以內的記錄", datetime(2025, 1, 8), datetime(2025, 1, 15)),
+        ("今年以来的记录", datetime(2025, 1, 1), datetime(2025, 1, 15)),
+        ("本周以来的记录", datetime(2025, 1, 13), datetime(2025, 1, 15)),
+        ("去年至今的记录", datetime(2024, 1, 1), datetime(2025, 1, 15)),
+        ("2024年以来的记录", datetime(2024, 1, 1), datetime(2025, 1, 15)),
+        ("2024年6月5日以来的记录", datetime(2024, 6, 5), datetime(2025, 1, 15)),
+        ("本周一以来的进展", datetime(2025, 1, 13), datetime(2025, 1, 15)),
+        ("昨晚以来的记录", datetime(2025, 1, 14), datetime(2025, 1, 15)),
+        ("上周末以来的记录", datetime(2025, 1, 11), datetime(2025, 1, 15)),
+        ("去年今天以来的记录", datetime(2024, 1, 15), datetime(2025, 1, 15)),
+        ("本季度以来的记录", datetime(2025, 1, 1), datetime(2025, 1, 15)),
+        ("去年第四季度以来的记录", datetime(2024, 10, 1), datetime(2025, 1, 15)),
+        ("去年底以来的记录", datetime(2024, 12, 1), datetime(2025, 1, 15)),
+        ("2024年6月下旬以来的活动", datetime(2024, 6, 21), datetime(2025, 1, 15)),
+        ("三天前以来的记录", datetime(2025, 1, 12), datetime(2025, 1, 15)),
+        ("三天前到現在的记录", datetime(2025, 1, 12), datetime(2025, 1, 15)),
+        ("三天前开始的记录", datetime(2025, 1, 12), datetime(2025, 1, 15)),
+        ("三天前開始的記錄", datetime(2025, 1, 12), datetime(2025, 1, 15)),
+        ("2024年6月開始的記錄", datetime(2024, 6, 1), datetime(2025, 1, 15)),
+        ("近三日的记录", datetime(2025, 1, 12), datetime(2025, 1, 15)),
+        ("去年迄今的记录", datetime(2024, 1, 1), datetime(2025, 1, 15)),
+        ("这两天的记录", datetime(2025, 1, 13), datetime(2025, 1, 15)),
+        ("这几天的记录", datetime(2025, 1, 10), datetime(2025, 1, 15)),
+        ("最近几天的记录", datetime(2025, 1, 10), datetime(2025, 1, 15)),
+        ("最近半个月记录", datetime(2024, 12, 31), datetime(2025, 1, 15)),
+        ("最近几个月的记录", datetime(2024, 8, 15), datetime(2025, 1, 15)),
+        ("最近两三天的记录", datetime(2025, 1, 12), datetime(2025, 1, 15)),
+        ("过去一两个月的记录", datetime(2024, 10, 15), datetime(2025, 1, 15)),
+        ("这两三天的记录", datetime(2025, 1, 12), datetime(2025, 1, 15)),
+        ("过去几个星期的记录", datetime(2024, 12, 11), datetime(2025, 1, 15)),
+        ("近半年记录", datetime(2024, 7, 15), datetime(2025, 1, 15)),
+        ("三天内的记录", datetime(2025, 1, 12), datetime(2025, 1, 15)),
+        ("一週內的記錄", datetime(2025, 1, 8), datetime(2025, 1, 15)),
+        ("半个月内的记录", datetime(2024, 12, 31), datetime(2025, 1, 15)),
+        ("过去24小时记录", datetime(2025, 1, 14), datetime(2025, 1, 15)),
+        ("過去24小時的記錄", datetime(2025, 1, 14), datetime(2025, 1, 15)),
+        ("過去24鐘頭的記錄", datetime(2025, 1, 14), datetime(2025, 1, 15)),
+        ("未来24小时计划", datetime(2025, 1, 15), datetime(2025, 1, 16)),
+        ("前三天的记录", datetime(2025, 1, 12), datetime(2025, 1, 15)),
+        ("前5天的记录", datetime(2025, 1, 10), datetime(2025, 1, 15)),
+        ("前两周的记录", datetime(2025, 1, 1), datetime(2025, 1, 15)),
+        ("前两个月的记录", datetime(2024, 11, 15), datetime(2025, 1, 15)),
+        ("最近一个月的记录", datetime(2024, 12, 15), datetime(2025, 1, 15)),
+        ("近三个月的记录", datetime(2024, 10, 15), datetime(2025, 1, 15)),
+        ("过去一年做了什么", datetime(2024, 1, 15), datetime(2025, 1, 15)),
+        ("未来一周的计划", datetime(2025, 1, 15), datetime(2025, 1, 22)),
+        ("未來三天計劃", datetime(2025, 1, 15), datetime(2025, 1, 18)),
+        ("未来几天计划", datetime(2025, 1, 15), datetime(2025, 1, 20)),
+        ("未来几个月计划", datetime(2025, 1, 15), datetime(2025, 6, 15)),
+        ("未来两三天计划", datetime(2025, 1, 15), datetime(2025, 1, 18)),
+        ("未来半年计划", datetime(2025, 1, 15), datetime(2025, 7, 15)),
+        ("接下来一个月的计划", datetime(2025, 1, 15), datetime(2025, 2, 15)),
+        ("接下来一两周计划", datetime(2025, 1, 15), datetime(2025, 2, 5)),
+        ("接下来几周的计划", datetime(2025, 1, 15), datetime(2025, 2, 19)),
+        ("未来一年做什么", datetime(2025, 1, 15), datetime(2026, 1, 15)),
+        ("两三天后提醒我", datetime(2025, 1, 17), datetime(2025, 1, 18)),
+        ("三四天后提醒我", datetime(2025, 1, 18), datetime(2025, 1, 19)),
+        ("几天后提醒我", datetime(2025, 1, 17), datetime(2025, 1, 20)),
+        ("明后天安排", datetime(2025, 1, 16), datetime(2025, 1, 17)),
+        ("明后两天安排", datetime(2025, 1, 16), datetime(2025, 1, 17)),
+        ("今明两天的记录", datetime(2025, 1, 15), datetime(2025, 1, 16)),
+        ("昨今两天的记录", datetime(2025, 1, 14), datetime(2025, 1, 15)),
+        ("本周六和周日的安排", datetime(2025, 1, 18), datetime(2025, 1, 19)),
+        ("上周六、周日做了什么", datetime(2025, 1, 11), datetime(2025, 1, 12)),
+        ("周六日安排", datetime(2025, 1, 18), datetime(2025, 1, 19)),
+        ("昨天到今天的记录", datetime(2025, 1, 14), datetime(2025, 1, 15)),
+        ("本周一到周三的会议", datetime(2025, 1, 13), datetime(2025, 1, 15)),
+        ("上周一到周三的会议", datetime(2025, 1, 6), datetime(2025, 1, 8)),
+        ("上周五到周日的会议", datetime(2025, 1, 10), datetime(2025, 1, 12)),
+        ("下周五到周一的安排", datetime(2025, 1, 24), datetime(2025, 1, 27)),
+        ("上周五到这周一的会议", datetime(2025, 1, 10), datetime(2025, 1, 13)),
+        ("上周周五到本周周一的会议", datetime(2025, 1, 10), datetime(2025, 1, 13)),
+        ("本周五到下周一的会议", datetime(2025, 1, 17), datetime(2025, 1, 20)),
+        ("周五到周一的安排", datetime(2025, 1, 17), datetime(2025, 1, 20)),
+        ("2024年6月至8月的活动", datetime(2024, 6, 1), datetime(2024, 8, 31)),
+        ("2024年6月5日到6月8日的活动", datetime(2024, 6, 5), datetime(2024, 6, 8)),
+        ("2024年6月5日至8日的活动", datetime(2024, 6, 5), datetime(2024, 6, 8)),
+        ("2024年6月5至8日的活动", datetime(2024, 6, 5), datetime(2024, 6, 8)),
+        ("2024年6月5-8日的活动", datetime(2024, 6, 5), datetime(2024, 6, 8)),
+        ("6月5日到6月8日的活动", datetime(2024, 6, 5), datetime(2024, 6, 8)),
+        ("6月5到8号的活动", datetime(2024, 6, 5), datetime(2024, 6, 8)),
+        ("去年到今年的记录", datetime(2024, 1, 1), datetime(2025, 12, 31)),
+    ],
+)
+def test_query_analyzer_chinese_rolling_windows(query_analyzer, query, start, end):
+    """Test Chinese rolling-window temporal expressions."""
+    reference_date = datetime(2025, 1, 15, 12, 0, 0)
+
+    analysis = query_analyzer.analyze(query, reference_date)
+
+    assert analysis.temporal_constraint is not None
+    assert analysis.temporal_constraint.start_date.date() == start.date()
+    assert analysis.temporal_constraint.end_date.date() == end.date()
+
+
+@pytest.mark.parametrize("query", ["三两天前提到的菜是什么"])
+def test_query_analyzer_chinese_exact_relative_boundaries(query_analyzer, query):
+    """Test malformed Chinese numerals are not truncated into exact relative rules."""
+    reference_date = datetime(2025, 1, 15, 12, 0, 0)
+
+    analysis = query_analyzer.analyze(query, reference_date)
+
+    assert analysis.temporal_constraint is not None
+    assert analysis.temporal_constraint.start_date.date() == datetime(2025, 1, 12).date()
+    assert analysis.temporal_constraint.end_date.date() == datetime(2025, 1, 14).date()
+
+
+@pytest.mark.parametrize(
+    "query",
+    [
+        "上周杰伦的歌",
+        "下周星驰电影",
+        "今年糕点",
+        "上个月亮很圆",
+        "下个月亮很圆",
+        "本月饼很好吃",
+        "周末端项目",
+        "明日方舟攻略",
+        "明日之后攻略",
+        "今日头条新闻",
+        "庆余年第一季剧情",
+        "后天免疫因素",
+        "会议之后三天发生了什么",
+        "每周末做什么",
+        "每个周末做什么",
+        "每年上半年计划",
+        "大大大后天要做什么",
+        "大大大前天提到的菜是什么",
+        "2024年6月前的记录",
+        "2024年6月份前的记录",
+        "2024年6月5日之前的记录",
+        "2024年前的记录",
+        "2026年后的计划",
+        "今年之前的记录",
+        "上周之前的记录",
+        "这个月以后的计划",
+        "明天起的计划",
+        "下周起的计划",
+        "三天后开始的计划",
+        "三天以前的记录",
+        "三天之前的记录",
+        "三年以前的记录",
+        "每周一开会",
+        "每周星期一开会",
+        "每个周一开会",
+        "每周一到周三开会",
+        "每个星期一至星期五开会",
+        "隔周一到周五排班",
+    ],
+)
+def test_query_analyzer_chinese_compound_word_false_positives(query_analyzer, query):
+    """Test Chinese compound-word prefixes do not fall through to dateparser false positives."""
+    reference_date = datetime(2025, 1, 15, 12, 0, 0)
+
+    analysis = query_analyzer.analyze(query, reference_date)
+
+    assert analysis.temporal_constraint is None
+
+
 def test_query_analyzer_couple_weeks_ago(query_analyzer):
    """Test extraction of 'a couple of weeks ago' colloquial expression."""
    reference_date = datetime(2025, 1, 15, 12, 0, 0)
@@ -0,0 +1,82 @@
+"""Responses drop null fields where it is wire-compatible to do so.
+
+`create_app` installs `ExcludeNoneRoute`, which enables `response_model_exclude_none`
+for every route whose response model has no required-and-nullable field. Routes whose
+model *does* have such a field (an omitted key would break strict generated clients) keep
+emitting nulls. These tests lock in that classification and the resulting serialization.
+"""
+
+from fastapi.encoders import jsonable_encoder
+from pydantic import BaseModel
+
+from hindsight_api.api.http import (
+    DocumentResponse,
+    ExcludeNoneRoute,
+    OperationResponse,
+    RecallResponse,
+    ReflectResponse,
+    RetainResponse,
+    WebhookDeliveryResponse,
+    WebhookResponse,
+    _response_model_has_required_nullable,
+)
+
+
+class _RequiredNullable(BaseModel):
+    value: str | None  # required (no default) AND nullable
+
+
+class _OptionalNullable(BaseModel):
+    value: str | None = None  # optional (has default)
+
+
+class _NestsRequiredNullable(BaseModel):
+    items: list[_RequiredNullable]
+
+
+def test_required_nullable_detection() -> None:
+    # Direct required-nullable field.
+    assert _response_model_has_required_nullable(_RequiredNullable) is True
+    # Optional (has default) is fine to drop.
+    assert _response_model_has_required_nullable(_OptionalNullable) is False
+    # Detection recurses through nested models and generic containers.
+    assert _response_model_has_required_nullable(_NestsRequiredNullable) is True
+    assert _response_model_has_required_nullable(list[_RequiredNullable]) is True
+    assert _response_model_has_required_nullable(_RequiredNullable | None) is True
+
+
+def test_high_traffic_responses_are_cleaned() -> None:
+    # These have only optional (defaulted) nullable fields -> safe to drop nulls.
+    for model in (RecallResponse, RetainResponse, ReflectResponse):
+        assert _response_model_has_required_nullable(model) is False
+
+
+def test_required_nullable_responses_are_preserved() -> None:
+    # These carry a required-nullable field (e.g. error_message, content_hash) that
+    # strict clients expect present -> must keep emitting nulls.
+    for model in (DocumentResponse, OperationResponse, WebhookResponse, WebhookDeliveryResponse):
+        assert _response_model_has_required_nullable(model) is True
+
+
+def _make_route(response_model: type[BaseModel]) -> ExcludeNoneRoute:
+    return ExcludeNoneRoute("/_t", endpoint=lambda: None, response_model=response_model)
+
+
+def test_route_class_sets_exclude_none_per_model() -> None:
+    assert _make_route(RecallResponse).response_model_exclude_none is True
+    assert _make_route(DocumentResponse).response_model_exclude_none is False
+
+
+def test_explicit_decorator_flag_is_respected() -> None:
+    # An explicit response_model_exclude_none on the decorator is not overridden.
+    route = ExcludeNoneRoute(
+        "/_t", endpoint=lambda: None, response_model=DocumentResponse, response_model_exclude_none=True
+    )
+    assert route.response_model_exclude_none is True
+
+
+def test_cleaned_response_omits_null_keys() -> None:
+    resp = RecallResponse(results=[], trace=None, entities=None, chunks=None, source_facts=None)
+    cleaned = jsonable_encoder(resp, exclude_none=True)
+    assert cleaned == {"results": []}
+    assert "trace" not in cleaned and "entities" not in cleaned
@@ -3205,7 +3205,7 @@ from unittest.mock import patch

 import pytest_asyncio

-from hindsight_api.engine.llm_wrapper import TokenUsage
+from hindsight_api.engine.response_models import TokenUsage
 from hindsight_api.engine.memory_engine import MemoryEngine
 from hindsight_api.engine.task_backend import SyncTaskBackend

@@ -0,0 +1,42 @@
+"""Tests for structured-delta prompt input budgeting."""
+
+from hindsight_api.engine.reflect.prompts import (
+    STRUCTURED_DELTA_SYSTEM_PROMPT,
+    _fit_structured_delta_prompt_parts,
+    build_structured_delta_prompt,
+)
+from hindsight_api.engine.reflect.tokenization import count_cl100k_tokens
+
+
+def test_build_structured_delta_prompt_truncates_huge_document():
+    huge_doc = (
+        '{"sections": [{"id": "s1", "heading": "H", "level": 1, "blocks": [{"type": "paragraph", "text": "'
+        + ("word " * 50_000)
+        + '"}]}]}'
+    )
+    prompt = build_structured_delta_prompt(
+        current_document_json=huge_doc,
+        candidate_markdown="short synthesis",
+        supporting_facts=[{"id": "1", "text": "new fact", "type": "world"}],
+        source_query="topic?",
+        max_input_tokens=4000,
+    )
+    total = count_cl100k_tokens(STRUCTURED_DELTA_SYSTEM_PROMPT) + count_cl100k_tokens(prompt)
+    assert total < 12_000
+    assert "truncated to fit the model" in prompt
+
+
+def test_fit_structured_delta_keeps_small_prompt_unchanged():
+    doc_out, cand_out, facts_out, truncated = _fit_structured_delta_prompt_parts(
+        source_query="q",
+        current_document_json='{"sections": []}',
+        candidate_markdown="hello",
+        facts_block="one line",
+        budget_hint="",
+        task_footer="## Task\nDo it.",
+        max_input_tokens=24_000,
+    )
+    assert not truncated
+    assert doc_out == '{"sections": []}'
+    assert cand_out == "hello"
+    assert facts_out == "one line"
--- a/Show More
+++ b/Show More