fix(opencode): fix message parsing, shared state, and post-compaction retain

Three bugs fixed: 1. msg.role → msg.info.role: OpenCode SDK wraps role inside info, so all messages were silently filtered out, breaking retain and recall (#941) 2. Move PluginState to module level so it persists across sessions instead of being recreated per plugin instantiation 3. Reset lastRetainedTurn after compaction so idle-retain resumes when the message list shrinks Co-Authored-By: Claude Opus 4.6 <[email protected]>
release(opencode): v0.1.2
2026-04-13 09:04:11 -07:00 · 2026-04-13 16:13:54 +02:00 · 2026-04-13 16:08:28 +02:00 · 2026-04-13 16:02:04 +02:00 · 2026-04-13 15:56:11 +02:00 · 2026-04-13 15:55:29 +02:00
568 changed files with 41028 additions and 8802 deletions
@@ -157,7 +157,16 @@ If any files in `hindsight-integrations/` were added or changed, verify:
 - **Release process** — check that the integration name is in the `VALID_INTEGRATIONS` array in `scripts/release-integration.sh`. If missing, flag it.
 - **Code standards** — the integration code must follow all Python style rules (type hints, no raw dicts, no tuple returns, etc.).

-### 10. Review against other coding standards
+### 10. Check MCP tool registration completeness
+
+If any new MCP tools were added or existing tools renamed in `hindsight-api-slim/hindsight_api/mcp_tools.py`:
+- **`_ALL_TOOLS` set** in `mcp_tools.py` — must include the new tool name
+- **`tools_to_register` default set** in `register_mcp_tools()` in `mcp_tools.py` — must include the new tool name
+- **`_SINGLE_BANK_TOOLS` set** in `hindsight-api-slim/hindsight_api/api/mcp.py` — must include the new tool if it is bank-scoped (not a bank-management tool like `list_banks`/`create_bank`)
+- **`MCP_TOOL_GROUPS`** in `hindsight-control-plane/src/components/bank-config-view.tsx` — must include the new tool in the appropriate group for the UI tool selector
+- **Tool count assertions** in tests (e.g., `test_mcp_tools.py`) — must be updated to reflect the new count
+
+### 11. Review against other coding standards

 Check the diff for violations of the standards listed above:
 - Python files at project root (not allowed)
@@ -169,7 +178,7 @@ Check the diff for violations of the standards listed above:
 - Premature abstractions or speculative helpers
 - Backwards-compatibility hacks (unused vars, re-exports, "removed" comments)

-### 11. Report findings
+### 12. Report findings

 Present a clear summary organized by severity:

@@ -82,6 +82,15 @@ jobs:
        node-version: '22'
        registry-url: 'https://registry.npmjs.org'

+    # Guard: fail fast if the integration's lockfile resolves any dep from a
+    # monorepo workspace (link=true) or a relative file path. The release
+    # runner has no pre-built workspace `dist/` so `npm run build` would
+    # later fail at tsc with "Cannot find module". See:
+    # https://github.com/vectorize-io/hindsight/issues/… (0.6.0 openclaw retry)
+    - name: Check integration lockfile
+      if: steps.type.outputs.type == 'typescript'
+      run: ./scripts/check-integration-lockfiles.sh
+
    - name: Install dependencies
      if: steps.type.outputs.type == 'typescript'
      working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
@@ -150,6 +150,55 @@ jobs:
        path: hindsight-clients/typescript/*.tgz
        retention-days: 1

+  release-hindsight-all-npm:
+    runs-on: ubuntu-latest
+    environment: npm
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v6
+      with:
+        node-version: '22'
+        registry-url: 'https://registry.npmjs.org'
+        cache: 'npm'
+        cache-dependency-path: package-lock.json
+
+    - name: Install dependencies
+      run: npm ci --workspace=hindsight-all-npm
+
+    - name: Build
+      run: npm run build --workspace=hindsight-all-npm
+
+    - name: Publish to npm
+      working-directory: ./hindsight-all-npm
+      run: |
+        set +e
+        OUTPUT=$(npm publish --access public 2>&1)
+        EXIT_CODE=$?
+        echo "$OUTPUT"
+        if [ $EXIT_CODE -ne 0 ]; then
+          if echo "$OUTPUT" | grep -q "cannot publish over"; then
+            echo "Package version already published, skipping..."
+            exit 0
+          fi
+          exit $EXIT_CODE
+        fi
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+    - name: Pack for GitHub release
+      working-directory: ./hindsight-all-npm
+      run: npm pack
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v7
+      with:
+        name: hindsight-all-npm
+        path: hindsight-all-npm/*.tgz
+        retention-days: 1
+
  release-control-plane:
    runs-on: ubuntu-latest
    environment: npm
@@ -407,7 +456,7 @@ jobs:

  create-github-release:
    runs-on: ubuntu-latest
-    needs: [release-python-packages, release-typescript-client, release-control-plane, release-rust-cli, release-docker-images, release-helm-chart]
+    needs: [release-python-packages, release-typescript-client, release-hindsight-all-npm, release-control-plane, release-rust-cli, release-docker-images, release-helm-chart]
    permissions:
      contents: write

@@ -436,6 +485,12 @@ jobs:
        name: control-plane
        path: ./artifacts/control-plane

+    - name: Download hindsight-embed npm wrapper
+      uses: actions/download-artifact@v8
+      with:
+        name: hindsight-all-npm
+        path: ./artifacts/hindsight-all-npm
+
    - name: Download Rust CLI (Linux)
      uses: actions/download-artifact@v8
      with:
@@ -472,6 +527,8 @@ jobs:
        cp artifacts/python-packages/hindsight-embed/dist/* release-assets/ || true
        # TypeScript client
        cp artifacts/typescript-client/*.tgz release-assets/ || true
+        # hindsight-embed npm wrapper
+        cp artifacts/hindsight-all-npm/*.tgz release-assets/ || true
        # Control Plane
        cp artifacts/control-plane/*.tgz release-assets/ || true
        # Rust CLI binaries
@@ -483,7 +540,7 @@ jobs:
        ls -la release-assets/

    - name: Create GitHub Release
-      uses: softprops/action-gh-release@v2
+      uses: softprops/action-gh-release@v3
      with:
        files: release-assets/*
        generate_release_notes: true
@@ -32,6 +32,7 @@ jobs:
      helm: ${{ steps.filter.outputs.helm }}
      docs: ${{ steps.filter.outputs.docs }}
      embed: ${{ steps.filter.outputs.embed }}
+      all-npm: ${{ steps.filter.outputs.all-npm }}
      hindsight-all: ${{ steps.filter.outputs.hindsight-all }}
      integration-tests: ${{ steps.filter.outputs.integration-tests }}
      integrations-openclaw: ${{ steps.filter.outputs.integrations-openclaw }}
@@ -43,9 +44,11 @@ jobs:
      integrations-litellm: ${{ steps.filter.outputs.integrations-litellm }}
      integrations-pydantic-ai: ${{ steps.filter.outputs.integrations-pydantic-ai }}
      integrations-ag2: ${{ steps.filter.outputs.integrations-ag2 }}
-      integrations-hermes: ${{ steps.filter.outputs.integrations-hermes }}
      integrations-llamaindex: ${{ steps.filter.outputs.integrations-llamaindex }}
      integrations-paperclip: ${{ steps.filter.outputs.integrations-paperclip }}
+      integrations-opencode: ${{ steps.filter.outputs.integrations-opencode }}
+      integrations-cloudflare-oauth-proxy: ${{ steps.filter.outputs.integrations-cloudflare-oauth-proxy }}
+      integrations-lockfiles: ${{ steps.filter.outputs.integrations-lockfiles }}
      dev: ${{ steps.filter.outputs.dev }}
      ci: ${{ steps.filter.outputs.ci }}
      # Secrets are available for internal PRs, pull_request_review, and workflow_dispatch.
@@ -92,6 +95,10 @@ jobs:
            - '*.md'
          embed:
            - 'hindsight-embed/**'
+          all-npm:
+            - 'hindsight-all-npm/**'
+            - 'package.json'
+            - 'package-lock.json'
          hindsight-all:
            - 'hindsight-all/**'
          integration-tests:
@@ -114,17 +121,46 @@ jobs:
            - 'hindsight-integrations/pydantic-ai/**'
          integrations-ag2:
            - 'hindsight-integrations/ag2/**'
-          integrations-hermes:
-            - 'hindsight-integrations/hermes/**'
          integrations-llamaindex:
            - 'hindsight-integrations/llamaindex/**'
          integrations-paperclip:
            - 'hindsight-integrations/paperclip/**'
+          integrations-opencode:
+            - 'hindsight-integrations/opencode/**'
+          integrations-cloudflare-oauth-proxy:
+            - 'hindsight-integrations/cloudflare-oauth-proxy/**'
+          integrations-lockfiles:
+            - 'hindsight-integrations/*/package-lock.json'
+            - 'hindsight-integrations/*/package.json'
+            - 'scripts/check-integration-lockfiles.sh'
          dev:
            - 'hindsight-dev/**'
          ci:
            - '.github/**'

+  # Fail fast if any hindsight-integrations/*/package-lock.json was regenerated
+  # from the monorepo root and ended up symlinked at a workspace path instead
+  # of the npm registry. That bit us on the 0.6.0 openclaw release — tsc in
+  # the release workflow couldn't find `@vectorize-io/hindsight-client`
+  # because its `resolved` url pointed at a workspace dir whose `dist/` was
+  # gitignored and unbuilt. Catching this at PR time means the release CI
+  # never hits that class of failure.
+  check-integration-lockfiles:
+    needs: [detect-changes]
+    if: >-
+      github.event_name != 'pull_request_review' &&
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-lockfiles == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Check integration lockfiles resolve from the npm registry
+      run: ./scripts/check-integration-lockfiles.sh
+
  build-api-python-versions:
    needs: [detect-changes]
    if: >-
@@ -183,12 +219,12 @@ jobs:
    - name: Build TypeScript client
      run: npm run build --workspace=hindsight-clients/typescript

-  build-openclaw-integration:
+  build-hindsight-all-npm:
    needs: [detect-changes]
    if: >-
      github.event_name != 'pull_request_review' &&
      (github.event_name == 'workflow_dispatch' ||
-      needs.detect-changes.outputs.integrations-openclaw == 'true' ||
+      needs.detect-changes.outputs.all-npm == 'true' ||
      needs.detect-changes.outputs.ci == 'true')
    runs-on: ubuntu-latest

@@ -201,18 +237,125 @@ jobs:
      uses: actions/setup-node@v6
      with:
        node-version: '22'
+        cache: 'npm'
+        cache-dependency-path: package-lock.json

    - name: Install dependencies
+      run: npm ci --workspace=hindsight-all-npm
+
+    - name: Run tests
+      run: npm test --workspace=hindsight-all-npm
+
+    - name: Build
+      run: npm run build --workspace=hindsight-all-npm
+
+  build-openclaw-integration:
+    needs: [detect-changes]
+    if: >-
+      github.event_name != 'pull_request_review' &&
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-openclaw == 'true' ||
+      needs.detect-changes.outputs.clients-ts == 'true' ||
+      needs.detect-changes.outputs.all-npm == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v6
+      with:
+        node-version: '22'
+        cache: 'npm'
+        cache-dependency-path: package-lock.json
+
+    # openclaw depends on two monorepo workspaces via `file:` deps:
+    # @vectorize-io/hindsight-client and @vectorize-io/hindsight-all. Their
+    # `dist/` directories are gitignored, so we must build them first.
+    # Otherwise vitest/tsc in openclaw fails with
+    # "Failed to resolve entry for package ..." on the value imports.
+    - name: Install root workspace dependencies
+      run: npm ci
+
+    - name: Build hindsight-client (openclaw dep)
+      run: npm run build --workspace=hindsight-clients/typescript
+
+    - name: Build hindsight-all-npm (openclaw dep)
+      run: npm run build --workspace=hindsight-all-npm
+
+    - name: Install openclaw dependencies
      working-directory: ./hindsight-integrations/openclaw
      run: npm ci

+    # Build must run before tests: one unit test in src/backfill.test.ts
+    # creates a symlink to `$cwd/dist/backfill.js` and calls realpathSync on
+    # it via isDirectExecution(). Without a populated dist/ the realpath call
+    # throws, both paths stay unresolved, and the equality assertion fails.
+    - name: Build
+      working-directory: ./hindsight-integrations/openclaw
+      run: npm run build
+
    - name: Run tests
      working-directory: ./hindsight-integrations/openclaw
      run: npm test

-    - name: Build
+  smoke-openclaw-install:
+    needs: [detect-changes, build-openclaw-integration]
+    if: >-
+      github.event_name != 'pull_request_review' &&
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-openclaw == 'true' ||
+      needs.detect-changes.outputs.clients-ts == 'true' ||
+      needs.detect-changes.outputs.all-npm == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v6
+      with:
+        node-version: '22'
+        cache: 'npm'
+        cache-dependency-path: package-lock.json
+
+    # Install the openclaw CLI globally. The smoke test exercises the real
+    # `openclaw plugins install` / `openclaw config set` / `openclaw plugins
+    # doctor` commands — not the in-repo integration tests — so a real CLI
+    # must be on PATH.
+    - name: Install openclaw CLI
+      run: npm install -g openclaw
+
+    - name: Verify openclaw CLI
+      run: openclaw --version
+
+    # openclaw depends on the workspace packages via published version
+    # ranges (^0.1.0 / ^0.5.0), not file: paths, so the smoke test's
+    # `openclaw plugins install <tarball>` resolves them straight from the
+    # npm registry. These builds are just for `npm pack` / local unit
+    # tests, not for resolving the plugin's runtime deps.
+    - name: Install root workspace dependencies
+      run: npm ci
+
+    - name: Build hindsight-client (openclaw dep)
+      run: npm run build --workspace=hindsight-clients/typescript
+
+    - name: Build hindsight-all-npm (openclaw dep)
+      run: npm run build --workspace=hindsight-all-npm
+
+    - name: Install openclaw dependencies
      working-directory: ./hindsight-integrations/openclaw
-      run: npm run build
+      run: npm ci
+
+    - name: Run openclaw install smoke test
+      working-directory: ./hindsight-integrations/openclaw
+      run: ./scripts/smoke-test.sh

  test-claude-code-integration:
    needs: [detect-changes]
@@ -329,6 +472,68 @@ jobs:
      working-directory: ./hindsight-integrations/ai-sdk
      run: npm run test:deno

+  test-opencode-integration:
+    needs: [detect-changes]
+    if: >-
+      github.event_name != 'pull_request_review' &&
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-opencode == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v6
+      with:
+        node-version: '22'
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/opencode
+      run: npm ci
+
+    - name: Run tests
+      working-directory: ./hindsight-integrations/opencode
+      run: npm test
+
+    - name: Build
+      working-directory: ./hindsight-integrations/opencode
+      run: npm run build
+
+  test-cloudflare-oauth-proxy-integration:
+    needs: [detect-changes]
+    if: >-
+      github.event_name != 'pull_request_review' &&
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.integrations-cloudflare-oauth-proxy == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v6
+      with:
+        node-version: '22'
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/cloudflare-oauth-proxy
+      run: npm ci
+
+    - name: Typecheck
+      working-directory: ./hindsight-integrations/cloudflare-oauth-proxy
+      run: npm run typecheck
+
+    - name: Run tests
+      working-directory: ./hindsight-integrations/cloudflare-oauth-proxy
+      run: npm test
+
  build-chat-integration:
    needs: [detect-changes]
    if: >-
@@ -1494,6 +1699,18 @@ jobs:
        print('Models downloaded successfully')
        "

+    # openclaw depends on @vectorize-io/hindsight-client and
+    # @vectorize-io/hindsight-all via `file:` — their `dist/` directories are
+    # gitignored and must be built before openclaw's npm ci copies them.
+    - name: Install root workspace dependencies
+      run: npm ci
+
+    - name: Build hindsight-client (openclaw dep)
+      run: npm run build --workspace=hindsight-clients/typescript
+
+    - name: Build hindsight-all-npm (openclaw dep)
+      run: npm run build --workspace=hindsight-all-npm
+
    - name: Install openclaw integration dependencies
      working-directory: ./hindsight-integrations/openclaw
      run: npm ci
@@ -1789,43 +2006,6 @@ jobs:
      working-directory: ./hindsight-integrations/pydantic-ai
      run: uv run pytest tests -v

-  test-hermes-integration:
-    needs: [detect-changes]
-    if: >-
-      github.event_name != 'pull_request_review' &&
-      (github.event_name == 'workflow_dispatch' ||
-      needs.detect-changes.outputs.integrations-hermes == 'true' ||
-      needs.detect-changes.outputs.ci == 'true')
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        ref: ${{ github.event.pull_request.head.sha || '' }}
-
-    - name: Install uv
-      uses: astral-sh/setup-uv@v7
-      with:
-        enable-cache: true
-        prune-cache: false
-
-    - name: Set up Python
-      uses: actions/setup-python@v6
-      with:
-        python-version-file: ".python-version"
-
-    - name: Build hermes integration
-      working-directory: ./hindsight-integrations/hermes
-      run: uv build
-
-    - name: Install dependencies
-      working-directory: ./hindsight-integrations/hermes
-      run: uv sync --frozen
-
-    - name: Run tests
-      working-directory: ./hindsight-integrations/hermes
-      run: uv run pytest tests -v
-
  test-llamaindex-integration:
    needs: [detect-changes]
    if: >-
@@ -2447,6 +2627,40 @@ jobs:
        cd hindsight-dev
        uv run check-openapi-compatibility /tmp/old-openapi.json ../hindsight-docs/static/openapi.json

+  check-cli-coverage:
+    needs: [detect-changes]
+    if: >-
+      github.event_name != 'pull_request_review' &&
+      (github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.core == 'true' ||
+      needs.detect-changes.outputs.cli == 'true' ||
+      needs.detect-changes.outputs.dev == 'true' ||
+      needs.detect-changes.outputs.ci == 'true')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha || '' }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version-file: ".python-version"
+
+    - name: Install hindsight-dev dependencies
+      run: |
+        cd hindsight-dev && uv sync --frozen --index-strategy unsafe-best-match
+
+    - name: Check CLI covers every OpenAPI operation
+      run: |
+        cd hindsight-dev
+        uv run cli-coverage-check
+
  # Report CI status back to the PR for pull_request_review events.
  # GitHub does not automatically link pull_request_review check runs to the PR,
  # so we create a commit status on the PR head SHA and post a comment.
@@ -2454,13 +2668,17 @@ jobs:
    if: github.event_name == 'pull_request_review' && github.event.review.state == 'approved' && always()
    needs:
      - detect-changes
+      - check-integration-lockfiles
      - build-api-python-versions
      - build-typescript-client
      - build-openclaw-integration
+      - smoke-openclaw-install
      - test-claude-code-integration
      - test-codex-integration
      - build-ai-sdk-integration
      - test-ai-sdk-integration-deno
+      - test-opencode-integration
+      - test-cloudflare-oauth-proxy-integration
      - build-chat-integration
      - test-paperclip-integration
      - build-control-plane
@@ -2481,7 +2699,6 @@ jobs:
      - test-crewai-integration
      - test-litellm-integration
      - test-pydantic-ai-integration
-      - test-hermes-integration
      - test-llamaindex-integration
      - test-pip-slim
      - test-embed
@@ -2490,6 +2707,7 @@ jobs:
      - test-upgrade
      - verify-generated-files
      - check-openapi-compatibility
+      - check-cli-coverage
    runs-on: ubuntu-latest
    permissions:
      statuses: write
@@ -222,6 +222,10 @@ Every new integration in `hindsight-integrations/` must satisfy all of the follo

 If any of these are missing, the integration is incomplete and must not be pushed or merged.

+### Changelogs
+
+Never add "Unreleased" entries to changelogs (e.g. `hindsight-docs/src/pages/changelog/**`). Changelog entries are written by the release script (`./scripts/release-integration.sh`) when a version is actually cut. If a bug fix or feature needs documenting before release, describe it in the PR/commit — the release tooling will surface it in the published changelog section.
+
 ### Adding New API Configuration Flags

 Configuration follows a hierarchical system: **Global (env vars) → Tenant (via extension) → Bank (database)**.
@@ -2,8 +2,8 @@ apiVersion: v2
 name: hindsight
 description: Hindsight helm chart
 type: application
-version: 0.4.22
-appVersion: "0.4.22"
+version: 0.5.1
+appVersion: "0.5.1"
 keywords:
  - ai
  - memory
@@ -0,0 +1,4 @@
+node_modules
+dist
+*.tgz
+.DS_Store
@@ -0,0 +1,80 @@
+# @vectorize-io/hindsight-all
+
+Node.js equivalent of the Python [`hindsight-all`](https://pypi.org/project/hindsight-all/) package — programmatic lifecycle manager for a local Hindsight daemon. Use this when you want to embed Hindsight in a Node application without hand-rolling subprocess management.
+
+This package deliberately does **not** ship an HTTP client. Once the daemon is running, talk to it with [`@vectorize-io/hindsight-client`](https://www.npmjs.com/package/@vectorize-io/hindsight-client) against `server.getBaseUrl()`. The two packages compose — one owns the daemon process, the other owns the HTTP API surface.
+
+## Requirements
+
+- **Node.js >= 22** — uses global `fetch` and `AbortSignal.timeout`.
+- **`uv` / `uvx`** on `PATH` — used to download and run the underlying `hindsight-embed` daemon on first use. Install via <https://docs.astral.sh/uv/>.
+
+## Install
+
+```bash
+npm install @vectorize-io/hindsight-all @vectorize-io/hindsight-client
+```
+
+## Example
+
+```ts
+import { HindsightServer, consoleLogger } from '@vectorize-io/hindsight-all';
+import { HindsightClient } from '@vectorize-io/hindsight-client';
+
+const server = new HindsightServer({
+  profile: 'my-app',
+  port: 9077,
+  env: {
+    HINDSIGHT_API_LLM_PROVIDER: 'anthropic',
+    HINDSIGHT_API_LLM_API_KEY: process.env.ANTHROPIC_API_KEY,
+    HINDSIGHT_API_LLM_MODEL: 'claude-sonnet-4-20250514',
+    HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT: '0',
+  },
+  logger: consoleLogger,
+});
+
+await server.start();
+
+const client = new HindsightClient({ baseUrl: server.getBaseUrl() });
+
+await client.retain('user-123', 'User prefers dark mode and concise answers.', {
+  documentId: 'pref-2026-04-01',
+});
+
+const recall = await client.recall('user-123', 'what are the user preferences?');
+console.log(recall.results);
+
+await server.stop();
+```
+
+For a remote Hindsight API, skip `HindsightServer` entirely and just point `HindsightClient` at the remote URL.
+
+## Open config — forward-compatible with new daemon flags
+
+`HindsightServerOptions` is designed so every new environment variable or CLI flag in the underlying Hindsight daemon can be used without waiting for a wrapper release:
+
+- **`env`** accepts an arbitrary `Record<string, string>`. Every entry is exported into the daemon process and written into the profile config via `--env KEY=VALUE`.
+- **`extraProfileCreateArgs`** / **`extraDaemonStartArgs`** append raw args to the respective commands.
+
+## Development against a local checkout
+
+If you're hacking on the Python `hindsight-embed` package in the same monorepo, point the server at the local path — it'll use `uv run --directory <path>` instead of `uvx`:
+
+```ts
+new HindsightServer({
+  embedPackagePath: '/path/to/hindsight-embed',
+  // ...
+});
+```
+
+## API surface
+
+- `HindsightServer` — daemon lifecycle (`start`, `stop`, `checkHealth`, `getBaseUrl`, `getProfile`).
+- `Logger` interface plus `silentLogger` (default) and `consoleLogger` helpers.
+- `getEmbedCommand(opts)` — low-level helper that returns the `[cmd, ...args]` tuple used to invoke the underlying Python CLI.
+
+For memory operations (retain, recall, reflect, bank management, stats) use [`@vectorize-io/hindsight-client`](https://www.npmjs.com/package/@vectorize-io/hindsight-client).
+
+## License
+
+MIT
@@ -0,0 +1,57 @@
+{
+  "name": "@vectorize-io/hindsight-all",
+  "version": "0.5.1",
+  "description": "Node.js programmatic lifecycle manager for Hindsight — embeds a local hindsight daemon in a Node application. Pair with @vectorize-io/hindsight-client for memory operations.",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "keywords": [
+    "hindsight",
+    "hindsight-all",
+    "memory",
+    "ai",
+    "agent",
+    "long-term-memory",
+    "llm",
+    "embedded-server"
+  ],
+  "author": "Vectorize <[email protected]>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vectorize-io/hindsight.git",
+    "directory": "hindsight-all-npm"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "clean": "rm -rf dist",
+    "test": "vitest run src",
+    "test:watch": "vitest src",
+    "prepublishOnly": "npm run clean && npm run build"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "tsup": "^8.5.1",
+    "typescript": "^5.7.0",
+    "vitest": "^4.1.2"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "overrides": {
+    "rollup": "^4.59.0",
+    "picomatch": ">=2.3.2 <3.0.0 || >=4.0.4",
+    "vite": ">=8.0.5"
+  }
+}
@@ -0,0 +1,32 @@
+import { describe, it, expect } from 'vitest';
+import { getEmbedCommand } from './command.js';
+
+describe('getEmbedCommand', () => {
+  it('defaults to uvx hindsight-embed@latest', () => {
+    expect(getEmbedCommand()).toEqual(['uvx', 'hindsight-embed@latest']);
+  });
+
+  it('honours an explicit version', () => {
+    expect(getEmbedCommand({ embedVersion: '0.5.0' })).toEqual(['uvx', '[email protected]']);
+  });
+
+  it('treats an empty version as latest', () => {
+    expect(getEmbedCommand({ embedVersion: '' })).toEqual(['uvx', 'hindsight-embed@latest']);
+  });
+
+  it('uses uv run --directory when a local path is given', () => {
+    expect(getEmbedCommand({ embedPackagePath: '/abs/path' })).toEqual([
+      'uv',
+      'run',
+      '--directory',
+      '/abs/path',
+      'hindsight-embed',
+    ]);
+  });
+
+  it('local path takes precedence over version', () => {
+    expect(
+      getEmbedCommand({ embedPackagePath: '/abs/path', embedVersion: '0.5.0' }),
+    ).toEqual(['uv', 'run', '--directory', '/abs/path', 'hindsight-embed']);
+  });
+});
@@ -0,0 +1,25 @@
+/**
+ * Resolve the command that invokes the `hindsight-embed` Python CLI.
+ *
+ * - If `embedPackagePath` is set, runs the package from a local checkout via
+ *   `uv run --directory <path> hindsight-embed`. Used for in-repo development.
+ * - Otherwise runs it via `uvx hindsight-embed@<version>` so no global install
+ *   is required.
+ *
+ * Returns the argv as `[command, ...baseArgs]` suitable for `spawn()` /
+ * `execFile()` (never shell-interpolated).
+ */
+export interface EmbedCommandOptions {
+  /** Version spec passed to uvx (e.g. "latest", "0.5.0"). Default: "latest". */
+  embedVersion?: string;
+  /** Local checkout path. When set, overrides `embedVersion` and uses `uv run`. */
+  embedPackagePath?: string;
+}
+
+export function getEmbedCommand(opts: EmbedCommandOptions = {}): string[] {
+  if (opts.embedPackagePath) {
+    return ['uv', 'run', '--directory', opts.embedPackagePath, 'hindsight-embed'];
+  }
+  const version = opts.embedVersion && opts.embedVersion.length > 0 ? opts.embedVersion : 'latest';
+  return ['uvx', `hindsight-embed@${version}`];
+}
@@ -0,0 +1,7 @@
+export { HindsightServer } from './server.js';
+export { getEmbedCommand } from './command.js';
+export { silentLogger, consoleLogger } from './logger.js';
+
+export type { Logger } from './logger.js';
+export type { EmbedCommandOptions } from './command.js';
+export type { HindsightServerOptions } from './types.js';
@@ -0,0 +1,29 @@
+/**
+ * Pluggable logger interface.
+ *
+ * This package does not own any logging infrastructure — consumers inject
+ * whatever they want (console, pino, openclaw's logger, a no-op). The default
+ * is silent so embedding this package never adds noise to an unrelated app.
+ */
+export interface Logger {
+  debug(msg: string): void;
+  info(msg: string): void;
+  warn(msg: string): void;
+  error(msg: string): void;
+}
+
+/** Logger that drops every call. Used when no logger is passed. */
+export const silentLogger: Logger = {
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+};
+
+/** Logger that writes to the standard console. Handy for CLIs and tests. */
+export const consoleLogger: Logger = {
+  debug: (msg) => console.debug(msg),
+  info: (msg) => console.log(msg),
+  warn: (msg) => console.warn(msg),
+  error: (msg) => console.error(msg),
+};
@@ -0,0 +1,35 @@
+import { describe, it, expect } from 'vitest';
+import { HindsightServer } from './server.js';
+
+describe('HindsightServer construction', () => {
+  it('defaults base URL to http://127.0.0.1:8888', () => {
+    const server = new HindsightServer();
+    expect(server.getBaseUrl()).toBe('http://127.0.0.1:8888');
+    expect(server.getProfile()).toBe('default');
+  });
+
+  it('honours custom profile, port, and host', () => {
+    const server = new HindsightServer({ profile: 'app', port: 9077, host: '0.0.0.0' });
+    expect(server.getProfile()).toBe('app');
+    expect(server.getBaseUrl()).toBe('http://0.0.0.0:9077');
+  });
+
+  it('accepts open env pass-through without complaining about unknown keys', () => {
+    const server = new HindsightServer({
+      env: {
+        HINDSIGHT_API_LLM_PROVIDER: 'openai',
+        HINDSIGHT_API_LLM_MODEL: 'gpt-4o-mini',
+        // A field that does not exist today — should still be accepted
+        HINDSIGHT_FUTURE_FLAG: 'enabled',
+      },
+    });
+    expect(server).toBeInstanceOf(HindsightServer);
+  });
+
+  it('exposes checkHealth that returns false when no daemon is running', async () => {
+    // Random high port that nothing is listening on.
+    const server = new HindsightServer({ port: 1, readyTimeoutMs: 100 });
+    const healthy = await server.checkHealth();
+    expect(healthy).toBe(false);
+  });
+});
@@ -0,0 +1,322 @@
+import { spawn } from 'child_process';
+import { getEmbedCommand } from './command.js';
+import { silentLogger } from './logger.js';
+import type { Logger } from './logger.js';
+import type { HindsightServerOptions } from './types.js';
+
+const DEFAULT_PORT = 8888;
+const DEFAULT_HOST = '127.0.0.1';
+const DEFAULT_PROFILE = 'default';
+const DEFAULT_READY_TIMEOUT_MS = 30_000;
+const DEFAULT_READY_POLL_INTERVAL_MS = 1_000;
+
+/**
+ * Manages the lifecycle of a local Hindsight daemon from a Node.js process.
+ *
+ * On {@link start}, this class:
+ *   1. Resolves the `hindsight-embed` command (via `uvx` or a local `uv run`).
+ *   2. Runs `profile create <name> --merge --port <port> [--env K=V ...]`
+ *      with every entry in {@link HindsightServerOptions.env} forwarded as
+ *      an `--env` flag.
+ *   3. Runs `daemon --profile <name> start` and waits for the start command
+ *      to exit.
+ *   4. Polls `http://host:port/health` until it returns `200` or the
+ *      `readyTimeoutMs` budget is exhausted.
+ *
+ * On {@link stop}, it runs `daemon --profile <name> stop` and returns once
+ * the command exits (or after a short grace period).
+ *
+ * This is the Node.js equivalent of the Python `hindsight-all` package's
+ * `HindsightServer`: a thin programmatic lifecycle wrapper around the
+ * Hindsight daemon. It does NOT ship an HTTP client — once `start()`
+ * resolves, use `@vectorize-io/hindsight-client` against `getBaseUrl()` for
+ * retain / recall / reflect.
+ *
+ * The class is deliberately transparent about the daemon: new CLI flags or
+ * environment variables never require a code change here — callers can pass
+ * them via `env`, `extraProfileCreateArgs`, or `extraDaemonStartArgs`.
+ */
+export class HindsightServer {
+  private readonly profile: string;
+  private readonly port: number;
+  private readonly host: string;
+  private readonly baseUrl: string;
+  private readonly embedVersion: string | undefined;
+  private readonly embedPackagePath: string | undefined;
+  private readonly userEnv: Record<string, string | undefined>;
+  private readonly extraProfileCreateArgs: string[];
+  private readonly extraDaemonStartArgs: string[];
+  private readonly platformCpuWorkaround: boolean;
+  private readonly readyTimeoutMs: number;
+  private readonly readyPollIntervalMs: number;
+  private readonly logger: Logger;
+
+  constructor(opts: HindsightServerOptions = {}) {
+    this.profile = opts.profile ?? DEFAULT_PROFILE;
+    this.port = opts.port ?? DEFAULT_PORT;
+    this.host = opts.host ?? DEFAULT_HOST;
+    this.baseUrl = `http://${this.host}:${this.port}`;
+    this.embedVersion = opts.embedVersion;
+    this.embedPackagePath = opts.embedPackagePath;
+    this.userEnv = opts.env ?? {};
+    this.extraProfileCreateArgs = opts.extraProfileCreateArgs ?? [];
+    this.extraDaemonStartArgs = opts.extraDaemonStartArgs ?? [];
+    this.platformCpuWorkaround = opts.platformCpuWorkaround ?? (process.platform === 'darwin');
+    this.readyTimeoutMs = opts.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS;
+    this.readyPollIntervalMs = opts.readyPollIntervalMs ?? DEFAULT_READY_POLL_INTERVAL_MS;
+    this.logger = opts.logger ?? silentLogger;
+  }
+
+  /** The base URL the daemon listens on (`http://host:port`). */
+  getBaseUrl(): string {
+    return this.baseUrl;
+  }
+
+  /** The profile name this server operates on. */
+  getProfile(): string {
+    return this.profile;
+  }
+
+  /**
+   * Ensure the daemon is configured and running. Idempotent — the underlying
+   * `profile create --merge` and `daemon start` commands tolerate re-runs.
+   */
+  async start(): Promise<void> {
+    this.logger.info(`[hindsight] starting daemon for profile "${this.profile}"`);
+
+    const env = this.buildEnv();
+    await this.configureProfile(env);
+    await this.startDaemon(env);
+    await this.waitForReady();
+
+    this.logger.info(`[hindsight] daemon ready at ${this.baseUrl}`);
+  }
+
+  /** Stop the daemon. Never throws — logs and resolves even on failure. */
+  async stop(): Promise<void> {
+    this.logger.info(`[hindsight] stopping daemon for profile "${this.profile}"`);
+
+    const [cmd, ...baseArgs] = getEmbedCommand({
+      embedVersion: this.embedVersion,
+      embedPackagePath: this.embedPackagePath,
+    });
+    const args = [...baseArgs, 'daemon', '--profile', this.profile, 'stop'];
+
+    const child = spawn(cmd, args, { stdio: 'pipe' });
+    this.pipeOutput(child, 'daemon.stop');
+
+    await new Promise<void>((resolve) => {
+      const timeout = setTimeout(() => {
+        this.logger.warn(`[hindsight] daemon stop timed out after 5s`);
+        resolve();
+      }, 5_000);
+      child.on('exit', () => {
+        clearTimeout(timeout);
+        this.logger.info(`[hindsight] daemon stopped`);
+        resolve();
+      });
+      child.on('error', (err) => {
+        clearTimeout(timeout);
+        this.logger.warn(`[hindsight] error stopping daemon: ${err.message}`);
+        resolve();
+      });
+    });
+  }
+
+  /** Probe `/health` once with a short timeout. */
+  async checkHealth(): Promise<boolean> {
+    try {
+      const res = await fetch(`${this.baseUrl}/health`, {
+        signal: AbortSignal.timeout(2_000),
+      });
+      return res.ok;
+    } catch {
+      return false;
+    }
+  }
+
+  // -------------------------------------------------------------------------
+  // Internal
+  // -------------------------------------------------------------------------
+
+  /**
+   * Merge the process env, the caller-supplied `env`, and (on macOS) the
+   * embeddings CPU workaround. Caller-supplied values always win over the
+   * workaround; undefined values are dropped.
+   */
+  private buildEnv(): NodeJS.ProcessEnv {
+    const merged: NodeJS.ProcessEnv = { ...process.env };
+
+    if (this.platformCpuWorkaround && process.platform === 'darwin') {
+      merged['HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU'] = '1';
+      merged['HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU'] = '1';
+    }
+
+    for (const [key, value] of Object.entries(this.userEnv)) {
+      if (value !== undefined) {
+        merged[key] = value;
+      }
+    }
+
+    return merged;
+  }
+
+  /**
+   * Run `profile create <name> --merge --port <port> [--env K=V ...]`.
+   * Every entry in the merged env that was passed via {@link userEnv} (or
+   * auto-applied by the CPU workaround) is forwarded as `--env`.
+   */
+  private async configureProfile(env: NodeJS.ProcessEnv): Promise<void> {
+    this.logger.info(`[hindsight] configuring profile "${this.profile}"`);
+
+    const [cmd, ...baseArgs] = getEmbedCommand({
+      embedVersion: this.embedVersion,
+      embedPackagePath: this.embedPackagePath,
+    });
+    const createArgs = [
+      ...baseArgs,
+      'profile',
+      'create',
+      this.profile,
+      '--merge',
+      '--port',
+      String(this.port),
+    ];
+
+    // Forward every env var that the caller intended for the daemon as --env.
+    // We only forward keys the caller explicitly set (userEnv) plus the CPU
+    // workaround values — not the entire process.env, to avoid leaking random
+    // host state into profile config.
+    const envForProfile = this.collectProfileEnv(env);
+    for (const [key, value] of Object.entries(envForProfile)) {
+      createArgs.push('--env', `${key}=${value}`);
+    }
+
+    createArgs.push(...this.extraProfileCreateArgs);
+
+    await this.runCommand(cmd, createArgs, env, 'profile.create');
+  }
+
+  /** Collect only the env vars that should be written into the profile file. */
+  private collectProfileEnv(env: NodeJS.ProcessEnv): Record<string, string> {
+    const out: Record<string, string> = {};
+
+    // 1. User-supplied env — always forwarded.
+    for (const [key, value] of Object.entries(this.userEnv)) {
+      if (value !== undefined) {
+        out[key] = value;
+      }
+    }
+
+    // 2. CPU workaround — only if auto-applied and not already overridden.
+    if (this.platformCpuWorkaround && process.platform === 'darwin') {
+      const cpuKeys = [
+        'HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU',
+        'HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU',
+      ];
+      for (const key of cpuKeys) {
+        if (!(key in out) && env[key] !== undefined) {
+          out[key] = env[key] as string;
+        }
+      }
+    }
+
+    return out;
+  }
+
+  private async startDaemon(env: NodeJS.ProcessEnv): Promise<void> {
+    const [cmd, ...baseArgs] = getEmbedCommand({
+      embedVersion: this.embedVersion,
+      embedPackagePath: this.embedPackagePath,
+    });
+    const args = [
+      ...baseArgs,
+      'daemon',
+      '--profile',
+      this.profile,
+      'start',
+      ...this.extraDaemonStartArgs,
+    ];
+
+    await this.runCommand(cmd, args, env, 'daemon.start');
+  }
+
+  /**
+   * Spawn `cmd` with `args`, pipe its output through the logger, and resolve
+   * once it exits with code 0. Rejects on non-zero exit or spawn error.
+   */
+  private async runCommand(
+    cmd: string,
+    args: string[],
+    env: NodeJS.ProcessEnv,
+    label: string,
+  ): Promise<void> {
+    const child = spawn(cmd, args, { stdio: 'pipe', env });
+    let output = '';
+    child.stdout?.on('data', (data: Buffer) => {
+      const text = data.toString();
+      output += text;
+      for (const line of text.trimEnd().split('\n')) {
+        if (line) this.logger.info(`[hindsight:${label}] ${line}`);
+      }
+    });
+    child.stderr?.on('data', (data: Buffer) => {
+      const text = data.toString();
+      output += text;
+      for (const line of text.trimEnd().split('\n')) {
+        if (line) this.logger.warn(`[hindsight:${label}] ${line}`);
+      }
+    });
+
+    await new Promise<void>((resolve, reject) => {
+      child.on('exit', (code) => {
+        if (code === 0) {
+          resolve();
+        } else {
+          reject(new Error(`${label} failed with code ${code}: ${output.trim()}`));
+        }
+      });
+      child.on('error', (err) => {
+        reject(new Error(`${label} failed to spawn: ${err.message}`, { cause: err }));
+      });
+    });
+  }
+
+  /** Stream a spawned child's stdout/stderr through the logger without blocking. */
+  private pipeOutput(child: ReturnType<typeof spawn>, label: string): void {
+    child.stdout?.on('data', (data: Buffer) => {
+      for (const line of data.toString().trimEnd().split('\n')) {
+        if (line) this.logger.info(`[hindsight:${label}] ${line}`);
+      }
+    });
+    child.stderr?.on('data', (data: Buffer) => {
+      for (const line of data.toString().trimEnd().split('\n')) {
+        if (line) this.logger.warn(`[hindsight:${label}] ${line}`);
+      }
+    });
+  }
+
+  /** Poll `/health` until it succeeds or `readyTimeoutMs` elapses. */
+  private async waitForReady(): Promise<void> {
+    const deadline = Date.now() + this.readyTimeoutMs;
+    let attempt = 0;
+    while (Date.now() < deadline) {
+      attempt++;
+      try {
+        const res = await fetch(`${this.baseUrl}/health`, {
+          signal: AbortSignal.timeout(this.readyPollIntervalMs),
+        });
+        if (res.ok) {
+          this.logger.debug(`[hindsight] health check passed (attempt ${attempt})`);
+          return;
+        }
+      } catch {
+        // expected while the daemon is still booting
+      }
+      await new Promise((resolve) => setTimeout(resolve, this.readyPollIntervalMs));
+    }
+    throw new Error(
+      `Hindsight daemon did not become ready within ${this.readyTimeoutMs}ms at ${this.baseUrl}`,
+    );
+  }
+}
@@ -0,0 +1,54 @@
+import type { Logger } from './logger.js';
+
+/**
+ * Options for {@link HindsightServer}.
+ *
+ * The server is intentionally thin and pass-through: anything configurable
+ * on the daemon side (env vars or CLI flags) can be set here without needing
+ * a new dedicated option. Use {@link env} for `HINDSIGHT_*` / `OPENAI_API_KEY` /
+ * custom provider settings, and the two `extra*` arrays to append raw CLI
+ * args to `profile create` or `daemon start`.
+ *
+ * For talking to the daemon after `start()`, use `@vectorize-io/hindsight-client`
+ * against `server.getBaseUrl()`. This package does not ship its own HTTP
+ * client.
+ */
+export interface HindsightServerOptions {
+  /** Profile name used for `--profile <name>` on every sub-command. Default: `"default"`. */
+  profile?: string;
+  /** TCP port the daemon listens on. Default: `8888`. */
+  port?: number;
+  /** Hostname the daemon binds to (for health checks). Default: `127.0.0.1`. */
+  host?: string;
+  /** Version of the underlying `hindsight-embed` PyPI package to run via `uvx`. Default: `"latest"`. */
+  embedVersion?: string;
+  /** Local path to a `hindsight-embed` checkout — takes precedence over `embedVersion`. */
+  embedPackagePath?: string;
+  /**
+   * Environment variables passed to the daemon process AND written into the
+   * profile via repeated `--env KEY=VALUE` flags. This is the preferred way
+   * to surface any `HINDSIGHT_API_*` / `HINDSIGHT_EMBED_*` setting — adding a
+   * new daemon env var never requires a wrapper update.
+   *
+   * Values of `undefined` are dropped (so you can spread conditionally).
+   */
+  env?: Record<string, string | undefined>;
+  /** Extra args appended verbatim to `hindsight-embed profile create <name> --merge ...`. */
+  extraProfileCreateArgs?: string[];
+  /** Extra args appended verbatim to `hindsight-embed daemon --profile <name> start ...`. */
+  extraDaemonStartArgs?: string[];
+  /**
+   * On macOS, automatically set
+   * `HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU=1` and
+   * `HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU=1` to avoid Metal/MPS crashes in
+   * daemon mode. Default: `true` on `darwin`, ignored elsewhere. Any value set
+   * explicitly in {@link env} wins over the auto-applied value.
+   */
+  platformCpuWorkaround?: boolean;
+  /** Max time (ms) to wait for `/health` to return 200. Default: `30_000`. */
+  readyTimeoutMs?: number;
+  /** Polling interval (ms) while waiting for `/health`. Default: `1_000`. */
+  readyPollIntervalMs?: number;
+  /** Optional pluggable logger. Default: silent. */
+  logger?: Logger;
+}
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ES2022",
+    "lib": ["ES2022"],
+    "moduleResolution": "node",
+    "declaration": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "src/**/*.test.ts"]
+}
@@ -0,0 +1,11 @@
+import { defineConfig } from 'tsup';
+
+export default defineConfig({
+  entry: ['src/index.ts'],
+  format: ['esm'],
+  dts: true,
+  outDir: 'dist',
+  clean: true,
+  sourcemap: true,
+  bundle: true,
+});
@@ -0,0 +1,8 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    include: ['src/**/*.test.ts'],
+    environment: 'node',
+  },
+});
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

 [project]
 name = "hindsight-all-slim"
-version = "0.4.22"
+version = "0.5.1"
 description = "Hindsight: Agent Memory That Works Like Human Memory - Slim All-in-One Bundle"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -190,12 +190,32 @@ class HindsightEmbedded:
        if self._closed:
            return

-        with self._lock:
+        acquired = self._lock.acquire(timeout=5.0)
+        if not acquired:
+            # Lock is held by another thread (e.g. _ensure_started).
+            # Mark closed to prevent new operations but skip shared-state
+            # teardown — the daemon's idle timeout handles the rest.
+            logger.warning(
+                "Cleanup lock acquisition timed out for profile '%s'; "
+                "marking closed, daemon will idle-stop on its own",
+                self.profile,
+            )
+            self._closed = True
+            return
+
+        try:
            if self._closed:
                return

            if self._client is not None:
-                self._client.close()
+                try:
+                    self._client.close()
+                except Exception:
+                    logger.debug(
+                        "Error closing client for profile '%s'",
+                        self.profile,
+                        exc_info=True,
+                    )
                self._client = None

            # Stop UI if it was started
@@ -209,6 +229,8 @@ class HindsightEmbedded:
                self._manager.stop(self.profile)

            self._closed = True
+        finally:
+            self._lock.release()

    def close(self, stop_daemon: bool = False):
        """
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"

 [project]
 name = "hindsight-all"
-version = "0.4.22"
+version = "0.5.1"
 description = "Hindsight: Agent Memory That Works Like Human Memory - All-in-One Bundle"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -20,6 +20,9 @@ hindsight-client = { workspace = true }
 hindsight-embed = { workspace = true }

 [project.optional-dependencies]
+local-llm = [
+    "hindsight-api-slim[local-llm]>=0.4.17",
+]
 test = [
    "pytest>=7.0.0",
    "pytest-asyncio>=0.21.0",
@@ -0,0 +1,56 @@
+"""
+Unit test for _cleanup lock timeout behavior.
+
+Verifies that _cleanup completes even when the lock is held by another thread,
+instead of hanging indefinitely (fixes #952).
+"""
+
+import threading
+import time
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+def test_cleanup_completes_when_lock_held():
+    """
+    _cleanup should complete (best-effort) even when self._lock is held
+    by another thread, e.g. during a long _ensure_started call.
+    """
+    with patch.dict("sys.modules", {
+        "hindsight_client": MagicMock(),
+        "hindsight_embed": MagicMock(),
+        "hindsight.api_namespaces": MagicMock(),
+    }):
+        from hindsight.embedded import HindsightEmbedded
+
+        client = HindsightEmbedded.__new__(HindsightEmbedded)
+        client.profile = "test"
+        client._lock = threading.Lock()
+        client._closed = False
+        client._client = None
+        client._started = False
+        client._ui = False
+
+        # Simulate another thread holding the lock
+        client._lock.acquire()
+
+        cleanup_done = threading.Event()
+
+        def run_cleanup():
+            client._cleanup()
+            cleanup_done.set()
+
+        t = threading.Thread(target=run_cleanup)
+        t.start()
+
+        # Cleanup should complete within the timeout (5s) + margin
+        assert cleanup_done.wait(timeout=8.0), (
+            "_cleanup hung instead of timing out on lock acquisition"
+        )
+
+        # Release the lock from the simulating thread
+        client._lock.release()
+        t.join(timeout=1.0)
+
+        assert client._closed, "Client should be marked as closed after cleanup"
@@ -46,4 +46,4 @@ __all__ = [
    "RemoteTEICrossEncoder",
    "LLMConfig",
 ]
-__version__ = "0.4.22"
+__version__ = "0.5.1"
@@ -0,0 +1,42 @@
+"""Merge 3 migration heads and add unit_entities composite index
+
+Revision ID: h3i4j5k6l7m8
+Revises: a4b5c6d7e8f9, c2d3e4f5g6h7, g2h3i4j5k6l7
+Create Date: 2026-04-07
+
+Merges three unmerged migration heads into one, and adds a composite index
+(entity_id, unit_id) on unit_entities for index-only scans in the LATERAL
+entity expansion query.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "h3i4j5k6l7m8"
+down_revision: str | Sequence[str] | None = ("a4b5c6d7e8f9", "c2d3e4f5g6h7", "g2h3i4j5k6l7")
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    # Composite index enables index-only scans for entity_id -> unit_id lookups
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS idx_unit_entities_entity_unit ON {schema}unit_entities (entity_id, unit_id)"
+    )
+    # Drop the now-redundant single-column index
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_unit_entities_entity")
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_unit_entities_entity_unit")
+    # Restore the single-column index
+    op.execute(f"CREATE INDEX IF NOT EXISTS idx_unit_entities_entity ON {schema}unit_entities (entity_id)")
@@ -463,6 +463,12 @@ class MemoryItem(BaseModel):
        description="Named retain strategy for this item. Overrides the bank's default strategy for this item only. "
        "Strategies are defined in the bank config under 'retain_strategies'.",
    )
+    update_mode: Literal["replace", "append"] | None = Field(
+        default=None,
+        description="How to handle an existing document with the same document_id. "
+        "'replace' (default) deletes old data and reprocesses from scratch. "
+        "'append' concatenates new content to the existing document text and reprocesses.",
+    )

    @field_validator("timestamp", mode="before")
    @classmethod
@@ -1661,7 +1667,9 @@ class BankTemplateConfig(BaseModel):
    disposition_skepticism: int | None = Field(default=None, ge=1, le=5, description="Skepticism trait (1-5)")
    disposition_literalism: int | None = Field(default=None, ge=1, le=5, description="Literalism trait (1-5)")
    disposition_empathy: int | None = Field(default=None, ge=1, le=5, description="Empathy trait (1-5)")
-    entity_labels: list[str] | None = Field(default=None, description="Controlled vocabulary for entity labels")
+    entity_labels: list[dict[str, Any]] | None = Field(
+        default=None, description="Controlled vocabulary for entity labels"
+    )
    entities_allow_free_form: bool | None = Field(
        default=None, description="Allow entities outside the label vocabulary"
    )
@@ -1792,6 +1800,150 @@ class BankTemplateImportResponse(BaseModel):
    dry_run: bool = Field(default=False, description="True if this was a validation-only run")


+def validate_bank_template(manifest: "BankTemplateManifest") -> list[str]:
+    """Validate a parsed manifest beyond Pydantic's structural checks.
+
+    Returns a list of human-readable error strings (e.g. invalid
+    extraction mode values, conflicting settings).
+    """
+    errors: list[str] = []
+    if manifest.bank:
+        bank = manifest.bank
+        if bank.retain_extraction_mode is not None:
+            valid_modes = ("concise", "verbose", "custom", "chunks")
+            if bank.retain_extraction_mode not in valid_modes:
+                errors.append(
+                    f"bank.retain_extraction_mode: must be one of {valid_modes}, got '{bank.retain_extraction_mode}'"
+                )
+        if bank.retain_custom_instructions and bank.retain_extraction_mode != "custom":
+            errors.append("bank.retain_custom_instructions: requires retain_extraction_mode='custom'")
+    if manifest.mental_models:
+        for i, mm in enumerate(manifest.mental_models):
+            if not mm.name.strip():
+                errors.append(f"mental_models[{i}].name: must not be empty")
+            if not mm.source_query.strip():
+                errors.append(f"mental_models[{i}].source_query: must not be empty")
+    if manifest.directives:
+        for i, d in enumerate(manifest.directives):
+            if not d.name.strip():
+                errors.append(f"directives[{i}].name: must not be empty")
+            if not d.content.strip():
+                errors.append(f"directives[{i}].content: must not be empty")
+    return errors
+
+
+async def apply_bank_template_manifest(
+    memory,
+    bank_id: str,
+    manifest: "BankTemplateManifest",
+    request_context: "RequestContext",
+) -> "BankTemplateImportResponse":
+    """Apply a validated BankTemplateManifest to an existing bank.
+
+    Shared by the /import endpoint and the default-template-on-create hook
+    driven by HINDSIGHT_API_DEFAULT_BANK_TEMPLATE. The bank MUST already
+    exist; caller is responsible for validation (Pydantic + validate_bank_template).
+    """
+    config_applied = False
+    if manifest.bank:
+        config_updates = manifest.bank.get_config_updates()
+        if config_updates:
+            await memory._config_resolver.update_bank_config(bank_id, config_updates, request_context)
+            config_applied = True
+
+    created_ids: list[str] = []
+    updated_ids: list[str] = []
+    operation_ids: list[str] = []
+
+    if manifest.mental_models:
+        # Fetch existing mental models to decide create vs update
+        existing = await memory.list_mental_models(bank_id=bank_id, request_context=request_context)
+        existing_by_id = {m["id"]: m for m in existing}
+
+        for mm in manifest.mental_models:
+            if mm.id in existing_by_id:
+                await memory.update_mental_model(
+                    bank_id=bank_id,
+                    mental_model_id=mm.id,
+                    name=mm.name,
+                    source_query=mm.source_query,
+                    max_tokens=mm.max_tokens,
+                    tags=mm.tags if mm.tags else None,
+                    trigger=mm.trigger.model_dump() if mm.trigger else None,
+                    request_context=request_context,
+                )
+                result = await memory.submit_async_refresh_mental_model(
+                    bank_id=bank_id,
+                    mental_model_id=mm.id,
+                    request_context=request_context,
+                )
+                operation_ids.append(result["operation_id"])
+                updated_ids.append(mm.id)
+            else:
+                mental_model = await memory.create_mental_model(
+                    bank_id=bank_id,
+                    name=mm.name,
+                    source_query=mm.source_query,
+                    content="Generating content...",
+                    mental_model_id=mm.id,
+                    tags=mm.tags if mm.tags else None,
+                    max_tokens=mm.max_tokens,
+                    trigger=mm.trigger.model_dump() if mm.trigger else None,
+                    request_context=request_context,
+                )
+                result = await memory.submit_async_refresh_mental_model(
+                    bank_id=bank_id,
+                    mental_model_id=mental_model["id"],
+                    request_context=request_context,
+                )
+                operation_ids.append(result["operation_id"])
+                created_ids.append(mm.id)
+
+    directives_created: list[str] = []
+    directives_updated: list[str] = []
+
+    if manifest.directives:
+        existing_directives = await memory.list_directives(
+            bank_id=bank_id, active_only=False, request_context=request_context
+        )
+        existing_by_name = {d["name"]: d for d in existing_directives}
+
+        for directive in manifest.directives:
+            if directive.name in existing_by_name:
+                await memory.update_directive(
+                    bank_id=bank_id,
+                    directive_id=existing_by_name[directive.name]["id"],
+                    content=directive.content,
+                    priority=directive.priority,
+                    is_active=directive.is_active,
+                    tags=directive.tags if directive.tags else None,
+                    request_context=request_context,
+                )
+                directives_updated.append(directive.name)
+            else:
+                await memory.create_directive(
+                    bank_id=bank_id,
+                    name=directive.name,
+                    content=directive.content,
+                    priority=directive.priority,
+                    is_active=directive.is_active,
+                    tags=directive.tags if directive.tags else None,
+                    request_context=request_context,
+                )
+                directives_created.append(directive.name)
+
+    return BankTemplateImportResponse(
+        bank_id=bank_id,
+        config_applied=config_applied,
+        mental_models_created=created_ids,
+        mental_models_updated=updated_ids,
+        directives_created=directives_created,
+        directives_updated=directives_updated,
+        operation_ids=operation_ids,
+        dry_run=False,
+    )
+
+
 class OperationResponse(BaseModel):
    """Response model for a single async operation."""

@@ -2677,6 +2829,8 @@ def _register_routes(app: FastAPI):
            return data
        except OperationValidationError as e:
            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
        except (AuthenticationError, HTTPException):
            raise
        except Exception as e:
@@ -3287,6 +3441,8 @@ def _register_routes(app: FastAPI):
            raise
        except OperationValidationError as e:
            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
        except Exception as e:
            import traceback

@@ -3320,6 +3476,8 @@ def _register_routes(app: FastAPI):
            raise
        except OperationValidationError as e:
            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
        except Exception as e:
            import traceback

@@ -3484,6 +3642,8 @@ def _register_routes(app: FastAPI):
            return {"status": "deleted"}
        except OperationValidationError as e:
            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
        except (AuthenticationError, HTTPException):
            raise
        except Exception as e:
@@ -4361,38 +4521,6 @@ def _register_routes(app: FastAPI):
    # Bank Template Import / Export
    # =====================================================================

-    def _validate_template(manifest: BankTemplateManifest) -> list[str]:
-        """Validate a parsed manifest beyond Pydantic's structural checks.
-
-        Returns a list of human-readable error strings (e.g. invalid
-        extraction mode values, conflicting settings).
-        """
-        errors: list[str] = []
-        if manifest.bank:
-            bank = manifest.bank
-            if bank.retain_extraction_mode is not None:
-                valid_modes = ("concise", "verbose", "custom", "chunks")
-                if bank.retain_extraction_mode not in valid_modes:
-                    errors.append(
-                        f"bank.retain_extraction_mode: must be one of {valid_modes}, "
-                        f"got '{bank.retain_extraction_mode}'"
-                    )
-            if bank.retain_custom_instructions and bank.retain_extraction_mode != "custom":
-                errors.append("bank.retain_custom_instructions: requires retain_extraction_mode='custom'")
-        if manifest.mental_models:
-            for i, mm in enumerate(manifest.mental_models):
-                if not mm.name.strip():
-                    errors.append(f"mental_models[{i}].name: must not be empty")
-                if not mm.source_query.strip():
-                    errors.append(f"mental_models[{i}].source_query: must not be empty")
-        if manifest.directives:
-            for i, d in enumerate(manifest.directives):
-                if not d.name.strip():
-                    errors.append(f"directives[{i}].name: must not be empty")
-                if not d.content.strip():
-                    errors.append(f"directives[{i}].content: must not be empty")
-        return errors
-
    @app.post(
        "/v1/default/banks/{bank_id}/import",
        response_model=BankTemplateImportResponse,
@@ -4428,7 +4556,7 @@ def _register_routes(app: FastAPI):
                )

            # Semantic validation beyond Pydantic structural checks
-            validation_errors = _validate_template(body)
+            validation_errors = validate_bank_template(body)
            if validation_errors:
                raise HTTPException(
                    status_code=400,
@@ -4446,107 +4574,11 @@ def _register_routes(app: FastAPI):
            # Ensure bank exists (auto-creates with defaults if needed)
            await app.state.memory.get_bank_profile(bank_id, request_context=request_context)

-            config_applied = False
-            if body.bank:
-                config_updates = body.bank.get_config_updates()
-                if config_updates:
-                    await app.state.memory._config_resolver.update_bank_config(bank_id, config_updates, request_context)
-                    config_applied = True
-
-            created_ids: list[str] = []
-            updated_ids: list[str] = []
-            operation_ids: list[str] = []
-
-            if body.mental_models:
-                # Fetch existing mental models to decide create vs update
-                existing = await app.state.memory.list_mental_models(bank_id=bank_id, request_context=request_context)
-                existing_by_id = {m["id"]: m for m in existing}
-
-                for mm in body.mental_models:
-                    if mm.id in existing_by_id:
-                        # Update existing mental model metadata
-                        await app.state.memory.update_mental_model(
-                            bank_id=bank_id,
-                            mental_model_id=mm.id,
-                            name=mm.name,
-                            source_query=mm.source_query,
-                            max_tokens=mm.max_tokens,
-                            tags=mm.tags if mm.tags else None,
-                            trigger=mm.trigger.model_dump() if mm.trigger else None,
-                            request_context=request_context,
-                        )
-                        # Schedule a refresh to regenerate content with updated query
-                        result = await app.state.memory.submit_async_refresh_mental_model(
-                            bank_id=bank_id,
-                            mental_model_id=mm.id,
-                            request_context=request_context,
-                        )
-                        operation_ids.append(result["operation_id"])
-                        updated_ids.append(mm.id)
-                    else:
-                        # Create new mental model
-                        mental_model = await app.state.memory.create_mental_model(
-                            bank_id=bank_id,
-                            name=mm.name,
-                            source_query=mm.source_query,
-                            content="Generating content...",
-                            mental_model_id=mm.id,
-                            tags=mm.tags if mm.tags else None,
-                            max_tokens=mm.max_tokens,
-                            trigger=mm.trigger.model_dump() if mm.trigger else None,
-                            request_context=request_context,
-                        )
-                        result = await app.state.memory.submit_async_refresh_mental_model(
-                            bank_id=bank_id,
-                            mental_model_id=mental_model["id"],
-                            request_context=request_context,
-                        )
-                        operation_ids.append(result["operation_id"])
-                        created_ids.append(mm.id)
-
-            directives_created: list[str] = []
-            directives_updated: list[str] = []
-
-            if body.directives:
-                # Fetch existing directives to decide create vs update (matched by name)
-                existing_directives = await app.state.memory.list_directives(
-                    bank_id=bank_id, active_only=False, request_context=request_context
-                )
-                existing_by_name = {d["name"]: d for d in existing_directives}
-
-                for directive in body.directives:
-                    if directive.name in existing_by_name:
-                        await app.state.memory.update_directive(
-                            bank_id=bank_id,
-                            directive_id=existing_by_name[directive.name]["id"],
-                            content=directive.content,
-                            priority=directive.priority,
-                            is_active=directive.is_active,
-                            tags=directive.tags if directive.tags else None,
-                            request_context=request_context,
-                        )
-                        directives_updated.append(directive.name)
-                    else:
-                        await app.state.memory.create_directive(
-                            bank_id=bank_id,
-                            name=directive.name,
-                            content=directive.content,
-                            priority=directive.priority,
-                            is_active=directive.is_active,
-                            tags=directive.tags if directive.tags else None,
-                            request_context=request_context,
-                        )
-                        directives_created.append(directive.name)
-
-            return BankTemplateImportResponse(
+            return await apply_bank_template_manifest(
+                memory=app.state.memory,
                bank_id=bank_id,
-                config_applied=config_applied,
-                mental_models_created=created_ids,
-                mental_models_updated=updated_ids,
-                directives_created=directives_created,
-                directives_updated=directives_updated,
-                operation_ids=operation_ids,
-                dry_run=False,
+                manifest=body,
+                request_context=request_context,
            )
        except ValueError as e:
            raise HTTPException(status_code=400, detail=str(e))
@@ -4948,7 +4980,9 @@ def _register_routes(app: FastAPI):
            from hindsight_api.engine.retain import bank_utils

            # Ensure the bank row exists before inserting into webhooks (FK constraint).
-            await bank_utils.get_bank_profile(pool, bank_id)
+            _, created = await bank_utils.get_or_create_bank_profile(pool, bank_id)
+            if created:
+                await app.state.memory._apply_default_bank_template(bank_id, request_context)

            webhook_id = uuid.uuid4()
            now = datetime.now(timezone.utc).isoformat()
@@ -5297,6 +5331,8 @@ def _register_routes(app: FastAPI):
                    content_dict["tags"] = item.tags
                if item.observation_scopes is not None:
                    content_dict["observation_scopes"] = item.observation_scopes
+                if item.update_mode is not None:
+                    content_dict["update_mode"] = item.update_mode
                strategy_groups[effective].append(content_dict)

            if request.async_:
@@ -97,6 +97,7 @@ def create_mcp_server(memory: MemoryEngine, multi_bank: bool = True) -> FastMCP:
    _SINGLE_BANK_TOOLS: frozenset[str] = frozenset(
        {
            "retain",
+            "sync_retain",
            "recall",
            "reflect",
            "list_mental_models",
@@ -194,6 +194,13 @@ ENV_RERANKER_COHERE_API_KEY = "HINDSIGHT_API_RERANKER_COHERE_API_KEY"
 ENV_RERANKER_COHERE_MODEL = "HINDSIGHT_API_RERANKER_COHERE_MODEL"
 ENV_RERANKER_COHERE_BASE_URL = "HINDSIGHT_API_RERANKER_COHERE_BASE_URL"

+# OpenRouter configuration (embeddings and reranker)
+ENV_OPENROUTER_API_KEY = "HINDSIGHT_API_OPENROUTER_API_KEY"
+ENV_EMBEDDINGS_OPENROUTER_API_KEY = "HINDSIGHT_API_EMBEDDINGS_OPENROUTER_API_KEY"
+ENV_EMBEDDINGS_OPENROUTER_MODEL = "HINDSIGHT_API_EMBEDDINGS_OPENROUTER_MODEL"
+ENV_RERANKER_OPENROUTER_API_KEY = "HINDSIGHT_API_RERANKER_OPENROUTER_API_KEY"
+ENV_RERANKER_OPENROUTER_MODEL = "HINDSIGHT_API_RERANKER_OPENROUTER_MODEL"
+
 # Deprecated: Legacy shared Cohere API key (for backward compatibility)
 ENV_COHERE_API_KEY = "HINDSIGHT_API_COHERE_API_KEY"

@@ -211,6 +218,7 @@ ENV_EMBEDDINGS_LITELLM_SDK_API_KEY = "HINDSIGHT_API_EMBEDDINGS_LITELLM_SDK_API_K
 ENV_EMBEDDINGS_LITELLM_SDK_MODEL = "HINDSIGHT_API_EMBEDDINGS_LITELLM_SDK_MODEL"
 ENV_EMBEDDINGS_LITELLM_SDK_API_BASE = "HINDSIGHT_API_EMBEDDINGS_LITELLM_SDK_API_BASE"
 ENV_EMBEDDINGS_LITELLM_SDK_OUTPUT_DIMENSIONS = "HINDSIGHT_API_EMBEDDINGS_LITELLM_SDK_OUTPUT_DIMENSIONS"
+ENV_EMBEDDINGS_LITELLM_SDK_ENCODING_FORMAT = "HINDSIGHT_API_EMBEDDINGS_LITELLM_SDK_ENCODING_FORMAT"
 ENV_RERANKER_LITELLM_SDK_API_KEY = "HINDSIGHT_API_RERANKER_LITELLM_SDK_API_KEY"
 ENV_RERANKER_LITELLM_SDK_MODEL = "HINDSIGHT_API_RERANKER_LITELLM_SDK_MODEL"
 ENV_RERANKER_LITELLM_SDK_API_BASE = "HINDSIGHT_API_RERANKER_LITELLM_SDK_API_BASE"
@@ -239,6 +247,11 @@ ENV_RERANKER_ZEROENTROPY_API_KEY = "HINDSIGHT_API_RERANKER_ZEROENTROPY_API_KEY"
 ENV_RERANKER_ZEROENTROPY_MODEL = "HINDSIGHT_API_RERANKER_ZEROENTROPY_MODEL"
 ENV_RERANKER_ZEROENTROPY_BASE_URL = "HINDSIGHT_API_RERANKER_ZEROENTROPY_BASE_URL"

+# SiliconFlow configuration (reranker only; Cohere-compatible /rerank endpoint)
+ENV_RERANKER_SILICONFLOW_API_KEY = "HINDSIGHT_API_RERANKER_SILICONFLOW_API_KEY"
+ENV_RERANKER_SILICONFLOW_MODEL = "HINDSIGHT_API_RERANKER_SILICONFLOW_MODEL"
+ENV_RERANKER_SILICONFLOW_BASE_URL = "HINDSIGHT_API_RERANKER_SILICONFLOW_BASE_URL"
+
 # Google Discovery Engine reranker configuration
 ENV_RERANKER_GOOGLE_MODEL = "HINDSIGHT_API_RERANKER_GOOGLE_MODEL"
 ENV_RERANKER_GOOGLE_PROJECT_ID = "HINDSIGHT_API_RERANKER_GOOGLE_PROJECT_ID"
@@ -257,11 +270,14 @@ ENV_MCP_ENABLED = "HINDSIGHT_API_MCP_ENABLED"
 ENV_MCP_ENABLED_TOOLS = "HINDSIGHT_API_MCP_ENABLED_TOOLS"
 ENV_MCP_STATELESS = "HINDSIGHT_API_MCP_STATELESS"
 ENV_ENABLE_BANK_CONFIG_API = "HINDSIGHT_API_ENABLE_BANK_CONFIG_API"
+ENV_DEFAULT_BANK_TEMPLATE = "HINDSIGHT_API_DEFAULT_BANK_TEMPLATE"
 ENV_GRAPH_RETRIEVER = "HINDSIGHT_API_GRAPH_RETRIEVER"
 ENV_RECALL_MAX_CONCURRENT = "HINDSIGHT_API_RECALL_MAX_CONCURRENT"
 ENV_RECALL_CONNECTION_BUDGET = "HINDSIGHT_API_RECALL_CONNECTION_BUDGET"
 ENV_RECALL_MAX_QUERY_TOKENS = "HINDSIGHT_API_RECALL_MAX_QUERY_TOKENS"
 ENV_MENTAL_MODEL_REFRESH_CONCURRENCY = "HINDSIGHT_API_MENTAL_MODEL_REFRESH_CONCURRENCY"
+ENV_LINK_EXPANSION_PER_ENTITY_LIMIT = "HINDSIGHT_API_LINK_EXPANSION_PER_ENTITY_LIMIT"
+ENV_LINK_EXPANSION_TIMEOUT = "HINDSIGHT_API_LINK_EXPANSION_TIMEOUT"

 # OpenTelemetry tracing configuration
 ENV_OTEL_TRACES_ENABLED = "HINDSIGHT_API_OTEL_TRACES_ENABLED"
@@ -334,6 +350,14 @@ ENV_WEBHOOK_SECRET = "HINDSIGHT_API_WEBHOOK_SECRET"
 ENV_WEBHOOK_EVENT_TYPES = "HINDSIGHT_API_WEBHOOK_EVENT_TYPES"
 ENV_WEBHOOK_DELIVERY_POLL_INTERVAL_SECONDS = "HINDSIGHT_API_WEBHOOK_DELIVERY_POLL_INTERVAL_SECONDS"

+# Built-in llama.cpp configuration (for provider=llamacpp)
+ENV_LLAMACPP_MODEL_PATH = "HINDSIGHT_API_LLAMACPP_MODEL_PATH"
+ENV_LLAMACPP_GPU_LAYERS = "HINDSIGHT_API_LLAMACPP_GPU_LAYERS"
+ENV_LLAMACPP_CONTEXT_SIZE = "HINDSIGHT_API_LLAMACPP_CONTEXT_SIZE"
+ENV_LLAMACPP_CHAT_FORMAT = "HINDSIGHT_API_LLAMACPP_CHAT_FORMAT"
+ENV_LLAMACPP_NO_GRAMMAR = "HINDSIGHT_API_LLAMACPP_NO_GRAMMAR"
+ENV_LLAMACPP_EXTRA_ARGS = "HINDSIGHT_API_LLAMACPP_EXTRA_ARGS"
+
 # Optimization flags
 ENV_SKIP_LLM_VERIFICATION = "HINDSIGHT_API_SKIP_LLM_VERIFICATION"
 ENV_LAZY_RERANKER = "HINDSIGHT_API_LAZY_RERANKER"
@@ -387,6 +411,7 @@ PROVIDER_DEFAULT_MODELS = {
    "groq": "openai/gpt-oss-120b",
    "minimax": "MiniMax-M2.7",
    "ollama": "gemma3:12b",
+    "llamacpp": "gemma-4-e2b-it",
    "lmstudio": "local-model",
    "vertexai": "google/gemini-2.5-flash-lite",
    "openai-codex": "gpt-5.2-codex",
@@ -396,8 +421,16 @@ PROVIDER_DEFAULT_MODELS = {
    "litellm": "gpt-4o-mini",
    "bedrock": "us.amazon.nova-2-lite-v1:0",
    "volcano": "doubao-pro-32k",
+    "openrouter": "qwen/qwen3.5-9b",
 }
 DEFAULT_LLM_MODEL = "gpt-4o-mini"  # Fallback if provider not in table
+# Built-in llama.cpp defaults
+DEFAULT_LLAMACPP_GPU_LAYERS = -1  # -1 = offload all layers to GPU (Metal/CUDA)
+DEFAULT_LLAMACPP_CONTEXT_SIZE = 8192
+DEFAULT_LLAMACPP_CHAT_FORMAT = None  # None = auto-detect from GGUF metadata
+DEFAULT_LLAMACPP_NO_GRAMMAR = False  # True = disable JSON grammar enforcement (faster but less reliable)
+DEFAULT_LLAMACPP_EXTRA_ARGS = None  # Space-separated extra CLI args for llama.cpp server
+
 DEFAULT_LLM_MAX_CONCURRENT = 32
 DEFAULT_LLM_MAX_RETRIES = 10  # Max retry attempts for LLM API calls
 DEFAULT_LLM_INITIAL_BACKOFF = 1.0  # Initial backoff in seconds for retry exponential backoff
@@ -440,8 +473,15 @@ DEFAULT_RERANKER_FLASHRANK_CACHE_DIR = None  # Use default cache directory
 DEFAULT_EMBEDDINGS_COHERE_MODEL = "embed-english-v3.0"
 DEFAULT_RERANKER_COHERE_MODEL = "rerank-english-v3.0"

+# OpenRouter defaults
+DEFAULT_EMBEDDINGS_OPENROUTER_MODEL = "perplexity/pplx-embed-v1-0.6b"
+DEFAULT_RERANKER_OPENROUTER_MODEL = "cohere/rerank-v3.5"
+
 DEFAULT_RERANKER_ZEROENTROPY_MODEL = "zerank-2"

+DEFAULT_RERANKER_SILICONFLOW_MODEL = "BAAI/bge-reranker-v2-m3"
+DEFAULT_RERANKER_SILICONFLOW_BASE_URL = "https://api.siliconflow.cn/v1"
+
 DEFAULT_RERANKER_GOOGLE_MODEL = "semantic-ranker-default-004"

 # Vector extension (pgvector, vchord, or pgvectorscale)
@@ -458,6 +498,7 @@ DEFAULT_RERANKER_LITELLM_MAX_TOKENS_PER_DOC: int | None = None

 # LiteLLM SDK defaults
 DEFAULT_EMBEDDINGS_LITELLM_SDK_MODEL = "cohere/embed-english-v3.0"
+DEFAULT_EMBEDDINGS_LITELLM_SDK_ENCODING_FORMAT = "float"
 DEFAULT_RERANKER_LITELLM_SDK_MODEL = "cohere/rerank-english-v3.0"

 DEFAULT_HOST = "0.0.0.0"
@@ -470,11 +511,14 @@ DEFAULT_MCP_ENABLED = True
 DEFAULT_MCP_ENABLED_TOOLS: list[str] | None = None  # None = all tools enabled
 DEFAULT_MCP_STATELESS = False  # False = stateful (supports SSE/GET); True = stateless (POST-only)
 DEFAULT_ENABLE_BANK_CONFIG_API = True
+DEFAULT_DEFAULT_BANK_TEMPLATE: dict | None = None  # BankTemplateManifest dict applied to newly-created banks
 DEFAULT_GRAPH_RETRIEVER = "link_expansion"
 DEFAULT_RECALL_MAX_CONCURRENT = 32  # Max concurrent recall operations per worker
 DEFAULT_RECALL_CONNECTION_BUDGET = 4  # Max concurrent DB connections per recall operation
 DEFAULT_RECALL_MAX_QUERY_TOKENS = 500  # Maximum tokens allowed in recall query
 DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY = 8  # Max concurrent mental model refreshes
+DEFAULT_LINK_EXPANSION_PER_ENTITY_LIMIT = 200  # Max target units per entity in graph expansion
+DEFAULT_LINK_EXPANSION_TIMEOUT = 10.0  # Timeout (seconds) for entity expansion query

 # Retain settings
 DEFAULT_RETAIN_MAX_COMPLETION_TOKENS = 64000  # Max tokens for fact extraction LLM call
@@ -642,6 +686,26 @@ def _get_default_model_for_provider(provider: str) -> str:
    return PROVIDER_DEFAULT_MODELS.get(provider.lower(), DEFAULT_LLM_MODEL)


+def _parse_default_bank_template(raw: str | None) -> dict | None:
+    """
+    Parse HINDSIGHT_API_DEFAULT_BANK_TEMPLATE as JSON.
+
+    The env var holds a BankTemplateManifest (JSON object) applied verbatim to
+    every newly-created bank. Full Pydantic validation is deferred to bank
+    creation time (to avoid pulling API models into config.py), but we fail
+    fast here if the value is not valid JSON or not a JSON object.
+    """
+    if raw is None or raw.strip() == "":
+        return DEFAULT_DEFAULT_BANK_TEMPLATE
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid {ENV_DEFAULT_BANK_TEMPLATE}: expected a JSON object, got invalid JSON: {e}") from e
+    if not isinstance(parsed, dict):
+        raise ValueError(f"Invalid {ENV_DEFAULT_BANK_TEMPLATE}: expected a JSON object, got {type(parsed).__name__}")
+    return parsed
+
+
@dataclass
 class HindsightConfig:
    """Configuration container for Hindsight API."""
@@ -677,6 +741,14 @@ class HindsightConfig:
    # Gemini safety settings (None = use Gemini defaults; list of dicts with category/threshold)
    llm_gemini_safety_settings: list | None

+    # Built-in llama.cpp configuration (for provider=llamacpp)
+    llamacpp_model_path: str | None  # Path to GGUF file (None = auto-download default)
+    llamacpp_gpu_layers: int  # -1 = all layers on GPU, 0 = CPU only
+    llamacpp_context_size: int  # Context window size
+    llamacpp_chat_format: str | None  # Chat template format (None = auto-detect from GGUF)
+    llamacpp_no_grammar: bool  # Disable JSON grammar enforcement (faster, less reliable)
+    llamacpp_extra_args: str | None  # Space-separated extra CLI args for llama.cpp server
+
    # Per-operation LLM configuration (None = use default LLM config)
    retain_llm_provider: str | None
    retain_llm_api_key: str | None
@@ -718,6 +790,8 @@ class HindsightConfig:
    embeddings_cohere_api_key: str | None
    embeddings_cohere_model: str
    embeddings_cohere_base_url: str | None
+    embeddings_openrouter_api_key: str | None
+    embeddings_openrouter_model: str
    embeddings_litellm_api_base: str
    embeddings_litellm_api_key: str | None
    embeddings_litellm_model: str
@@ -725,6 +799,7 @@ class HindsightConfig:
    embeddings_litellm_sdk_model: str
    embeddings_litellm_sdk_api_base: str | None
    embeddings_litellm_sdk_output_dimensions: int | None
+    embeddings_litellm_sdk_encoding_format: str | None
    # Gemini/Vertex AI embeddings
    embeddings_gemini_api_key: str | None
    embeddings_gemini_model: str
@@ -749,6 +824,8 @@ class HindsightConfig:
    reranker_cohere_api_key: str | None
    reranker_cohere_model: str
    reranker_cohere_base_url: str | None
+    reranker_openrouter_api_key: str | None
+    reranker_openrouter_model: str
    reranker_litellm_api_base: str
    reranker_litellm_api_key: str | None
    reranker_litellm_model: str
@@ -759,6 +836,9 @@ class HindsightConfig:
    reranker_zeroentropy_api_key: str | None
    reranker_zeroentropy_model: str
    reranker_zeroentropy_base_url: str | None
+    reranker_siliconflow_api_key: str | None
+    reranker_siliconflow_model: str
+    reranker_siliconflow_base_url: str
    reranker_google_model: str
    reranker_google_project_id: str | None
    reranker_google_service_account_key: str | None
@@ -773,6 +853,9 @@ class HindsightConfig:
    mcp_enabled_tools: list[str] | None  # None = all tools; explicit list = allowlist
    mcp_stateless: bool  # True = stateless HTTP (POST-only); False = stateful (supports GET/SSE)
    enable_bank_config_api: bool
+    # Default bank template (static, server-level only). When set, the manifest is applied
+    # to every newly-created bank, overriding the env/config defaults for any fields it sets.
+    default_bank_template: dict | None

    # Recall
    graph_retriever: str
@@ -780,6 +863,8 @@ class HindsightConfig:
    recall_connection_budget: int
    recall_max_query_tokens: int
    mental_model_refresh_concurrency: int
+    link_expansion_per_entity_limit: int
+    link_expansion_timeout: float

    # Retain settings
    retain_max_completion_tokens: int
@@ -910,6 +995,7 @@ class HindsightConfig:
        "reranker_tei_base_url",
        "reranker_cohere_base_url",
        "reranker_zeroentropy_base_url",
+        "reranker_siliconflow_base_url",
        # Service Account Keys
        "llm_vertexai_service_account_key",
        "embeddings_vertexai_service_account_key",
@@ -1092,6 +1178,14 @@ class HindsightConfig:
            or DEFAULT_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY,
            # Gemini safety settings (JSON-encoded list of {category, threshold} dicts)
            llm_gemini_safety_settings=json.loads(os.getenv(ENV_LLM_GEMINI_SAFETY_SETTINGS, "null")),
+            # Built-in llama.cpp configuration
+            llamacpp_model_path=os.getenv(ENV_LLAMACPP_MODEL_PATH) or None,
+            llamacpp_gpu_layers=int(os.getenv(ENV_LLAMACPP_GPU_LAYERS, str(DEFAULT_LLAMACPP_GPU_LAYERS))),
+            llamacpp_context_size=int(os.getenv(ENV_LLAMACPP_CONTEXT_SIZE, str(DEFAULT_LLAMACPP_CONTEXT_SIZE))),
+            llamacpp_chat_format=os.getenv(ENV_LLAMACPP_CHAT_FORMAT) or DEFAULT_LLAMACPP_CHAT_FORMAT,
+            llamacpp_no_grammar=os.getenv(ENV_LLAMACPP_NO_GRAMMAR, str(DEFAULT_LLAMACPP_NO_GRAMMAR)).lower()
+            in ("true", "1"),
+            llamacpp_extra_args=os.getenv(ENV_LLAMACPP_EXTRA_ARGS) or DEFAULT_LLAMACPP_EXTRA_ARGS,
            # Per-operation LLM config (None = use default)
            retain_llm_provider=os.getenv(ENV_RETAIN_LLM_PROVIDER) or None,
            retain_llm_api_key=os.getenv(ENV_RETAIN_LLM_API_KEY) or None,
@@ -1180,6 +1274,11 @@ class HindsightConfig:
            embeddings_cohere_api_key=os.getenv(ENV_EMBEDDINGS_COHERE_API_KEY) or os.getenv(ENV_COHERE_API_KEY),
            embeddings_cohere_model=os.getenv(ENV_EMBEDDINGS_COHERE_MODEL, DEFAULT_EMBEDDINGS_COHERE_MODEL),
            embeddings_cohere_base_url=os.getenv(ENV_EMBEDDINGS_COHERE_BASE_URL) or None,
+            # OpenRouter embeddings (with fallback to shared OpenRouter key, then LLM key)
+            embeddings_openrouter_api_key=os.getenv(ENV_EMBEDDINGS_OPENROUTER_API_KEY)
+            or os.getenv(ENV_OPENROUTER_API_KEY)
+            or os.getenv(ENV_LLM_API_KEY),
+            embeddings_openrouter_model=os.getenv(ENV_EMBEDDINGS_OPENROUTER_MODEL, DEFAULT_EMBEDDINGS_OPENROUTER_MODEL),
            # LiteLLM embeddings (with backward-compatible fallback to shared config)
            embeddings_litellm_api_base=os.getenv(ENV_EMBEDDINGS_LITELLM_API_BASE)
            or os.getenv(ENV_LITELLM_API_BASE, DEFAULT_LITELLM_API_BASE),
@@ -1194,6 +1293,9 @@ class HindsightConfig:
            embeddings_litellm_sdk_output_dimensions=int(v)
            if (v := os.getenv(ENV_EMBEDDINGS_LITELLM_SDK_OUTPUT_DIMENSIONS))
            else None,
+            embeddings_litellm_sdk_encoding_format=os.getenv(
+                ENV_EMBEDDINGS_LITELLM_SDK_ENCODING_FORMAT, DEFAULT_EMBEDDINGS_LITELLM_SDK_ENCODING_FORMAT
+            ),
            # Gemini/Vertex AI embeddings (with fallback to LLM keys)
            embeddings_gemini_api_key=os.getenv(ENV_EMBEDDINGS_GEMINI_API_KEY) or os.getenv(ENV_LLM_API_KEY),
            embeddings_gemini_model=os.getenv(ENV_EMBEDDINGS_GEMINI_MODEL, DEFAULT_EMBEDDINGS_GEMINI_MODEL),
@@ -1241,6 +1343,11 @@ class HindsightConfig:
            reranker_cohere_api_key=os.getenv(ENV_RERANKER_COHERE_API_KEY) or os.getenv(ENV_COHERE_API_KEY),
            reranker_cohere_model=os.getenv(ENV_RERANKER_COHERE_MODEL, DEFAULT_RERANKER_COHERE_MODEL),
            reranker_cohere_base_url=os.getenv(ENV_RERANKER_COHERE_BASE_URL) or None,
+            # OpenRouter reranker (with fallback to shared OpenRouter key, then LLM key)
+            reranker_openrouter_api_key=os.getenv(ENV_RERANKER_OPENROUTER_API_KEY)
+            or os.getenv(ENV_OPENROUTER_API_KEY)
+            or os.getenv(ENV_LLM_API_KEY),
+            reranker_openrouter_model=os.getenv(ENV_RERANKER_OPENROUTER_MODEL, DEFAULT_RERANKER_OPENROUTER_MODEL),
            # LiteLLM reranker (with backward-compatible fallback to shared config)
            reranker_litellm_api_base=os.getenv(ENV_RERANKER_LITELLM_API_BASE)
            or os.getenv(ENV_LITELLM_API_BASE, DEFAULT_LITELLM_API_BASE),
@@ -1257,6 +1364,12 @@ class HindsightConfig:
            reranker_zeroentropy_api_key=os.getenv(ENV_RERANKER_ZEROENTROPY_API_KEY),
            reranker_zeroentropy_model=os.getenv(ENV_RERANKER_ZEROENTROPY_MODEL, DEFAULT_RERANKER_ZEROENTROPY_MODEL),
            reranker_zeroentropy_base_url=os.getenv(ENV_RERANKER_ZEROENTROPY_BASE_URL) or None,
+            # SiliconFlow reranker (Cohere-compatible /rerank endpoint)
+            reranker_siliconflow_api_key=os.getenv(ENV_RERANKER_SILICONFLOW_API_KEY),
+            reranker_siliconflow_model=os.getenv(ENV_RERANKER_SILICONFLOW_MODEL, DEFAULT_RERANKER_SILICONFLOW_MODEL),
+            reranker_siliconflow_base_url=os.getenv(
+                ENV_RERANKER_SILICONFLOW_BASE_URL, DEFAULT_RERANKER_SILICONFLOW_BASE_URL
+            ),
            # Google Discovery Engine reranker (with fallback to LLM Vertex AI keys)
            reranker_google_model=os.getenv(ENV_RERANKER_GOOGLE_MODEL, DEFAULT_RERANKER_GOOGLE_MODEL),
            reranker_google_project_id=os.getenv(ENV_RERANKER_GOOGLE_PROJECT_ID)
@@ -1276,6 +1389,7 @@ class HindsightConfig:
            mcp_stateless=os.getenv(ENV_MCP_STATELESS, str(DEFAULT_MCP_STATELESS)).lower() == "true",
            enable_bank_config_api=os.getenv(ENV_ENABLE_BANK_CONFIG_API, str(DEFAULT_ENABLE_BANK_CONFIG_API)).lower()
            == "true",
+            default_bank_template=_parse_default_bank_template(os.getenv(ENV_DEFAULT_BANK_TEMPLATE)),
            # Recall
            graph_retriever=os.getenv(ENV_GRAPH_RETRIEVER, DEFAULT_GRAPH_RETRIEVER),
            recall_max_concurrent=int(os.getenv(ENV_RECALL_MAX_CONCURRENT, str(DEFAULT_RECALL_MAX_CONCURRENT))),
@@ -1286,6 +1400,10 @@ class HindsightConfig:
            mental_model_refresh_concurrency=int(
                os.getenv(ENV_MENTAL_MODEL_REFRESH_CONCURRENCY, str(DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY))
            ),
+            link_expansion_per_entity_limit=int(
+                os.getenv(ENV_LINK_EXPANSION_PER_ENTITY_LIMIT, str(DEFAULT_LINK_EXPANSION_PER_ENTITY_LIMIT))
+            ),
+            link_expansion_timeout=float(os.getenv(ENV_LINK_EXPANSION_TIMEOUT, str(DEFAULT_LINK_EXPANSION_TIMEOUT))),
            # Optimization flags
            skip_llm_verification=os.getenv(ENV_SKIP_LLM_VERIFICATION, "false").lower() == "true",
            lazy_reranker=os.getenv(ENV_LAZY_RERANKER, "false").lower() == "true",
@@ -239,6 +239,15 @@ class ConfigResolver:
                logger.warning(f"Failed to check permissions for bank {bank_id}: {e}")
                # Continue without permission check (fail open for backward compatibility)

+        # Validate entity_labels structure
+        if "entity_labels" in normalized_updates and normalized_updates["entity_labels"] is not None:
+            from .engine.retain.entity_labels import parse_entity_labels
+
+            try:
+                parse_entity_labels(normalized_updates["entity_labels"])
+            except Exception as e:
+                raise ValueError(f"Invalid entity_labels format: {e}")
+
        # Validate retain_strategies: reject empty string keys
        if "retain_strategies" in normalized_updates and normalized_updates["retain_strategies"]:
            empty_keys = [k for k in normalized_updates["retain_strategies"] if not str(k).strip()]
@@ -30,6 +30,8 @@ from ..config import (
    DEFAULT_RERANKER_LOCAL_MODEL,
    DEFAULT_RERANKER_LOCAL_TRUST_REMOTE_CODE,
    DEFAULT_RERANKER_PROVIDER,
+    DEFAULT_RERANKER_SILICONFLOW_BASE_URL,
+    DEFAULT_RERANKER_SILICONFLOW_MODEL,
    DEFAULT_RERANKER_TEI_BATCH_SIZE,
    DEFAULT_RERANKER_TEI_MAX_CONCURRENT,
    DEFAULT_RERANKER_ZEROENTROPY_MODEL,
@@ -44,6 +46,7 @@ from ..config import (
    ENV_RERANKER_LOCAL_MODEL,
    ENV_RERANKER_LOCAL_TRUST_REMOTE_CODE,
    ENV_RERANKER_PROVIDER,
+    ENV_RERANKER_SILICONFLOW_API_KEY,
    ENV_RERANKER_TEI_BATCH_SIZE,
    ENV_RERANKER_TEI_MAX_CONCURRENT,
    ENV_RERANKER_TEI_URL,
@@ -518,6 +521,84 @@ class RemoteTEICrossEncoder(CrossEncoderModel):
        return await self._predict_async(pairs)


+class _CohereCompatibleRerankClient:
+    """
+    Internal HTTP client for Cohere-compatible /rerank endpoints.
+
+    Shared by all providers that speak the Cohere rerank wire format —
+    {model, query, documents[, top_n]} request and
+    {results: [{index, relevance_score}, ...]} response. This covers
+    SiliconFlow, ZeroEntropy, Jina, Voyage, BGE self-hosted, and Cohere
+    itself when reached via a custom base_url (e.g. Azure AI Foundry).
+
+    Not a CrossEncoderModel — providers compose it and expose their own
+    provider_name / initialization logging.
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        model: str,
+        rerank_url: str,
+        timeout: float = 60.0,
+        include_top_n: bool = True,
+    ):
+        self.api_key = api_key
+        self.model = model
+        self.rerank_url = rerank_url
+        self.timeout = timeout
+        self.include_top_n = include_top_n
+        self._async_client: httpx.AsyncClient | None = None
+
+    async def initialize(self) -> None:
+        if self._async_client is not None:
+            return
+        self._async_client = httpx.AsyncClient(
+            timeout=self.timeout,
+            headers={
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+            },
+        )
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+        if self._async_client is None:
+            raise RuntimeError("Reranker not initialized. Call initialize() first.")
+
+        if not pairs:
+            return []
+
+        query_groups: dict[str, list[tuple[int, str]]] = {}
+        for idx, (query, text) in enumerate(pairs):
+            query_groups.setdefault(query, []).append((idx, text))
+
+        all_scores = [0.0] * len(pairs)
+
+        for query, indexed_texts in query_groups.items():
+            texts = [text for _, text in indexed_texts]
+            indices = [idx for idx, _ in indexed_texts]
+
+            body: dict[str, object] = {
+                "model": self.model,
+                "query": query,
+                "documents": texts,
+                "return_documents": False,
+            }
+            if self.include_top_n:
+                body["top_n"] = len(texts)
+
+            response = await self._async_client.post(self.rerank_url, json=body)
+            response.raise_for_status()
+            result = response.json()
+
+            for item in result.get("results", []):
+                original_idx = item["index"]
+                score = item["relevance_score"]
+                all_scores[indices[original_idx]] = score
+
+        return all_scores
+
+
 class CohereCrossEncoder(CrossEncoderModel):
    """
    Cohere cross-encoder implementation using the Cohere Rerank API.
@@ -546,7 +627,20 @@ class CohereCrossEncoder(CrossEncoderModel):
        self.base_url = base_url
        self.timeout = timeout
        self._client = None
-        self._httpx_client: httpx.Client | None = None
+        # Used when base_url is set (Azure AI Foundry and other Cohere-compatible hosts).
+        # Azure endpoints already include the full invoke path, so rerank_url == base_url
+        # and top_n is omitted to match the existing Azure contract.
+        self._http_client: _CohereCompatibleRerankClient | None = (
+            _CohereCompatibleRerankClient(
+                api_key=api_key,
+                model=model,
+                rerank_url=base_url,
+                timeout=timeout,
+                include_top_n=False,
+            )
+            if base_url
+            else None
+        )

    @property
    def provider_name(self) -> str:
@@ -554,23 +648,15 @@ class CohereCrossEncoder(CrossEncoderModel):

    async def initialize(self) -> None:
        """Initialize the Cohere client."""
-        if self._client is not None or self._httpx_client is not None:
+        if self._client is not None or (self._http_client and self._http_client._async_client):
            return

        base_url_msg = f" at {self.base_url}" if self.base_url else ""
        logger.info(f"Reranker: initializing Cohere provider with model {self.model}{base_url_msg}")

-        if self.base_url:
-            # For custom endpoints (Azure AI Foundry), use httpx directly to avoid SDK path appending
-            # Azure endpoints already include the full path (e.g., /models/.../invoke)
-            self._httpx_client = httpx.Client(
-                timeout=self.timeout,
-                headers={
-                    "Authorization": f"Bearer {self.api_key}",
-                    "Content-Type": "application/json",
-                },
-            )
-            logger.info("Reranker: Cohere provider initialized (using httpx for custom endpoint)")
+        if self._http_client is not None:
+            await self._http_client.initialize()
+            logger.info("Reranker: Cohere provider initialized (Cohere-compatible HTTP endpoint)")
        else:
            # For native Cohere API, use the official SDK
            try:
@@ -591,25 +677,24 @@ class CohereCrossEncoder(CrossEncoderModel):
        Returns:
            List of relevance scores
        """
-        if self._client is None and self._httpx_client is None:
+        if self._client is None and self._http_client is None:
            raise RuntimeError("Reranker not initialized. Call initialize() first.")

        if not pairs:
            return []

-        # Run sync Cohere API calls in thread pool
-        loop = asyncio.get_event_loop()
-        return await loop.run_in_executor(None, self._predict_sync, pairs)
+        if self._http_client is not None:
+            return await self._http_client.predict(pairs)

-    def _predict_sync(self, pairs: list[tuple[str, str]]) -> list[float]:
-        """Synchronous predict implementation for Cohere API."""
-        # Group pairs by query for efficient batching
-        # Cohere rerank expects one query with multiple documents
+        # Run sync Cohere SDK calls in thread pool
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, self._predict_sync_sdk, pairs)
+
+    def _predict_sync_sdk(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """Synchronous predict using the native Cohere SDK."""
        query_groups: dict[str, list[tuple[int, str]]] = {}
        for idx, (query, text) in enumerate(pairs):
-            if query not in query_groups:
-                query_groups[query] = []
-            query_groups[query].append((idx, text))
+            query_groups.setdefault(query, []).append((idx, text))

        all_scores = [0.0] * len(pairs)

@@ -617,40 +702,17 @@ class CohereCrossEncoder(CrossEncoderModel):
            texts = [text for _, text in indexed_texts]
            indices = [idx for idx, _ in indexed_texts]

-            if self._httpx_client:
-                # Direct HTTP request for custom endpoints (Azure AI Foundry)
-                response = self._httpx_client.post(
-                    self.base_url,
-                    json={
-                        "model": self.model,
-                        "query": query,
-                        "documents": texts,
-                        "return_documents": False,
-                    },
-                )
-                response.raise_for_status()
-                result = response.json()
+            response = self._client.rerank(
+                query=query,
+                documents=texts,
+                model=self.model,
+                return_documents=False,
+            )

-                # Map scores back to original positions
-                # Azure Cohere response format: {"results": [{"index": 0, "relevance_score": 0.9}, ...]}
-                for item in result.get("results", []):
-                    original_idx = item["index"]
-                    score = item["relevance_score"]
-                    all_scores[indices[original_idx]] = score
-            else:
-                # Native Cohere SDK for standard API
-                response = self._client.rerank(
-                    query=query,
-                    documents=texts,
-                    model=self.model,
-                    return_documents=False,
-                )
-
-                # Map scores back to original positions
-                for result in response.results:
-                    original_idx = result.index
-                    score = result.relevance_score
-                    all_scores[indices[original_idx]] = score
+            for result in response.results:
+                original_idx = result.index
+                score = result.relevance_score
+                all_scores[indices[original_idx]] = score

        return all_scores

@@ -673,89 +735,70 @@ class ZeroEntropyCrossEncoder(CrossEncoderModel):
        base_url: str | None = None,
        timeout: float = 60.0,
    ):
-        """
-        Initialize ZeroEntropy cross-encoder client.
-
-        Args:
-            api_key: ZeroEntropy API key
-            model: ZeroEntropy rerank model name (default: zerank-2)
-            base_url: Custom base URL for ZeroEntropy-compatible API (e.g., mock server or proxy)
-            timeout: Request timeout in seconds (default: 60.0)
-        """
-        self.api_key = api_key
        self.model = model
        self.base_url = base_url.rstrip("/") if base_url else self.DEFAULT_BASE_URL
-        self.rerank_url = f"{self.base_url}{self.RERANK_PATH}"
-        self.timeout = timeout
-        self._async_client: httpx.AsyncClient | None = None
+        self._client = _CohereCompatibleRerankClient(
+            api_key=api_key,
+            model=model,
+            rerank_url=f"{self.base_url}{self.RERANK_PATH}",
+            timeout=timeout,
+        )

    @property
    def provider_name(self) -> str:
        return "zeroentropy"

    async def initialize(self) -> None:
-        """Initialize the async HTTP client."""
-        if self._async_client is not None:
+        if self._client._async_client is not None:
            return
-
        logger.info(f"Reranker: initializing ZeroEntropy provider with model {self.model}")
-        self._async_client = httpx.AsyncClient(
-            timeout=self.timeout,
-            headers={
-                "Authorization": f"Bearer {self.api_key}",
-                "Content-Type": "application/json",
-            },
-        )
+        await self._client.initialize()
        logger.info("Reranker: ZeroEntropy provider initialized")

    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
-        """
-        Score query-document pairs using the ZeroEntropy Rerank API.
+        return await self._client.predict(pairs)

-        Args:
-            pairs: List of (query, document) tuples to score

-        Returns:
-            List of relevance scores
-        """
-        if self._async_client is None:
-            raise RuntimeError("Reranker not initialized. Call initialize() first.")
+class SiliconFlowCrossEncoder(CrossEncoderModel):
+    """
+    SiliconFlow cross-encoder implementation.

-        if not pairs:
-            return []
+    SiliconFlow (https://siliconflow.cn) exposes a Cohere-compatible /rerank
+    endpoint. Shares the HTTP client with ZeroEntropy/Cohere-custom-endpoint
+    via _CohereCompatibleRerankClient.
+    """

-        # Group pairs by query for efficient batching
-        query_groups: dict[str, list[tuple[int, str]]] = {}
-        for idx, (query, text) in enumerate(pairs):
-            if query not in query_groups:
-                query_groups[query] = []
-            query_groups[query].append((idx, text))
+    RERANK_PATH = "/rerank"

-        all_scores = [0.0] * len(pairs)
+    def __init__(
+        self,
+        api_key: str,
+        model: str = DEFAULT_RERANKER_SILICONFLOW_MODEL,
+        base_url: str = DEFAULT_RERANKER_SILICONFLOW_BASE_URL,
+        timeout: float = 60.0,
+    ):
+        self.model = model
+        self.base_url = base_url.rstrip("/")
+        self._client = _CohereCompatibleRerankClient(
+            api_key=api_key,
+            model=model,
+            rerank_url=f"{self.base_url}{self.RERANK_PATH}",
+            timeout=timeout,
+        )

-        for query, indexed_texts in query_groups.items():
-            texts = [text for _, text in indexed_texts]
-            indices = [idx for idx, _ in indexed_texts]
+    @property
+    def provider_name(self) -> str:
+        return "siliconflow"

-            response = await self._async_client.post(
-                self.rerank_url,
-                json={
-                    "model": self.model,
-                    "query": query,
-                    "documents": texts,
-                    "top_n": len(texts),
-                },
-            )
-            response.raise_for_status()
-            result = response.json()
+    async def initialize(self) -> None:
+        if self._client._async_client is not None:
+            return
+        logger.info(f"Reranker: initializing SiliconFlow provider at {self.base_url} with model {self.model}")
+        await self._client.initialize()
+        logger.info("Reranker: SiliconFlow provider initialized")

-            # Map scores back to original positions
-            for item in result.get("results", []):
-                original_idx = item["index"]
-                score = item["relevance_score"]
-                all_scores[indices[original_idx]] = score
-
-        return all_scores
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+        return await self._client.predict(pairs)


 class RRFPassthroughCrossEncoder(CrossEncoderModel):
@@ -1207,14 +1250,31 @@ class JinaMLXCrossEncoder(CrossEncoderModel):
        if self._reranker is not None:
            return

+        # Pre-warm transformers.AutoTokenizer to fully populate the transformers
+        # namespace before mlx_lm imports it. transformers 5.x uses _LazyModule,
+        # which has an unguarded window where `from transformers import AutoTokenizer`
+        # raises ImportError if another thread is concurrently initializing the
+        # namespace (e.g. embeddings init in an executor thread).
+        # See: https://github.com/vectorize-io/hindsight/issues/994
+        import transformers
+
+        _ = transformers.AutoTokenizer
+
        try:
            import mlx.core  # noqa: F401
            import mlx_lm  # noqa: F401
-        except ImportError:
+        except ImportError as exc:
+            # Only swallow "package not installed" errors. Anything else (e.g. a
+            # transitive import failure inside mlx_lm) must surface verbatim so
+            # the real cause is debuggable instead of being masked by a generic
+            # "install mlx" message.
+            msg = str(exc)
+            if "mlx" not in msg and "mlx_lm" not in msg:
+                raise
            raise ImportError(
                "mlx and mlx-lm are required for JinaMLXCrossEncoder. "
                "Install with: pip install mlx>=0.31.0 mlx-lm>=0.31.1 safetensors>=0.6.2"
-            )
+            ) from exc

        loop = asyncio.get_event_loop()
        await loop.run_in_executor(None, self._load_model)
@@ -1468,6 +1528,18 @@ def create_cross_encoder_from_env() -> CrossEncoderModel:
            model=config.reranker_cohere_model,
            base_url=config.reranker_cohere_base_url,
        )
+    elif provider == "openrouter":
+        api_key = config.reranker_openrouter_api_key
+        if not api_key:
+            raise ValueError(
+                "HINDSIGHT_API_RERANKER_OPENROUTER_API_KEY, HINDSIGHT_API_OPENROUTER_API_KEY, "
+                f"or HINDSIGHT_API_LLM_API_KEY is required when {ENV_RERANKER_PROVIDER} is 'openrouter'"
+            )
+        return CohereCrossEncoder(
+            api_key=api_key,
+            model=config.reranker_openrouter_model,
+            base_url="https://openrouter.ai/api/v1/rerank",
+        )
    elif provider == "flashrank":
        model = os.environ.get(ENV_RERANKER_FLASHRANK_MODEL, DEFAULT_RERANKER_FLASHRANK_MODEL)
        cache_dir = os.environ.get(ENV_RERANKER_FLASHRANK_CACHE_DIR, DEFAULT_RERANKER_FLASHRANK_CACHE_DIR)
@@ -1501,6 +1573,17 @@ def create_cross_encoder_from_env() -> CrossEncoderModel:
            api_key=api_key,
            model=config.reranker_zeroentropy_model,
        )
+    elif provider == "siliconflow":
+        api_key = config.reranker_siliconflow_api_key
+        if not api_key:
+            raise ValueError(
+                f"{ENV_RERANKER_SILICONFLOW_API_KEY} is required when {ENV_RERANKER_PROVIDER} is 'siliconflow'"
+            )
+        return SiliconFlowCrossEncoder(
+            api_key=api_key,
+            model=config.reranker_siliconflow_model,
+            base_url=config.reranker_siliconflow_base_url,
+        )
    elif provider == "google":
        project_id = config.reranker_google_project_id
        if not project_id:
@@ -1519,5 +1602,5 @@ def create_cross_encoder_from_env() -> CrossEncoderModel:
        return JinaMLXCrossEncoder()
    else:
        raise ValueError(
-            f"Unknown reranker provider: {provider}. Supported: 'local', 'tei', 'cohere', 'zeroentropy', 'google', 'flashrank', 'litellm', 'litellm-sdk', 'rrf', 'jina-mlx'"
+            f"Unknown reranker provider: {provider}. Supported: 'local', 'tei', 'cohere', 'zeroentropy', 'siliconflow', 'google', 'flashrank', 'litellm', 'litellm-sdk', 'rrf', 'jina-mlx'"
        )
@@ -757,6 +757,7 @@ class LiteLLMSDKEmbeddings(Embeddings):
        output_dimensions: int | None = None,
        batch_size: int = 100,
        timeout: float = 60.0,
+        encoding_format: str | None = "float",
    ):
        """
        Initialize LiteLLM SDK embeddings client.
@@ -768,6 +769,8 @@ class LiteLLMSDKEmbeddings(Embeddings):
            output_dimensions: Optional output embedding dimensions (provider-dependent)
            batch_size: Maximum batch size for embedding requests (default: 100)
            timeout: Request timeout in seconds (default: 60.0)
+            encoding_format: Encoding format for embeddings (default: "float").
+                Set to None or empty string to omit (needed for Voyage AI, Gemini).
        """
        self.api_key = api_key
        self.model = model
@@ -775,6 +778,7 @@ class LiteLLMSDKEmbeddings(Embeddings):
        self.output_dimensions = output_dimensions
        self.batch_size = batch_size
        self.timeout = timeout
+        self.encoding_format = encoding_format or None
        self._litellm = None  # Will be set during initialization
        self._dimension: int | None = None

@@ -810,8 +814,9 @@ class LiteLLMSDKEmbeddings(Embeddings):
                "model": self.model,
                "input": ["test"],
                "api_key": self.api_key,
-                "encoding_format": "float",
            }
+            if self.encoding_format:
+                embed_kwargs["encoding_format"] = self.encoding_format
            if self.api_base:
                embed_kwargs["api_base"] = self.api_base
            if self.output_dimensions is not None:
@@ -859,8 +864,9 @@ class LiteLLMSDKEmbeddings(Embeddings):
                    "model": self.model,
                    "input": batch,
                    "api_key": self.api_key,
-                    "encoding_format": "float",
                }
+                if self.encoding_format:
+                    embed_kwargs["encoding_format"] = self.encoding_format
                if self.api_base:
                    embed_kwargs["api_base"] = self.api_base
                if self.output_dimensions is not None:
@@ -1095,6 +1101,18 @@ def create_embeddings_from_env() -> Embeddings:
        model = os.environ.get(ENV_EMBEDDINGS_OPENAI_MODEL, DEFAULT_EMBEDDINGS_OPENAI_MODEL)
        base_url = os.environ.get(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None
        return OpenAIEmbeddings(api_key=api_key, model=model, base_url=base_url)
+    elif provider == "openrouter":
+        api_key = config.embeddings_openrouter_api_key
+        if not api_key:
+            raise ValueError(
+                "HINDSIGHT_API_EMBEDDINGS_OPENROUTER_API_KEY, HINDSIGHT_API_OPENROUTER_API_KEY, "
+                f"or {ENV_LLM_API_KEY} is required when {ENV_EMBEDDINGS_PROVIDER} is 'openrouter'"
+            )
+        return OpenAIEmbeddings(
+            api_key=api_key,
+            model=config.embeddings_openrouter_model,
+            base_url="https://openrouter.ai/api/v1",
+        )
    elif provider == "cohere":
        api_key = config.embeddings_cohere_api_key
        if not api_key:
@@ -1121,6 +1139,7 @@ def create_embeddings_from_env() -> Embeddings:
            model=config.embeddings_litellm_sdk_model,
            api_base=config.embeddings_litellm_sdk_api_base,
            output_dimensions=config.embeddings_litellm_sdk_output_dimensions,
+            encoding_format=config.embeddings_litellm_sdk_encoding_format,
        )
    elif provider == "google":
        vertexai_project_id = config.embeddings_vertexai_project_id
@@ -122,6 +122,7 @@ _PROVIDERS_WITHOUT_API_KEY = frozenset(
    {
        "ollama",
        "lmstudio",
+        "llamacpp",
        "openai-codex",
        "claude-code",
        "mock",
@@ -178,6 +179,7 @@ def create_llm_provider(
        CodexLLM,
        GeminiLLM,
        LiteLLMLLM,
+        LlamaCppLLM,
        MockLLM,
        NoneLLM,
        OpenAICompatibleLLM,
@@ -263,7 +265,25 @@ def create_llm_provider(
            reasoning_effort=reasoning_effort,
        )

-    elif provider_lower in ("openai", "groq", "ollama", "lmstudio", "minimax", "volcano"):
+    elif provider_lower == "llamacpp":
+        from ..config import get_config
+
+        config = get_config()
+        return LlamaCppLLM(
+            provider=provider,
+            api_key=api_key,
+            base_url=base_url,
+            model=model,
+            reasoning_effort=reasoning_effort,
+            model_path=config.llamacpp_model_path,
+            gpu_layers=config.llamacpp_gpu_layers,
+            context_size=config.llamacpp_context_size,
+            chat_format=config.llamacpp_chat_format,
+            no_grammar=config.llamacpp_no_grammar,
+            extra_args=config.llamacpp_extra_args,
+        )
+
+    elif provider_lower in ("openai", "groq", "ollama", "lmstudio", "minimax", "volcano", "openrouter"):
        return OpenAICompatibleLLM(
            provider=provider,
            api_key=api_key,
@@ -333,6 +353,7 @@ class LLMProvider:
            "gemini",
            "anthropic",
            "lmstudio",
+            "llamacpp",
            "vertexai",
            "openai-codex",
            "claude-code",
@@ -342,6 +363,7 @@ class LLMProvider:
            "litellm",
            "bedrock",
            "volcano",
+            "openrouter",
        ]
        if self.provider not in valid_providers:
            raise ValueError(f"Invalid LLM provider: {self.provider}. Must be one of: {', '.join(valid_providers)}")
@@ -356,6 +378,8 @@ class LLMProvider:
                self.base_url = "http://localhost:1234/v1"
            elif self.provider == "minimax":
                self.base_url = "https://api.minimax.io/v1"
+            elif self.provider == "openrouter":
+                self.base_url = "https://openrouter.ai/api/v1"

        # Prepare Vertex AI config (if applicable)
        vertexai_project_id = None
@@ -512,6 +536,15 @@ class LLMProvider:
            OutputTooLongError: If output exceeds token limits.
            Exception: Re-raises API errors after retries exhausted.
        """
+        # Stage breadcrumb so the worker log shows which LLM call a task is
+        # currently inside; the stage_age field then reveals long JSON-schema
+        # retry loops (e.g. a small model that can't satisfy strict_schema).
+        # No-op outside a worker context.
+        from ..worker.stage import set_stage
+
+        structured = "+structured" if response_format is not None else ""
+        set_stage(f"llm.{self.provider}.{scope}{structured}")
+
        async with _global_llm_semaphore:
            # Delegate to provider implementation
            result = await self._provider_impl.call(
@@ -568,6 +601,10 @@ class LLMProvider:
        Returns:
            LLMToolCallResult with content and/or tool_calls.
        """
+        from ..worker.stage import set_stage
+
+        set_stage(f"llm.{self.provider}.{scope}+tools")
+
        async with _global_llm_semaphore:
            # Delegate to provider implementation
            result = await self._provider_impl.call_with_tools(
@@ -711,8 +748,9 @@ class LLMProvider:
        return ConfiguredLLMProvider(self, config.llm_gemini_safety_settings)

    async def cleanup(self) -> None:
-        """Clean up resources."""
-        pass
+        """Clean up resources (e.g. stop llamacpp subprocess)."""
+        if self._provider_impl:
+            await self._provider_impl.cleanup()

    @classmethod
    def from_env(cls) -> "LLMProvider":
@@ -29,6 +29,7 @@ from ..metrics import get_metrics_collector
 from ..tracing import create_operation_span
 from ..utils import mask_network_location
 from ..worker.exceptions import RetryTaskAt
+from ..worker.stage import set_stage
 from .audit import AuditLogger, audit_context
 from .db_budget import budgeted_operation
 from .operation_metadata import (
@@ -1090,6 +1091,9 @@ class MemoryEngine(MemoryEngineInterface):
            self._audit_logger, task_type or "unknown", "system", bank_id, request=task_dict
        ) as audit_entry:
            try:
+                # Stage breadcrumb for the worker poller's WORKER_TASK log line.
+                # No-op outside a worker context.
+                set_stage(f"task.{task_type}")
                if task_type == "batch_retain":
                    await self._handle_batch_retain(task_dict)
                elif task_type == "file_convert_retain":
@@ -1140,6 +1144,26 @@ class MemoryEngine(MemoryEngineInterface):
                    logger.error(f"Not retrying task {task_type} (non-retryable), marking as failed")
                    if operation_id:
                        await self._mark_operation_failed(operation_id, str(e), error_traceback)
+                elif isinstance(e, asyncpg.exceptions.IntegrityConstraintViolationError):
+                    # Non-retryable: deterministic Postgres integrity violations
+                    # (UniqueViolationError, ForeignKeyViolationError, CheckViolationError,
+                    # NotNullViolationError, ExclusionViolationError) will never succeed on
+                    # retry — the offending row state is already committed. Retrying just
+                    # burns worker capacity. See vectorize-io/hindsight#980.
+                    logger.error(
+                        f"Not retrying task {task_type} (integrity violation, deterministic): {type(e).__name__}"
+                    )
+                    if task_type == "consolidation" and operation_id:
+                        await self._fire_consolidation_webhook(
+                            bank_id=task_dict.get("bank_id", ""),
+                            operation_id=operation_id,
+                            status="failed",
+                            result=None,
+                            error_message=str(e),
+                            schema=schema,
+                        )
+                    if operation_id:
+                        await self._mark_operation_failed(operation_id, str(e), error_traceback)
                else:
                    if task_type == "consolidation" and operation_id:
                        # Fire failure webhook (non-transactional — operation not yet marked failed;
@@ -1923,6 +1947,18 @@ class MemoryEngine(MemoryEngineInterface):

        self._initialized = False

+        # Clean up LLM providers (e.g. stop llamacpp subprocess)
+        for llm_config in (
+            self._llm_config,
+            self._retain_llm_config,
+            self._reflect_llm_config,
+            self._consolidation_llm_config,
+        ):
+            try:
+                await llm_config.cleanup()
+            except Exception as e:
+                logger.warning(f"Error cleaning up LLM provider: {e}")
+
        # Stop pg0 if we started it
        if self._pg0 is not None:
            logger.info("Stopping pg0...")
@@ -2146,6 +2182,11 @@ class MemoryEngine(MemoryEngineInterface):
                f"Each content item in a batch must have a unique document_id to avoid race conditions."
            )

+        # Validate update_mode=append requires document_id
+        for item in contents:
+            if item.get("update_mode") == "append" and not item.get("document_id"):
+                raise ValueError("update_mode='append' requires a document_id")
+
        # Auto-chunk large batches by token count to avoid timeouts and memory issues
        # Calculate total token count
        total_tokens = sum(count_tokens(item.get("content", "")) for item in contents)
@@ -2726,8 +2767,11 @@ class MemoryEngine(MemoryEngineInterface):
        pool = await self._get_pool()
        recall_start = time.time()

-        # Buffer logs for clean output in concurrent scenarios
-        recall_id = f"{bank_id[:8]}-{int(time.time() * 1000) % 100000}"
+        # Buffer logs for clean output in concurrent scenarios.
+        # Include a uuid suffix so two recalls on the same bank within the
+        # same millisecond don't collide on the budgeted_operation key
+        # (`recall-{recall_id}`), which would raise "Operation ... already exists".
+        recall_id = f"{bank_id[:8]}-{int(time.time() * 1000) % 100000}-{uuid.uuid4().hex[:6]}"
        log_buffer = []
        tags_info = f", tags={tags}, tags_match={tags_match}" if tags else ""
        log_buffer.append(
@@ -2748,7 +2792,8 @@ class MemoryEngine(MemoryEngineInterface):
            embedding_span.set_attribute("hindsight.query", query[:100])

            try:
-                query_embedding = embedding_utils.generate_embedding(self.embeddings, query)
+                query_embeddings = await embedding_utils.generate_embeddings_batch(self.embeddings, [query])
+                query_embedding = query_embeddings[0]
                step_duration = time.time() - step_start
                log_buffer.append(f"  [1] Generate query embedding: {step_duration:.3f}s")
            finally:
@@ -3069,8 +3114,13 @@ class MemoryEngine(MemoryEngineInterface):

            # Step 4.5: Combine cross-encoder score with retrieval signals via multiplicative boosts.
            # See apply_combined_scoring for the full rationale and formula.
+            # is_passthrough_reranker tells the scoring code to seed CE scores
+            # from RRF rank — only meaningful when the configured reranker is
+            # the slim/passthrough one that returns a constant score per pair.
            if scored_results:
-                apply_combined_scoring(scored_results, now=utcnow())
+                ce = reranker_instance.cross_encoder
+                is_passthrough = ce is not None and ce.provider_name == "rrf"
+                apply_combined_scoring(scored_results, now=utcnow(), is_passthrough_reranker=is_passthrough)
                scored_results.sort(key=lambda x: x.weight, reverse=True)
                log_buffer.append("  [4.6] Combined scoring: ce * recency_boost(0.2) * temporal_boost(0.2)")

@@ -3792,7 +3842,14 @@ class MemoryEngine(MemoryEngineInterface):

        Returns:
            Dictionary with deletion result
+
+        Raises:
+            ValueError: If unit_id is not a valid UUID
        """
+        try:
+            unit_uuid = uuid.UUID(unit_id)
+        except ValueError:
+            raise ValueError(f"Invalid unit_id: '{unit_id}' is not a valid UUID")
        await self._authenticate_tenant(request_context)
        pool = await self._get_pool()
        invalidated_obs = 0
@@ -3802,7 +3859,7 @@ class MemoryEngine(MemoryEngineInterface):
                # Get bank_id and fact_type before deletion
                row = await conn.fetchrow(
                    f"SELECT bank_id, fact_type FROM {fq_table('memory_units')} WHERE id = $1",
-                    unit_id,
+                    str(unit_uuid),
                )
                bank_id = row["bank_id"] if row else None
                fact_type = row["fact_type"] if row else None
@@ -4697,7 +4754,14 @@ class MemoryEngine(MemoryEngineInterface):

        Returns:
            Dict with memory unit data or None if not found
+
+        Raises:
+            ValueError: If memory_id is not a valid UUID
        """
+        try:
+            memory_uuid = uuid.UUID(memory_id)
+        except ValueError:
+            raise ValueError(f"Invalid memory_id: '{memory_id}' is not a valid UUID")
        await self._authenticate_tenant(request_context)
        if self._operation_validator:
            from hindsight_api.extensions import BankReadContext
@@ -4715,7 +4779,7 @@ class MemoryEngine(MemoryEngineInterface):
                FROM {fq_table("memory_units")}
                WHERE id = $1 AND bank_id = $2
                """,
-                memory_id,
+                str(memory_uuid),
                bank_id,
            )

@@ -5122,7 +5186,13 @@ class MemoryEngine(MemoryEngineInterface):
            ctx = BankReadContext(bank_id=bank_id, operation="get_bank_profile", request_context=request_context)
            await self._validate_operation(self._operation_validator.validate_bank_read(ctx))
        pool = await self._get_pool()
-        profile = await bank_utils.get_bank_profile(pool, bank_id)
+        profile, created = await bank_utils.get_or_create_bank_profile(pool, bank_id)
+
+        # Apply HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to freshly-created banks. Done
+        # before reading the resolved config below so the template's overrides
+        # (e.g. reflect_mission, dispositions) are visible on this very call.
+        if created:
+            await self._apply_default_bank_template(bank_id, request_context)

        # reflect_mission and disposition in config take precedence over the legacy DB columns
        config_dict = await self._config_resolver.get_bank_config(bank_id, request_context)
@@ -5147,6 +5217,62 @@ class MemoryEngine(MemoryEngineInterface):
            "mission": mission,
        }

+    async def _apply_default_bank_template(
+        self,
+        bank_id: str,
+        request_context: "RequestContext",
+    ) -> None:
+        """Apply HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to a freshly-created bank.
+
+        No-op if the env var is unset. A malformed default template is logged
+        and swallowed here rather than raised, so a bad server-level setting
+        cannot wedge bank creation across all callers. Misconfiguration is
+        still surfaced loudly via `logger.error`.
+        """
+        from ..config import get_config
+
+        template_dict = get_config().default_bank_template
+        if not template_dict:
+            return
+
+        # Lazy import to avoid a cycle (http.py imports memory_engine).
+        from pydantic import ValidationError
+
+        from hindsight_api.api.http import (
+            BankTemplateManifest,
+            apply_bank_template_manifest,
+            validate_bank_template,
+        )
+
+        try:
+            manifest = BankTemplateManifest.model_validate(template_dict)
+        except ValidationError as e:
+            errors = [f"{'.'.join(str(loc) for loc in err['loc'])}: {err['msg']}" for err in e.errors()]
+            logger.error(
+                "HINDSIGHT_API_DEFAULT_BANK_TEMPLATE failed schema validation "
+                f"and will be ignored for bank '{bank_id}': {'; '.join(errors)}"
+            )
+            return
+
+        semantic_errors = validate_bank_template(manifest)
+        if semantic_errors:
+            logger.error(
+                "HINDSIGHT_API_DEFAULT_BANK_TEMPLATE failed semantic validation "
+                f"and will be ignored for bank '{bank_id}': {'; '.join(semantic_errors)}"
+            )
+            return
+
+        try:
+            await apply_bank_template_manifest(
+                memory=self,
+                bank_id=bank_id,
+                manifest=manifest,
+                request_context=request_context,
+            )
+            logger.info(f"Applied HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to newly-created bank '{bank_id}'")
+        except Exception as e:
+            logger.error(f"Failed to apply HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to bank '{bank_id}': {e}")
+
    async def update_bank_disposition(
        self,
        bank_id: str,
@@ -6497,6 +6623,7 @@ class MemoryEngine(MemoryEngineInterface):

        Returns None if the mental model is not found.
        Returns a list of history entries (most recent first), each with previous_content and changed_at.
+
        """
        await self._authenticate_tenant(request_context)
        pool = await self._get_pool()
@@ -7757,7 +7884,9 @@ class MemoryEngine(MemoryEngineInterface):

        # Ensure the bank row exists before inserting async_operations (which now has a FK).
        # Banks are created lazily on first retain, but the FK requires the row to exist first.
-        await bank_utils.get_bank_profile(pool, bank_id)
+        _, created = await bank_utils.get_or_create_bank_profile(pool, bank_id)
+        if created:
+            await self._apply_default_bank_template(bank_id, request_context)

        # Create typed metadata for parent operation
        parent_metadata = BatchRetainParentMetadata(
@@ -9,6 +9,7 @@ from .claude_code_llm import ClaudeCodeLLM
 from .codex_llm import CodexLLM
 from .gemini_llm import GeminiLLM
 from .litellm_llm import LiteLLMLLM
+from .llamacpp_llm import LlamaCppLLM
 from .mock_llm import MockLLM
 from .none_llm import NoneLLM
 from .openai_compatible_llm import OpenAICompatibleLLM
@@ -18,6 +19,7 @@ __all__ = [
    "ClaudeCodeLLM",
    "CodexLLM",
    "GeminiLLM",
+    "LlamaCppLLM",
    "LiteLLMLLM",
    "MockLLM",
    "NoneLLM",
@@ -23,6 +23,7 @@ from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError
 from hindsight_api.engine.llm_wrapper import parse_llm_json
 from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
 from hindsight_api.metrics import get_metrics_collector
+from hindsight_api.worker.stage import set_stage

 logger = logging.getLogger(__name__)

@@ -242,6 +243,8 @@ class GeminiLLM(LLMInterface):
        last_exception = None

        for attempt in range(max_retries + 1):
+            if attempt > 0:
+                set_stage(f"llm.gemini.{scope}.attempt={attempt + 1}/{max_retries + 1}")
            try:
                response = await asyncio.wait_for(
                    self._client.aio.models.generate_content(
@@ -527,6 +530,8 @@ class GeminiLLM(LLMInterface):

        last_exception = None
        for attempt in range(max_retries + 1):
+            if attempt > 0:
+                set_stage(f"llm.gemini.tools.attempt={attempt + 1}/{max_retries + 1}")
            try:
                response = await asyncio.wait_for(
                    self._client.aio.models.generate_content(
@@ -21,6 +21,7 @@ from typing import Any
 from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError
 from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
 from hindsight_api.metrics import get_metrics_collector
+from hindsight_api.worker.stage import set_stage

 logger = logging.getLogger(__name__)

@@ -141,6 +142,8 @@ class LiteLLMLLM(LLMInterface):
        last_exception = None

        for attempt in range(max_retries + 1):
+            if attempt > 0:
+                set_stage(f"llm.litellm.{scope}.attempt={attempt + 1}/{max_retries + 1}")
            try:
                response = await self._litellm.acompletion(**call_kwargs)

@@ -283,6 +286,8 @@ class LiteLLMLLM(LLMInterface):

        last_exception = None
        for attempt in range(max_retries + 1):
+            if attempt > 0:
+                set_stage(f"llm.litellm.tools.attempt={attempt + 1}/{max_retries + 1}")
            try:
                response = await self._litellm.acompletion(**call_kwargs)

@@ -0,0 +1,428 @@
+"""
+Built-in llama.cpp LLM provider for fully offline operation.
+
+Manages a llama-cpp-python server as a subprocess, downloads GGUF models
+from HuggingFace on first use, and delegates inference to the OpenAI-compatible API.
+
+Usage:
+    HINDSIGHT_API_LLM_PROVIDER=llamacpp
+    HINDSIGHT_API_LLAMACPP_MODEL_PATH=~/.hindsight/models/gemma-4-E2B-it-Q4_K_M.gguf
+    HINDSIGHT_API_LLAMACPP_GPU_LAYERS=-1  # -1 = all layers on GPU
+    HINDSIGHT_API_LLAMACPP_CONTEXT_SIZE=8192
+"""
+
+import asyncio
+import logging
+import os
+import signal
+import socket
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+from hindsight_api.engine.llm_interface import LLMInterface
+from hindsight_api.engine.response_models import LLMToolCallResult
+
+logger = logging.getLogger(__name__)
+
+# Default GGUF model for offline mode
+DEFAULT_LLAMACPP_HF_REPO = "bartowski/google_gemma-4-E2B-it-GGUF"
+DEFAULT_LLAMACPP_HF_FILENAME = "google_gemma-4-E2B-it-Q4_K_M.gguf"
+DEFAULT_LLAMACPP_MODEL_ALIAS = "gemma-4-e2b-it"
+
+MODELS_DIR = Path.home() / ".hindsight" / "models"
+
+# Singleton server instance — shared across all LlamaCppLLM instances
+# (retain, reflect, consolidation each create their own LLMProvider,
+# but they should all share one llama.cpp server process)
+_shared_server: "LlamaCppServer | None" = None
+_shared_server_lock = asyncio.Lock()
+
+
+def _find_free_port() -> int:
+    """Find a free TCP port on localhost."""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+def _download_default_model() -> Path:
+    """Download the default GGUF model from HuggingFace if not already cached.
+
+    Returns:
+        Path to the downloaded GGUF file.
+    """
+    try:
+        from huggingface_hub import hf_hub_download
+    except ImportError:
+        raise ImportError(
+            "huggingface-hub is required for automatic model download. "
+            "Install with: pip install 'hindsight-api-slim[local-llm]'"
+        )
+
+    MODELS_DIR.mkdir(parents=True, exist_ok=True)
+    target = MODELS_DIR / DEFAULT_LLAMACPP_HF_FILENAME
+
+    if target.exists():
+        logger.info(f"Using cached model: {target}")
+        return target
+
+    logger.info(
+        f"Downloading {DEFAULT_LLAMACPP_HF_FILENAME} from {DEFAULT_LLAMACPP_HF_REPO} (~3.5 GB, first run only)..."
+    )
+
+    downloaded = hf_hub_download(
+        repo_id=DEFAULT_LLAMACPP_HF_REPO,
+        filename=DEFAULT_LLAMACPP_HF_FILENAME,
+        local_dir=str(MODELS_DIR),
+    )
+
+    logger.info(f"Model downloaded: {downloaded}")
+    return Path(downloaded)
+
+
+def _resolve_model_path(model_path: str | None) -> Path:
+    """Resolve the model path, downloading the default if needed.
+
+    Args:
+        model_path: Explicit path to a GGUF file, or None to use the default.
+
+    Returns:
+        Resolved Path to the GGUF file.
+    """
+    if model_path:
+        p = Path(model_path).expanduser()
+        if not p.exists():
+            raise FileNotFoundError(
+                f"GGUF model not found: {p}\n"
+                f"Set HINDSIGHT_API_LLAMACPP_MODEL_PATH to a valid .gguf file, "
+                f"or remove the setting to auto-download the default model."
+            )
+        return p
+
+    return _download_default_model()
+
+
+class LlamaCppServer:
+    """Manages a llama-cpp-python OpenAI-compatible server as a subprocess."""
+
+    def __init__(
+        self,
+        model_path: Path,
+        port: int,
+        gpu_layers: int = -1,
+        context_size: int = 8192,
+        chat_format: str | None = None,
+        extra_args: str | None = None,
+    ):
+        self.model_path = model_path
+        self.port = port
+        self.gpu_layers = gpu_layers
+        self.context_size = context_size
+        self.chat_format = chat_format
+        self.extra_args = extra_args
+        self._process: subprocess.Popen | None = None
+
+    @property
+    def base_url(self) -> str:
+        return f"http://127.0.0.1:{self.port}/v1"
+
+    async def start(self) -> None:
+        """Start the llama.cpp server subprocess."""
+        cmd = [
+            sys.executable,
+            "-m",
+            "llama_cpp.server",
+            "--model",
+            str(self.model_path),
+            "--host",
+            "127.0.0.1",
+            "--port",
+            str(self.port),
+            "--n_gpu_layers",
+            str(self.gpu_layers),
+            "--n_ctx",
+            str(self.context_size),
+            "--flash_attn",
+            "true",
+            "--n_batch",
+            "2048",
+            # Prompt cache: reuse KV cache for repeated system prompts
+            "--cache",
+            "true",
+        ]
+        # Only pass chat_format if explicitly set (most GGUF models have it embedded)
+        if self.chat_format:
+            cmd.extend(["--chat_format", self.chat_format])
+        # User-provided extra args (e.g. "--type_k 1 --type_v 1 --n_threads 8")
+        if self.extra_args:
+            cmd.extend(self.extra_args.split())
+
+        logger.info(f"Starting llama.cpp server: {' '.join(cmd)}")
+
+        # Write stderr to a log file to avoid pipe buffer deadlock
+        # (llama.cpp outputs a lot of model metadata on stderr during loading)
+        self._log_path = MODELS_DIR / "llamacpp_server.log"
+        self._log_file = open(self._log_path, "w")
+
+        self._process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.DEVNULL,
+            stderr=self._log_file,
+            # Ensure the subprocess is killed when the parent exits
+            preexec_fn=os.setsid if hasattr(os, "setsid") else None,
+        )
+
+        # Wait for the server to be ready
+        await self._wait_for_ready()
+
+    async def _wait_for_ready(self, timeout: float = 120.0) -> None:
+        """Wait for the llama.cpp server to accept connections."""
+        import httpx
+
+        start = time.monotonic()
+        url = f"http://127.0.0.1:{self.port}/v1/models"
+        last_log = start
+
+        while time.monotonic() - start < timeout:
+            # Check if process died
+            if self._process and self._process.poll() is not None:
+                stderr = ""
+                try:
+                    stderr = self._log_path.read_text()[-2000:]
+                except Exception:
+                    pass
+                raise RuntimeError(f"llama.cpp server exited with code {self._process.returncode}.\nstderr: {stderr}")
+
+            try:
+                async with httpx.AsyncClient() as client:
+                    resp = await client.get(url, timeout=5.0)
+                    if resp.status_code == 200:
+                        logger.info(f"llama.cpp server ready on port {self.port}")
+                        return
+            except (httpx.ConnectError, httpx.TimeoutException, httpx.ConnectTimeout):
+                pass
+
+            # Log progress every 15s
+            now = time.monotonic()
+            if now - last_log > 15:
+                elapsed = int(now - start)
+                logger.info(f"Waiting for llama.cpp server to load model... ({elapsed}s)")
+                last_log = now
+
+            await asyncio.sleep(1.0)
+
+        # Timeout — read the log to help debug
+        stderr = ""
+        try:
+            stderr = self._log_path.read_text()[-2000:]
+        except Exception:
+            pass
+        raise TimeoutError(
+            f"llama.cpp server did not become ready within {timeout}s.\n"
+            f"Check model compatibility and available memory.\n"
+            f"Server log: {stderr}"
+        )
+
+    async def stop(self) -> None:
+        """Stop the llama.cpp server subprocess."""
+        if self._process is None:
+            return
+
+        logger.info("Stopping llama.cpp server...")
+        try:
+            # Send SIGTERM to the process group
+            if hasattr(os, "killpg"):
+                os.killpg(os.getpgid(self._process.pid), signal.SIGTERM)
+            else:
+                self._process.terminate()
+
+            # Wait up to 10s for graceful shutdown
+            try:
+                self._process.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                if hasattr(os, "killpg"):
+                    os.killpg(os.getpgid(self._process.pid), signal.SIGKILL)
+                else:
+                    self._process.kill()
+                self._process.wait(timeout=5)
+        except (ProcessLookupError, OSError):
+            pass  # Process already exited
+        finally:
+            self._process = None
+            if hasattr(self, "_log_file") and self._log_file:
+                self._log_file.close()
+                self._log_file = None
+            logger.info("llama.cpp server stopped")
+
+
+class LlamaCppLLM(LLMInterface):
+    """
+    Built-in llama.cpp provider.
+
+    Manages a llama-cpp-python server subprocess and delegates to OpenAICompatibleLLM
+    for actual inference calls. Handles model downloading and server lifecycle.
+    """
+
+    def __init__(
+        self,
+        provider: str,
+        api_key: str,
+        base_url: str,
+        model: str,
+        reasoning_effort: str = "low",
+        model_path: str | None = None,
+        gpu_layers: int = -1,
+        context_size: int = 8192,
+        chat_format: str | None = None,
+        no_grammar: bool = False,
+        extra_args: str | None = None,
+        **kwargs: Any,
+    ):
+        super().__init__(
+            provider=provider,
+            api_key=api_key or "llamacpp",
+            base_url=base_url or "",
+            model=model or DEFAULT_LLAMACPP_MODEL_ALIAS,
+            reasoning_effort=reasoning_effort,
+        )
+        self._model_path_str = model_path
+        self._gpu_layers = gpu_layers
+        self._context_size = context_size
+        self._chat_format = chat_format
+        self._no_grammar = no_grammar
+        self._extra_args = extra_args
+        self._server: LlamaCppServer | None = None
+        self._delegate: Any = None  # OpenAICompatibleLLM, created after server starts
+        self._initialized = False
+
+    async def _ensure_initialized(self) -> None:
+        """Lazy initialization: download model + start shared server on first use."""
+        if self._initialized:
+            return
+
+        global _shared_server
+
+        from .openai_compatible_llm import OpenAICompatibleLLM
+
+        async with _shared_server_lock:
+            if _shared_server is None:
+                # Resolve and potentially download the model
+                model_path = _resolve_model_path(self._model_path_str)
+                logger.info(f"Using GGUF model: {model_path}")
+
+                # Start the shared llama.cpp server
+                port = _find_free_port()
+                _shared_server = LlamaCppServer(
+                    model_path=model_path,
+                    port=port,
+                    gpu_layers=self._gpu_layers,
+                    context_size=self._context_size,
+                    chat_format=self._chat_format,
+                    extra_args=self._extra_args,
+                )
+                await _shared_server.start()
+
+        self._server = _shared_server
+
+        # Create the delegate that talks to the shared server's OpenAI-compatible API
+        if self._no_grammar:
+            logger.info("Grammar enforcement disabled (HINDSIGHT_API_LLAMACPP_NO_GRAMMAR=true)")
+        self._delegate = OpenAICompatibleLLM(
+            provider="llamacpp",
+            api_key="llamacpp",
+            base_url=self._server.base_url,
+            model=self.model,
+            reasoning_effort=self.reasoning_effort,
+        )
+
+        self._initialized = True
+
+    async def verify_connection(self) -> None:
+        """Verify the llama.cpp server is running and can generate text."""
+        await self._ensure_initialized()
+        # Make a simple test call to verify the model can actually generate
+        await self._delegate.call(
+            messages=[{"role": "user", "content": "Say 'ok'"}],
+            max_completion_tokens=10,
+            max_retries=2,
+            initial_backoff=0.5,
+            max_backoff=2.0,
+            scope="verification",
+        )
+        logger.info("llama.cpp LLM verification passed")
+
+    async def call(
+        self,
+        messages: list[dict[str, str]],
+        response_format: Any | None = None,
+        max_completion_tokens: int | None = None,
+        temperature: float | None = None,
+        scope: str = "memory",
+        max_retries: int = 10,
+        initial_backoff: float = 1.0,
+        max_backoff: float = 60.0,
+        skip_validation: bool = False,
+        strict_schema: bool = False,
+        return_usage: bool = False,
+    ) -> Any:
+        """Delegate call to the OpenAI-compatible API."""
+        await self._ensure_initialized()
+        return await self._delegate.call(
+            messages=messages,
+            response_format=response_format,
+            max_completion_tokens=max_completion_tokens,
+            temperature=temperature,
+            scope=scope,
+            max_retries=max_retries,
+            initial_backoff=initial_backoff,
+            max_backoff=max_backoff,
+            skip_validation=skip_validation,
+            strict_schema=strict_schema,
+            return_usage=return_usage,
+        )
+
+    async def call_with_tools(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]],
+        max_completion_tokens: int | None = None,
+        temperature: float | None = None,
+        scope: str = "tools",
+        max_retries: int = 5,
+        initial_backoff: float = 1.0,
+        max_backoff: float = 30.0,
+        tool_choice: str | dict[str, Any] = "auto",
+    ) -> LLMToolCallResult:
+        """Delegate tool calls to the OpenAI-compatible API."""
+        await self._ensure_initialized()
+        return await self._delegate.call_with_tools(
+            messages=messages,
+            tools=tools,
+            max_completion_tokens=max_completion_tokens,
+            temperature=temperature,
+            scope=scope,
+            max_retries=max_retries,
+            initial_backoff=initial_backoff,
+            max_backoff=max_backoff,
+            tool_choice=tool_choice,
+        )
+
+    async def cleanup(self) -> None:
+        """Stop the shared llama.cpp server."""
+        global _shared_server
+
+        if self._delegate:
+            await self._delegate.cleanup()
+            self._delegate = None
+
+        # Stop the shared server (only the first cleanup call actually stops it)
+        async with _shared_server_lock:
+            if _shared_server is not None:
+                await _shared_server.stop()
+                _shared_server = None
+
+        self._server = None
+        self._initialized = False
@@ -33,6 +33,7 @@ from hindsight_api.config import DEFAULT_LLM_TIMEOUT, ENV_LLM_TIMEOUT
 from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError
 from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
 from hindsight_api.metrics import get_metrics_collector
+from hindsight_api.worker.stage import set_stage

 logger = logging.getLogger(__name__)

@@ -100,7 +101,7 @@ class OpenAICompatibleLLM(LLMInterface):
        super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs)

        # Validate provider
-        valid_providers = ["openai", "groq", "ollama", "lmstudio", "minimax", "volcano"]
+        valid_providers = ["openai", "groq", "ollama", "lmstudio", "llamacpp", "minimax", "volcano", "openrouter"]
        if self.provider not in valid_providers:
            raise ValueError(f"OpenAICompatibleLLM only supports: {', '.join(valid_providers)}. Got: {self.provider}")

@@ -114,13 +115,15 @@ class OpenAICompatibleLLM(LLMInterface):
                self.base_url = "http://localhost:1234/v1"
            elif self.provider == "minimax":
                self.base_url = "https://api.minimax.io/v1"
+            elif self.provider == "openrouter":
+                self.base_url = "https://openrouter.ai/api/v1"

        # For ollama/lmstudio, use dummy key if not provided
        if self.provider in ("ollama", "lmstudio") and not self.api_key:
            self.api_key = "local"

        # Validate API key for cloud providers
-        if self.provider in ("openai", "groq", "minimax") and not self.api_key:
+        if self.provider in ("openai", "groq", "minimax", "openrouter") and not self.api_key:
            raise ValueError(f"API key is required for {self.provider}")

        # Service tier configuration (from config, not env vars)
@@ -194,16 +197,29 @@ class OpenAICompatibleLLM(LLMInterface):
    def _max_tokens_param_name(self) -> str:
        """Return the correct parameter name for limiting response tokens.

-        Native OpenAI and Groq accept 'max_completion_tokens'. Mistral and other
-        OpenAI-compatible endpoints that haven't adopted the newer parameter name
-        require 'max_tokens'. Using a custom base_url with the openai provider
-        signals a third-party compatible API, so fall back to 'max_tokens'.
+        Native OpenAI, Azure OpenAI, Groq, and llamacpp accept 'max_completion_tokens'.
+        Mistral and other OpenAI-compatible endpoints that haven't adopted the newer
+        parameter name require 'max_tokens', so when the openai provider is configured
+        with a non-Azure custom base_url we fall back to the widely-supported
+        'max_tokens'.
+
+        Reasoning models (GPT-5, o1, o3) only accept 'max_completion_tokens' and reject
+        'max_tokens' outright, so they always use the new parameter name regardless of
+        base_url.
        """
-        # Native OpenAI (no custom base URL) and Groq use max_completion_tokens
-        if self.provider == "groq":
+        # Reasoning models (GPT-5, o1, o3, ...) only accept max_completion_tokens.
+        # Azure OpenAI + GPT-5 is the canonical example: issue #978.
+        if self._supports_reasoning_model():
+            return "max_completion_tokens"
+        # Native OpenAI (no custom base URL), Groq, and llamacpp use max_completion_tokens
+        if self.provider in ("groq", "llamacpp"):
            return "max_completion_tokens"
        if self.provider == "openai" and not self.base_url:
            return "max_completion_tokens"
+        # Azure OpenAI is fully OpenAI-API-compatible — detect it by hostname so users
+        # can keep provider=openai + an Azure base_url (the documented setup).
+        if self.provider == "openai" and self.base_url and ".openai.azure.com" in self.base_url:
+            return "max_completion_tokens"
        # openai with custom base_url, ollama, lmstudio, minimax, volcano —
        # use the widely-supported max_tokens
        return "max_tokens"
@@ -335,13 +351,23 @@ class OpenAICompatibleLLM(LLMInterface):
                        first_msg = call_params["messages"][0]
                        if isinstance(first_msg, dict) and isinstance(first_msg.get("content"), str):
                            first_msg["content"] = schema_msg + "\n\n" + first_msg["content"]
-                if self.provider not in ("lmstudio", "ollama", "volcano"):
-                    # LM Studio, Ollama and Volcano don't support json_object response format reliably
+                # Providers that skip json_object grammar enforcement
+                skip_grammar = self.provider in ("lmstudio", "ollama", "volcano")
+                if self.provider == "llamacpp":
+                    from hindsight_api.config import get_config
+
+                    skip_grammar = get_config().llamacpp_no_grammar
+                if not skip_grammar:
                    call_params["response_format"] = {"type": "json_object"}

        last_exception = None

        for attempt in range(max_retries + 1):
+            # Surface attempt count in worker stage so JSON-schema retry loops
+            # are visible from logs (small models on strict structured output
+            # often loop here). Cheap no-op outside worker context.
+            if attempt > 0:
+                set_stage(f"llm.{self.provider}.{scope}.attempt={attempt + 1}/{max_retries + 1}")
            try:
                if response_format is not None:
                    response = await self._client.chat.completions.create(**call_params)
@@ -609,6 +635,8 @@ class OpenAICompatibleLLM(LLMInterface):
        last_exception = None

        for attempt in range(max_retries + 1):
+            if attempt > 0:
+                set_stage(f"llm.{self.provider}.tools.attempt={attempt + 1}/{max_retries + 1}")
            try:
                response = await self._client.chat.completions.create(**call_params)

@@ -758,6 +786,8 @@ class OpenAICompatibleLLM(LLMInterface):

        async with httpx.AsyncClient(timeout=300.0) as client:
            for attempt in range(max_retries + 1):
+                if attempt > 0:
+                    set_stage(f"llm.ollama_native.{scope}.attempt={attempt + 1}/{max_retries + 1}")
                try:
                    response = await client.post(native_url, json=payload)
                    response.raise_for_status()
@@ -9,6 +9,7 @@ Implements hierarchical retrieval:

 import logging
 import uuid
+from dataclasses import replace
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any

@@ -162,13 +163,18 @@ async def tool_search_observations(
    if include_source_facts and source_facts_max_tokens > 0:
        recall_kwargs["max_source_facts_tokens"] = source_facts_max_tokens

+    # Use an internal request context so this recall is not billed as a
+    # user-facing operation. The reflect caller is already billed for the
+    # overall reflect operation; double-billing the sub-recalls would
+    # overcharge the customer.
+    internal_ctx = replace(request_context, internal=True)
    result = await memory_engine.recall_async(
        bank_id=bank_id,
        query=query,
        fact_type=["observation"],
        max_tokens=max_tokens,
        enable_trace=False,
-        request_context=request_context,
+        request_context=internal_ctx,
        tags=tags,
        tags_match=tags_match,
        tag_groups=tag_groups,
@@ -233,13 +239,14 @@ async def tool_recall(
    # Only world/experience are valid for raw recall (observation is handled by search_observations)
    recall_fact_type = [ft for ft in (fact_types or ["experience", "world"]) if ft in ("world", "experience")]
    include_chunks = True
+    internal_ctx = replace(request_context, internal=True)
    result = await memory_engine.recall_async(
        bank_id=bank_id,
        query=query,
        fact_type=recall_fact_type,
        max_tokens=max_tokens,
        enable_trace=False,
-        request_context=request_context,
+        request_context=internal_ctx,
        tags=tags,
        tags_match=tags_match,
        tag_groups=tag_groups,
@@ -113,6 +113,22 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
    Returns:
        BankProfile with name, typed DispositionTraits, and mission
    """
+    profile, _ = await get_or_create_bank_profile(pool, bank_id)
+    return profile
+
+
+async def get_or_create_bank_profile(pool, bank_id: str) -> tuple[BankProfile, bool]:
+    """
+    Get bank profile, auto-creating with defaults if it doesn't exist.
+
+    Same as get_bank_profile, but also returns a flag indicating whether the
+    bank was freshly created on this call. Used by the memory engine to apply
+    the HINDSIGHT_API_DEFAULT_BANK_TEMPLATE hook on first bank creation.
+
+    Returns:
+        Tuple of (BankProfile, created) where created is True if the bank
+        did not exist before this call.
+    """
    async with acquire_with_retry(pool) as conn:
        # Try to get existing bank
        row = await conn.fetchrow(
@@ -129,10 +145,13 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
            if isinstance(disposition_data, str):
                disposition_data = json.loads(disposition_data)

-            return BankProfile(
-                name=row["name"],
-                disposition=DispositionTraits(**disposition_data),
-                mission=row["mission"] or "",
+            return (
+                BankProfile(
+                    name=row["name"],
+                    disposition=DispositionTraits(**disposition_data),
+                    mission=row["mission"] or "",
+                ),
+                False,
            )

        # Bank doesn't exist, create with defaults.
@@ -153,11 +172,15 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
            internal_id,
        )

-        if inserted:
+        created = inserted is not None
+        if created:
            # Fresh insert — create per-bank vector indexes (instant on empty bank)
            await create_bank_vector_indexes(conn, bank_id, str(internal_id))

-        return BankProfile(name=bank_id, disposition=DispositionTraits(**DEFAULT_DISPOSITION), mission="")
+        return (
+            BankProfile(name=bank_id, disposition=DispositionTraits(**DEFAULT_DISPOSITION), mission=""),
+            created,
+        )


 async def update_bank_disposition(pool, bank_id: str, disposition: dict[str, int]) -> None:
@@ -100,11 +100,20 @@ async def store_chunks_batch(conn, bank_id: str, document_id: str, chunks: list[
        content_hashes.append(compute_chunk_hash(chunk.chunk_text))
        chunk_id_map[chunk.chunk_index] = chunk_id

-    # Batch insert all chunks
+    # Batch upsert all chunks. ON CONFLICT makes this idempotent: re-submitting
+    # a retain under the same document_id (the pattern in vectorize-io/hindsight#977)
+    # may produce chunk_ids that already exist when upstream cascade-delete or
+    # delta-retain paths don't run (or race with a concurrent task). Overwriting
+    # is the correct behavior per the document_id grouping semantics — the caller
+    # intends this chunk to hold the latest content at that (document_id, index).
    await conn.execute(
        f"""
        INSERT INTO {fq_table("chunks")} (chunk_id, document_id, bank_id, chunk_text, chunk_index, content_hash)
        SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::integer[], $6::text[])
+        ON CONFLICT (chunk_id) DO UPDATE SET
+            chunk_text = EXCLUDED.chunk_text,
+            chunk_index = EXCLUDED.chunk_index,
+            content_hash = EXCLUDED.content_hash
        """,
        chunk_ids,
        [document_id] * len(chunk_texts),
@@ -909,6 +909,7 @@ def _build_user_message(
    event_date: datetime | None,
    context: str,
    metadata: dict[str, str] | None = None,
+    agent_name: str | None = None,
 ) -> str:
    """Build user message for fact extraction."""
    from .orchestrator import parse_datetime_flexible
@@ -927,11 +928,15 @@ def _build_user_message(
        metadata_lines = "\n".join(f"  {k}: {v}" for k, v in metadata.items())
        metadata_section = f"\nMetadata:\n{metadata_lines}"

+    narrator_section = ""
+    if agent_name:
+        narrator_section = f'\nNarrator: {agent_name} (AI agent — first-person statements like "I did X" are the agent\'s own actions; classify as "assistant")'
+
    return f"""Extract facts from the following text chunk.

 Chunk: {chunk_index + 1}/{total_chunks}
 Event Date: {event_date_str}
-Context: {sanitized_context}{metadata_section}
+Context: {sanitized_context}{metadata_section}{narrator_section}

 Text:
 {sanitized_chunk}"""
@@ -995,7 +1000,7 @@ async def _extract_facts_from_chunk(
    extract_causal_links = config.retain_extract_causal_links

    # Build user message using helper function
-    user_message = _build_user_message(chunk, chunk_index, total_chunks, event_date, context, metadata)
+    user_message = _build_user_message(chunk, chunk_index, total_chunks, event_date, context, metadata, agent_name)

    # Retry logic for JSON validation errors
    # Use retain-specific overrides if set, otherwise fall back to global LLM config
@@ -1632,7 +1637,13 @@ async def extract_facts_from_contents_batch_api(

            # Build user message using helper function
            user_message = _build_user_message(
-                chunk, chunk_index_in_content, len(chunks), item.event_date, item.context, item.metadata or None
+                chunk,
+                chunk_index_in_content,
+                len(chunks),
+                item.event_date,
+                item.context,
+                item.metadata or None,
+                agent_name,
            )

            # Build request body using helper function
@@ -17,6 +17,23 @@ from .types import ProcessedFact
 logger = logging.getLogger(__name__)


+async def get_document_content(
+    conn,
+    bank_id: str,
+    document_id: str,
+) -> str | None:
+    """Fetch the original_text of an existing document.
+
+    Returns None if the document does not exist.
+    """
+    row = await conn.fetchval(
+        f"SELECT original_text FROM {fq_table('documents')} WHERE id = $1 AND bank_id = $2",
+        document_id,
+        bank_id,
+    )
+    return row
+
+
 async def insert_facts_batch(
    conn, bank_id: str, facts: list[ProcessedFact], document_id: str | None = None
 ) -> list[str]:
@@ -739,17 +739,10 @@ async def compute_semantic_links_ann(
        return []

    import time as time_mod
-    import uuid as uuid_mod

    ann_start = time_mod.time()
    links = []

-    # Lower ef_search for retain ANN — default 400 is tuned for recall precision
-    # but at 164k units each HNSW probe takes 94ms. ef_search=60 gives 2.7ms/probe
-    # (35x faster) with sufficient accuracy for top-50 semantic link creation.
-    # Reset after to avoid polluting the connection pool for recall queries.
-    await conn.execute("SET hnsw.ef_search = 60")
-
    logger.debug(f"[ANN] Starting: {len(unit_ids)} seeds, top_k={top_k}")

    # Build per-unit fact_types (default to 'world' if not provided)
@@ -760,54 +753,71 @@ async def compute_semantic_links_ann(
    # sequential-scan every HNSW probe result against the array, destroying
    # performance (67s for 8k seeds). Self-links are harmless (ON CONFLICT DO
    # NOTHING handles duplicates in memory_links).
-    t_setup = time_mod.time()
-    await conn.execute("CREATE TEMP TABLE IF NOT EXISTS _ann_seeds (unit_id text, emb_text text, fact_type text)")
-    await conn.execute("TRUNCATE _ann_seeds")
+    #
+    # The entire CREATE TEMP TABLE → COPY → SELECT sequence MUST run inside a
+    # single transaction. Callers may connect through pgBouncer in `transaction`
+    # pool mode, in which case the backend is only pinned to the client for the
+    # duration of a transaction. Outside a transaction, pgBouncer can rebind
+    # the client to a different backend between statements, and the temp table
+    # (which is session-scoped to its creating backend) becomes invisible.
+    # The observed failure mode was an intermittent
+    # `relation "_ann_seeds" does not exist` on the second statement.
+    #
+    # Using ON COMMIT DROP + SET LOCAL also means we don't have to remember to
+    # manually drop the temp table or reset hnsw.ef_search — the transaction
+    # end handles both.
+    rows: list = []
+    async with conn.transaction():
+        # Transaction-local ef_search. Default 400 is tuned for recall precision
+        # but at 164k units each HNSW probe takes 94ms. ef_search=60 gives 2.7ms
+        # per probe (35x faster) with sufficient accuracy for top-50 semantic
+        # link creation. SET LOCAL auto-reverts at commit, so we don't pollute
+        # the pool for subsequent recall queries.
+        await conn.execute("SET LOCAL hnsw.ef_search = 60")

-    records = [
-        (uid, emb if isinstance(emb, str) else str(emb), ft) for uid, emb, ft in zip(unit_ids, embeddings, fact_types)
-    ]
-    await conn.copy_records_to_table("_ann_seeds", records=records, columns=["unit_id", "emb_text", "fact_type"])
-    logger.debug(f"[ANN] Temp table setup: {time_mod.time() - t_setup:.3f}s ({len(records)} seeds)")
+        t_setup = time_mod.time()
+        await conn.execute("CREATE TEMP TABLE _ann_seeds (unit_id text, emb_text text, fact_type text) ON COMMIT DROP")

-    # Run one ANN query per fact_type so each uses the right HNSW index.
-    rows = []
-    active_types = set(fact_types)
-    for fact_type in active_types:
-        t_query = time_mod.time()
-        seed_count = sum(1 for ft in fact_types if ft == fact_type)
-        logger.debug(f"[ANN] Querying fact_type={fact_type}: {seed_count} seeds")
-        ft_rows = await conn.fetch(
-            f"""
-            SELECT s.unit_id       AS from_id,
-                   n.id::text      AS to_id,
-                   n.similarity
-            FROM _ann_seeds s
-            CROSS JOIN LATERAL (
-                SELECT mu.id,
-                       1 - (mu.embedding <=> s.emb_text::vector) AS similarity
-                FROM {fq_table("memory_units")} mu
-                WHERE mu.bank_id = $1
-                  AND mu.fact_type = $2
-                  AND mu.embedding IS NOT NULL
-                ORDER BY mu.embedding <=> s.emb_text::vector
-                LIMIT $3
-            ) n
-            WHERE s.fact_type = $2
-            """,
-            bank_id,
-            fact_type,
-            top_k,
-            timeout=300,  # ANN on large banks can take minutes
-        )
-        logger.debug(f"[ANN] fact_type={fact_type}: {len(ft_rows)} rows in {time_mod.time() - t_query:.3f}s")
-        rows.extend(ft_rows)
+        records = [
+            (uid, emb if isinstance(emb, str) else str(emb), ft)
+            for uid, emb, ft in zip(unit_ids, embeddings, fact_types)
+        ]
+        await conn.copy_records_to_table("_ann_seeds", records=records, columns=["unit_id", "emb_text", "fact_type"])
+        logger.debug(f"[ANN] Temp table setup: {time_mod.time() - t_setup:.3f}s ({len(records)} seeds)")

-    # Clean up temp table (no ON COMMIT DROP since we're not in a transaction)
-    await conn.execute("DROP TABLE IF EXISTS _ann_seeds")
-
-    # Reset ef_search to default so the pooled connection doesn't affect recall queries
-    await conn.execute("RESET hnsw.ef_search")
+        # Run one ANN query per fact_type so each uses the right HNSW index.
+        active_types = set(fact_types)
+        for fact_type in active_types:
+            t_query = time_mod.time()
+            seed_count = sum(1 for ft in fact_types if ft == fact_type)
+            logger.debug(f"[ANN] Querying fact_type={fact_type}: {seed_count} seeds")
+            ft_rows = await conn.fetch(
+                f"""
+                SELECT s.unit_id       AS from_id,
+                       n.id::text      AS to_id,
+                       n.similarity
+                FROM _ann_seeds s
+                CROSS JOIN LATERAL (
+                    SELECT mu.id,
+                           1 - (mu.embedding <=> s.emb_text::vector) AS similarity
+                    FROM {fq_table("memory_units")} mu
+                    WHERE mu.bank_id = $1
+                      AND mu.fact_type = $2
+                      AND mu.embedding IS NOT NULL
+                    ORDER BY mu.embedding <=> s.emb_text::vector
+                    LIMIT $3
+                ) n
+                WHERE s.fact_type = $2
+                """,
+                bank_id,
+                fact_type,
+                top_k,
+                timeout=300,  # ANN on large banks can take minutes
+            )
+            logger.debug(f"[ANN] fact_type={fact_type}: {len(ft_rows)} rows in {time_mod.time() - t_query:.3f}s")
+            rows.extend(ft_rows)
+    # Transaction commits here. _ann_seeds is dropped (ON COMMIT DROP).
+    # hnsw.ef_search reverts (SET LOCAL).

    for row in rows:
        sim = float(min(1.0, max(0.0, row["similarity"])))
@@ -14,6 +14,7 @@ from collections.abc import Awaitable, Callable
 from datetime import UTC, datetime
 from typing import Any

+from ...worker.stage import set_stage
 from ..db_utils import acquire_with_retry
 from ..memory_engine import fq_table
 from . import bank_utils
@@ -133,6 +134,7 @@ async def _pre_resolve_phase1(
    Running these outside the transaction avoids holding row locks during
    slow reads, eliminating TimeoutErrors under concurrent load.
    """
+    set_stage("retain.phase1.resolve")
    from .link_utils import compute_semantic_links_ann

    user_entities_per_content = {idx: content.entities for idx, content in enumerate(contents) if content.entities}
@@ -238,6 +240,7 @@ async def _insert_facts_and_links(
    only the unit_entities INSERT (FK to memory_units) stays in the transaction.
    Entity link building is deferred to Phase 3 (post-transaction, best-effort).
    """
+    set_stage("retain.phase2.insert_facts")
    unit_ids = await fact_storage.insert_facts_batch(conn, bank_id, processed_facts)
    step_start = time.time()
    log_buffer.append(f"  Insert facts: {len(unit_ids)} units in {time.time() - step_start:.3f}s")
@@ -322,6 +325,7 @@ async def _build_and_insert_entity_links_phase3(
    Entity links are for UI graph visualization only — retrieval uses
    the unit_entities self-join instead.
    """
+    set_stage("retain.phase3.entity_links")
    p3_unit_ids = phase3_ctx.unit_ids
    p3_resolved = phase3_ctx.resolved_entity_ids
    p3_entity_to_unit = phase3_ctx.entity_to_unit
@@ -367,6 +371,7 @@ async def _extract_and_embed(
    Returns:
        Tuple of (extracted_facts, processed_facts, chunks_metadata, usage)
    """
+    set_stage("retain.extract_and_embed")
    step_start = time.time()
    extracted_facts, chunks, usage = await fact_extraction.extract_facts_from_contents(
        contents, llm_config, agent_name, config, pool, operation_id, schema
@@ -523,6 +528,35 @@ async def retain_batch(
            except Exception:
                logger.warning("Failed to persist generated document_id", exc_info=True)

+    # --- Append mode: prepend existing document content to new content ---
+    # When update_mode="append", fetch the existing document text and prepend it
+    # so the full document is reprocessed (delta retain will skip unchanged chunks).
+    update_mode = None
+    for item in contents_dicts:
+        item_mode = item.get("update_mode")
+        if item_mode:
+            update_mode = item_mode
+            break
+
+    if update_mode == "append" and effective_doc_id and is_first_batch:
+        async with acquire_with_retry(pool) as conn:
+            existing_text = await fact_storage.get_document_content(conn, bank_id, effective_doc_id)
+        if existing_text:
+            # Prepend existing text as a new content item at the beginning
+            existing_content: RetainContentDict = {"content": existing_text}
+            # Copy context/tags from first item for consistency
+            first = contents_dicts[0]
+            if first.get("context"):
+                existing_content["context"] = first["context"]
+            if first.get("tags"):
+                existing_content["tags"] = first["tags"]
+            contents_dicts = [existing_content, *contents_dicts]
+            # Rebuild contents list to match
+            contents = _build_contents(contents_dicts, document_tags)
+            log_buffer.append(
+                f"[append] Prepended {len(existing_text):,} chars from existing document {effective_doc_id}"
+            )
+
    # --- Delta retain: check if we can skip unchanged chunks ---
    if is_first_batch:
        delta_result = await _try_delta_retain(
@@ -1522,7 +1556,12 @@ def _map_results_to_contents(
    """Map created unit IDs back to original content items."""
    facts_by_content: dict[int, list[int]] = {i: [] for i in range(len(contents))}
    for i, fact in enumerate(extracted_facts):
-        facts_by_content[fact.content_index].append(i)
+        # Normalize content_index: some LLM providers return 1-indexed values.
+        # Clamp to valid range to prevent KeyError.
+        idx = fact.content_index
+        if idx < 0 or idx >= len(contents):
+            idx = min(max(idx, 0), len(contents) - 1) if len(contents) > 0 else 0
+        facts_by_content[idx].append(i)

    result_unit_ids = []
    unit_idx = 0
@@ -25,6 +25,9 @@ class RetainContentDict(TypedDict, total=False):
        observation_scopes: How to scope observations for consolidation (optional).
            "per_tag" runs one pass per individual tag; "combined" (default) runs a
            single pass with all tags; a list[list[str]] specifies exact passes.
+        update_mode: How to handle existing documents with the same document_id (optional).
+            "replace" (default) deletes old data and reprocesses. "append" concatenates
+            new content to the existing document and reprocesses.
    """

    content: str  # Required
@@ -37,6 +40,7 @@ class RetainContentDict(TypedDict, total=False):
    observation_scopes: (
        Literal["per_tag", "combined", "all_combinations"] | list[list[str]]
    )  # Observation scopes for consolidation
+    update_mode: Literal["replace", "append"]


@dataclass
@@ -6,25 +6,30 @@ stored in memory_links:

 1. Entity links  — query-time self-join through unit_entities. Score = number of distinct
                   shared entities between the seed set and each candidate, computed via
-                   COUNT(DISTINCT entity_id). More accurate than precomputed entity links.
+                   COUNT(DISTINCT entity_id). Uses a LATERAL per-entity cap
+                   (graph_per_entity_limit, default 200) to prevent high-fanout entities
+                   from exploding the self-join intermediate rows.
 2. Semantic links — precomputed kNN graph (each new fact linked to its top-5 most
                    similar existing facts at insert time, similarity >= 0.7). Checked
                    in both directions since the graph is not symmetric. Score = weight.
 3. Causal links  — explicit causal chains (causes/caused_by/enables/prevents).
                   Score = weight + 1.0 (boosted as highest-quality signal).

-All three signals are bounded at retain time, so no LATERAL fan-out caps are needed
-at query time. Each expansion is a simple aggregation over a small result set.
+Entity expansion is bounded by graph_per_entity_limit (LATERAL cap per entity).
+A timeout fallback (graph_expansion_timeout) drops entity expansion entirely if the
+query still exceeds the budget.

 For non-observation fact types the three expansions are issued as a single CTE query
 (one roundtrip, one connection) with a `source` discriminator column so the Python
 merge step can apply per-signal score transformations.
 """

+import asyncio
 import logging
 import math
 import time

+from ...config import get_config
 from ..db_utils import acquire_with_retry
 from ..memory_engine import fq_table
 from .graph_retrieval import GraphRetriever
@@ -262,35 +267,48 @@ class LinkExpansionRetriever(GraphRetriever):
                    idx_memory_links_to_type_weight (to_unit_id, link_type, weight DESC)
                    → replaces costly BitmapAnd of two separate scans
        """
+        config = get_config()
        ml = fq_table("memory_links")
        mu = fq_table("memory_units")
        ue = fq_table("unit_entities")

+        per_entity_limit = config.link_expansion_per_entity_limit
+
+        # Entity CTE with LATERAL fanout cap.
+        # Every seed entity (including high-frequency ones) is kept, but each
+        # entity's expansion is capped to per_entity_limit target units.  The
+        # LATERAL subquery orders by unit_id DESC so the most recently inserted
+        # units are preferred (a recency proxy that is free — it rides the PK
+        # index with no extra sort).
        entity_cte = f"""
+            seed_entities AS (
+                SELECT DISTINCT ue.entity_id
+                FROM {ue} ue
+                WHERE ue.unit_id = ANY($1::uuid[])
+            ),
            entity_expanded AS (
-                -- Entity co-occurrence via unit_entities self-join.
-                -- Finds units sharing entities with seeds at query time — more accurate
-                -- than precomputed entity links (no stale 50-neighbor cap).
-                -- Score = COUNT(DISTINCT shared entities), mapped to [0,1] via tanh.
                SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
                       mu.occurred_end, mu.mentioned_at,
                       mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count,
-                       COUNT(DISTINCT ue_seed.entity_id)::float AS score,
+                       COUNT(DISTINCT se.entity_id)::float AS score,
                       'entity'::text AS source
-                FROM {ue} ue_seed
-                JOIN {ue} ue_target ON ue_seed.entity_id = ue_target.entity_id
-                JOIN {mu} mu ON mu.id = ue_target.unit_id
-                WHERE ue_seed.unit_id = ANY($1::uuid[])
-                  AND ue_target.unit_id != ALL($1::uuid[])
-                  AND mu.fact_type = $2
+                FROM seed_entities se
+                CROSS JOIN LATERAL (
+                    SELECT ue_target.unit_id
+                    FROM {ue} ue_target
+                    WHERE ue_target.entity_id = se.entity_id
+                      AND ue_target.unit_id != ALL($1::uuid[])
+                    ORDER BY ue_target.unit_id DESC
+                    LIMIT {per_entity_limit}
+                ) t
+                JOIN {mu} mu ON mu.id = t.unit_id
+                WHERE mu.fact_type = $2
                GROUP BY mu.id
                ORDER BY score DESC
                LIMIT $3
            )"""

-        all_rows = await conn.fetch(
-            f"""
-            WITH {entity_cte},
+        semantic_causal_cte = f"""
            semantic_expanded AS (
                -- Semantic kNN: both outgoing (seeds → their kNN at insert time) and
                -- incoming (facts inserted after seeds that found seeds as kNN).
@@ -350,18 +368,37 @@ class LinkExpansionRetriever(GraphRetriever):
                  AND mu.fact_type = $2
                ORDER BY mu.id, ml.weight DESC
                LIMIT $3
-            )
+            )"""
+
+        full_query = f"""
+            WITH {entity_cte},
+            {semantic_causal_cte}
            SELECT * FROM entity_expanded
            UNION ALL
            SELECT * FROM semantic_expanded
            UNION ALL
            SELECT * FROM causal_expanded
-            """,
-            seed_ids,
-            fact_type,
-            budget,
-            self.causal_weight_threshold,
-        )
+            """
+
+        params = [seed_ids, fact_type, budget, self.causal_weight_threshold]
+
+        try:
+            all_rows = await asyncio.wait_for(
+                conn.fetch(full_query, *params),
+                timeout=config.link_expansion_timeout,
+            )
+        except asyncio.TimeoutError:
+            logger.warning(
+                f"[LinkExpansion] Entity expansion timed out after {config.link_expansion_timeout}s "
+                f"for fact_type={fact_type}, falling back to semantic+causal only"
+            )
+            fallback_query = f"""
+                WITH {semantic_causal_cte}
+                SELECT * FROM semantic_expanded
+                UNION ALL
+                SELECT * FROM causal_expanded
+                """
+            all_rows = await conn.fetch(fallback_query, *params)

        entity_rows = [r for r in all_rows if r["source"] == "entity"]
        semantic_rows = [r for r in all_rows if r["source"] == "semantic"]
@@ -401,17 +438,31 @@ class LinkExpansionRetriever(GraphRetriever):
                f"{len(source_ids_found)} source_memory_ids found"
            )

+        config = get_config()
        ue = fq_table("unit_entities")
+        per_entity_limit = config.link_expansion_per_entity_limit

        connected_sources_cte = f"""
-            connected_sources AS (
-                -- Find sources sharing entities with seed observation sources
-                -- via unit_entities self-join (query-time, no precomputed links needed).
-                SELECT DISTINCT ue_target.unit_id AS source_id
+            source_entities AS (
+                SELECT DISTINCT ue_seed.entity_id
                FROM seed_sources ss
                JOIN {ue} ue_seed ON ue_seed.unit_id = ss.source_id
-                JOIN {ue} ue_target ON ue_seed.entity_id = ue_target.entity_id
-                WHERE ue_target.unit_id != ss.source_id
+            ),
+            connected_sources AS (
+                -- Find sources sharing entities with seed observation sources
+                -- via LATERAL-capped self-join (prevents hub entity fanout).
+                SELECT DISTINCT t.unit_id AS source_id
+                FROM source_entities se
+                CROSS JOIN LATERAL (
+                    SELECT ue_target.unit_id
+                    FROM {ue} ue_target
+                    WHERE ue_target.entity_id = se.entity_id
+                    ORDER BY ue_target.unit_id DESC
+                    LIMIT {per_entity_limit}
+                ) t
+                WHERE NOT EXISTS (
+                    SELECT 1 FROM seed_sources ss WHERE ss.source_id = t.unit_id
+                )
            )"""

        entity_rows = await conn.fetch(
@@ -23,6 +23,7 @@ def apply_combined_scoring(
    recency_alpha: float = _RECENCY_ALPHA,
    temporal_alpha: float = _TEMPORAL_ALPHA,
    proof_count_alpha: float = _PROOF_COUNT_ALPHA,
+    is_passthrough_reranker: bool = False,
 ) -> None:
    """Apply combined scoring to a list of ScoredResults in-place.

@@ -60,6 +61,42 @@ def apply_combined_scoring(
    if now.tzinfo is None:
        now = now.replace(tzinfo=UTC)

+    # When the configured cross-encoder is a passthrough (e.g.
+    # RRFPassthroughCrossEncoder used by slim deployments), every
+    # cross_encoder_score_normalized is identical and provides no relevance
+    # signal. In that case the multiplicative recency / temporal / proof_count
+    # boosts below become the *only* ranking signal — making the final order a
+    # pure recency sort regardless of how relevant a candidate actually is.
+    #
+    # Detect that case and seed cross_encoder_score_normalized from the RRF
+    # rank instead, so the boosts modulate a meaningful base score rather than
+    # replacing it. This is a no-op for real cross-encoders, which produce
+    # diverse scores.
+    # When the reranker is a passthrough (e.g. RRFPassthroughCrossEncoder used
+    # by slim deployments), every cross_encoder_score_normalized is identical
+    # and provides no relevance signal. The multiplicative recency / temporal /
+    # proof_count boosts below would then become the *only* ranking signal,
+    # making the final order a pure recency sort regardless of how relevant a
+    # candidate actually is.
+    #
+    # Seed cross_encoder_score_normalized from the RRF rank instead, so the
+    # boosts modulate a meaningful base score. Caller passes is_passthrough
+    # explicitly because "all scores identical" is too fragile a heuristic —
+    # a real reranker can also tie scores (especially in tests with synthetic
+    # data) and we'd corrupt legitimate single-result reranks.
+    if is_passthrough_reranker and scored_results:
+        n = len(scored_results)
+        sorted_by_rrf = sorted(
+            scored_results,
+            key=lambda s: getattr(getattr(s, "candidate", None), "rrf_score", 0.0),
+            reverse=True,
+        )
+        denom = max(1, n - 1)
+        for new_rank, sr in enumerate(sorted_by_rrf):
+            # Map rank → [0.1, 1.0] so the recency boost can still nudge
+            # ordering between adjacent candidates without overpowering RRF.
+            sr.cross_encoder_score_normalized = 1.0 - (0.9 * new_rank / denom)
+
    for sr in scored_results:
        # Recency: linear decay over 365 days → [0.1, 1.0]; neutral 0.5 if no date.
        sr.recency = 0.5
@@ -62,13 +62,14 @@ def format_facts_for_prompt(facts: list[MemoryFact]) -> str:
        if fact.context:
            fact_obj["context"] = fact.context

-        # Add occurred_start if available (when the fact occurred)
-        if fact.occurred_start:
-            occurred_start = fact.occurred_start
-            if isinstance(occurred_start, str):
-                fact_obj["occurred_start"] = occurred_start
-            elif isinstance(occurred_start, datetime):
-                fact_obj["occurred_start"] = occurred_start.strftime("%Y-%m-%d %H:%M:%S")
+        # Add temporal fields if available
+        for field_name in ("occurred_start", "occurred_end", "mentioned_at"):
+            value = getattr(fact, field_name, None)
+            if value:
+                if isinstance(value, str):
+                    fact_obj[field_name] = value
+                elif isinstance(value, datetime):
+                    fact_obj[field_name] = value.strftime("%Y-%m-%d %H:%M:%S")

        formatted.append(fact_obj)

@@ -29,6 +29,7 @@ from hindsight_api.models import RequestContext
 _ALL_TOOLS: frozenset[str] = frozenset(
    {
        "retain",
+        "sync_retain",
        "recall",
        "reflect",
        "list_banks",
@@ -139,6 +140,7 @@ def build_content_dict(
    metadata: dict[str, str] | None = None,
    document_id: str | None = None,
    strategy: str | None = None,
+    update_mode: str | None = None,
 ) -> tuple[dict[str, Any], str | None]:
    """Build a content dict for retain operations.

@@ -150,6 +152,7 @@ def build_content_dict(
        metadata: Optional key-value metadata to attach to the memory
        document_id: Optional document ID to associate the memory with
        strategy: Optional named retain strategy override (e.g., 'exact', 'verbose')
+        update_mode: How to handle existing documents ('replace' or 'append')

    Returns:
        Tuple of (content_dict, error_message). error_message is None if successful.
@@ -184,6 +187,8 @@ def build_content_dict(
        content_dict["document_id"] = document_id
    if strategy is not None:
        content_dict["strategy"] = strategy
+    if update_mode is not None:
+        content_dict["update_mode"] = update_mode

    return content_dict, None

@@ -202,6 +207,7 @@ def register_mcp_tools(
    """
    tools_to_register = config.tools or {
        "retain",
+        "sync_retain",
        "recall",
        "reflect",
        "list_banks",
@@ -235,6 +241,9 @@ def register_mcp_tools(
    if "retain" in tools_to_register:
        _register_retain(mcp, memory, config)

+    if "sync_retain" in tools_to_register:
+        _register_sync_retain(mcp, memory, config)
+
    if "recall" in tools_to_register:
        _register_recall(mcp, memory, config)

@@ -539,6 +548,7 @@ def _register_retain(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig)
            document_id: str | None = None,
            bank_id: str | None = None,
            strategy: str | None = None,
+            update_mode: str | None = None,
        ) -> dict:
            """
            Args:
@@ -550,12 +560,15 @@ def _register_retain(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig)
                document_id: Optional document ID to associate this memory with
                bank_id: Optional bank to store in (defaults to session bank). Use for cross-bank operations.
                strategy: Optional named retain strategy (e.g., 'exact' for verbatim storage). Strategies are defined in the bank config.
+                update_mode: How to handle existing documents with the same document_id. 'replace' (default) or 'append' (concatenates new content to existing).
            """
            target_bank = bank_id or config.bank_id_resolver()
            if target_bank is None:
                return {"status": "error", "message": "No bank_id configured"}

-            content_dict, error = build_content_dict(content, context, timestamp, tags, metadata, document_id, strategy)
+            content_dict, error = build_content_dict(
+                content, context, timestamp, tags, metadata, document_id, strategy, update_mode
+            )
            if error:
                return {"status": "error", "message": error}

@@ -590,6 +603,7 @@ def _register_retain(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig)
            metadata: dict[str, str] | None = None,
            document_id: str | None = None,
            strategy: str | None = None,
+            update_mode: str | None = None,
        ) -> dict:
            """
            Args:
@@ -600,12 +614,15 @@ def _register_retain(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig)
                metadata: Optional key-value metadata to attach (e.g., {'source': 'slack', 'channel': 'general'})
                document_id: Optional document ID to associate this memory with
                strategy: Optional named retain strategy (e.g., 'exact' for verbatim storage). Strategies are defined in the bank config.
+                update_mode: How to handle existing documents with the same document_id. 'replace' (default) or 'append' (concatenates new content to existing).
            """
            target_bank = config.bank_id_resolver()
            if target_bank is None:
                return {"status": "error", "message": "No bank_id configured"}

-            content_dict, error = build_content_dict(content, context, timestamp, tags, metadata, document_id, strategy)
+            content_dict, error = build_content_dict(
+                content, context, timestamp, tags, metadata, document_id, strategy, update_mode
+            )
            if error:
                return {"status": "error", "message": error}

@@ -630,6 +647,124 @@ def _register_retain(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig)
                return {"status": "error", "message": str(e)}


+def _register_sync_retain(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig) -> None:
+    """Register the sync_retain tool (synchronous retain that waits for completion)."""
+
+    if config.include_bank_id_param:
+
+        @mcp.tool()
+        async def sync_retain(
+            content: str,
+            context: str = "general",
+            timestamp: str | None = None,
+            tags: list[str] | None = None,
+            metadata: dict[str, str] | None = None,
+            document_id: str | None = None,
+            bank_id: str | None = None,
+            strategy: str | None = None,
+        ) -> dict:
+            """Store information to long-term memory and wait for completion.
+
+            Unlike retain (which is asynchronous), this tool blocks until the memory
+            is fully stored and immediately available for recall.
+
+            Args:
+                content: The fact/memory to store (be specific and include relevant details)
+                context: Category for the memory (e.g., 'preferences', 'work', 'hobbies', 'family'). Default: 'general'
+                timestamp: When this event/fact occurred (ISO format, e.g., '2024-01-15T10:30:00Z'). Useful for timeline tracking.
+                tags: Optional tags for scoped visibility filtering (e.g., ['project:alpha', 'user:123'])
+                metadata: Optional key-value metadata to attach (e.g., {'source': 'slack', 'channel': 'general'})
+                document_id: Optional document ID to associate this memory with
+                bank_id: Optional bank to store in (defaults to session bank). Use for cross-bank operations.
+                strategy: Optional named retain strategy (e.g., 'exact' for verbatim storage). Strategies are defined in the bank config.
+            """
+            target_bank = bank_id or config.bank_id_resolver()
+            if target_bank is None:
+                return {"status": "error", "message": "No bank_id configured"}
+
+            content_dict, error = build_content_dict(content, context, timestamp, tags, metadata, document_id, strategy)
+            if error:
+                return {"status": "error", "message": error}
+
+            request_context = _get_request_context(config)
+
+            try:
+                result = await memory.retain_batch_async(
+                    bank_id=target_bank,
+                    contents=[content_dict],
+                    request_context=request_context,
+                    strategy=content_dict.pop("strategy", None),
+                )
+                memory_ids = [uid for batch in result for uid in batch]
+                return {
+                    "status": "completed",
+                    "message": "Memory stored successfully",
+                    "memory_ids": memory_ids,
+                }
+            except OperationValidationError as e:
+                logger.warning(f"Sync retain rejected: {e}")
+                return {"status": "error", "message": str(e)}
+            except Exception as e:
+                logger.error(f"Error in sync retain: {e}", exc_info=True)
+                return {"status": "error", "message": str(e)}
+
+    else:
+
+        @mcp.tool()
+        async def sync_retain(
+            content: str,
+            context: str = "general",
+            timestamp: str | None = None,
+            tags: list[str] | None = None,
+            metadata: dict[str, str] | None = None,
+            document_id: str | None = None,
+            strategy: str | None = None,
+        ) -> dict:
+            """Store information to long-term memory and wait for completion.
+
+            Unlike retain (which is asynchronous), this tool blocks until the memory
+            is fully stored and immediately available for recall.
+
+            Args:
+                content: The fact/memory to store (be specific and include relevant details)
+                context: Category for the memory (e.g., 'preferences', 'work', 'hobbies', 'family'). Default: 'general'
+                timestamp: When this event/fact occurred (ISO format, e.g., '2024-01-15T10:30:00Z'). Useful for timeline tracking.
+                tags: Optional tags for scoped visibility filtering (e.g., ['project:alpha', 'user:123'])
+                metadata: Optional key-value metadata to attach (e.g., {'source': 'slack', 'channel': 'general'})
+                document_id: Optional document ID to associate this memory with
+                strategy: Optional named retain strategy (e.g., 'exact' for verbatim storage). Strategies are defined in the bank config.
+            """
+            target_bank = config.bank_id_resolver()
+            if target_bank is None:
+                return {"status": "error", "message": "No bank_id configured"}
+
+            content_dict, error = build_content_dict(content, context, timestamp, tags, metadata, document_id, strategy)
+            if error:
+                return {"status": "error", "message": error}
+
+            request_context = _get_request_context(config)
+
+            try:
+                result = await memory.retain_batch_async(
+                    bank_id=target_bank,
+                    contents=[content_dict],
+                    request_context=request_context,
+                    strategy=content_dict.pop("strategy", None),
+                )
+                memory_ids = [uid for batch in result for uid in batch]
+                return {
+                    "status": "completed",
+                    "message": "Memory stored successfully",
+                    "memory_ids": memory_ids,
+                }
+            except OperationValidationError as e:
+                logger.warning(f"Sync retain rejected: {e}")
+                return {"status": "error", "message": str(e)}
+            except Exception as e:
+                logger.error(f"Error in sync retain: {e}", exc_info=True)
+                return {"status": "error", "message": str(e)}
+
+
 def _register_recall(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig) -> None:
    """Register the recall tool."""
    description = config.recall_description or DEFAULT_MCP_RECALL_DESCRIPTION
@@ -6,15 +6,17 @@ FOR UPDATE SKIP LOCKED for safe concurrent claiming.
 """

 import asyncio
+import io
 import json
 import logging
 import time
 import traceback
 from collections.abc import Awaitable, Callable
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any

 from .exceptions import RetryTaskAt
+from .stage import StageHolder, bind_holder

 if TYPE_CHECKING:
    import asyncpg
@@ -26,6 +28,31 @@ logger = logging.getLogger(__name__)
 # Progress logging interval in seconds
 PROGRESS_LOG_INTERVAL = 30

+# Stuck-task stack-dump thresholds (seconds). Each task gets one stack dump
+# per threshold it crosses (5min, 10min, 20min, 40min, 80min...).
+STUCK_STACK_INITIAL_THRESHOLD_S = 300
+STUCK_STACK_MAX_THRESHOLD_S = 3600 * 6  # cap doubling at 6h
+
+
+@dataclass
+class ActiveTaskInfo:
+    """Tracking info for an in-flight worker task.
+
+    Carries everything the periodic stats / stuck-task logger needs
+    so it can render a useful per-task line without touching the DB.
+    """
+
+    op_type: str
+    bank_id: str
+    schema: str | None
+    bg_task: "asyncio.Task[Any]"
+    started_at: float
+    stage_holder: StageHolder
+    # Largest stuck-stack threshold (seconds) for which we've already
+    # dumped a stack trace; used to suppress repeated dumps.
+    last_stack_dump_threshold: int = 0
+    task_type: str = ""
+

 def fq_table(table: str, schema: str | None = None) -> str:
    """Get fully-qualified table name with optional schema prefix."""
@@ -99,8 +126,8 @@ class WorkerPoller:
        self._in_flight_lock = asyncio.Lock()
        self._last_progress_log = 0.0
        self._tasks_completed_since_log = 0
-        # Track active tasks locally: operation_id -> (op_type, bank_id, schema, asyncio.Task)
-        self._active_tasks: dict[str, tuple[str, str, str | None, asyncio.Task]] = {}
+        # Track active tasks locally: operation_id -> ActiveTaskInfo
+        self._active_tasks: dict[str, ActiveTaskInfo] = {}
        # Track in-flight tasks by operation type
        self._in_flight_by_type: dict[str, int] = {}

@@ -116,17 +143,25 @@ class WorkerPoller:
        """
        Calculate available slots for claiming tasks.

+        Consolidation has a reserved pool of ``consolidation_max_slots`` within
+        ``max_slots``. Non-consolidation tasks may use at most
+        ``max_slots - consolidation_max_slots`` slots, leaving the remainder
+        always available for consolidation. This prevents consolidation from
+        being starved when retain throughput continuously saturates the queue.
+
        Returns:
-            (total_available, consolidation_available) tuple
+            (non_consolidation_available, consolidation_available) tuple
        """
        async with self._in_flight_lock:
            total_in_flight = self._in_flight_count
            consolidation_in_flight = self._in_flight_by_type.get("consolidation", 0)

-        total_available = max(0, self._max_slots - total_in_flight)
+        non_consolidation_in_flight = max(0, total_in_flight - consolidation_in_flight)
+        non_consolidation_max = max(0, self._max_slots - self._consolidation_max_slots)
+        non_consolidation_available = max(0, non_consolidation_max - non_consolidation_in_flight)
        consolidation_available = max(0, self._consolidation_max_slots - consolidation_in_flight)

-        return total_available, consolidation_available
+        return non_consolidation_available, consolidation_available

    async def wait_for_active_tasks(self, timeout: float = 10.0) -> bool:
        """
@@ -164,40 +199,40 @@ class WorkerPoller:
        Returns:
            List of ClaimedTask objects containing operation_id, task_dict, and schema
        """
-        # Calculate available slots
-        total_available, consolidation_available = await self._get_available_slots()
+        # Calculate available slots (independent pools after reservation)
+        non_consolidation_available, consolidation_available = await self._get_available_slots()

-        if total_available <= 0:
+        if non_consolidation_available <= 0 and consolidation_available <= 0:
            return []

        schemas = await self._get_schemas()
        all_tasks: list[ClaimedTask] = []
-        remaining_total = total_available
+        remaining_non_consolidation = non_consolidation_available
        remaining_consolidation = consolidation_available

        for schema in schemas:
-            if remaining_total <= 0:
+            if remaining_non_consolidation <= 0 and remaining_consolidation <= 0:
                break

-            tasks = await self._claim_batch_for_schema(schema, remaining_total, remaining_consolidation)
+            tasks = await self._claim_batch_for_schema(schema, remaining_non_consolidation, remaining_consolidation)

-            # Update remaining slots based on what was claimed
            for task in tasks:
                op_type = task.task_dict.get("operation_type", "unknown")
                if op_type == "consolidation":
                    remaining_consolidation -= 1
+                else:
+                    remaining_non_consolidation -= 1

            all_tasks.extend(tasks)
-            remaining_total -= len(tasks)

        return all_tasks

    async def _claim_batch_for_schema(
-        self, schema: str | None, limit: int, consolidation_limit: int
+        self, schema: str | None, non_consolidation_limit: int, consolidation_limit: int
    ) -> list[ClaimedTask]:
        """Claim tasks from a specific schema respecting slot limits."""
        try:
-            return await self._claim_batch_for_schema_inner(schema, limit, consolidation_limit)
+            return await self._claim_batch_for_schema_inner(schema, non_consolidation_limit, consolidation_limit)
        except Exception as e:
            # Format schema for logging: custom schemas in quotes, None as-is
            schema_display = f'"{schema}"' if schema else str(schema)
@@ -205,37 +240,38 @@ class WorkerPoller:
            return []

    async def _claim_batch_for_schema_inner(
-        self, schema: str | None, limit: int, consolidation_limit: int
+        self, schema: str | None, non_consolidation_limit: int, consolidation_limit: int
    ) -> list[ClaimedTask]:
-        """Inner implementation for claiming tasks from a specific schema with slot limits."""
+        """Inner implementation for claiming tasks from a specific schema with slot limits.
+
+        Non-consolidation and consolidation pools are independent: each is bounded by
+        its own limit and they do not borrow from each other.
+        """
        table = fq_table("async_operations", schema)

        async with self._pool.acquire() as conn:
            async with conn.transaction():
-                # Strategy: Claim non-consolidation tasks first, then consolidation up to limit
+                # 1. Claim non-consolidation tasks
+                non_consolidation_rows = []
+                if non_consolidation_limit > 0:
+                    non_consolidation_rows = await conn.fetch(
+                        f"""
+                        SELECT operation_id, task_payload, retry_count
+                        FROM {table}
+                        WHERE status = 'pending'
+                          AND task_payload IS NOT NULL
+                          AND operation_type != 'consolidation'
+                          AND (next_retry_at IS NULL OR next_retry_at <= NOW())
+                        ORDER BY created_at
+                        LIMIT $1
+                        FOR UPDATE SKIP LOCKED
+                        """,
+                        non_consolidation_limit,
+                    )

-                # 1. Claim non-consolidation tasks (up to limit)
-                non_consolidation_rows = await conn.fetch(
-                    f"""
-                    SELECT operation_id, task_payload, retry_count
-                    FROM {table}
-                    WHERE status = 'pending'
-                      AND task_payload IS NOT NULL
-                      AND operation_type != 'consolidation'
-                      AND (next_retry_at IS NULL OR next_retry_at <= NOW())
-                    ORDER BY created_at
-                    LIMIT $1
-                    FOR UPDATE SKIP LOCKED
-                    """,
-                    limit,
-                )
-
-                claimed_count = len(non_consolidation_rows)
-                remaining_limit = limit - claimed_count
-
-                # 2. Claim consolidation tasks (up to consolidation_limit and remaining_limit)
+                # 2. Claim consolidation tasks from their reserved pool
                consolidation_rows = []
-                if consolidation_limit > 0 and remaining_limit > 0:
+                if consolidation_limit > 0:
                    consolidation_rows = await conn.fetch(
                        f"""
                        SELECT operation_id, task_payload, retry_count
@@ -254,16 +290,17 @@ class WorkerPoller:
                        LIMIT $1
                        FOR UPDATE SKIP LOCKED
                        """,
-                        min(consolidation_limit, remaining_limit),
+                        consolidation_limit,
                    )

-                all_rows = non_consolidation_rows + consolidation_rows
+                tagged_rows = [(row, False) for row in non_consolidation_rows] + [
+                    (row, True) for row in consolidation_rows
+                ]

-                if not all_rows:
+                if not tagged_rows:
                    return []

-                # Claim the tasks by updating status and worker_id
-                operation_ids = [row["operation_id"] for row in all_rows]
+                operation_ids = [row["operation_id"] for row, _ in tagged_rows]
                await conn.execute(
                    f"""
                    UPDATE {table}
@@ -274,12 +311,16 @@ class WorkerPoller:
                    operation_ids,
                )

-                # Parse and return task payloads with schema context
                result = []
-                for row in all_rows:
+                for row, is_consolidation in tagged_rows:
                    task_dict = json.loads(row["task_payload"])
                    task_dict["_retry_count"] = row["retry_count"]
                    task_dict["_operation_id"] = str(row["operation_id"])
+                    # The DB row knows the operation_type, but the JSON payload may not
+                    # carry it. Inject it so in-flight tracking and slot accounting
+                    # (which key off task_dict["operation_type"]) work correctly.
+                    if is_consolidation:
+                        task_dict["operation_type"] = "consolidation"
                    result.append(
                        ClaimedTask(
                            operation_id=str(row["operation_id"]),
@@ -426,12 +467,27 @@ class WorkerPoller:
        operation_type = task.task_dict.get("operation_type", "unknown")
        bank_id = task.task_dict.get("bank_id", "unknown")

-        # Create background task
-        bg_task = asyncio.create_task(self._execute_task_inner(task))
+        # Stage holder is updated by engine code via stage.set_stage(); the
+        # poller reads it during periodic logging to surface what each
+        # in-flight task is doing.
+        holder = StageHolder(stage=f"queued.{task_type}")
+
+        # Create background task. The holder is passed in and bound to the
+        # task's own contextvar scope inside _execute_task_inner so engine
+        # code running under that task sees it via stage.set_stage().
+        bg_task = asyncio.create_task(self._execute_task_inner(task, holder))

        # Track this task as active
        async with self._in_flight_lock:
-            self._active_tasks[task.operation_id] = (task_type, bank_id, task.schema, bg_task)
+            self._active_tasks[task.operation_id] = ActiveTaskInfo(
+                op_type=operation_type,
+                bank_id=bank_id,
+                schema=task.schema,
+                bg_task=bg_task,
+                started_at=time.monotonic(),
+                stage_holder=holder,
+                task_type=task_type,
+            )
            self._in_flight_count += 1
            self._in_flight_by_type[operation_type] = self._in_flight_by_type.get(operation_type, 0) + 1

@@ -450,7 +506,7 @@ class WorkerPoller:
                    if self._in_flight_by_type[operation_type] == 0:
                        del self._in_flight_by_type[operation_type]

-    async def _execute_task_inner(self, task: ClaimedTask):
+    async def _execute_task_inner(self, task: ClaimedTask, holder: StageHolder | None = None):
        """Inner task execution with retry/fail handling.

        Tasks that want to be retried raise RetryTaskAt; the poller sets next_retry_at
@@ -461,6 +517,14 @@ class WorkerPoller:
        task_type = task.task_dict.get("type", "unknown")
        bank_id = task.task_dict.get("bank_id", "unknown")

+        # Bind the stage holder in this task's own contextvar scope so engine
+        # code running under us can update it via stage.set_stage(). If holder
+        # is None (legacy / direct invocation), set_stage becomes a no-op.
+        if holder is not None:
+            bind_holder(holder)
+            holder.stage = f"executor.{task_type}"
+            holder.updated_at = time.monotonic()
+
        try:
            schema_info = f", schema={task.schema}" if task.schema else ""
            logger.debug(f"Executing task {task.operation_id} (type={task_type}, bank={bank_id}{schema_info})")
@@ -683,7 +747,7 @@ class WorkerPoller:
        while asyncio.get_event_loop().time() - start_time < timeout:
            async with self._in_flight_lock:
                in_flight = self._in_flight_count
-                active_task_objects = [task_info[3] for task_info in self._active_tasks.values()]
+                active_task_objects = [info.bg_task for info in self._active_tasks.values()]

            if in_flight == 0:
                logger.info(f"Worker {self._worker_id} graceful shutdown complete")
@@ -701,12 +765,19 @@ class WorkerPoller:

        # Cancel remaining tasks
        async with self._in_flight_lock:
-            for operation_id, (_, _, _, bg_task) in list(self._active_tasks.items()):
-                if not bg_task.done():
-                    bg_task.cancel()
+            for operation_id, info in list(self._active_tasks.items()):
+                if not info.bg_task.done():
+                    info.bg_task.cancel()

    async def _log_progress_if_due(self):
-        """Log progress stats every PROGRESS_LOG_INTERVAL seconds."""
+        """Log progress stats every PROGRESS_LOG_INTERVAL seconds.
+
+        Emits four kinds of lines:
+          * [WORKER_STATS]  - aggregate slots / pool / global pending counts
+          * [WORKER_TASK]   - one line per in-flight task with age + stage
+          * [STUCK_STACK]   - async stack trace for tasks past stuck thresholds
+          * [DB_WAITS]      - any non-idle hindsight session waiting on a lock
+        """
        now = time.time()
        if now - self._last_progress_log < PROGRESS_LOG_INTERVAL:
            return
@@ -721,13 +792,15 @@ class WorkerPoller:
                active_tasks = dict(self._active_tasks)

            consolidation_count = in_flight_by_type.get("consolidation", 0)
-            available_slots = self._max_slots - in_flight
-            available_consolidation_slots = self._consolidation_max_slots - consolidation_count
+            non_consolidation_in_flight = max(0, in_flight - consolidation_count)
+            non_consolidation_max = max(0, self._max_slots - self._consolidation_max_slots)
+            available_slots = max(0, non_consolidation_max - non_consolidation_in_flight)
+            available_consolidation_slots = max(0, self._consolidation_max_slots - consolidation_count)

-            # Build local processing breakdown
+            # Build local processing breakdown (aggregate counts)
            task_groups: dict[tuple[str, str], int] = {}
-            for op_type, bank_id, _, _ in active_tasks.values():
-                key = (op_type, bank_id)
+            for info in active_tasks.values():
+                key = (info.op_type, info.bank_id)
                task_groups[key] = task_groups.get(key, 0) + 1

            processing_info = [f"{op}:{bank}({cnt})" for (op, bank), cnt in task_groups.items()]
@@ -765,6 +838,11 @@ class WorkerPoller:
                    other_workers.append(f"{wid}:{cnt}")
            others_str = ", ".join(other_workers) if other_workers else "none"

+            # asyncpg pool stats - exhaustion presents as "everything slow",
+            # making it invisible without this line.
+            pool_str = self._format_pool_stats()
+            proc_str = self._format_proc_stats()
+
            # Display None as "default" in logs
            schemas_str = ", ".join(s if s else "default" for s in schemas)
            logger.info(
@@ -773,12 +851,173 @@ class WorkerPoller:
                f"available={available_slots} (consolidation={available_consolidation_slots}) | "
                f"global: pending={global_pending} (schemas: {schemas_str}) | "
                f"others: {others_str} | "
+                f"pool: {pool_str} | "
+                f"proc: {proc_str} | "
                f"my_active: {processing_str}"
            )

+            # Per-task lines, sorted oldest-first so stuck tasks bubble to the top.
+            self._log_per_task_lines(active_tasks, now=time.monotonic())
+
+            # DB lock waits - separate from per-task lines because a single
+            # blocking session can wedge many tasks.
+            await self._log_db_waits()
+
        except Exception as e:
            logger.debug(f"Failed to log progress stats: {e}")

+    def _format_proc_stats(self) -> str:
+        """Render lightweight process memory stats. Returns 'unavailable' if introspection fails."""
+        try:
+            import resource
+
+            # ru_maxrss is bytes on macOS, kilobytes on Linux. Detect by checking platform.
+            import sys
+
+            usage = resource.getrusage(resource.RUSAGE_SELF)
+            rss = usage.ru_maxrss
+            if sys.platform != "darwin":
+                rss *= 1024  # Linux reports KB
+            rss_mb = rss / (1024 * 1024)
+            return f"rss_mb={rss_mb:.0f}"
+        except Exception as e:
+            logger.debug(f"Process stats unavailable: {e}")
+            return "unavailable"
+
+    def _format_pool_stats(self) -> str:
+        """Render asyncpg pool stats. Returns 'unavailable' if pool can't be introspected."""
+        pool = self._pool
+        try:
+            # asyncpg.Pool exposes _holders / _queue internally; fall back gracefully
+            # to public methods if the layout ever changes.
+            size = pool.get_size() if hasattr(pool, "get_size") else len(getattr(pool, "_holders", []))
+            free = pool.get_idle_size() if hasattr(pool, "get_idle_size") else None
+            min_size = pool.get_min_size() if hasattr(pool, "get_min_size") else None
+            max_size = pool.get_max_size() if hasattr(pool, "get_max_size") else None
+            queue = getattr(pool, "_queue", None)
+            waiters = queue.qsize() if queue is not None and hasattr(queue, "qsize") else None
+
+            parts = [f"size={size}"]
+            if min_size is not None and max_size is not None:
+                parts.append(f"limits={min_size}-{max_size}")
+            if free is not None:
+                parts.append(f"idle={free}")
+                parts.append(f"in_use={size - free}")
+            if waiters is not None:
+                parts.append(f"waiters={waiters}")
+            return " ".join(parts)
+        except Exception as e:
+            logger.debug(f"Pool stats unavailable: {e}")
+            return "unavailable"
+
+    def _log_per_task_lines(self, active_tasks: dict[str, ActiveTaskInfo], now: float) -> None:
+        """Emit one [WORKER_TASK] line per in-flight task and dump stuck stacks.
+
+        Sorted by age desc so the oldest (most likely stuck) tasks appear first.
+        """
+        if not active_tasks:
+            return
+
+        # Sort by age descending; tie-break on op_id for determinism.
+        ordered = sorted(
+            active_tasks.items(),
+            key=lambda kv: (now - kv[1].started_at, kv[0]),
+            reverse=True,
+        )
+
+        for op_id, info in ordered:
+            age_s = now - info.started_at
+            holder = info.stage_holder
+            stage = holder.stage if holder is not None else "unknown"
+            stage_age_s = (now - holder.updated_at) if holder is not None else 0.0
+            stuck_marker = "[STUCK?] " if age_s >= STUCK_STACK_INITIAL_THRESHOLD_S else ""
+            schema_part = f" schema={info.schema}" if info.schema else ""
+            logger.info(
+                f"[WORKER_TASK] {stuck_marker}op={op_id} type={info.task_type} "
+                f"op_type={info.op_type} bank={info.bank_id}{schema_part} "
+                f"age={age_s:.0f}s stage={stage} stage_age={stage_age_s:.0f}s"
+            )
+
+            self._maybe_dump_stuck_stack(op_id, info, age_s)
+
+    def _maybe_dump_stuck_stack(self, op_id: str, info: ActiveTaskInfo, age_s: float) -> None:
+        """Dump a coroutine stack for tasks that crossed a stuck threshold.
+
+        Each task gets one dump per threshold (5min, 10min, 20min, 40min...),
+        gated by `info.last_stack_dump_threshold` so logs don't flood for tasks
+        that legitimately take a long time (large LLM jobs, schema-retry loops).
+        """
+        if age_s < STUCK_STACK_INITIAL_THRESHOLD_S:
+            return
+
+        # Find the largest doubling-threshold that the task has crossed.
+        threshold = STUCK_STACK_INITIAL_THRESHOLD_S
+        crossed = STUCK_STACK_INITIAL_THRESHOLD_S
+        while threshold <= age_s and threshold <= STUCK_STACK_MAX_THRESHOLD_S:
+            crossed = threshold
+            threshold *= 2
+
+        if crossed <= info.last_stack_dump_threshold:
+            return
+
+        info.last_stack_dump_threshold = crossed
+
+        try:
+            buf = io.StringIO()
+            info.bg_task.print_stack(file=buf, limit=15)
+            stage = info.stage_holder.stage if info.stage_holder else "unknown"
+            logger.warning(
+                f"[STUCK_STACK] op={op_id} type={info.task_type} bank={info.bank_id} "
+                f"age={age_s:.0f}s threshold={crossed}s stage={stage}\n{buf.getvalue()}"
+            )
+        except Exception as e:
+            # Stack capture is best-effort - never crash the polling loop over it.
+            logger.debug(f"Failed to capture stack for {op_id}: {e}")
+
+    async def _log_db_waits(self) -> None:
+        """Log any non-idle hindsight session that's waiting on a lock or other resource.
+
+        Catches the case where a coroutine appears 'fine' from Python's perspective
+        but is blocked on a Postgres row lock - which is exactly how the 3-phase
+        retain pipeline deadlock would present.
+        """
+        try:
+            async with self._pool.acquire() as conn:
+                rows = await conn.fetch(
+                    """
+                    SELECT
+                        pid,
+                        application_name,
+                        wait_event_type,
+                        wait_event,
+                        state,
+                        EXTRACT(EPOCH FROM (now() - query_start))::int AS age_s,
+                        LEFT(query, 200) AS query
+                    FROM pg_stat_activity
+                    WHERE datname = current_database()
+                      AND state IS NOT NULL
+                      AND state != 'idle'
+                      AND wait_event IS NOT NULL
+                      AND wait_event_type NOT IN ('Activity', 'Client')
+                    ORDER BY age_s DESC NULLS LAST
+                    LIMIT 20
+                    """
+                )
+        except Exception as e:
+            # pg_stat_activity may be restricted on managed Postgres - degrade silently.
+            logger.debug(f"DB waits query failed: {e}")
+            return
+
+        if not rows:
+            return
+
+        for r in rows:
+            logger.info(
+                f"[DB_WAITS] pid={r['pid']} app={r['application_name']} "
+                f"wait={r['wait_event_type']}.{r['wait_event']} state={r['state']} "
+                f"age={r['age_s']}s query={r['query']!r}"
+            )
+
    @property
    def worker_id(self) -> str:
        """Get the worker ID."""
@@ -0,0 +1,59 @@
+"""Stage breadcrumbs for in-flight worker tasks.
+
+The worker poller binds a `StageHolder` to each task's contextvar scope.
+Engine code calls `set_stage("retain.facts.llm")` at phase boundaries; the
+poller reads the holder periodically to surface what each in-flight task is
+currently doing in `WORKER_STATS` / `WORKER_TASK` log lines.
+
+Outside a worker context the contextvar is unset and `set_stage` is a no-op,
+so engine code is safe to call from sync HTTP requests, tests, or the CLI
+without any setup.
+"""
+
+from __future__ import annotations
+
+import time
+from contextvars import ContextVar
+from dataclasses import dataclass, field
+
+
+@dataclass
+class StageHolder:
+    """Mutable container for the current task's stage label."""
+
+    stage: str = "init"
+    updated_at: float = field(default_factory=time.monotonic)
+
+
+_current_holder: ContextVar[StageHolder | None] = ContextVar("hindsight_stage_holder", default=None)
+
+
+def bind_holder(holder: StageHolder):
+    """Bind a holder to the current async context.
+
+    Must be called from inside the task coroutine itself (not from the
+    spawning code) so the binding lives in the task's own contextvar scope.
+
+    Returns the token that can be passed to `_current_holder.reset()` if
+    the binding ever needs to be unwound.
+    """
+    return _current_holder.set(holder)
+
+
+def set_stage(name: str) -> None:
+    """Update the current task's stage label.
+
+    No-op when called outside a worker task context (e.g. from a sync HTTP
+    request, a test, or the CLI). Cheap enough to call per-phase.
+    """
+    holder = _current_holder.get()
+    if holder is None:
+        return
+    holder.stage = name
+    holder.updated_at = time.monotonic()
+
+
+def get_stage() -> str | None:
+    """Return the current stage label, or None if no holder is bound."""
+    holder = _current_holder.get()
+    return holder.stage if holder is not None else None
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"

 [project]
 name = "hindsight-api-slim"
-version = "0.4.22"
+version = "0.5.1"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -21,7 +21,7 @@ dependencies = [
    "sqlalchemy>=2.0.44",
    "alembic>=1.17.1",
    "pgvector>=0.4.1",
-    "greenlet>=3.2.4",
+    "greenlet>=3.2.4,<3.4.0",  # 3.4.0 lacks arm64 wheels for manylinux_2_41
    "psycopg2-binary>=2.9.11",
    "tiktoken>=0.12.0",
    "httpx>=0.27.0",
@@ -40,7 +40,7 @@ dependencies = [
    "anthropic>=0.40.0",
    "typer>=0.9.0",
    "cohere>=5.0.0",
-    "litellm>=1.0.0,<=1.82.6", # 1.82.7+ contains a supply chain attack (malicious .pth credential stealer)
+    "litellm>=1.83.0", # 1.82.7/1.82.8 had a supply chain compromise (yanked); 1.83.0+ also fixes GHSA-jjhc-v7c2-5hh6 / GHSA-53mr-6c8q-9789
    "markitdown[pdf,docx,pptx,xlsx,xls]>=0.1.4", # File to markdown conversion
    "obstore>=0.4.0", # S3/GCS/Azure object storage client (Rust-backed)
    "winloop>=0.1.0; sys_platform == 'win32'",
@@ -78,6 +78,11 @@ local-ml = [
    "mlx-lm>=0.31.1",
    "safetensors>=0.6.2",
 ]
+local-llm = [
+    # Built-in llama.cpp inference for fully offline operation
+    "llama-cpp-python[server]>=0.3.0",
+    "huggingface-hub>=0.20.0",
+]
 embedded-db = [
    "pg0-embedded>=0.11.0",
 ]
@@ -34,7 +34,12 @@ async def test_submit_async_retain_includes_document_tags_in_task_payload():
    contents = [{"content": "Async retain payload test."}]
    document_tags = ["scope:tools", "user:alice"]

-    with patch("hindsight_api.engine.memory_engine.bank_utils.get_bank_profile", new_callable=AsyncMock):
+    # Return (profile, created=False) so the default-template-on-create hook is skipped.
+    with patch(
+        "hindsight_api.engine.memory_engine.bank_utils.get_or_create_bank_profile",
+        new_callable=AsyncMock,
+        return_value=(MagicMock(), False),
+    ):
        result = await MemoryEngine.submit_async_retain(
            engine,
            bank_id="bank-1",
@@ -598,3 +598,182 @@ class TestExport:
        assert resp.status_code == 200
        data = resp.json()
        assert data["version"] == "1"
+
+
+class TestDefaultBankTemplateEnvVar:
+    """Tests for HINDSIGHT_API_DEFAULT_BANK_TEMPLATE — a server-level env var
+    whose manifest is applied automatically to every newly-created bank."""
+
+    @pytest.fixture
+    def default_template(self):
+        return {
+            "version": "1",
+            "bank": {
+                "reflect_mission": "default-env-mission",
+                "retain_extraction_mode": "verbose",
+                "disposition_empathy": 5,
+                "disposition_skepticism": 1,
+            },
+            "mental_models": [
+                {
+                    "id": "default-env-model",
+                    "name": "Default Env Model",
+                    "source_query": "What is the default?",
+                },
+            ],
+            "directives": [
+                {
+                    "name": "Default Env Directive",
+                    "content": "Follow the default behavior.",
+                    "priority": 7,
+                },
+            ],
+        }
+
+    @pytest.fixture
+    def _patched_default_template(self, monkeypatch, default_template):
+        """Install the default template on the already-initialized global config.
+
+        We can't rely on env-var resolution here: MemoryEngine (and its
+        ConfigResolver) snapshot the global config at fixture init time.
+        Patching the field directly keeps the test deterministic while still
+        exercising the same code path that reads `get_config().default_bank_template`.
+        """
+        from hindsight_api.config import _get_raw_config
+
+        raw = _get_raw_config()
+        monkeypatch.setattr(raw, "default_bank_template", default_template)
+        yield default_template
+
+    @pytest.mark.asyncio
+    async def test_default_template_applied_on_new_bank(
+        self, api_client, bank_id, _patched_default_template
+    ):
+        """Creating a new bank applies the default template (config + mental models + directives)."""
+        # Trigger bank auto-creation via GET profile
+        resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
+        assert resp.status_code == 200
+
+        # Config from template should be present as bank overrides
+        config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config")
+        assert config_resp.status_code == 200
+        overrides = config_resp.json()["overrides"]
+        assert overrides["reflect_mission"] == "default-env-mission"
+        assert overrides["retain_extraction_mode"] == "verbose"
+        assert overrides["disposition_empathy"] == 5
+        assert overrides["disposition_skepticism"] == 1
+
+        # Mental model from template should exist
+        mm_resp = await api_client.get(f"/v1/default/banks/{bank_id}/mental-models/default-env-model")
+        assert mm_resp.status_code == 200
+        assert mm_resp.json()["name"] == "Default Env Model"
+
+        # Directive from template should exist
+        dir_resp = await api_client.get(f"/v1/default/banks/{bank_id}/directives")
+        assert dir_resp.status_code == 200
+        names = [d["name"] for d in dir_resp.json()["items"]]
+        assert "Default Env Directive" in names
+
+    @pytest.mark.asyncio
+    async def test_default_template_overrides_env_config_defaults(
+        self, api_client, bank_id, monkeypatch, default_template
+    ):
+        """Fields set by the default template override server-level env-var defaults.
+
+        We point both HINDSIGHT_API_RETAIN_EXTRACTION_MODE (env) and the
+        default template at different values, then confirm the template wins
+        via the per-bank config overrides layer (highest precedence).
+        """
+        from hindsight_api.config import _get_raw_config
+
+        raw = _get_raw_config()
+        # Simulate an env-level default of "concise", overridden by a template that sets "verbose".
+        monkeypatch.setattr(raw, "retain_extraction_mode", "concise")
+        monkeypatch.setattr(raw, "default_bank_template", default_template)
+
+        resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
+        assert resp.status_code == 200
+
+        config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config")
+        overrides = config_resp.json()["overrides"]
+        # Template value wins at the bank-override layer.
+        assert overrides["retain_extraction_mode"] == "verbose"
+
+    @pytest.mark.asyncio
+    async def test_default_template_not_reapplied_on_existing_bank(
+        self, api_client, bank_id, _patched_default_template
+    ):
+        """Template only applies on FIRST creation; subsequent puts are no-ops."""
+        # First hit creates the bank and applies the template
+        resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
+        assert resp.status_code == 200
+
+        # User explicitly overrides a template-set field
+        patch_resp = await api_client.patch(
+            f"/v1/default/banks/{bank_id}/config",
+            json={"updates": {"reflect_mission": "user-override"}},
+        )
+        assert patch_resp.status_code == 200
+
+        # Second put — template must NOT be reapplied (would clobber the override)
+        resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
+        assert resp.status_code == 200
+
+        config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config")
+        assert config_resp.json()["overrides"]["reflect_mission"] == "user-override"
+
+    @pytest.mark.asyncio
+    async def test_default_template_unset_is_noop(self, api_client, bank_id):
+        """With the env var unset (fixture default), bank creation behaves as before."""
+        resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
+        assert resp.status_code == 200
+
+        # No template = no overrides
+        config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config")
+        assert config_resp.json()["overrides"] == {}
+
+    @pytest.mark.asyncio
+    async def test_default_template_malformed_is_swallowed(
+        self, api_client, bank_id, monkeypatch
+    ):
+        """A malformed default template is logged and ignored — bank creation still succeeds."""
+        from hindsight_api.config import _get_raw_config
+
+        raw = _get_raw_config()
+        # Wrong version number fails Pydantic validation.
+        monkeypatch.setattr(raw, "default_bank_template", {"version": "999"})
+
+        resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
+        # Bank creation must not fail even though the template is broken.
+        assert resp.status_code == 200
+
+    def test_parse_default_bank_template_valid_json(self, monkeypatch):
+        """_parse_default_bank_template parses a valid JSON object env var."""
+        from hindsight_api.config import _parse_default_bank_template
+
+        parsed = _parse_default_bank_template('{"version": "1", "bank": {"disposition_empathy": 4}}')
+        assert parsed == {"version": "1", "bank": {"disposition_empathy": 4}}
+
+    def test_parse_default_bank_template_none_or_empty(self):
+        """Unset / empty env var resolves to None."""
+        from hindsight_api.config import _parse_default_bank_template
+
+        assert _parse_default_bank_template(None) is None
+        assert _parse_default_bank_template("") is None
+        assert _parse_default_bank_template("   ") is None
+
+    def test_parse_default_bank_template_invalid_json_raises(self):
+        """Invalid JSON fails fast with a clear error."""
+        from hindsight_api.config import _parse_default_bank_template
+
+        with pytest.raises(ValueError, match="HINDSIGHT_API_DEFAULT_BANK_TEMPLATE"):
+            _parse_default_bank_template("not-json")
+
+    def test_parse_default_bank_template_non_object_raises(self):
+        """Non-object JSON (e.g. array, string) fails fast."""
+        from hindsight_api.config import _parse_default_bank_template
+
+        with pytest.raises(ValueError, match="expected a JSON object"):
+            _parse_default_bank_template("[1, 2, 3]")
+        with pytest.raises(ValueError, match="expected a JSON object"):
+            _parse_default_bank_template('"just a string"')
@@ -0,0 +1,149 @@
+"""
+Regression tests for chunk_storage.store_chunks_batch idempotency.
+
+Covers vectorize-io/hindsight#977: re-submitting a retain under the same
+document_id must not fail with ``UniqueViolationError`` on ``pk_chunks``.
+The upstream retain paths (cascade delete on first batch, delta retain)
+should usually prevent a chunk_id collision, but any bug in those paths
+used to surface as a raw Postgres constraint violation. ``store_chunks_batch``
+is now idempotent: inserting the same ``chunk_id`` twice overwrites the
+existing row rather than raising.
+"""
+
+from datetime import datetime, timezone
+
+import pytest
+
+from hindsight_api.engine.retain import chunk_storage
+from hindsight_api.engine.retain.types import ChunkMetadata
+
+
+def _ts() -> float:
+    return datetime.now(timezone.utc).timestamp()
+
+
+async def _seed_bank_and_document(conn, bank_id: str, document_id: str) -> None:
+    """Insert the minimum rows required for the chunks FK to pass."""
+    await conn.execute(
+        "INSERT INTO banks (bank_id, name) VALUES ($1, $2) ON CONFLICT DO NOTHING",
+        bank_id,
+        bank_id,
+    )
+    await conn.execute(
+        """
+        INSERT INTO documents (id, bank_id, original_text, content_hash)
+        VALUES ($1, $2, $3, $4)
+        ON CONFLICT (id, bank_id) DO NOTHING
+        """,
+        document_id,
+        bank_id,
+        "seed",
+        "seed-hash",
+    )
+
+
+@pytest.mark.asyncio
+async def test_store_chunks_batch_is_idempotent_for_same_chunk_id(memory):
+    """
+    Regression for #977.
+
+    Directly exercises the chunk insert path: inserting a ChunkMetadata with
+    a chunk_index that already exists (i.e., the same chunk_id) must not
+    raise. The new content should overwrite the old one.
+    """
+    bank_id = f"test_chunk_upsert_{_ts()}"
+    document_id = "doc-upsert-regression"
+
+    pool = await memory._get_pool()
+    try:
+        async with pool.acquire() as conn:
+            await _seed_bank_and_document(conn, bank_id, document_id)
+
+            # First insert — fresh chunks at indices 0, 1, 2.
+            v1 = [
+                ChunkMetadata(chunk_text="alpha", fact_count=1, content_index=0, chunk_index=0),
+                ChunkMetadata(chunk_text="beta", fact_count=1, content_index=0, chunk_index=1),
+                ChunkMetadata(chunk_text="gamma", fact_count=1, content_index=0, chunk_index=2),
+            ]
+            v1_map = await chunk_storage.store_chunks_batch(conn, bank_id, document_id, v1)
+            assert set(v1_map.keys()) == {0, 1, 2}
+
+            # Second insert — overlapping chunk_index (1 and 2) with new text,
+            # plus a fresh chunk at index 3. Before the fix this raised
+            # asyncpg.exceptions.UniqueViolationError on pk_chunks; after the
+            # fix the conflicting rows are overwritten and the new one is
+            # inserted.
+            v2 = [
+                ChunkMetadata(chunk_text="beta-updated", fact_count=1, content_index=0, chunk_index=1),
+                ChunkMetadata(chunk_text="gamma-updated", fact_count=1, content_index=0, chunk_index=2),
+                ChunkMetadata(chunk_text="delta", fact_count=1, content_index=0, chunk_index=3),
+            ]
+            v2_map = await chunk_storage.store_chunks_batch(conn, bank_id, document_id, v2)
+            assert set(v2_map.keys()) == {1, 2, 3}
+
+            # Verify the stored state matches the upserted content.
+            rows = await conn.fetch(
+                """
+                SELECT chunk_index, chunk_text, content_hash
+                FROM chunks
+                WHERE document_id = $1 AND bank_id = $2
+                ORDER BY chunk_index
+                """,
+                document_id,
+                bank_id,
+            )
+            by_index = {row["chunk_index"]: row for row in rows}
+
+            assert set(by_index.keys()) == {0, 1, 2, 3}, (
+                "Expected four chunks total after upsert (0 untouched, 1-2 overwritten, 3 new)"
+            )
+            assert by_index[0]["chunk_text"] == "alpha", "Untouched chunk must be preserved"
+            assert by_index[1]["chunk_text"] == "beta-updated", "Conflicting chunk must be overwritten"
+            assert by_index[2]["chunk_text"] == "gamma-updated", "Conflicting chunk must be overwritten"
+            assert by_index[3]["chunk_text"] == "delta", "New chunk must be inserted"
+
+            # content_hash should reflect the new text, not the original.
+            assert by_index[1]["content_hash"] == chunk_storage.compute_chunk_hash("beta-updated")
+            assert by_index[2]["content_hash"] == chunk_storage.compute_chunk_hash("gamma-updated")
+    finally:
+        async with pool.acquire() as conn:
+            await conn.execute("DELETE FROM chunks WHERE bank_id = $1", bank_id)
+            await conn.execute("DELETE FROM documents WHERE bank_id = $1", bank_id)
+            await conn.execute("DELETE FROM banks WHERE bank_id = $1", bank_id)
+
+
+@pytest.mark.asyncio
+async def test_store_chunks_batch_second_call_with_identical_payload(memory):
+    """
+    The exact #977 shape: ``store_chunks_batch`` called twice with the same
+    chunks must succeed both times (the second call is a no-op in terms of
+    stored content, but must not raise).
+    """
+    bank_id = f"test_chunk_upsert_identical_{_ts()}"
+    document_id = "doc-upsert-identical"
+
+    pool = await memory._get_pool()
+    try:
+        async with pool.acquire() as conn:
+            await _seed_bank_and_document(conn, bank_id, document_id)
+
+            chunks = [
+                ChunkMetadata(chunk_text=f"chunk-{i}", fact_count=1, content_index=0, chunk_index=i)
+                for i in range(5)
+            ]
+
+            await chunk_storage.store_chunks_batch(conn, bank_id, document_id, chunks)
+            # Second call with identical chunks — must not raise.
+            await chunk_storage.store_chunks_batch(conn, bank_id, document_id, chunks)
+
+            count = await conn.fetchval(
+                "SELECT COUNT(*) FROM chunks WHERE document_id = $1 AND bank_id = $2",
+                document_id,
+                bank_id,
+            )
+            assert count == 5, "Second identical insert should not duplicate rows"
+    finally:
+        async with pool.acquire() as conn:
+            await conn.execute("DELETE FROM chunks WHERE bank_id = $1", bank_id)
+            await conn.execute("DELETE FROM documents WHERE bank_id = $1", bank_id)
+            await conn.execute("DELETE FROM banks WHERE bank_id = $1", bank_id)
@@ -5,7 +5,7 @@ Tests the Cohere cross-encoder implementation, including Azure AI Foundry endpoi
 """

 import os
-from unittest.mock import MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch

 import httpx
 import pytest
@@ -28,7 +28,7 @@ class TestCohereCrossEncoder:
        assert encoder.api_key == "test_key"
        assert encoder.model == "rerank-english-v3.0"
        assert encoder._client is None
-        assert encoder._httpx_client is None
+        assert encoder._http_client is None

        # Mock the cohere import
        mock_cohere = MagicMock()
@@ -36,7 +36,7 @@ class TestCohereCrossEncoder:
        with patch.dict("sys.modules", {"cohere": mock_cohere}):
            await encoder.initialize()
            assert encoder._client is not None
-            assert encoder._httpx_client is None
+            assert encoder._http_client is None
            mock_cohere.Client.assert_called_once_with(api_key="test_key", timeout=60.0)

    @pytest.mark.asyncio
@@ -52,9 +52,14 @@ class TestCohereCrossEncoder:

        await encoder.initialize()

-        assert encoder._httpx_client is not None
+        assert encoder._http_client is not None
        assert encoder._client is None
-        assert isinstance(encoder._httpx_client, httpx.Client)
+        assert isinstance(encoder._http_client._async_client, httpx.AsyncClient)
+        assert encoder._http_client.include_top_n is False
+        assert (
+            encoder._http_client.rerank_url
+            == "https://my-endpoint.inference.ai.azure.com/models/cohere-rerank-v3-english/invoke"
+        )

    @pytest.mark.asyncio
    async def test_initialization_missing_package(self):
@@ -150,7 +155,7 @@ class TestCohereCrossEncoder:

        await encoder.initialize()

-        # Mock httpx response
+        # Mock async httpx response
        mock_response = MagicMock()
        mock_response.json.return_value = {
            "results": [
@@ -159,8 +164,9 @@ class TestCohereCrossEncoder:
                {"index": 2, "relevance_score": 0.5},
            ]
        }
+        mock_response.raise_for_status = MagicMock()

-        encoder._httpx_client.post = MagicMock(return_value=mock_response)
+        encoder._http_client._async_client.post = AsyncMock(return_value=mock_response)

        pairs = [
            ("What is Python?", "Python is a programming language"),
@@ -174,13 +180,15 @@ class TestCohereCrossEncoder:
        assert scores == [0.9, 0.7, 0.5]

        # Verify httpx.post was called with correct URL and payload
-        encoder._httpx_client.post.assert_called_once()
-        call_args = encoder._httpx_client.post.call_args
+        encoder._http_client._async_client.post.assert_called_once()
+        call_args = encoder._http_client._async_client.post.call_args
        assert call_args[0][0] == "https://my-endpoint.inference.ai.azure.com/models/cohere-rerank-v3-english/invoke"
        assert call_args.kwargs["json"]["model"] == "cohere-rerank-v3-english"
        assert call_args.kwargs["json"]["query"] == "What is Python?"
        assert len(call_args.kwargs["json"]["documents"]) == 3
        assert call_args.kwargs["json"]["return_documents"] is False
+        # Azure endpoints expect no top_n in the body
+        assert "top_n" not in call_args.kwargs["json"]

    @pytest.mark.asyncio
    async def test_predict_multiple_queries(self):
@@ -281,7 +289,7 @@ class TestCohereCrossEncoder:
            request=MagicMock(),
            response=MagicMock(status_code=404),
        )
-        encoder._httpx_client.post = MagicMock(return_value=mock_response)
+        encoder._http_client._async_client.post = AsyncMock(return_value=mock_response)

        pairs = [("What is Python?", "Python is a programming language")]

@@ -7,7 +7,6 @@ relevance score, independent of the cross-encoder model's score calibration.
 """

 from datetime import datetime, timedelta, timezone
-from unittest.mock import MagicMock

 import pytest

@@ -23,13 +22,18 @@ def _make_result(
    occurred_start: datetime | None = None,
    temporal_proximity: float | None = None,
 ) -> ScoredResult:
-    retrieval = MagicMock(spec=RetrievalResult)
-    retrieval.occurred_start = occurred_start
-    retrieval.temporal_proximity = temporal_proximity
+    retrieval = RetrievalResult(
+        id="test",
+        text="test",
+        fact_type="world",
+        occurred_start=occurred_start,
+        temporal_proximity=temporal_proximity,
+    )

-    candidate = MagicMock(spec=MergedCandidate)
-    candidate.retrieval = retrieval
-    candidate.rrf_score = 0.05
+    candidate = MergedCandidate(
+        retrieval=retrieval,
+        rrf_score=0.05,
+    )

    return ScoredResult(
        candidate=candidate,
@@ -15,7 +15,12 @@ import pytest
 from sqlalchemy import create_engine, text

 from hindsight_api import MemoryEngine, RequestContext
-from hindsight_api.engine.cross_encoder import CohereCrossEncoder, LocalSTCrossEncoder, ZeroEntropyCrossEncoder
+from hindsight_api.engine.cross_encoder import (
+    CohereCrossEncoder,
+    LocalSTCrossEncoder,
+    SiliconFlowCrossEncoder,
+    ZeroEntropyCrossEncoder,
+)
 from hindsight_api.engine.embeddings import CohereEmbeddings, LocalSTEmbeddings, OpenAIEmbeddings
 from hindsight_api.engine.query_analyzer import DateparserQueryAnalyzer
 from hindsight_api.engine.task_backend import SyncTaskBackend
@@ -739,3 +744,58 @@ class TestZeroEntropyCrossEncoder:
        assert all(isinstance(s, float) for s in scores)
        # The first result should be most relevant
        assert scores[0] > scores[2], "Direct answer should score higher than unrelated text"
+
+
+# =============================================================================
+# SiliconFlow Reranker Tests
+# =============================================================================
+
+
+def has_siliconflow_api_key() -> bool:
+    """Check if SiliconFlow API key is available."""
+    return bool(os.environ.get("SILICONFLOW_API_KEY"))
+
+
+def get_siliconflow_api_key() -> str:
+    """Get SiliconFlow API key from environment."""
+    return os.environ.get("SILICONFLOW_API_KEY", "")
+
+
+@pytest.fixture(scope="module")
+def siliconflow_cross_encoder():
+    """Create SiliconFlow cross-encoder instance."""
+    if not has_siliconflow_api_key():
+        pytest.skip("SiliconFlow API key not available (set SILICONFLOW_API_KEY)")
+
+    cross_encoder = SiliconFlowCrossEncoder(
+        api_key=get_siliconflow_api_key(),
+        model="BAAI/bge-reranker-v2-m3",
+    )
+    loop = asyncio.new_event_loop()
+    try:
+        loop.run_until_complete(cross_encoder.initialize())
+    finally:
+        loop.close()
+    return cross_encoder
+
+
+class TestSiliconFlowCrossEncoder:
+    """Tests for SiliconFlow cross-encoder/reranker."""
+
+    def test_siliconflow_cross_encoder_initialization(self, siliconflow_cross_encoder):
+        """Test that SiliconFlow cross-encoder initializes correctly."""
+        assert siliconflow_cross_encoder.provider_name == "siliconflow"
+
+    @pytest.mark.asyncio
+    async def test_siliconflow_cross_encoder_predict(self, siliconflow_cross_encoder):
+        """Test that SiliconFlow cross-encoder can score pairs."""
+        pairs = [
+            ("What is the capital of France?", "Paris is the capital of France."),
+            ("What is the capital of France?", "The Eiffel Tower is in Paris."),
+            ("What is the capital of France?", "Python is a programming language."),
+        ]
+        scores = await siliconflow_cross_encoder.predict(pairs)
+
+        assert len(scores) == 3
+        assert all(isinstance(s, float) for s in scores)
+        assert scores[0] > scores[2], "Direct answer should score higher than unrelated text"
@@ -0,0 +1,65 @@
+"""
+Tests for format_facts_for_prompt in think_utils.
+"""
+
+import json
+
+from hindsight_api.engine.response_models import MemoryFact
+from hindsight_api.engine.search.think_utils import format_facts_for_prompt
+
+
+def test_format_facts_includes_temporal_fields():
+    """All temporal fields (occurred_start, occurred_end, mentioned_at) should appear in the JSON."""
+    facts = [
+        MemoryFact(
+            id="fact-1",
+            text="Team offsite in February",
+            fact_type="experience",
+            occurred_start="2024-02-01T00:00:00Z",
+            occurred_end="2024-02-28T23:59:59Z",
+            mentioned_at="2024-03-05T10:00:00Z",
+        )
+    ]
+    result = json.loads(format_facts_for_prompt(facts))
+    assert len(result) == 1
+    assert result[0]["text"] == "Team offsite in February"
+    assert result[0]["occurred_start"] == "2024-02-01T00:00:00Z"
+    assert result[0]["occurred_end"] == "2024-02-28T23:59:59Z"
+    assert result[0]["mentioned_at"] == "2024-03-05T10:00:00Z"
+
+
+def test_format_facts_omits_null_temporal_fields():
+    """Null temporal fields should not appear in the JSON."""
+    facts = [
+        MemoryFact(
+            id="fact-2",
+            text="The sky is blue",
+            fact_type="world",
+        )
+    ]
+    result = json.loads(format_facts_for_prompt(facts))
+    assert len(result) == 1
+    assert "occurred_start" not in result[0]
+    assert "occurred_end" not in result[0]
+    assert "mentioned_at" not in result[0]
+
+
+def test_format_facts_partial_temporal_fields():
+    """Only non-null temporal fields should appear."""
+    facts = [
+        MemoryFact(
+            id="fact-3",
+            text="Meeting happened",
+            fact_type="experience",
+            occurred_start="2024-06-01T09:00:00Z",
+        )
+    ]
+    result = json.loads(format_facts_for_prompt(facts))
+    assert result[0]["occurred_start"] == "2024-06-01T09:00:00Z"
+    assert "occurred_end" not in result[0]
+    assert "mentioned_at" not in result[0]
+
+
+def test_format_facts_empty_list():
+    """Empty list should return '[]'."""
+    assert format_facts_for_prompt([]) == "[]"
@@ -0,0 +1,211 @@
+"""
+Tests for LATERAL entity fanout cap in graph expansion.
+
+Verifies that the per-entity LIMIT in _expand_combined prevents high-fanout
+entities from exploding the self-join, while still returning entity-based
+graph results.
+"""
+
+import asyncio
+from datetime import datetime, timezone
+
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_high_fanout_entity_returns_results(memory, request_context):
+    """
+    A high-fanout entity (appearing in many facts) should still produce
+    graph retrieval results — the LATERAL cap limits rows per entity but
+    does not drop the entity entirely.
+    """
+    bank_id = f"test_fanout_cap_{datetime.now(timezone.utc).timestamp()}"
+
+    try:
+        # Create many facts sharing one common entity ("Acme Corp") plus
+        # a few with a unique entity so we can query for the unique one
+        # and verify graph expansion finds siblings via "Acme Corp".
+        contents = [
+            # Target: unique entity "Zara" shares "Acme Corp" with the rest
+            {
+                "content": "Zara joined Acme Corp as a senior engineer last month",
+                "context": "hr update",
+                "entities": [{"text": "Zara"}, {"text": "Acme Corp"}],
+            },
+        ]
+        # Add many facts that all share "Acme Corp" — creates a high-fanout entity
+        for i in range(60):
+            contents.append(
+                {
+                    "content": f"Employee {i} completed onboarding at Acme Corp in department {i % 5}",
+                    "context": "hr update",
+                    "entities": [{"text": f"Employee {i}"}, {"text": "Acme Corp"}],
+                }
+            )
+
+        await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=contents,
+            request_context=request_context,
+        )
+
+        from hindsight_api.engine.memory_engine import Budget
+
+        # Query for "Zara" — semantic search finds Zara's fact as a seed,
+        # then graph expansion should find other Acme Corp facts via the
+        # shared entity, even though "Acme Corp" has 60+ mentions.
+        result = await memory.recall_async(
+            bank_id=bank_id,
+            query="Zara",
+            budget=Budget.HIGH,
+            max_tokens=4096,
+            enable_trace=True,
+            request_context=request_context,
+            _quiet=True,
+        )
+
+        assert result.results is not None
+        assert len(result.results) > 0
+
+        # Verify graph retrieval ran and found results
+        retrieval_results = result.trace.get("retrieval_results", [])
+        graph_results = [r for r in retrieval_results if r.get("method_name") == "graph"]
+        assert len(graph_results) > 0, "Graph retrieval should have run"
+
+        # At least one graph result should contain Acme Corp content
+        # (found via shared entity, not just semantic similarity)
+        all_texts = [r.text for r in result.results]
+        acme_found = any("Acme Corp" in t for t in all_texts)
+        assert acme_found, "Should find Acme Corp facts via entity graph expansion"
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_entity_expansion_timeout_fallback(memory, request_context):
+    """
+    When graph_expansion_timeout is set very low, entity expansion should
+    time out gracefully and fall back to semantic+causal links only,
+    rather than failing the entire recall.
+    """
+    bank_id = f"test_timeout_fallback_{datetime.now(timezone.utc).timestamp()}"
+
+    try:
+        await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {
+                    "content": "Alice works on the backend API at TechCorp",
+                    "context": "team info",
+                    "entities": [{"text": "Alice"}, {"text": "TechCorp"}],
+                },
+                {
+                    "content": "Bob maintains the frontend at TechCorp",
+                    "context": "team info",
+                    "entities": [{"text": "Bob"}, {"text": "TechCorp"}],
+                },
+            ],
+            request_context=request_context,
+        )
+
+        from hindsight_api.config import _get_raw_config
+        from hindsight_api.engine.memory_engine import Budget
+
+        config = _get_raw_config()
+        original_timeout = config.link_expansion_timeout
+
+        try:
+            # Set an impossibly low timeout to force the fallback path
+            config.link_expansion_timeout = 0.0001
+
+            result = await memory.recall_async(
+                bank_id=bank_id,
+                query="Alice",
+                budget=Budget.MID,
+                max_tokens=2048,
+                enable_trace=True,
+                request_context=request_context,
+                _quiet=True,
+            )
+
+            # Recall should succeed even when entity expansion times out
+            assert result.results is not None
+            assert len(result.results) > 0
+
+            # Alice should still be found via semantic search
+            result_texts = [r.text for r in result.results]
+            alice_found = any("Alice" in t for t in result_texts)
+            assert alice_found, "Should find Alice via semantic search despite graph timeout"
+        finally:
+            config.link_expansion_timeout = original_timeout
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_per_entity_limit_caps_expansion(memory, request_context):
+    """
+    With graph_per_entity_limit set to a small value, entity expansion should
+    still work but return fewer results from high-fanout entities.
+    """
+    bank_id = f"test_per_entity_limit_{datetime.now(timezone.utc).timestamp()}"
+
+    try:
+        # Create facts with a shared entity
+        contents = [
+            {
+                "content": "Lead engineer Dana oversees the Widgets project at MegaCorp",
+                "context": "project info",
+                "entities": [{"text": "Dana"}, {"text": "MegaCorp"}],
+            },
+        ]
+        for i in range(30):
+            contents.append(
+                {
+                    "content": f"MegaCorp hired contractor {i} for the Q4 push",
+                    "context": "hiring info",
+                    "entities": [{"text": f"Contractor {i}"}, {"text": "MegaCorp"}],
+                }
+            )
+
+        await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=contents,
+            request_context=request_context,
+        )
+
+        from hindsight_api.config import _get_raw_config
+        from hindsight_api.engine.memory_engine import Budget
+
+        config = _get_raw_config()
+        original_limit = config.link_expansion_per_entity_limit
+
+        try:
+            # Set a very small per-entity limit
+            config.link_expansion_per_entity_limit = 5
+
+            result = await memory.recall_async(
+                bank_id=bank_id,
+                query="Dana",
+                budget=Budget.HIGH,
+                max_tokens=4096,
+                enable_trace=True,
+                request_context=request_context,
+                _quiet=True,
+            )
+
+            # Recall should succeed with the cap
+            assert result.results is not None
+            assert len(result.results) > 0
+
+            # Graph retrieval should have run
+            retrieval_results = result.trace.get("retrieval_results", [])
+            graph_results = [r for r in retrieval_results if r.get("method_name") == "graph"]
+            assert len(graph_results) > 0, "Graph retrieval should have run"
+        finally:
+            config.link_expansion_per_entity_limit = original_limit
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
@@ -209,6 +209,47 @@ async def test_config_validation_rejects_static_fields(memory, request_context):
        await memory.delete_bank(bank_id, request_context=request_context)


+@pytest.mark.asyncio
+async def test_config_validation_rejects_malformed_entity_labels(memory, request_context):
+    """Test that passing strings instead of LabelGroup dicts to entity_labels raises ValueError.
+
+    Regression test for the fix in PR #902: entity_labels PATCH must validate the
+    format before saving to prevent silent corruption that previously caused 500s on
+    subsequent retain calls (reported in issue #946).
+    """
+    bank_id = "test-entity-labels-validation"
+
+    try:
+        await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        resolver = ConfigResolver(pool=memory._pool)
+
+        # String list instead of LabelGroup dicts must raise ValueError, not silently accept.
+        # Previously this produced HTTP 200, then 500 on the next retain call (issue #946).
+        with pytest.raises(ValueError, match="Invalid entity_labels format"):
+            await resolver.update_bank_config(
+                bank_id,
+                {"entity_labels": ["person", "client", "tool"]},
+            )
+
+        # The correct LabelGroup format must succeed
+        await resolver.update_bank_config(
+            bank_id,
+            {
+                "entity_labels": [
+                    {
+                        "key": "kind",
+                        "type": "value",
+                        "values": [{"value": "person"}, {"value": "client"}],
+                    }
+                ]
+            },
+        )
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
@pytest.mark.asyncio
 async def test_config_freshness_across_updates(memory, request_context):
    """Test that config changes are immediately visible (no stale cache)."""
@@ -394,10 +435,16 @@ async def test_config_get_bank_config_no_static_or_credential_fields_leak(memory

        # SECURITY: Verify specific sensitive fields are NOT present
        sensitive_fields = [
-            "database_url", "api_port", "host", "worker_count",  # Infrastructure
-            "llm_api_key", "llm_base_url",  # Credentials
-            "retain_llm_api_key", "reflect_llm_api_key",  # More credentials
-            "llm_provider", "llm_model",  # Not configurable (need presets)
+            "database_url",
+            "api_port",
+            "host",
+            "worker_count",  # Infrastructure
+            "llm_api_key",
+            "llm_base_url",  # Credentials
+            "retain_llm_api_key",
+            "reflect_llm_api_key",  # More credentials
+            "llm_provider",
+            "llm_model",  # Not configurable (need presets)
        ]
        for field in sensitive_fields:
            assert field not in config, (
@@ -0,0 +1,184 @@
+"""
+Regression tests for vectorize-io/hindsight#980.
+
+Deterministic Postgres integrity-constraint violations (UniqueViolationError,
+ForeignKeyViolationError, CheckViolationError, NotNullViolationError,
+ExclusionViolationError) must NOT be retried by the worker — they will never
+succeed on retry, and retrying just burns worker capacity for ~3 minutes
+(3 retries × 60s) before finally giving up.
+
+These tests verify that ``MemoryEngine.execute_task`` classifies
+``asyncpg.exceptions.IntegrityConstraintViolationError`` as non-retryable
+and marks the operation as failed on the first occurrence.
+"""
+
+import json
+import uuid
+from unittest.mock import AsyncMock, patch
+
+import asyncpg
+import pytest
+
+from hindsight_api.worker.exceptions import RetryTaskAt
+
+
+async def _ensure_bank(pool, bank_id: str) -> None:
+    """Upsert a minimal bank row so FK on async_operations passes."""
+    await pool.execute(
+        "INSERT INTO banks (bank_id, name) VALUES ($1, $2) ON CONFLICT DO NOTHING",
+        bank_id,
+        bank_id,
+    )
+
+
+async def _create_pending_operation(pool, bank_id: str, operation_id: uuid.UUID) -> None:
+    """Insert a pending batch_retain operation row for the test."""
+    payload = json.dumps(
+        {
+            "type": "batch_retain",
+            "operation_id": str(operation_id),
+            "bank_id": bank_id,
+            "contents": [{"content": "test", "document_id": "doc-1"}],
+        }
+    )
+    await pool.execute(
+        """
+        INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
+        VALUES ($1, $2, 'retain', 'pending', $3::jsonb)
+        """,
+        operation_id,
+        bank_id,
+        payload,
+    )
+
+
+@pytest.mark.asyncio
+async def test_unique_violation_marks_failed_without_retry(memory):
+    """
+    UniqueViolationError must mark the operation as failed immediately, not
+    raise RetryTaskAt. This is the primary symptom from #977: re-submitting
+    retain caused PK collisions that the poller retried ~3 times before
+    giving up. With #980's fix, the first collision fails the task.
+    """
+    bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+    operation_id = uuid.uuid4()
+
+    pool = await memory._get_pool()
+    await _ensure_bank(pool, bank_id)
+    await _create_pending_operation(pool, bank_id, operation_id)
+
+    # Synthesize a real asyncpg UniqueViolationError the way the server would
+    # raise it (matches the error observed in the bug report's logs).
+    unique_violation = asyncpg.exceptions.UniqueViolationError(
+        'duplicate key value violates unique constraint "pk_chunks"'
+    )
+
+    task_dict = {
+        "type": "batch_retain",
+        "operation_id": str(operation_id),
+        "bank_id": bank_id,
+        "contents": [{"content": "test", "document_id": "doc-1"}],
+    }
+
+    # Force _handle_batch_retain to raise the integrity error, isolating the
+    # execute_task exception-classification path.
+    with patch.object(memory, "_handle_batch_retain", side_effect=unique_violation):
+        # Must not raise RetryTaskAt — the whole point of the fix.
+        try:
+            await memory.execute_task(task_dict)
+        except RetryTaskAt as exc:
+            pytest.fail(
+                f"IntegrityConstraintViolationError must not be retried, but execute_task raised {exc!r}"
+            )
+
+    # The operation must be marked 'failed' (not left pending / retrying).
+    row = await pool.fetchrow(
+        "SELECT status, error_message FROM async_operations WHERE operation_id = $1",
+        operation_id,
+    )
+    assert row is not None, "Operation row disappeared"
+    assert row["status"] == "failed", (
+        f"Expected status='failed' after integrity violation, got {row['status']!r}"
+    )
+    assert row["error_message"] is not None
+    assert "pk_chunks" in row["error_message"]
+
+    # Cleanup
+    await pool.execute("DELETE FROM async_operations WHERE operation_id = $1", operation_id)
+    await pool.execute("DELETE FROM banks WHERE bank_id = $1", bank_id)
+
+
+@pytest.mark.asyncio
+async def test_foreign_key_violation_also_not_retried(memory):
+    """
+    All subclasses of IntegrityConstraintViolationError are non-retryable —
+    verify ForeignKeyViolationError is classified the same way as
+    UniqueViolationError.
+    """
+    bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+    operation_id = uuid.uuid4()
+
+    pool = await memory._get_pool()
+    await _ensure_bank(pool, bank_id)
+    await _create_pending_operation(pool, bank_id, operation_id)
+
+    fk_violation = asyncpg.exceptions.ForeignKeyViolationError(
+        "insert or update on table \"memory_units\" violates foreign key constraint \"fk_bank\""
+    )
+
+    task_dict = {
+        "type": "batch_retain",
+        "operation_id": str(operation_id),
+        "bank_id": bank_id,
+        "contents": [{"content": "test", "document_id": "doc-1"}],
+    }
+
+    with patch.object(memory, "_handle_batch_retain", side_effect=fk_violation):
+        try:
+            await memory.execute_task(task_dict)
+        except RetryTaskAt as exc:
+            pytest.fail(
+                f"ForeignKeyViolationError must not be retried, but execute_task raised {exc!r}"
+            )
+
+    row = await pool.fetchrow(
+        "SELECT status FROM async_operations WHERE operation_id = $1",
+        operation_id,
+    )
+    assert row["status"] == "failed"
+
+    await pool.execute("DELETE FROM async_operations WHERE operation_id = $1", operation_id)
+    await pool.execute("DELETE FROM banks WHERE bank_id = $1", bank_id)
+
+
+@pytest.mark.asyncio
+async def test_non_integrity_error_still_retried(memory):
+    """
+    Sanity check: non-integrity errors (network errors, timeouts, value errors)
+    should STILL use the existing retry path — i.e., raise RetryTaskAt when
+    ``_retry_count < 3``. Only integrity violations are the new non-retryable
+    class.
+    """
+    bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+    operation_id = uuid.uuid4()
+
+    pool = await memory._get_pool()
+    await _ensure_bank(pool, bank_id)
+    await _create_pending_operation(pool, bank_id, operation_id)
+
+    task_dict = {
+        "type": "batch_retain",
+        "operation_id": str(operation_id),
+        "bank_id": bank_id,
+        "contents": [{"content": "test", "document_id": "doc-1"}],
+        # _retry_count = 0 (first attempt), so the existing retry path should fire.
+    }
+
+    transient_error = RuntimeError("transient connection blip")
+
+    with patch.object(memory, "_handle_batch_retain", side_effect=transient_error):
+        with pytest.raises(RetryTaskAt):
+            await memory.execute_task(task_dict)
+
+    await pool.execute("DELETE FROM async_operations WHERE operation_id = $1", operation_id)
+    await pool.execute("DELETE FROM banks WHERE bank_id = $1", bank_id)
@@ -0,0 +1,78 @@
+"""
+Regression test for the JinaMLXCrossEncoder import-error handling.
+
+See: https://github.com/vectorize-io/hindsight/issues/994
+
+Before the fix, the bare `except ImportError` around `import mlx_lm` masked
+*any* ImportError raised transitively during mlx_lm's own initialization
+(e.g. transformers 5.x's _LazyModule race producing
+`ImportError: cannot import name 'AutoTokenizer' from 'transformers'`),
+replacing it with a misleading "install mlx" message.
+
+These tests verify:
+1. A transitive ImportError raised from inside mlx_lm surfaces verbatim.
+2. A genuine "package not installed" ImportError still produces the install hint.
+"""
+
+import sys
+import types
+from unittest.mock import patch
+
+import pytest
+
+from hindsight_api.engine.cross_encoder import JinaMLXCrossEncoder
+
+
+def _stub_mlx_modules() -> dict[str, types.ModuleType]:
+    """Stub mlx + mlx.core so `import mlx.core` succeeds even without mlx installed."""
+    import importlib.machinery
+
+    mlx = types.ModuleType("mlx")
+    mlx.__spec__ = importlib.machinery.ModuleSpec("mlx", loader=None)
+    mlx_core = types.ModuleType("mlx.core")
+    mlx_core.__spec__ = importlib.machinery.ModuleSpec("mlx.core", loader=None)
+    mlx.core = mlx_core
+    return {"mlx": mlx, "mlx.core": mlx_core}
+
+
+@pytest.mark.asyncio
+async def test_initialize_surfaces_transitive_import_error():
+    """A transformers-lazy-load-style failure must propagate, not be masked."""
+    encoder = JinaMLXCrossEncoder()
+
+    real_import = __import__
+
+    def fake_import(name, *args, **kwargs):
+        if name == "mlx_lm" or name.startswith("mlx_lm."):
+            raise ImportError("cannot import name 'AutoTokenizer' from 'transformers'")
+        return real_import(name, *args, **kwargs)
+
+    sys.modules.pop("mlx_lm", None)
+
+    with patch.dict(sys.modules, _stub_mlx_modules()):
+        with patch("builtins.__import__", side_effect=fake_import):
+            with pytest.raises(ImportError, match="AutoTokenizer"):
+                await encoder.initialize()
+
+
+@pytest.mark.asyncio
+async def test_initialize_reports_install_hint_when_mlx_missing():
+    """A genuine 'package not installed' error still gets the friendly install hint."""
+    encoder = JinaMLXCrossEncoder()
+
+    real_import = __import__
+
+    def fake_import(name, *args, **kwargs):
+        if name == "mlx_lm" or name.startswith("mlx_lm."):
+            raise ImportError("No module named 'mlx_lm'")
+        if name == "mlx" or name.startswith("mlx."):
+            raise ImportError("No module named 'mlx'")
+        return real_import(name, *args, **kwargs)
+
+    sys.modules.pop("mlx_lm", None)
+    sys.modules.pop("mlx", None)
+    sys.modules.pop("mlx.core", None)
+
+    with patch("builtins.__import__", side_effect=fake_import):
+        with pytest.raises(ImportError, match="mlx and mlx-lm are required"):
+            await encoder.initialize()
@@ -2,12 +2,14 @@
 import numpy as np
 import pytest
 from datetime import datetime, timezone, timedelta
+from unittest.mock import AsyncMock, MagicMock

 from hindsight_api.engine.retain.link_utils import (
    _normalize_datetime,
    _cap_links_per_unit,
    compute_temporal_links,
    compute_temporal_query_bounds,
+    compute_semantic_links_ann,
    compute_semantic_links_within_batch,
    MAX_TEMPORAL_LINKS_PER_UNIT,
 )
@@ -388,3 +390,145 @@ class TestComputeSemanticLinksWithinBatch:
            assert link_type == "semantic"
            assert 0.0 <= weight <= 1.0
            assert entity_id is None
+
+
+class TestComputeSemanticLinksAnnPgBouncerSafety:
+    """Regression tests ensuring compute_semantic_links_ann stays in a single
+    transaction so that the `_ann_seeds` temp table remains visible when the
+    caller's connection goes through pgBouncer in `transaction` pool mode.
+
+    In pgBouncer transaction mode, the backend is only pinned to the client
+    for the duration of an actual PostgreSQL transaction. Outside a
+    transaction, consecutive statements can land on different backends, and
+    session-scoped temp tables (which are bound to the backend that created
+    them) become invisible. The observed failure mode was an intermittent
+    `relation "_ann_seeds" does not exist` on the statement immediately
+    following the CREATE TEMP TABLE.
+    """
+
+    @pytest.fixture
+    def mock_conn(self):
+        """An asyncpg-like connection mock with an async `transaction()`
+        context manager and awaitable execute/fetch/copy helpers."""
+        conn = MagicMock()
+
+        txn_cm = MagicMock()
+        txn_cm.__aenter__ = AsyncMock(return_value=None)
+        txn_cm.__aexit__ = AsyncMock(return_value=None)
+        conn.transaction = MagicMock(return_value=txn_cm)
+
+        conn.execute = AsyncMock()
+        conn.copy_records_to_table = AsyncMock()
+        conn.fetch = AsyncMock(return_value=[])
+        return conn
+
+    @pytest.mark.asyncio
+    async def test_empty_inputs_skip_transaction(self, mock_conn):
+        """No seeds -> no work, no transaction, no temp-table churn."""
+        result = await compute_semantic_links_ann(
+            conn=mock_conn,
+            bank_id="bank-1",
+            unit_ids=[],
+            embeddings=[],
+        )
+        assert result == []
+        mock_conn.transaction.assert_not_called()
+        mock_conn.execute.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_runs_inside_a_transaction(self, mock_conn):
+        """The full CREATE TEMP TABLE -> COPY -> SELECT sequence must happen
+        inside a single `async with conn.transaction():` block."""
+        emb = [0.1] * 384
+        await compute_semantic_links_ann(
+            conn=mock_conn,
+            bank_id="bank-1",
+            unit_ids=["u1", "u2"],
+            embeddings=[emb, emb],
+            fact_types=["world", "world"],
+        )
+
+        # Transaction context manager was entered.
+        mock_conn.transaction.assert_called_once()
+        txn_cm = mock_conn.transaction.return_value
+        txn_cm.__aenter__.assert_awaited_once()
+        txn_cm.__aexit__.assert_awaited_once()
+
+    @pytest.mark.asyncio
+    async def test_temp_table_uses_on_commit_drop(self, mock_conn):
+        """The CREATE TEMP TABLE statement must use ON COMMIT DROP so the
+        table is transaction-scoped. Without ON COMMIT DROP the table would
+        be session-scoped and would not survive pgBouncer backend rebinding
+        between transactions."""
+        emb = [0.1] * 384
+        await compute_semantic_links_ann(
+            conn=mock_conn,
+            bank_id="bank-1",
+            unit_ids=["u1"],
+            embeddings=[emb],
+            fact_types=["world"],
+        )
+
+        executed_sql = [call.args[0] for call in mock_conn.execute.call_args_list]
+        create_statements = [s for s in executed_sql if "CREATE TEMP TABLE" in s]
+        assert len(create_statements) == 1, "Should create _ann_seeds exactly once"
+        assert "_ann_seeds" in create_statements[0]
+        assert "ON COMMIT DROP" in create_statements[0], (
+            "CREATE TEMP TABLE must use ON COMMIT DROP so the table is cleaned "
+            "up at transaction end and is transaction-scoped"
+        )
+
+        # Must not use IF NOT EXISTS — the table is fresh each transaction.
+        assert "IF NOT EXISTS" not in create_statements[0], (
+            "With ON COMMIT DROP the table is always fresh at transaction start, "
+            "so IF NOT EXISTS is both unnecessary and misleading (suggests the "
+            "table might persist across transactions)"
+        )
+
+    @pytest.mark.asyncio
+    async def test_no_manual_drop_or_truncate(self, mock_conn):
+        """With ON COMMIT DROP we must not re-add manual TRUNCATE or DROP
+        statements — they were the source of the original pgBouncer bug."""
+        emb = [0.1] * 384
+        await compute_semantic_links_ann(
+            conn=mock_conn,
+            bank_id="bank-1",
+            unit_ids=["u1"],
+            embeddings=[emb],
+            fact_types=["world"],
+        )
+
+        executed_sql = [call.args[0] for call in mock_conn.execute.call_args_list]
+        assert not any("TRUNCATE _ann_seeds" in s for s in executed_sql), (
+            "TRUNCATE is unnecessary with ON COMMIT DROP and was previously "
+            "the statement that failed with 'relation does not exist' when "
+            "pgBouncer rebound the backend"
+        )
+        assert not any("DROP TABLE" in s and "_ann_seeds" in s for s in executed_sql), (
+            "Explicit DROP is unnecessary with ON COMMIT DROP"
+        )
+
+    @pytest.mark.asyncio
+    async def test_uses_set_local_for_ef_search(self, mock_conn):
+        """hnsw.ef_search must be set with SET LOCAL so the change is scoped
+        to the transaction. Without SET LOCAL, the setting would leak onto
+        the pooled backend and affect subsequent recall queries that land
+        on the same backend."""
+        emb = [0.1] * 384
+        await compute_semantic_links_ann(
+            conn=mock_conn,
+            bank_id="bank-1",
+            unit_ids=["u1"],
+            embeddings=[emb],
+            fact_types=["world"],
+        )
+
+        executed_sql = [call.args[0] for call in mock_conn.execute.call_args_list]
+        ef_statements = [s for s in executed_sql if "hnsw.ef_search" in s]
+        assert ef_statements, "ef_search must be tuned down for retain ANN"
+        for stmt in ef_statements:
+            assert stmt.strip().startswith("SET LOCAL"), (
+                f"hnsw.ef_search must use SET LOCAL, got: {stmt}"
+            )
+        # And there must not be a RESET — SET LOCAL handles it at commit.
+        assert not any("RESET hnsw.ef_search" in s for s in executed_sql)
@@ -345,6 +345,65 @@ class TestLiteLLMSDKEmbeddings:
            assert encode_call_args.kwargs["api_base"] == "https://custom.api.com"
            assert encode_call_args.kwargs["dimensions"] == 768

+    async def test_encoding_format_default_is_float(self, mock_litellm):
+        """Test that encoding_format defaults to 'float' for backwards compatibility."""
+        with patch(
+            "builtins.__import__",
+            side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args),
+        ):
+            emb = LiteLLMSDKEmbeddings(
+                api_key="test_key",
+                model="cohere/embed-english-v3.0",
+            )
+            await emb.initialize()
+
+            init_call_args = mock_litellm.aembedding.call_args
+            assert init_call_args.kwargs["encoding_format"] == "float"
+
+            mock_litellm.embedding.return_value.data = [{"embedding": [0.1] * 768, "index": 0}]
+            emb.encode(["test"])
+
+            encode_call_args = mock_litellm.embedding.call_args
+            assert encode_call_args.kwargs["encoding_format"] == "float"
+
+    async def test_encoding_format_omitted_when_none(self, mock_litellm):
+        """Test that encoding_format is omitted when set to None (for Voyage AI, Gemini)."""
+        with patch(
+            "builtins.__import__",
+            side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args),
+        ):
+            emb = LiteLLMSDKEmbeddings(
+                api_key="test_key",
+                model="voyage/voyage-4-large",
+                encoding_format=None,
+            )
+            await emb.initialize()
+
+            init_call_args = mock_litellm.aembedding.call_args
+            assert "encoding_format" not in init_call_args.kwargs
+
+            mock_litellm.embedding.return_value.data = [{"embedding": [0.1] * 768, "index": 0}]
+            emb.encode(["test"])
+
+            encode_call_args = mock_litellm.embedding.call_args
+            assert "encoding_format" not in encode_call_args.kwargs
+
+    async def test_encoding_format_omitted_when_empty_string(self, mock_litellm):
+        """Test that encoding_format is omitted when set to empty string."""
+        with patch(
+            "builtins.__import__",
+            side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args),
+        ):
+            emb = LiteLLMSDKEmbeddings(
+                api_key="test_key",
+                model="gemini/gemini-embedding-2-preview",
+                encoding_format="",
+            )
+            await emb.initialize()
+
+            init_call_args = mock_litellm.aembedding.call_args
+            assert "encoding_format" not in init_call_args.kwargs
+
    async def test_openai_invalid_output_dimensions_raises(self, mock_litellm):
        """Invalid dimensions fail during initialize() (probe call), not per HTTP request.

@@ -342,7 +342,8 @@ class TestMentalModelToolRegistration:
        assert "update_bank" in tools
        assert "delete_bank" in tools
        assert "clear_memories" in tools
-        assert len(tools) == 29
+        assert "sync_retain" in tools
+        assert len(tools) == 30


@pytest.fixture
@@ -1107,12 +1108,67 @@ class TestMemoryBrowsingTools:
        assert '"deleted"' in result
        assert mock_memory.delete_memory_unit.call_args.kwargs["unit_id"] == "mem-1"

+    async def test_get_memory_invalid_uuid(self, mock_memory):
+        mock_memory.get_memory_unit.side_effect = ValueError("Invalid memory_id: 'nonexistent' is not a valid UUID")
+        mcp = _make_mcp_server(mock_memory, {"get_memory"}, include_bank_id=True)
+        result = await _tools(mcp)["get_memory"].fn(memory_id="nonexistent")
+        assert "not a valid UUID" in result
+
+    async def test_get_memory_invalid_uuid_single_bank(self, mock_memory):
+        mock_memory.get_memory_unit.side_effect = ValueError("Invalid memory_id: 'bad' is not a valid UUID")
+        mcp = _make_mcp_server(mock_memory, {"get_memory"}, include_bank_id=False)
+        result = await _tools(mcp)["get_memory"].fn(memory_id="bad")
+        assert "not a valid UUID" in result["error"]
+
+    async def test_delete_memory_invalid_uuid(self, mock_memory):
+        mock_memory.delete_memory_unit.side_effect = ValueError("Invalid unit_id: 'bad' is not a valid UUID")
+        mcp = _make_mcp_server(mock_memory, {"delete_memory"}, include_bank_id=True)
+        result = await _tools(mcp)["delete_memory"].fn(memory_id="bad")
+        assert "not a valid UUID" in result
+
    async def test_list_memories_single_bank(self, mock_memory):
        mcp = _make_mcp_server(mock_memory, {"list_memories"}, include_bank_id=False)
        result = await _tools(mcp)["list_memories"].fn()
        assert isinstance(result, dict)


+# =========================================================================
+# Sync Retain Tool Tests
+# =========================================================================
+
+
+@pytest.mark.asyncio
+class TestSyncRetainTool:
+    async def test_sync_retain_basic(self, mock_memory):
+        mock_memory.retain_batch_async.return_value = [["unit-1", "unit-2"]]
+        mcp = _make_mcp_server(mock_memory, {"sync_retain"}, include_bank_id=True)
+        result = await _tools(mcp)["sync_retain"].fn(content="test memory")
+        assert result["status"] == "completed"
+        assert result["memory_ids"] == ["unit-1", "unit-2"]
+
+    async def test_sync_retain_single_bank(self, mock_memory):
+        mock_memory.retain_batch_async.return_value = [["unit-1"]]
+        mcp = _make_mcp_server(mock_memory, {"sync_retain"}, include_bank_id=False)
+        result = await _tools(mcp)["sync_retain"].fn(content="test memory")
+        assert result["status"] == "completed"
+        assert result["memory_ids"] == ["unit-1"]
+
+    async def test_sync_retain_with_tags(self, mock_memory):
+        mock_memory.retain_batch_async.return_value = [["unit-1"]]
+        mcp = _make_mcp_server(mock_memory, {"sync_retain"}, include_bank_id=True)
+        result = await _tools(mcp)["sync_retain"].fn(content="test", tags=["project:alpha"])
+        assert result["status"] == "completed"
+        call_kwargs = mock_memory.retain_batch_async.call_args.kwargs
+        assert call_kwargs["contents"][0]["tags"] == ["project:alpha"]
+
+    async def test_sync_retain_error(self, mock_memory):
+        mock_memory.retain_batch_async.side_effect = Exception("DB error")
+        mcp = _make_mcp_server(mock_memory, {"sync_retain"}, include_bank_id=True)
+        result = await _tools(mcp)["sync_retain"].fn(content="test")
+        assert result["status"] == "error"
+        assert "DB error" in result["message"]
+
+
 # =========================================================================
 # Document Tool Tests
 # =========================================================================
@@ -0,0 +1,72 @@
+"""
+Tests for OpenAICompatibleLLM._max_tokens_param_name.
+
+Regression coverage for issue #978: Azure OpenAI + GPT-5 models were failing with
+"'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead."
+because PR #858 started sending 'max_tokens' whenever the openai provider had a
+custom base_url. Reasoning models only accept 'max_completion_tokens', and Azure
+OpenAI is fully OpenAI-API-compatible, so both cases must keep using the new
+parameter name.
+"""
+
+from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
+
+
+def _make(provider: str, model: str, base_url: str = "") -> OpenAICompatibleLLM:
+    return OpenAICompatibleLLM(
+        provider=provider,
+        api_key="test-key",
+        base_url=base_url,
+        model=model,
+    )
+
+
+class TestMaxTokensParamName:
+    def test_native_openai_uses_max_completion_tokens(self):
+        llm = _make("openai", "gpt-4o-mini")
+        assert llm._max_tokens_param_name() == "max_completion_tokens"
+
+    def test_openai_custom_base_url_falls_back_to_max_tokens(self):
+        """Mistral/Together-style OpenAI-compatible endpoints need max_tokens (PR #858)."""
+        llm = _make("openai", "mistral-large-latest", base_url="https://api.mistral.ai/v1")
+        assert llm._max_tokens_param_name() == "max_tokens"
+
+    def test_azure_openai_uses_max_completion_tokens(self):
+        """Regression for #978: Azure is fully OpenAI-API-compatible, not a third-party clone."""
+        llm = _make(
+            "openai",
+            "gpt-4o-mini",
+            base_url="https://my-resource.openai.azure.com/openai/v1/",
+        )
+        assert llm._max_tokens_param_name() == "max_completion_tokens"
+
+    def test_reasoning_model_always_uses_max_completion_tokens(self):
+        """Regression for #978: GPT-5/o1/o3 reject max_tokens outright, base_url must not matter."""
+        # Azure + GPT-5 (exact reporter setup)
+        azure_gpt5 = _make(
+            "openai",
+            "gpt-5.4-nano",
+            base_url="https://my-resource.openai.azure.com/openai/v1/",
+        )
+        assert azure_gpt5._max_tokens_param_name() == "max_completion_tokens"
+
+        # Even a Mistral-style custom base_url must not downgrade a reasoning model
+        for model in ("gpt-5", "gpt-5-mini", "o1-mini", "o3", "deepseek-r1"):
+            llm = _make("openai", model, base_url="https://some-proxy.example.com/v1")
+            assert llm._max_tokens_param_name() == "max_completion_tokens", model
+
+    def test_groq_uses_max_completion_tokens(self):
+        llm = _make("groq", "openai/gpt-oss-120b", base_url="https://api.groq.com/openai/v1")
+        assert llm._max_tokens_param_name() == "max_completion_tokens"
+
+    def test_llamacpp_uses_max_completion_tokens(self):
+        llm = _make("llamacpp", "some-model", base_url="http://localhost:8080/v1")
+        assert llm._max_tokens_param_name() == "max_completion_tokens"
+
+    def test_ollama_uses_max_tokens(self):
+        llm = _make("ollama", "gemma3:12b", base_url="http://localhost:11434/v1")
+        assert llm._max_tokens_param_name() == "max_tokens"
+
+    def test_lmstudio_uses_max_tokens(self):
+        llm = _make("lmstudio", "openai/gpt-oss-20b", base_url="http://localhost:1234/v1")
+        assert llm._max_tokens_param_name() == "max_tokens"
@@ -0,0 +1,80 @@
+"""Regression test for #972: reflect sub-recalls must be marked internal.
+
+When reflect calls search_observations or recall, the sub-recalls must use
+``request_context.internal=True`` to avoid double-billing.  The reflect caller
+is already billed for the overall operation; sub-recalls are implementation
+details that should not generate additional billing events.
+"""
+
+from dataclasses import dataclass, field
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from hindsight_api.engine.reflect.tools import tool_recall, tool_search_observations
+from hindsight_api.engine.response_models import RecallResult
+
+
+@dataclass
+class _FakeRequestContext:
+    """Dataclass stand-in matching the fields used by ``dataclasses.replace``."""
+
+    api_key: str | None = None
+    api_key_id: str | None = None
+    tenant_id: str | None = None
+    internal: bool = False
+    mcp_authenticated: bool = False
+    user_initiated: bool = False
+    allowed_bank_ids: list[str] | None = None
+
+
+def _mock_engine():
+    engine = MagicMock()
+    engine.recall_async = AsyncMock(
+        return_value=RecallResult(results=[], source_facts={})
+    )
+    return engine
+
+
+class TestReflectInternalBilling:
+    """Verify that reflect sub-recalls are marked internal (#972)."""
+
+    @pytest.mark.asyncio
+    async def test_search_observations_marks_recall_internal(self):
+        engine = _mock_engine()
+        ctx = _FakeRequestContext(api_key="k", internal=False)
+
+        await tool_search_observations(engine, "bank-1", "query", ctx)
+
+        engine.recall_async.assert_called_once()
+        passed_ctx = engine.recall_async.call_args.kwargs["request_context"]
+        assert passed_ctx.internal is True, "sub-recall must be internal"
+
+    @pytest.mark.asyncio
+    async def test_search_observations_preserves_original_context(self):
+        engine = _mock_engine()
+        ctx = _FakeRequestContext(api_key="k", internal=False)
+
+        await tool_search_observations(engine, "bank-1", "query", ctx)
+
+        assert ctx.internal is False, "original context must not be mutated"
+
+    @pytest.mark.asyncio
+    async def test_recall_marks_recall_internal(self):
+        engine = _mock_engine()
+        ctx = _FakeRequestContext(api_key="k", internal=False)
+
+        await tool_recall(engine, "bank-1", "query", ctx)
+
+        engine.recall_async.assert_called_once()
+        passed_ctx = engine.recall_async.call_args.kwargs["request_context"]
+        assert passed_ctx.internal is True, "sub-recall must be internal"
+
+    @pytest.mark.asyncio
+    async def test_recall_preserves_original_context(self):
+        engine = _mock_engine()
+        ctx = _FakeRequestContext(api_key="k", internal=False)
+
+        await tool_recall(engine, "bank-1", "query", ctx)
+
+        assert ctx.internal is False, "original context must not be mutated"
@@ -11,6 +11,7 @@ import pytest

 from hindsight_api.engine.reflect.tools import tool_search_observations
 from hindsight_api.engine.response_models import RecallResult
+from hindsight_api.models import RequestContext


 def _make_mock_engine(recall_result=None):
@@ -24,7 +25,11 @@ def _make_mock_engine(recall_result=None):

@pytest.fixture
 def mock_request_context():
-    return MagicMock()
+    # Use a real dataclass instance — tool_search_observations calls
+    # dataclasses.replace(request_context, internal=True), which fails on
+    # MagicMock. The fields don't matter for these tests; we only inspect
+    # the kwargs passed to the mocked recall_async.
+    return RequestContext()


 class TestSearchObservationsSourceFacts:
@@ -14,24 +14,19 @@ UTC = timezone.utc
 def create_mock_scored_result(proof_count: int | None = None, ce_score: float = 0.8) -> ScoredResult:
    """Helper to create a minimal ScoredResult suitable for scoring tests."""
    retrieval = RetrievalResult(
-        id=uuid4(),
+        id=str(uuid4()),
        text="Test mock fact",
        fact_type="observation" if proof_count is not None else "world",
-        document_id=uuid4(),
-        chunk_id=uuid4(),
-        embedding=[0.1]*384,
-        similarity=0.9,
+        document_id=str(uuid4()),
+        chunk_id=str(uuid4()),
        proof_count=proof_count,
-        # Default neutral dates for testing so only proof_count changes score
-        occurred_start=datetime.now(UTC),
-        occurred_end=datetime.now(UTC)
+        # Use None for neutral recency so only proof_count changes score
+        occurred_start=None,
+        occurred_end=None
    )
    candidate = MergedCandidate(
-        id=retrieval.id,
        retrieval=retrieval,
-        semantic_rank=1,
-        bm25_rank=1,
-        rrf_score=0.1
+        rrf_score=0.1,
    )
    return ScoredResult(
        candidate=candidate,
@@ -0,0 +1,240 @@
+"""
+Tests for retain update_mode='append' — appends new content to existing documents.
+"""
+
+import logging
+from datetime import datetime, timezone
+
+import pytest
+
+from hindsight_api.engine.memory_engine import Budget
+
+logger = logging.getLogger(__name__)
+
+
+def _ts():
+    return datetime.now(timezone.utc).timestamp()
+
+
+@pytest.mark.asyncio
+async def test_append_mode_concatenates_content(memory, request_context):
+    """
+    When update_mode='append', new content should be appended to the existing
+    document and the full document should be reprocessed. Facts from both
+    old and new content should be recallable.
+    """
+    bank_id = f"test_append_{_ts()}"
+    document_id = "conversation-append"
+
+    try:
+        # First retain — initial content
+        v1_units = await memory.retain_async(
+            bank_id=bank_id,
+            content="Alice works at Google as a software engineer.",
+            context="team info",
+            document_id=document_id,
+            request_context=request_context,
+        )
+        assert len(v1_units) > 0, "v1 should create facts"
+
+        doc_v1 = await memory.get_document(document_id, bank_id, request_context=request_context)
+        v1_text = doc_v1["original_text"]
+        assert "Alice works at Google" in v1_text
+
+        # Second retain with append — add new content
+        v2_units = await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {
+                    "content": "Bob works at Microsoft as a data scientist.",
+                    "context": "team info",
+                    "document_id": document_id,
+                    "update_mode": "append",
+                }
+            ],
+            request_context=request_context,
+        )
+
+        # Verify document now contains both old and new content
+        doc_v2 = await memory.get_document(document_id, bank_id, request_context=request_context)
+        v2_text = doc_v2["original_text"]
+        assert "Alice works at Google" in v2_text, "Original content should be preserved"
+        assert "Bob works at Microsoft" in v2_text, "New content should be appended"
+
+        # Verify facts from both old and new content are recallable
+        result_alice = await memory.recall_async(
+            bank_id=bank_id,
+            query="Where does Alice work?",
+            budget=Budget.MID,
+            max_tokens=1000,
+            request_context=request_context,
+        )
+        assert len(result_alice.results) > 0, "Should recall facts about Alice"
+
+        result_bob = await memory.recall_async(
+            bank_id=bank_id,
+            query="Where does Bob work?",
+            budget=Budget.MID,
+            max_tokens=1000,
+            request_context=request_context,
+        )
+        assert len(result_bob.results) > 0, "Should recall facts about Bob"
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_append_mode_no_existing_document(memory, request_context):
+    """
+    When update_mode='append' but no existing document exists,
+    it should behave like a normal retain (no content to prepend).
+    """
+    bank_id = f"test_append_new_{_ts()}"
+    document_id = "new-doc-append"
+
+    try:
+        units = await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {
+                    "content": "Charlie is a product manager at Stripe.",
+                    "context": "team info",
+                    "document_id": document_id,
+                    "update_mode": "append",
+                }
+            ],
+            request_context=request_context,
+        )
+
+        assert len(units) > 0, "Should create facts even with no existing document"
+        # Flatten if nested
+        flat_units = units[0] if units and isinstance(units[0], list) else units
+        assert len(flat_units) > 0
+
+        doc = await memory.get_document(document_id, bank_id, request_context=request_context)
+        assert "Charlie is a product manager" in doc["original_text"]
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_append_mode_requires_document_id(memory, request_context):
+    """update_mode='append' without document_id should raise ValueError."""
+    bank_id = f"test_append_no_docid_{_ts()}"
+
+    with pytest.raises(ValueError, match="update_mode='append' requires a document_id"):
+        await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {
+                    "content": "Some content",
+                    "update_mode": "append",
+                }
+            ],
+            request_context=request_context,
+        )
+
+
+@pytest.mark.asyncio
+async def test_append_mode_multiple_appends(memory, request_context):
+    """Multiple appends should accumulate content over successive retains."""
+    bank_id = f"test_multi_append_{_ts()}"
+    document_id = "multi-append-doc"
+
+    try:
+        # Initial retain
+        await memory.retain_async(
+            bank_id=bank_id,
+            content="Day 1: Alice joined the team.",
+            context="journal",
+            document_id=document_id,
+            request_context=request_context,
+        )
+
+        # First append
+        await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {
+                    "content": "Day 2: Alice completed her onboarding.",
+                    "context": "journal",
+                    "document_id": document_id,
+                    "update_mode": "append",
+                }
+            ],
+            request_context=request_context,
+        )
+
+        # Second append
+        await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {
+                    "content": "Day 3: Alice shipped her first feature.",
+                    "context": "journal",
+                    "document_id": document_id,
+                    "update_mode": "append",
+                }
+            ],
+            request_context=request_context,
+        )
+
+        # Verify all content is present
+        doc = await memory.get_document(document_id, bank_id, request_context=request_context)
+        text = doc["original_text"]
+        assert "Day 1" in text, "Original content should be present"
+        assert "Day 2" in text, "First append should be present"
+        assert "Day 3" in text, "Second append should be present"
+
+        # All days should be recallable
+        result = await memory.recall_async(
+            bank_id=bank_id,
+            query="What happened on Alice's first days?",
+            budget=Budget.MID,
+            max_tokens=1000,
+            request_context=request_context,
+        )
+        assert len(result.results) > 0, "Should recall facts from all appends"
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_replace_mode_is_default(memory, request_context):
+    """Without update_mode (or update_mode='replace'), retain should replace content."""
+    bank_id = f"test_replace_default_{_ts()}"
+    document_id = "replace-doc"
+
+    try:
+        await memory.retain_async(
+            bank_id=bank_id,
+            content="Alice works at Google.",
+            context="team info",
+            document_id=document_id,
+            request_context=request_context,
+        )
+
+        # Retain again without update_mode — should replace
+        await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=[
+                {
+                    "content": "Bob works at Microsoft.",
+                    "context": "team info",
+                    "document_id": document_id,
+                }
+            ],
+            request_context=request_context,
+        )
+
+        doc = await memory.get_document(document_id, bank_id, request_context=request_context)
+        text = doc["original_text"]
+        # With replace, only new content should remain
+        assert "Bob works at Microsoft" in text, "New content should be present"
+        assert "Alice works at Google" not in text, "Old content should be replaced"
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
@@ -0,0 +1,71 @@
+"""Unit tests for the worker stage breadcrumb module."""
+
+import asyncio
+
+import pytest
+
+from hindsight_api.worker.stage import StageHolder, bind_holder, get_stage, set_stage
+
+
+def test_set_stage_is_noop_without_holder():
+    # No holder bound in this context: must not raise, and get_stage returns None.
+    set_stage("anything")
+    assert get_stage() is None
+
+
+@pytest.mark.asyncio
+async def test_holder_bound_inside_task_is_visible_to_called_code():
+    holder = StageHolder()
+
+    async def inner():
+        # The poller binds the holder from inside the task coroutine so it
+        # lives in that task's contextvar scope; mirror that here.
+        bind_holder(holder)
+        set_stage("phase1")
+        # Engine code further down the call stack reads via set_stage.
+        set_stage("phase2")
+        assert get_stage() == "phase2"
+
+    await asyncio.create_task(inner())
+
+    # Holder is mutable: the spawning context sees the latest stage written
+    # by the child task without needing access to the contextvar.
+    assert holder.stage == "phase2"
+
+
+@pytest.mark.asyncio
+async def test_holder_does_not_leak_across_tasks():
+    # Each asyncio.create_task copies the parent's context. Binding inside
+    # one task must not affect a sibling task's view.
+    holder_a = StageHolder()
+    holder_b = StageHolder()
+
+    async def task_a():
+        bind_holder(holder_a)
+        set_stage("a")
+
+    async def task_b():
+        bind_holder(holder_b)
+        set_stage("b")
+
+    await asyncio.gather(asyncio.create_task(task_a()), asyncio.create_task(task_b()))
+
+    assert holder_a.stage == "a"
+    assert holder_b.stage == "b"
+    # Outside both tasks, no holder is bound.
+    assert get_stage() is None
+
+
+@pytest.mark.asyncio
+async def test_set_stage_updates_timestamp():
+    holder = StageHolder()
+
+    async def inner():
+        bind_holder(holder)
+        first = holder.updated_at
+        # asyncio.sleep guarantees monotonic clock advances on next set.
+        await asyncio.sleep(0.01)
+        set_stage("next")
+        assert holder.updated_at > first
+
+    await asyncio.create_task(inner())
@@ -217,6 +217,7 @@ class TestWorkerPoller:
            worker_id="test-worker-1",
            executor=lambda x: None,
            max_slots=3,  # Limit to 3 concurrent tasks
+            consolidation_max_slots=0,  # No reservation; all 3 slots available for non-consolidation
        )

        claimed = await poller.claim_batch()
@@ -1363,7 +1364,7 @@ async def test_worker_slot_limits_enforced(pool, clean_operations):
        executor=controlled_executor,
        poll_interval_ms=50,
        max_slots=3,  # Only allow 3 concurrent tasks
-        consolidation_max_slots=1,
+        consolidation_max_slots=0,  # No consolidation reservation; all 3 slots available for retain
    )

    # Submit 10 tasks
@@ -1428,6 +1429,106 @@ async def test_worker_slot_limits_enforced(pool, clean_operations):
            pass


+async def test_consolidation_slots_reserved_when_retain_saturates(pool, clean_operations):
+    """Regression: consolidation must not be starved when retain saturates the queue.
+
+    With ``max_slots=5`` and ``consolidation_max_slots=2``, retain tasks may use at
+    most 3 concurrent slots, leaving 2 slots reserved for consolidation. Without
+    the reservation (issue #1006), a continuous stream of retain tasks would fill
+    every slot and consolidation would never run.
+    """
+    from hindsight_api.worker.poller import WorkerPoller
+
+    started: dict[str, str] = {}  # op_id -> op_type
+    finish_events: dict[str, asyncio.Event] = {}
+
+    async def blocking_executor(task_dict: dict):
+        op_id = task_dict["operation_id"]
+        started[op_id] = task_dict.get("operation_type", "unknown")
+        event = asyncio.Event()
+        finish_events[op_id] = event
+        await event.wait()
+
+    poller = WorkerPoller(
+        pool=pool,
+        worker_id="test-worker-consolidation-reservation",
+        executor=blocking_executor,
+        poll_interval_ms=50,
+        max_slots=5,
+        consolidation_max_slots=2,
+    )
+
+    bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
+    await _ensure_bank(pool, bank_id)
+
+    # Submit 10 retain tasks first — these should be claimed up to the
+    # non-consolidation cap (max_slots - consolidation_max_slots = 3).
+    for _ in range(10):
+        op_id = uuid.uuid4()
+        payload = json.dumps(
+            {"type": "test", "operation_type": "retain", "operation_id": str(op_id), "bank_id": bank_id}
+        )
+        await pool.execute(
+            """
+            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
+            VALUES ($1, $2, 'retain', 'pending', $3::jsonb)
+            """,
+            op_id,
+            bank_id,
+            payload,
+        )
+
+    # Submit 1 consolidation task. Note the payload deliberately omits operation_type
+    # to verify the poller injects it from the DB column.
+    consolidation_op_id = uuid.uuid4()
+    consolidation_payload = json.dumps({"type": "test", "operation_id": str(consolidation_op_id), "bank_id": bank_id})
+    await pool.execute(
+        """
+        INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
+        VALUES ($1, $2, 'consolidation', 'pending', $3::jsonb)
+        """,
+        consolidation_op_id,
+        bank_id,
+        consolidation_payload,
+    )
+
+    poll_task = asyncio.create_task(poller.run())
+
+    try:
+        # Wait for the worker to fill its slots: 3 retain + 1 consolidation = 4 active.
+        for _ in range(200):
+            if len(started) >= 4:
+                break
+            await asyncio.sleep(0.01)
+
+        retain_started = [op for op, t in started.items() if t == "retain"]
+        consolidation_started = [op for op, t in started.items() if t == "consolidation"]
+
+        assert len(retain_started) == 3, (
+            f"Retain should be capped at max_slots - consolidation_max_slots = 3, "
+            f"got {len(retain_started)}"
+        )
+        assert len(consolidation_started) == 1, (
+            f"Consolidation should claim its reserved slot even while retain saturates, "
+            f"got {len(consolidation_started)}"
+        )
+        assert str(consolidation_op_id) in consolidation_started
+
+        # In-flight tracking must record the consolidation task under the right key,
+        # otherwise the consolidation pool accounting drifts on subsequent claims.
+        async with poller._in_flight_lock:
+            assert poller._in_flight_by_type.get("consolidation", 0) == 1
+
+    finally:
+        for event in finish_events.values():
+            event.set()
+        await poller.shutdown_graceful(timeout=2.0)
+        try:
+            await asyncio.wait_for(poll_task, timeout=1.0)
+        except asyncio.CancelledError:
+            pass
+
+
 class TestMarkFailedParentPropagation:
    """Tests for _mark_failed parent propagation in WorkerPoller.

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

 [project]
 name = "hindsight-api"
-version = "0.4.22"
+version = "0.5.1"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -0,0 +1,90 @@
+# Hindsight CLI ↔ OpenAPI coverage manifest.
+#
+# The CI job `cli-coverage-check` (see hindsight-dev/hindsight_dev/cli_coverage_check.py)
+# enforces both endpoint-level and parameter-level coverage:
+#
+#  1. Every operationId in hindsight-docs/static/openapi.json must be either
+#     called from hindsight-cli/src/**/*.rs (the progenitor-generated client
+#     methods are named identically to the operationId) or listed under
+#     [skip] below with a reason.
+#
+#  2. For each operation with a JSON request body, every top-level property
+#     of that body must be either present in hindsight-cli/src/main.rs as a
+#     clap command variant field (`field_name: <type>`) or a `long = "..."`
+#     attribute, OR listed under [fields.<operation_id>] below with a reason.
+#
+# Skip entries should explain *why* the field/operation is not exposed (e.g.
+# flattened into several CLI flags, complex nested struct, available via a
+# different subcommand).
+
+# ---------------------------------------------------------------------------
+# Operation-level skips
+# ---------------------------------------------------------------------------
+[skip]
+# (empty — every operation is currently wired)
+
+# ---------------------------------------------------------------------------
+# Per-operation parameter skips
+# ---------------------------------------------------------------------------
+
+[fields.add_bank_background]
+update_disposition = "Exposed inverted as --no-update-disposition on `bank background`."
+
+[fields.create_or_update_bank]
+disposition = "Flattened into --skepticism / --literalism / --empathy on `bank create`."
+disposition_skepticism = "Covered by --skepticism; the flat form is an API alias."
+disposition_literalism = "Covered by --literalism; the flat form is an API alias."
+disposition_empathy = "Covered by --empathy; the flat form is an API alias."
+background = "Set via the dedicated `bank background` subcommand."
+reflect_mission = "Set via `bank set-config --reflect-mission`."
+retain_mission = "Set via `bank set-config --retain-mission`."
+retain_extraction_mode = "Set via `bank set-config --retain-extraction-mode`."
+retain_custom_instructions = "Set via `bank set-config` (hierarchical config)."
+retain_chunk_size = "Set via `bank set-config` (hierarchical config)."
+enable_observations = "Set via `bank set-config` (hierarchical config)."
+observations_mission = "Set via `bank set-config --observations-mission`."
+
+[fields.update_bank]
+disposition = "Flattened into --skepticism / --literalism / --empathy on `bank update`."
+disposition_skepticism = "Covered by --skepticism; the flat form is an API alias."
+disposition_literalism = "Covered by --literalism; the flat form is an API alias."
+disposition_empathy = "Covered by --empathy; the flat form is an API alias."
+background = "Set via the dedicated `bank background` subcommand."
+reflect_mission = "Set via `bank set-config --reflect-mission`."
+retain_mission = "Set via `bank set-config --retain-mission`."
+retain_extraction_mode = "Set via `bank set-config --retain-extraction-mode`."
+retain_custom_instructions = "Set via `bank set-config` (hierarchical config)."
+retain_chunk_size = "Set via `bank set-config` (hierarchical config)."
+enable_observations = "Set via `bank set-config` (hierarchical config)."
+observations_mission = "Set via `bank set-config --observations-mission`."
+
+[fields.update_bank_disposition]
+disposition = "Flattened into --skepticism / --literalism / --empathy on `bank set-disposition`."
+
+[fields.update_bank_config]
+updates = "Flattened into per-setting flags (--llm-provider, --llm-model, etc) on `bank set-config`."
+
+[fields.create_webhook]
+http_config = "Advanced HTTP customisation (headers/method/timeout/params) is not exposed in the CLI yet; use the JSON API if needed."
+
+[fields.update_webhook]
+http_config = "Advanced HTTP customisation (headers/method/timeout/params) is not exposed in the CLI yet; use the JSON API if needed."
+
+[fields.recall_memories]
+types = "CLI exposes this as --fact-type (the schema property is named `types` but it holds fact types)."
+include = "Flattened into --include-chunks / --chunk-max-tokens (facts are always included)."
+tag_groups = "Complex nested tag filter not yet exposed in the CLI; use --tags / --tags-match for simple cases."
+
+[fields.reflect]
+include = "Flattened into --include-facts and related flags."
+response_schema = "Exposed as --schema (path to a JSON schema file)."
+tag_groups = "Complex nested tag filter not yet exposed in the CLI; use --tags / --tags-match for simple cases."
+
+[fields.retain_memories]
+items = "Constructed from the single positional content argument on `memory retain`."
+
+[fields.create_mental_model]
+trigger = "Exposed as --trigger-refresh-after-consolidation on `mental-model create` (other nested trigger fields like fact_types/tag_groups are not exposed yet)."
+
+[fields.update_mental_model]
+trigger = "Exposed as --trigger-refresh-after-consolidation on `mental-model update` (other nested trigger fields like fact_types/tag_groups are not exposed yet)."
@@ -1,6 +1,6 @@
 [package]
 name = "hindsight-cli"
-version = "0.4.22"
+version = "0.5.1"
 edition = "2021"
 authors = ["Hindsight Team"]
 description = "A beautiful CLI for Hindsight - semantic memory system"
@@ -118,6 +118,41 @@ run_test "clear memories" "$HINDSIGHT_CLI" memory clear "$TEST_BANK" || FAILED=1
 # Test 15: List operations
 run_test "list operations" "$HINDSIGHT_CLI" operation list "$TEST_BANK" || FAILED=1

+# --- Coverage-critical commands (added to ensure CLI exercises every endpoint) ---
+
+# Test: Set disposition directly (PUT /profile)
+run_test "bank set-disposition" "$HINDSIGHT_CLI" bank set-disposition "$TEST_BANK" \
+    --skepticism 3 --literalism 3 --empathy 3 || FAILED=1
+
+# Test: Recover consolidation (no-op when nothing stalled, but exercises the endpoint)
+run_test "bank consolidation-recover" "$HINDSIGHT_CLI" bank consolidation-recover "$TEST_BANK" || FAILED=1
+
+# Test: Bank template schema
+run_test "bank template-schema" "$HINDSIGHT_CLI" bank template-schema -o json || FAILED=1
+
+# Test: Export bank template
+run_test "bank export-template" "$HINDSIGHT_CLI" bank export-template "$TEST_BANK" -o json || FAILED=1
+
+# Test: Audit log list + stats
+run_test "audit list" "$HINDSIGHT_CLI" audit list "$TEST_BANK" -o json || FAILED=1
+run_test "audit stats" "$HINDSIGHT_CLI" audit stats "$TEST_BANK" -o json || FAILED=1
+
+# Test: Webhook lifecycle (list / create / update / deliveries / delete)
+run_test "webhook list (empty)" "$HINDSIGHT_CLI" webhook list "$TEST_BANK" -o json || FAILED=1
+
+WEBHOOK_OUT=$("$HINDSIGHT_CLI" webhook create "$TEST_BANK" https://example.invalid/hook -o json 2>/tmp/cli-test-output.txt || true)
+if echo "$WEBHOOK_OUT" | grep -q '"id"'; then
+    echo "Testing: webhook create... OK"
+    WEBHOOK_ID=$(echo "$WEBHOOK_OUT" | sed -n 's/.*"id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1)
+    run_test "webhook update" "$HINDSIGHT_CLI" webhook update "$TEST_BANK" "$WEBHOOK_ID" --enabled false || FAILED=1
+    run_test "webhook deliveries" "$HINDSIGHT_CLI" webhook deliveries "$TEST_BANK" "$WEBHOOK_ID" -o json || FAILED=1
+    run_test "webhook delete" "$HINDSIGHT_CLI" webhook delete "$TEST_BANK" "$WEBHOOK_ID" -y || FAILED=1
+else
+    echo "Testing: webhook create... FAILED"
+    cat /tmp/cli-test-output.txt | sed 's/^/    /'
+    FAILED=1
+fi
+
 # Test 16: Delete bank
 run_test "delete bank" "$HINDSIGHT_CLI" bank delete "$TEST_BANK" -y || FAILED=1

@@ -0,0 +1,118 @@
+//! Audit log commands.
+
+use anyhow::Result;
+
+use crate::api::ApiClient;
+use crate::output::{self, OutputFormat};
+use crate::ui;
+
+/// List audit log entries for a bank
+#[allow(clippy::too_many_arguments)]
+pub fn list(
+    client: &ApiClient,
+    bank_id: &str,
+    action: Option<String>,
+    transport: Option<String>,
+    start_date: Option<String>,
+    end_date: Option<String>,
+    limit: Option<u64>,
+    offset: Option<u64>,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Fetching audit logs..."))
+    } else {
+        None
+    };
+
+    let response = client.list_audit_logs(
+        bank_id,
+        action.as_deref(),
+        transport.as_deref(),
+        start_date.as_deref(),
+        end_date.as_deref(),
+        limit,
+        offset,
+        verbose,
+    );
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let result = response?;
+    if output_format == OutputFormat::Pretty {
+        ui::print_section_header(&format!("Audit logs: {}", bank_id));
+        println!(
+            "  {} {} ({} total)",
+            ui::dim("Showing:"),
+            result.items.len(),
+            result.total
+        );
+        println!();
+        if result.items.is_empty() {
+            println!("  {}", ui::dim("No audit log entries."));
+        } else {
+            for entry in &result.items {
+                let started = entry.started_at.as_deref().unwrap_or("-");
+                let duration = entry
+                    .duration_ms
+                    .map(|d| format!("{}ms", d))
+                    .unwrap_or_else(|| "-".to_string());
+                println!(
+                    "  {} {} [{}] {}",
+                    ui::dim(started),
+                    ui::gradient_start(&entry.action),
+                    entry.transport,
+                    duration
+                );
+            }
+        }
+    } else {
+        output::print_output(&result, output_format)?;
+    }
+    Ok(())
+}
+
+/// Get audit log statistics for a bank
+pub fn stats(
+    client: &ApiClient,
+    bank_id: &str,
+    action: Option<String>,
+    period: Option<String>,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Fetching audit log stats..."))
+    } else {
+        None
+    };
+
+    let response = client.audit_log_stats(bank_id, action.as_deref(), period.as_deref(), verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let result = response?;
+    if output_format == OutputFormat::Pretty {
+        ui::print_section_header(&format!("Audit stats: {}", bank_id));
+        println!("  {} {}", ui::dim("Period:"), result.period);
+        println!("  {} {}", ui::dim("Start:"), result.start);
+        println!("  {} {}", ui::dim("Bucket:"), result.trunc);
+        println!();
+        if result.buckets.is_empty() {
+            println!("  {}", ui::dim("No activity in this period."));
+        } else {
+            for bucket in &result.buckets {
+                let json = serde_json::to_value(bucket)?;
+                println!("  {}", json);
+            }
+        }
+    } else {
+        output::print_output(&result, output_format)?;
+    }
+    Ok(())
+}
@@ -1,7 +1,7 @@
-use anyhow::{anyhow, Result};
 use crate::api::ApiClient;
 use crate::output::{self, OutputFormat};
 use crate::ui;
+use anyhow::{anyhow, Result};

 pub fn list(client: &ApiClient, verbose: bool, output_format: OutputFormat) -> Result<()> {
    let spinner = if output_format == OutputFormat::Pretty {
@@ -32,11 +32,16 @@ pub fn list(client: &ApiClient, verbose: bool, output_format: OutputFormat) -> R
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

-pub fn disposition(client: &ApiClient, bank_id: &str, verbose: bool, output_format: OutputFormat) -> Result<()> {
+pub fn disposition(
+    client: &ApiClient,
+    bank_id: &str,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
    let spinner = if output_format == OutputFormat::Pretty {
        Some(ui::create_spinner("Fetching disposition..."))
    } else {
@@ -58,11 +63,16 @@ pub fn disposition(client: &ApiClient, bank_id: &str, verbose: bool, output_form
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

-pub fn stats(client: &ApiClient, bank_id: &str, verbose: bool, output_format: OutputFormat) -> Result<()> {
+pub fn stats(
+    client: &ApiClient,
+    bank_id: &str,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
    let spinner = if output_format == OutputFormat::Pretty {
        Some(ui::create_spinner("Fetching statistics..."))
    } else {
@@ -80,9 +90,21 @@ pub fn stats(client: &ApiClient, bank_id: &str, verbose: bool, output_format: Ou
            if output_format == OutputFormat::Pretty {
                ui::print_section_header(&format!("Statistics: {}", bank_id));

-                println!("  {} {}", ui::dim("memory units:"), ui::gradient_start(&stats.total_nodes.to_string()));
-                println!("  {} {}", ui::dim("links:"), ui::gradient_mid(&stats.total_links.to_string()));
-                println!("  {} {}", ui::dim("documents:"), ui::gradient_end(&stats.total_documents.to_string()));
+                println!(
+                    "  {} {}",
+                    ui::dim("memory units:"),
+                    ui::gradient_start(&stats.total_nodes.to_string())
+                );
+                println!(
+                    "  {} {}",
+                    ui::dim("links:"),
+                    ui::gradient_mid(&stats.total_links.to_string())
+                );
+                println!(
+                    "  {} {}",
+                    ui::dim("documents:"),
+                    ui::gradient_end(&stats.total_documents.to_string())
+                );
                println!();

                println!("{}", ui::gradient_text("─── Memory Units by Type ───"));
@@ -90,7 +112,11 @@ pub fn stats(client: &ApiClient, bank_id: &str, verbose: bool, output_format: Ou
                fact_types.sort_by_key(|(k, _)| *k);
                for (i, (fact_type, count)) in fact_types.iter().enumerate() {
                    let t = i as f32 / fact_types.len().max(1) as f32;
-                    println!("  {:<10} {}", fact_type, ui::gradient(&count.to_string(), t));
+                    println!(
+                        "  {:<10} {}",
+                        fact_type,
+                        ui::gradient(&count.to_string(), t)
+                    );
                }
                println!();

@@ -99,7 +125,11 @@ pub fn stats(client: &ApiClient, bank_id: &str, verbose: bool, output_format: Ou
                link_types.sort_by_key(|(k, _)| *k);
                for (i, (link_type, count)) in link_types.iter().enumerate() {
                    let t = i as f32 / link_types.len().max(1) as f32;
-                    println!("  {:<10} {}", link_type, ui::gradient(&count.to_string(), t));
+                    println!(
+                        "  {:<10} {}",
+                        link_type,
+                        ui::gradient(&count.to_string(), t)
+                    );
                }
                println!();

@@ -108,7 +138,11 @@ pub fn stats(client: &ApiClient, bank_id: &str, verbose: bool, output_format: Ou
                fact_type_links.sort_by_key(|(k, _)| *k);
                for (i, (fact_type, count)) in fact_type_links.iter().enumerate() {
                    let t = i as f32 / fact_type_links.len().max(1) as f32;
-                    println!("  {:<10} {}", fact_type, ui::gradient(&count.to_string(), t));
+                    println!(
+                        "  {:<10} {}",
+                        fact_type,
+                        ui::gradient(&count.to_string(), t)
+                    );
                }
                println!();

@@ -141,11 +175,17 @@ pub fn stats(client: &ApiClient, bank_id: &str, verbose: bool, output_format: Ou
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

-pub fn update_name(client: &ApiClient, bank_id: &str, name: &str, verbose: bool, output_format: OutputFormat) -> Result<()> {
+pub fn update_name(
+    client: &ApiClient,
+    bank_id: &str,
+    name: &str,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
    let spinner = if output_format == OutputFormat::Pretty {
        Some(ui::create_spinner("Updating bank name..."))
    } else {
@@ -167,7 +207,7 @@ pub fn update_name(client: &ApiClient, bank_id: &str, name: &str, verbose: bool,
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

@@ -177,7 +217,7 @@ pub fn update_background(
    content: &str,
    no_update_disposition: bool,
    verbose: bool,
-    output_format: OutputFormat
+    output_format: OutputFormat,
 ) -> Result<()> {
    let current_profile = if !no_update_disposition {
        client.get_profile(bank_id, verbose).ok()
@@ -204,9 +244,10 @@ pub fn update_background(
                println!("\n{}", profile.mission);

                if !no_update_disposition {
-                    if let (Some(old_p), Some(new_p)) =
-                        (current_profile.as_ref().map(|p| p.disposition.clone()), &profile.disposition)
-                    {
+                    if let (Some(old_p), Some(new_p)) = (
+                        current_profile.as_ref().map(|p| p.disposition.clone()),
+                        &profile.disposition,
+                    ) {
                        println!("\nDisposition changes:");
                        println!("  Skepticism:  {} → {}", old_p.skepticism, new_p.skepticism);
                        println!("  Literalism:  {} → {}", old_p.literalism, new_p.literalism);
@@ -218,7 +259,7 @@ pub fn update_background(
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

@@ -329,7 +370,12 @@ pub fn update(
    verbose: bool,
    output_format: OutputFormat,
 ) -> Result<()> {
-    if name.is_none() && mission_text.is_none() && skepticism.is_none() && literalism.is_none() && empathy.is_none() {
+    if name.is_none()
+        && mission_text.is_none()
+        && skepticism.is_none()
+        && literalism.is_none()
+        && empathy.is_none()
+    {
        anyhow::bail!("At least one field must be provided (--name, --mission, --skepticism, --literalism, --empathy)");
    }

@@ -407,20 +453,27 @@ pub fn graph(
            if output_format == OutputFormat::Pretty {
                ui::print_section_header(&format!("Memory Graph: {}", bank_id));

-                println!("  {} {}", ui::dim("Nodes:"), ui::gradient_start(&result.nodes.len().to_string()));
-                println!("  {} {}", ui::dim("Edges:"), ui::gradient_end(&result.edges.len().to_string()));
+                println!(
+                    "  {} {}",
+                    ui::dim("Nodes:"),
+                    ui::gradient_start(&result.nodes.len().to_string())
+                );
+                println!(
+                    "  {} {}",
+                    ui::dim("Edges:"),
+                    ui::gradient_end(&result.edges.len().to_string())
+                );
                println!();

                // Show sample of nodes
                if !result.nodes.is_empty() {
                    println!("{}", ui::gradient_text("─── Sample Nodes ───"));
                    for node in result.nodes.iter().take(5) {
-                        let fact_type = node.get("type")
-                            .and_then(|v| v.as_str())
-                            .unwrap_or("unknown");
-                        let id = node.get("id")
+                        let fact_type = node
+                            .get("type")
                            .and_then(|v| v.as_str())
                            .unwrap_or("unknown");
+                        let id = node.get("id").and_then(|v| v.as_str()).unwrap_or("unknown");
                        println!("  {} [{}]", ui::dim(id), fact_type);
                        if let Some(text) = node.get("text").and_then(|v| v.as_str()) {
                            let preview: String = text.chars().take(60).collect();
@@ -429,12 +482,18 @@ pub fn graph(
                        }
                    }
                    if result.nodes.len() > 5 {
-                        println!("  {} more...", ui::dim(&format!("+ {}", result.nodes.len() - 5)));
+                        println!(
+                            "  {} more...",
+                            ui::dim(&format!("+ {}", result.nodes.len() - 5))
+                        );
                    }
                    println!();
                }

-                println!("{}", ui::dim("Use JSON output for full graph data: -o json"));
+                println!(
+                    "{}",
+                    ui::dim("Use JSON output for full graph data: -o json")
+                );
            } else {
                output::print_output(&result, output_format)?;
            }
@@ -449,7 +508,7 @@ pub fn delete(
    bank_id: &str,
    yes: bool,
    verbose: bool,
-    output_format: OutputFormat
+    output_format: OutputFormat,
 ) -> Result<()> {
    // Confirmation prompt unless -y flag is used
    if !yes && output_format == OutputFormat::Pretty {
@@ -494,7 +553,7 @@ pub fn delete(
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

@@ -527,7 +586,11 @@ pub fn consolidate(
                ui::print_success("Consolidation triggered");
                println!("  {} {}", ui::dim("Operation ID:"), operation_id);
                if result.deduplicated {
-                    println!("  {} {}", ui::dim("Note:"), "Reusing existing pending consolidation task");
+                    println!(
+                        "  {} {}",
+                        ui::dim("Note:"),
+                        "Reusing existing pending consolidation task"
+                    );
                }
            } else {
                output::print_output(&result, output_format)?;
@@ -544,7 +607,13 @@ pub fn consolidate(
            // Poll for completion
            if output_format == OutputFormat::Pretty {
                println!();
-                println!("{}", ui::dim(&format!("Polling every {}s for completion...", poll_interval)));
+                println!(
+                    "{}",
+                    ui::dim(&format!(
+                        "Polling every {}s for completion...",
+                        poll_interval
+                    ))
+                );
            }

            let start = std::time::Instant::now();
@@ -561,7 +630,10 @@ pub fn consolidate(
                        match op.map(|o| o.status.as_str()) {
                            Some("completed") => {
                                if output_format == OutputFormat::Pretty {
-                                    ui::print_success(&format!("Consolidation completed ({}s)", elapsed));
+                                    ui::print_success(&format!(
+                                        "Consolidation completed ({}s)",
+                                        elapsed
+                                    ));
                                }
                                break;
                            }
@@ -571,7 +643,10 @@ pub fn consolidate(
                                    .map(|s| s.as_str())
                                    .unwrap_or("Unknown error");
                                if output_format == OutputFormat::Pretty {
-                                    ui::print_error(&format!("Consolidation failed: {}", error_msg));
+                                    ui::print_error(&format!(
+                                        "Consolidation failed: {}",
+                                        error_msg
+                                    ));
                                }
                                std::process::exit(1);
                            }
@@ -582,7 +657,10 @@ pub fn consolidate(
                            }
                            None => {
                                if output_format == OutputFormat::Pretty {
-                                    ui::print_warning(&format!("Operation {} not found in list", operation_id));
+                                    ui::print_warning(&format!(
+                                        "Operation {} not found in list",
+                                        operation_id
+                                    ));
                                }
                                break;
                            }
@@ -732,37 +810,67 @@ pub fn set_config(
    let mut updates: HashMap<String, serde_json::Value> = HashMap::new();

    if let Some(provider) = llm_provider {
-        updates.insert("llm_provider".to_string(), serde_json::Value::String(provider));
+        updates.insert(
+            "llm_provider".to_string(),
+            serde_json::Value::String(provider),
+        );
    }
    if let Some(model) = llm_model {
        updates.insert("llm_model".to_string(), serde_json::Value::String(model));
    }
    if let Some(api_key) = llm_api_key {
-        updates.insert("llm_api_key".to_string(), serde_json::Value::String(api_key));
+        updates.insert(
+            "llm_api_key".to_string(),
+            serde_json::Value::String(api_key),
+        );
    }
    if let Some(base_url) = llm_base_url {
-        updates.insert("llm_base_url".to_string(), serde_json::Value::String(base_url));
+        updates.insert(
+            "llm_base_url".to_string(),
+            serde_json::Value::String(base_url),
+        );
    }
    if let Some(mission) = retain_mission {
-        updates.insert("retain_mission".to_string(), serde_json::Value::String(mission));
+        updates.insert(
+            "retain_mission".to_string(),
+            serde_json::Value::String(mission),
+        );
    }
    if let Some(mode) = retain_extraction_mode {
-        updates.insert("retain_extraction_mode".to_string(), serde_json::Value::String(mode));
+        updates.insert(
+            "retain_extraction_mode".to_string(),
+            serde_json::Value::String(mode),
+        );
    }
    if let Some(mission) = observations_mission {
-        updates.insert("observations_mission".to_string(), serde_json::Value::String(mission));
+        updates.insert(
+            "observations_mission".to_string(),
+            serde_json::Value::String(mission),
+        );
    }
    if let Some(mission) = reflect_mission {
-        updates.insert("reflect_mission".to_string(), serde_json::Value::String(mission));
+        updates.insert(
+            "reflect_mission".to_string(),
+            serde_json::Value::String(mission),
+        );
    }
    if let Some(skepticism) = disposition_skepticism {
-        updates.insert("disposition_skepticism".to_string(), serde_json::Value::Number(skepticism.into()));
+        updates.insert(
+            "disposition_skepticism".to_string(),
+            serde_json::Value::Number(skepticism.into()),
+        );
    }
    if let Some(literalism) = disposition_literalism {
-        updates.insert("disposition_literalism".to_string(), serde_json::Value::Number(literalism.into()));
+        updates.insert(
+            "disposition_literalism".to_string(),
+            serde_json::Value::Number(literalism.into()),
+        );
    }
    if let Some(empathy) = disposition_empathy {
-        updates.insert("disposition_empathy".to_string(), serde_json::Value::Number(empathy.into()));
+        updates.insert(
+            "disposition_empathy".to_string(),
+            serde_json::Value::Number(empathy.into()),
+        );
    }

    if updates.is_empty() {
@@ -832,7 +940,10 @@ pub fn reset_config(
    match response {
        Ok(result) => {
            if output_format == OutputFormat::Pretty {
-                ui::print_success(&format!("Configuration reset to defaults for bank '{}'", bank_id));
+                ui::print_success(&format!(
+                    "Configuration reset to defaults for bank '{}'",
+                    bank_id
+                ));
            } else {
                output::print_output(&result, output_format)?;
            }
@@ -841,3 +952,188 @@ pub fn reset_config(
        Err(e) => Err(e),
    }
 }
+
+/// Set disposition traits (skepticism, literalism, empathy) via PUT /profile
+pub fn set_disposition(
+    client: &ApiClient,
+    bank_id: &str,
+    skepticism: u64,
+    literalism: u64,
+    empathy: u64,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Updating disposition..."))
+    } else {
+        None
+    };
+
+    let response =
+        client.update_bank_disposition(bank_id, skepticism, literalism, empathy, verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let profile = response?;
+    if output_format == OutputFormat::Pretty {
+        ui::print_success(&format!("Disposition updated for bank '{}'", bank_id));
+        ui::print_disposition(&profile);
+    } else {
+        output::print_output(&profile, output_format)?;
+    }
+    Ok(())
+}
+
+/// Recover from a stalled consolidation
+pub fn consolidation_recover(
+    client: &ApiClient,
+    bank_id: &str,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Recovering consolidation..."))
+    } else {
+        None
+    };
+
+    let response = client.recover_consolidation(bank_id, verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let result = response?;
+    if output_format == OutputFormat::Pretty {
+        ui::print_success(&format!("Consolidation recovered for bank '{}'", bank_id));
+        let json = serde_json::to_value(&result)?;
+        println!(
+            "  {}",
+            serde_json::to_string_pretty(&json).unwrap_or_default()
+        );
+    } else {
+        output::print_output(&result, output_format)?;
+    }
+    Ok(())
+}
+
+/// Export a bank template manifest (bank config + mental models + directives)
+pub fn export_template(
+    client: &ApiClient,
+    bank_id: &str,
+    out_path: Option<std::path::PathBuf>,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Exporting bank template..."))
+    } else {
+        None
+    };
+
+    let response = client.export_bank_template(bank_id, verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let manifest = response?;
+    let json = serde_json::to_string_pretty(&manifest)?;
+
+    if let Some(path) = out_path {
+        std::fs::write(&path, &json)
+            .map_err(|e| anyhow!("Failed to write {}: {}", path.display(), e))?;
+        if output_format == OutputFormat::Pretty {
+            ui::print_success(&format!("Template written to {}", path.display()));
+        }
+    } else if output_format == OutputFormat::Pretty {
+        println!("{}", json);
+    } else {
+        output::print_output(&manifest, output_format)?;
+    }
+    Ok(())
+}
+
+/// Import a bank template manifest from a JSON file
+pub fn import_template(
+    client: &ApiClient,
+    bank_id: &str,
+    manifest_path: &std::path::Path,
+    dry_run: bool,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    let raw = std::fs::read_to_string(manifest_path)
+        .map_err(|e| anyhow!("Failed to read {}: {}", manifest_path.display(), e))?;
+    let manifest: serde_json::Value = serde_json::from_str(&raw)
+        .map_err(|e| anyhow!("Invalid JSON in {}: {}", manifest_path.display(), e))?;
+
+    let spinner = if output_format == OutputFormat::Pretty {
+        let msg = if dry_run {
+            "Validating bank template (dry run)..."
+        } else {
+            "Importing bank template..."
+        };
+        Some(ui::create_spinner(msg))
+    } else {
+        None
+    };
+
+    let response = client.import_bank_template(bank_id, &manifest, dry_run, verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let result = response?;
+    if output_format == OutputFormat::Pretty {
+        if dry_run {
+            ui::print_success(&format!("Template for bank '{}' validated", bank_id));
+        } else {
+            ui::print_success(&format!("Template imported into bank '{}'", bank_id));
+        }
+        println!("  directives created: {:?}", result.directives_created);
+        println!("  directives updated: {:?}", result.directives_updated);
+        println!(
+            "  mental models created: {:?}",
+            result.mental_models_created
+        );
+        println!(
+            "  mental models updated: {:?}",
+            result.mental_models_updated
+        );
+        println!("  config applied: {}", result.config_applied);
+    } else {
+        output::print_output(&result, output_format)?;
+    }
+    Ok(())
+}
+
+/// Fetch the bank template JSON schema
+pub fn template_schema(
+    client: &ApiClient,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Fetching template schema..."))
+    } else {
+        None
+    };
+
+    let response = client.get_bank_template_schema(verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let schema = response?;
+    if output_format == OutputFormat::Pretty {
+        println!("{}", serde_json::to_string_pretty(&schema)?);
+    } else {
+        output::print_output(&schema, output_format)?;
+    }
+    Ok(())
+}
@@ -99,11 +99,13 @@ pub fn get(
 }

 /// Create a new directive
+#[allow(clippy::too_many_arguments)]
 pub fn create(
    client: &ApiClient,
    bank_id: &str,
    name: &str,
    content: &str,
+    priority: i64,
    verbose: bool,
    output_format: OutputFormat,
 ) -> Result<()> {
@@ -117,7 +119,7 @@ pub fn create(
        name: name.to_string(),
        content: content.to_string(),
        is_active: true,
-        priority: 0,
+        priority,
        tags: vec![],
    };

@@ -143,6 +145,7 @@ pub fn create(
 }

 /// Update a directive
+#[allow(clippy::too_many_arguments)]
 pub fn update(
    client: &ApiClient,
    bank_id: &str,
@@ -150,11 +153,14 @@ pub fn update(
    name: Option<String>,
    content: Option<String>,
    is_active: Option<bool>,
+    priority: Option<i64>,
    verbose: bool,
    output_format: OutputFormat,
 ) -> Result<()> {
-    if name.is_none() && content.is_none() && is_active.is_none() {
-        anyhow::bail!("At least one of --name, --content, or --is-active must be provided");
+    if name.is_none() && content.is_none() && is_active.is_none() && priority.is_none() {
+        anyhow::bail!(
+            "At least one of --name, --content, --is-active, or --priority must be provided"
+        );
    }

    let spinner = if output_format == OutputFormat::Pretty {
@@ -167,7 +173,7 @@ pub fn update(
        name,
        content,
        is_active,
-        priority: None,
+        priority,
        tags: None,
    };

@@ -1,9 +1,9 @@
-use anyhow::Result;
-use chrono::{Duration as ChronoDuration, NaiveDate, Utc};
-use std::collections::BTreeMap;
 use crate::api::ApiClient;
 use crate::output::{self, OutputFormat};
 use crate::ui;
+use anyhow::Result;
+use chrono::{Duration as ChronoDuration, NaiveDate, Utc};
+use std::collections::BTreeMap;

 pub fn list(
    client: &ApiClient,
@@ -26,7 +26,13 @@ pub fn list(
        None
    };

-    let response = client.list_documents(agent_id, query.as_deref(), Some(limit), Some(offset), verbose);
+    let response = client.list_documents(
+        agent_id,
+        query.as_deref(),
+        Some(limit),
+        Some(offset),
+        verbose,
+    );

    if let Some(mut sp) = spinner {
        sp.finish();
@@ -35,13 +41,25 @@ pub fn list(
    match response {
        Ok(docs_response) => {
            if output_format == OutputFormat::Pretty {
-                ui::print_info(&format!("Documents for bank '{}' (total: {})", agent_id, docs_response.total));
+                ui::print_info(&format!(
+                    "Documents for bank '{}' (total: {})",
+                    agent_id, docs_response.total
+                ));
                for doc in &docs_response.items {
                    let id = doc.get("id").and_then(|v| v.as_str()).unwrap_or("unknown");
-                    let created = doc.get("created_at").and_then(|v| v.as_str()).unwrap_or("unknown");
-                    let updated = doc.get("updated_at").and_then(|v| v.as_str()).unwrap_or("unknown");
+                    let created = doc
+                        .get("created_at")
+                        .and_then(|v| v.as_str())
+                        .unwrap_or("unknown");
+                    let updated = doc
+                        .get("updated_at")
+                        .and_then(|v| v.as_str())
+                        .unwrap_or("unknown");
                    let text_len = doc.get("text_length").and_then(|v| v.as_i64()).unwrap_or(0);
-                    let mem_count = doc.get("memory_unit_count").and_then(|v| v.as_i64()).unwrap_or(0);
+                    let mem_count = doc
+                        .get("memory_unit_count")
+                        .and_then(|v| v.as_i64())
+                        .unwrap_or(0);

                    println!("\n  Document ID: {}", id);
                    println!("    Created: {}", created);
@@ -54,7 +72,7 @@ pub fn list(
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

@@ -87,9 +105,7 @@ fn list_with_date(
    let mut filtered_count = 0;

    for doc in all_docs {
-        let created_at = doc.get("created_at")
-            .and_then(|v| v.as_str())
-            .unwrap_or("");
+        let created_at = doc.get("created_at").and_then(|v| v.as_str()).unwrap_or("");

        // Parse the date part (YYYY-MM-DD) from created_at
        let doc_date = created_at.split('T').next().unwrap_or("");
@@ -126,7 +142,10 @@ fn list_with_date(
            println!("  {} ({} documents)", date_str, docs.len());
            for doc in docs {
                let id = doc.get("id").and_then(|v| v.as_str()).unwrap_or("unknown");
-                let mem_count = doc.get("memory_unit_count").and_then(|v| v.as_i64()).unwrap_or(0);
+                let mem_count = doc
+                    .get("memory_unit_count")
+                    .and_then(|v| v.as_i64())
+                    .unwrap_or(0);
                println!("    - {} ({} memories)", id, mem_count);
            }
            println!();
@@ -224,7 +243,7 @@ pub fn get(
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

@@ -260,6 +279,45 @@ pub fn delete(
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }
+
+/// Update a document (currently only supports replacing tags)
+pub fn update(
+    client: &ApiClient,
+    bank_id: &str,
+    document_id: &str,
+    tags: Option<Vec<String>>,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    if tags.is_none() {
+        anyhow::bail!("At least one of --tags must be provided");
+    }
+
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Updating document..."))
+    } else {
+        None
+    };
+
+    let response = client.update_document(bank_id, document_id, tags, verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let result = response?;
+    if output_format == OutputFormat::Pretty {
+        ui::print_success(&format!("Document '{}' updated", document_id));
+        let json = serde_json::to_value(&result)?;
+        println!(
+            "  {}",
+            serde_json::to_string_pretty(&json).unwrap_or_default()
+        );
+    } else {
+        output::print_output(&result, output_format)?;
+    }
+    Ok(())
+}
@@ -3,13 +3,16 @@ use std::fs;
 use std::path::PathBuf;
 use walkdir::WalkDir;

-use crate::api::{ApiClient, RecallRequest, ReflectRequest, MemoryItem, RetainRequest};
+use crate::api::{ApiClient, MemoryItem, RecallRequest, ReflectRequest, RetainRequest};
 use crate::config;
 use crate::output::{self, OutputFormat};
 use crate::ui;

 // Import types from generated client
-use hindsight_client::types::{Budget, ChunkIncludeOptions, FactsIncludeOptions, IncludeOptions, ReflectIncludeOptions, TagsMatch};
+use hindsight_client::types::{
+    Budget, ChunkIncludeOptions, FactsIncludeOptions, IncludeOptions, ReflectIncludeOptions,
+    TagsMatch,
+};
 use serde::Deserialize;
 use serde_json;

@@ -18,7 +21,7 @@ use serde_json;
 struct MemoryUnitDetail {
    id: String,
    text: String,
-    #[serde(rename = "type")]
+    #[serde(rename = "fact_type")]
    type_: Option<String>,
    document_id: Option<String>,
    context: Option<String>,
@@ -45,7 +48,12 @@ fn parse_budget(budget: &str) -> Budget {

 // Helper function to parse tags_match string to TagsMatch enum
 fn parse_tags_match(tags_match: &Option<String>) -> TagsMatch {
-    match tags_match.as_deref().unwrap_or("any").to_lowercase().as_str() {
+    match tags_match
+        .as_deref()
+        .unwrap_or("any")
+        .to_lowercase()
+        .as_str()
+    {
        "all" => TagsMatch::All,
        "any_strict" => TagsMatch::AnyStrict,
        "all_strict" => TagsMatch::AllStrict,
@@ -86,25 +94,30 @@ pub fn list(
    match response {
        Ok(result) => {
            if output_format == OutputFormat::Pretty {
-                ui::print_section_header(&format!("Memories: {} (showing {}-{})", bank_id, offset + 1, offset + result.items.len() as i64));
+                ui::print_section_header(&format!(
+                    "Memories: {} (showing {}-{})",
+                    bank_id,
+                    offset + 1,
+                    offset + result.items.len() as i64
+                ));

                if result.items.is_empty() {
                    println!("  {}", ui::dim("No memories found."));
                } else {
                    for item in &result.items {
-                        let fact_type = item.get("type")
+                        let fact_type = item
+                            .get("fact_type")
                            .and_then(|v| v.as_str())
                            .unwrap_or("unknown");
                        let type_t = match fact_type {
                            "world" => 0.0,
                            "experience" => 0.5,
                            "opinion" => 1.0,
+                            "observation" => 0.25,
                            _ => 0.5,
                        };

-                        let id = item.get("id")
-                            .and_then(|v| v.as_str())
-                            .unwrap_or("unknown");
+                        let id = item.get("id").and_then(|v| v.as_str()).unwrap_or("unknown");

                        println!(
                            "  {} {}",
@@ -167,12 +180,17 @@ pub fn get(
                    "world" => 0.0,
                    "experience" => 0.5,
                    "opinion" => 1.0,
+                    "observation" => 0.25,
                    _ => 0.5,
                };

                ui::print_section_header(&format!("Memory: {}", memory_id));

-                println!("  {} {}", ui::dim("Type:"), ui::gradient(&fact_type.to_uppercase(), type_t));
+                println!(
+                    "  {} {}",
+                    ui::dim("Type:"),
+                    ui::gradient(&fact_type.to_uppercase(), type_t)
+                );
                println!("  {} {}", ui::dim("ID:"), result.id);

                if let Some(doc_id) = &result.document_id {
@@ -234,12 +252,9 @@ pub fn get(
 fn is_supported_file(path: &std::path::Path) -> bool {
    const SUPPORTED_EXTENSIONS: &[&str] = &[
        // Documents
-        "pdf", "docx", "doc", "pptx", "ppt", "xlsx", "xls",
-        // Images (OCR)
-        "jpg", "jpeg", "png", "gif", "bmp", "webp", "tiff",
-        // Web / markup
-        "html", "htm",
-        // Text / data
+        "pdf", "docx", "doc", "pptx", "ppt", "xlsx", "xls", // Images (OCR)
+        "jpg", "jpeg", "png", "gif", "bmp", "webp", "tiff", // Web / markup
+        "html", "htm", // Text / data
        "txt", "md", "csv", "json", "yaml", "yml", "toml", "xml", "rst", "adoc", "log",
        // Audio (transcription)
        "mp3", "wav", "ogg", "flac",
@@ -250,6 +265,7 @@ fn is_supported_file(path: &std::path::Path) -> bool {
        .unwrap_or(false)
 }

+#[allow(clippy::too_many_arguments)]
 pub fn recall(
    client: &ApiClient,
    agent_id: &str,
@@ -262,6 +278,7 @@ pub fn recall(
    chunk_max_tokens: i64,
    tags: Vec<String>,
    tags_match: Option<String>,
+    query_timestamp: Option<String>,
    verbose: bool,
    output_format: OutputFormat,
 ) -> Result<()> {
@@ -286,11 +303,15 @@ pub fn recall(

    let request = RecallRequest {
        query,
-        types: if fact_type.is_empty() { None } else { Some(fact_type) },
+        types: if fact_type.is_empty() {
+            None
+        } else {
+            Some(fact_type)
+        },
        budget: Some(parse_budget(&budget)),
        max_tokens,
        trace,
-        query_timestamp: None,
+        query_timestamp,
        include,
        tags: if tags.is_empty() { None } else { Some(tags) },
        tags_match: parse_tags_match(&tags_match),
@@ -312,10 +333,11 @@ pub fn recall(
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

+#[allow(clippy::too_many_arguments)]
 pub fn reflect(
    client: &ApiClient,
    agent_id: &str,
@@ -327,6 +349,9 @@ pub fn reflect(
    tags: Vec<String>,
    tags_match: Option<String>,
    include_facts: bool,
+    fact_types: Option<Vec<String>>,
+    exclude_mental_models: bool,
+    exclude_mental_model_ids: Option<Vec<String>>,
    verbose: bool,
    output_format: OutputFormat,
 ) -> Result<()> {
@@ -340,8 +365,9 @@ pub fn reflect(
    let response_schema = if let Some(path) = schema_path {
        let schema_content = fs::read_to_string(&path)
            .with_context(|| format!("Failed to read schema file: {}", path.display()))?;
-        let schema: serde_json::Map<String, serde_json::Value> = serde_json::from_str(&schema_content)
-            .with_context(|| format!("Failed to parse JSON schema from: {}", path.display()))?;
+        let schema: serde_json::Map<String, serde_json::Value> =
+            serde_json::from_str(&schema_content)
+                .with_context(|| format!("Failed to parse JSON schema from: {}", path.display()))?;
        Some(schema)
    } else {
        None
@@ -356,6 +382,21 @@ pub fn reflect(
        None
    };

+    // Map the CLI fact-type strings (world, experience, observation) into the
+    // generated FactTypesItem enum. Unknown values are dropped — the server
+    // would reject them anyway.
+    let mapped_fact_types = fact_types.as_ref().map(|types| {
+        types
+            .iter()
+            .filter_map(|t| match t.to_lowercase().as_str() {
+                "world" => Some(hindsight_client::types::FactTypesItem::World),
+                "experience" => Some(hindsight_client::types::FactTypesItem::Experience),
+                "observation" => Some(hindsight_client::types::FactTypesItem::Observation),
+                _ => None,
+            })
+            .collect::<Vec<_>>()
+    });
+
    let request = ReflectRequest {
        query,
        budget: Some(parse_budget(&budget)),
@@ -366,9 +407,9 @@ pub fn reflect(
        tags: if tags.is_empty() { None } else { Some(tags) },
        tags_match: parse_tags_match(&tags_match),
        tag_groups: None,
-        fact_types: None,
-        exclude_mental_models: false,
-        exclude_mental_model_ids: None,
+        fact_types: mapped_fact_types,
+        exclude_mental_models,
+        exclude_mental_model_ids,
    };

    let response = client.reflect(agent_id, &request, verbose);
@@ -386,10 +427,11 @@ pub fn reflect(
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

+#[allow(clippy::too_many_arguments)]
 pub fn retain(
    client: &ApiClient,
    agent_id: &str,
@@ -397,6 +439,7 @@ pub fn retain(
    doc_id: Option<String>,
    context: Option<String>,
    r#async: bool,
+    document_tags: Option<Vec<String>>,
    verbose: bool,
    output_format: OutputFormat,
 ) -> Result<()> {
@@ -418,12 +461,13 @@ pub fn retain(
        tags: None,
        observation_scopes: None,
        strategy: None,
+        update_mode: None,
    };

    let request = RetainRequest {
        items: vec![item],
        async_: r#async,
-        document_tags: None,
+        document_tags,
    };

    let response = client.retain(agent_id, &request, r#async, verbose);
@@ -450,7 +494,7 @@ pub fn retain(
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

@@ -617,7 +661,7 @@ pub fn delete(
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

@@ -687,10 +731,85 @@ pub fn clear(
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

+/// Get the observation history for a memory unit
+pub fn history(
+    client: &ApiClient,
+    bank_id: &str,
+    memory_id: &str,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Fetching observation history..."))
+    } else {
+        None
+    };
+
+    let response = client.get_observation_history(bank_id, memory_id, verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let result = response?;
+    if output_format == OutputFormat::Pretty {
+        println!("{}", serde_json::to_string_pretty(&result)?);
+    } else {
+        output::print_output(&result, output_format)?;
+    }
+    Ok(())
+}
+
+/// Clear the observations attached to a specific memory unit
+pub fn clear_observations(
+    client: &ApiClient,
+    bank_id: &str,
+    memory_id: &str,
+    yes: bool,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    if !yes && output_format == OutputFormat::Pretty {
+        let msg = format!(
+            "Clear observations for memory '{}'? They will be re-derived on next consolidation.",
+            memory_id
+        );
+        if !ui::prompt_confirmation(&msg)? {
+            ui::print_info("Operation cancelled");
+            return Ok(());
+        }
+    }
+
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Clearing observations..."))
+    } else {
+        None
+    };
+
+    let response = client.clear_memory_observations(bank_id, memory_id, verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let result = response?;
+    if output_format == OutputFormat::Pretty {
+        ui::print_success(&format!("Cleared observations for memory '{}'", memory_id));
+        let json = serde_json::to_value(&result)?;
+        println!(
+            "  {}",
+            serde_json::to_string_pretty(&json).unwrap_or_default()
+        );
+    } else {
+        output::print_output(&result, output_format)?;
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -699,8 +818,17 @@ mod tests {
    #[test]
    fn test_is_supported_file_text_extensions() {
        let supported = [
-            "file.txt", "file.md", "file.json", "file.yaml", "file.yml",
-            "file.toml", "file.xml", "file.csv", "file.log", "file.rst", "file.adoc",
+            "file.txt",
+            "file.md",
+            "file.json",
+            "file.yaml",
+            "file.yml",
+            "file.toml",
+            "file.xml",
+            "file.csv",
+            "file.log",
+            "file.rst",
+            "file.adoc",
        ];
        for filename in supported {
            assert!(
@@ -714,9 +842,16 @@ mod tests {
    #[test]
    fn test_is_supported_file_binary_extensions() {
        let supported = [
-            "file.pdf", "file.docx", "file.pptx", "file.xlsx",
-            "file.png", "file.jpg", "file.jpeg", "file.gif",
-            "file.mp3", "file.wav",
+            "file.pdf",
+            "file.docx",
+            "file.pptx",
+            "file.xlsx",
+            "file.png",
+            "file.jpg",
+            "file.jpeg",
+            "file.gif",
+            "file.mp3",
+            "file.wav",
        ];
        for filename in supported {
            assert!(
@@ -738,9 +873,7 @@ mod tests {

    #[test]
    fn test_is_supported_file_unsupported_extensions() {
-        let unsupported = [
-            "file.exe", "file.bin", "file.zip", "file.tar", "file.gz",
-        ];
+        let unsupported = ["file.exe", "file.bin", "file.zip", "file.tar", "file.gz"];
        for filename in unsupported {
            assert!(
                !is_supported_file(Path::new(filename)),
@@ -95,12 +95,16 @@ pub fn get(
 }

 /// Create a new mental model
+#[allow(clippy::too_many_arguments)]
 pub fn create(
    client: &ApiClient,
    bank_id: &str,
    name: &str,
    source_query: &str,
    id: Option<&str>,
+    tags: Vec<String>,
+    max_tokens: i64,
+    trigger_refresh_after_consolidation: bool,
    verbose: bool,
    output_format: OutputFormat,
 ) -> Result<()> {
@@ -110,13 +114,28 @@ pub fn create(
        None
    };

+    // Only send a trigger when the user opted in, so the server's default
+    // behaviour is preserved otherwise.
+    let trigger = if trigger_refresh_after_consolidation {
+        Some(types::MentalModelTriggerInput {
+            refresh_after_consolidation: true,
+            exclude_mental_models: false,
+            exclude_mental_model_ids: None,
+            fact_types: None,
+            tag_groups: None,
+            tags_match: None,
+        })
+    } else {
+        None
+    };
+
    let request = types::CreateMentalModelRequest {
        id: id.map(|s| s.to_string()),
        name: name.to_string(),
        source_query: source_query.to_string(),
-        max_tokens: 2048,
-        tags: vec![],
-        trigger: None,
+        max_tokens,
+        tags,
+        trigger,
    };

    let response = client.create_mental_model(bank_id, &request, verbose);
@@ -139,16 +158,29 @@ pub fn create(
 }

 /// Update a mental model
+#[allow(clippy::too_many_arguments)]
 pub fn update(
    client: &ApiClient,
    bank_id: &str,
    mental_model_id: &str,
    name: Option<String>,
+    source_query: Option<String>,
+    max_tokens: Option<i64>,
+    tags: Option<Vec<String>>,
+    trigger_refresh_after_consolidation: Option<bool>,
    verbose: bool,
    output_format: OutputFormat,
 ) -> Result<()> {
-    if name.is_none() {
-        anyhow::bail!("--name must be provided");
+    if name.is_none()
+        && source_query.is_none()
+        && max_tokens.is_none()
+        && tags.is_none()
+        && trigger_refresh_after_consolidation.is_none()
+    {
+        anyhow::bail!(
+            "At least one of --name, --source-query, --max-tokens, --tags, or \
+             --trigger-refresh-after-consolidation must be provided"
+        );
    }

    let spinner = if output_format == OutputFormat::Pretty {
@@ -157,12 +189,23 @@ pub fn update(
        None
    };

+    // Only build a trigger override when the user actually passed the flag;
+    // sending None leaves the existing trigger config untouched on the server.
+    let trigger = trigger_refresh_after_consolidation.map(|refresh| types::MentalModelTriggerInput {
+        refresh_after_consolidation: refresh,
+        exclude_mental_models: false,
+        exclude_mental_model_ids: None,
+        fact_types: None,
+        tag_groups: None,
+        tags_match: None,
+    });
+
    let request = types::UpdateMentalModelRequest {
        name,
-        source_query: None,
-        max_tokens: None,
-        tags: None,
-        trigger: None,
+        source_query,
+        max_tokens,
+        tags,
+        trigger,
    };

    let response = client.update_mental_model(bank_id, mental_model_id, &request, verbose);
@@ -1,3 +1,4 @@
+pub mod audit;
 pub mod bank;
 pub mod chunk;
 pub mod directive;
@@ -6,6 +7,7 @@ pub mod entity;
 pub mod explore;
 pub mod health;
 pub mod memory;
-pub mod operation;
 pub mod mental_model;
+pub mod operation;
 pub mod tag;
+pub mod webhook;
@@ -1,7 +1,7 @@
-use anyhow::Result;
 use crate::api::ApiClient;
 use crate::output::{self, OutputFormat};
 use crate::ui;
+use anyhow::Result;

 pub fn list(
    client: &ApiClient,
@@ -27,7 +27,10 @@ pub fn list(
                if ops_response.operations.is_empty() {
                    ui::print_info("No operations found");
                } else {
-                    ui::print_info(&format!("Found {} operation(s)", ops_response.operations.len()));
+                    ui::print_info(&format!(
+                        "Found {} operation(s)",
+                        ops_response.operations.len()
+                    ));
                    for op in &ops_response.operations {
                        println!("\n  Operation ID: {}", op.id);
                        println!("    Type: {}", op.task_type);
@@ -43,7 +46,7 @@ pub fn list(
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }

@@ -128,6 +131,40 @@ pub fn cancel(
            }
            Ok(())
        }
-        Err(e) => Err(e)
+        Err(e) => Err(e),
    }
 }
+
+/// Retry a failed async operation
+pub fn retry(
+    client: &ApiClient,
+    agent_id: &str,
+    operation_id: &str,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Retrying operation..."))
+    } else {
+        None
+    };
+
+    let response = client.retry_operation(agent_id, operation_id, verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let result = response?;
+    if output_format == OutputFormat::Pretty {
+        ui::print_success(&format!("Operation '{}' retried", operation_id));
+        let json = serde_json::to_value(&result)?;
+        println!(
+            "  {}",
+            serde_json::to_string_pretty(&json).unwrap_or_default()
+        );
+    } else {
+        output::print_output(&result, output_format)?;
+    }
+    Ok(())
+}
@@ -0,0 +1,249 @@
+//! Webhook commands for managing event delivery hooks.
+
+use anyhow::Result;
+
+use crate::api::ApiClient;
+use crate::output::{self, OutputFormat};
+use crate::ui;
+
+use hindsight_client::types;
+
+/// List webhooks for a bank
+pub fn list(
+    client: &ApiClient,
+    bank_id: &str,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Fetching webhooks..."))
+    } else {
+        None
+    };
+
+    let response = client.list_webhooks(bank_id, verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let result = response?;
+    if output_format == OutputFormat::Pretty {
+        ui::print_section_header(&format!("Webhooks: {}", bank_id));
+        if result.items.is_empty() {
+            println!("  {}", ui::dim("No webhooks configured."));
+        } else {
+            for wh in &result.items {
+                let status = if wh.enabled {
+                    ui::gradient_start("enabled")
+                } else {
+                    ui::dim("disabled")
+                };
+                println!("  {} [{}] {}", ui::gradient_start(&wh.id), status, wh.url);
+                if !wh.event_types.is_empty() {
+                    println!("    events: {}", wh.event_types.join(", "));
+                }
+                println!();
+            }
+        }
+    } else {
+        output::print_output(&result, output_format)?;
+    }
+    Ok(())
+}
+
+/// Create a new webhook
+#[allow(clippy::too_many_arguments)]
+pub fn create(
+    client: &ApiClient,
+    bank_id: &str,
+    url: &str,
+    event_types: Vec<String>,
+    enabled: bool,
+    secret: Option<String>,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Creating webhook..."))
+    } else {
+        None
+    };
+
+    let effective_events = if event_types.is_empty() {
+        vec!["consolidation.completed".to_string()]
+    } else {
+        event_types
+    };
+
+    let request = types::CreateWebhookRequest {
+        enabled,
+        event_types: effective_events,
+        http_config: None,
+        secret,
+        url: url.to_string(),
+    };
+
+    let response = client.create_webhook(bank_id, &request, verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let wh = response?;
+    if output_format == OutputFormat::Pretty {
+        ui::print_success(&format!("Webhook '{}' created", wh.id));
+        println!("  URL: {}", wh.url);
+        println!("  Events: {}", wh.event_types.join(", "));
+    } else {
+        output::print_output(&wh, output_format)?;
+    }
+    Ok(())
+}
+
+/// Update a webhook
+#[allow(clippy::too_many_arguments)]
+pub fn update(
+    client: &ApiClient,
+    bank_id: &str,
+    webhook_id: &str,
+    url: Option<String>,
+    event_types: Option<Vec<String>>,
+    enabled: Option<bool>,
+    secret: Option<String>,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    if url.is_none() && event_types.is_none() && enabled.is_none() && secret.is_none() {
+        anyhow::bail!(
+            "At least one of --url, --event-types, --enabled, or --secret must be provided"
+        );
+    }
+
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Updating webhook..."))
+    } else {
+        None
+    };
+
+    let request = types::UpdateWebhookRequest {
+        enabled,
+        event_types,
+        http_config: None,
+        secret,
+        url,
+    };
+
+    let response = client.update_webhook(bank_id, webhook_id, &request, verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let wh = response?;
+    if output_format == OutputFormat::Pretty {
+        ui::print_success(&format!("Webhook '{}' updated", wh.id));
+    } else {
+        output::print_output(&wh, output_format)?;
+    }
+    Ok(())
+}
+
+/// Delete a webhook
+pub fn delete(
+    client: &ApiClient,
+    bank_id: &str,
+    webhook_id: &str,
+    yes: bool,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    if !yes && output_format == OutputFormat::Pretty {
+        let message = format!(
+            "Are you sure you want to delete webhook '{}'? This cannot be undone.",
+            webhook_id
+        );
+        if !ui::prompt_confirmation(&message)? {
+            ui::print_info("Operation cancelled");
+            return Ok(());
+        }
+    }
+
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Deleting webhook..."))
+    } else {
+        None
+    };
+
+    let response = client.delete_webhook(bank_id, webhook_id, verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let result = response?;
+    if output_format == OutputFormat::Pretty {
+        if result.success {
+            ui::print_success(&format!("Webhook '{}' deleted", webhook_id));
+        } else {
+            ui::print_error("Failed to delete webhook");
+        }
+    } else {
+        output::print_output(&result, output_format)?;
+    }
+    Ok(())
+}
+
+/// List recent delivery attempts for a webhook
+pub fn deliveries(
+    client: &ApiClient,
+    bank_id: &str,
+    webhook_id: &str,
+    cursor: Option<String>,
+    limit: Option<i64>,
+    verbose: bool,
+    output_format: OutputFormat,
+) -> Result<()> {
+    let spinner = if output_format == OutputFormat::Pretty {
+        Some(ui::create_spinner("Fetching deliveries..."))
+    } else {
+        None
+    };
+
+    let response =
+        client.list_webhook_deliveries(bank_id, webhook_id, cursor.as_deref(), limit, verbose);
+
+    if let Some(mut sp) = spinner {
+        sp.finish();
+    }
+
+    let result = response?;
+    if output_format == OutputFormat::Pretty {
+        ui::print_section_header(&format!("Deliveries for {}", webhook_id));
+        if result.items.is_empty() {
+            println!("  {}", ui::dim("No delivery attempts recorded."));
+        } else {
+            for d in &result.items {
+                println!(
+                    "  {} [{}] {} — attempts: {}",
+                    ui::gradient_start(&d.id),
+                    d.event_type,
+                    d.last_response_status
+                        .map(|s| s.to_string())
+                        .unwrap_or_else(|| "-".to_string()),
+                    d.attempts
+                );
+                if let Some(err) = &d.last_error {
+                    println!("    {} {}", ui::dim("error:"), err);
+                }
+            }
+            if let Some(cursor) = &result.next_cursor {
+                println!();
+                println!("  {} {}", ui::dim("next cursor:"), cursor);
+            }
+        }
+    } else {
+        output::print_output(&result, output_format)?;
+    }
+    Ok(())
+}
@@ -84,6 +84,8 @@ pub fn print_fact(fact: &RecallResult, _show_activation: bool) {
    let type_t = match fact_type {
        "world" => 0.0,
        "agent" => 0.5,
+        "experience" => 0.5,
+        "observation" => 0.25,
        "opinion" => 1.0,
        _ => 0.5,
    };
@@ -7,7 +7,7 @@ info:
    name: Apache 2.0
    url: https://www.apache.org/licenses/LICENSE-2.0.html
  title: Hindsight HTTP API
-  version: 0.4.22
+  version: 0.5.1
 servers:
 - url: /
 paths:
@@ -3570,7 +3570,7 @@ components:
          type: integer
        entity_labels:
          items:
-            type: string
+            additionalProperties: {}
          nullable: true
          type: array
        entities_allow_free_form:
@@ -4784,6 +4784,12 @@ components:
        strategy:
          nullable: true
          type: string
+        update_mode:
+          enum:
+          - replace
+          - append
+          nullable: true
+          type: string
      required:
      - content
      title: MemoryItem
@@ -3,7 +3,7 @@ Hindsight HTTP API

 HTTP API for Hindsight

-API version: 0.4.22
+API version: 0.5.1
 */

 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -3,7 +3,7 @@ Hindsight HTTP API

 HTTP API for Hindsight

-API version: 0.4.22
+API version: 0.5.1
 */

 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -3,7 +3,7 @@ Hindsight HTTP API

 HTTP API for Hindsight

-API version: 0.4.22
+API version: 0.5.1
 */

 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -3,7 +3,7 @@ Hindsight HTTP API

 HTTP API for Hindsight

-API version: 0.4.22
+API version: 0.5.1
 */

 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -3,7 +3,7 @@ Hindsight HTTP API

 HTTP API for Hindsight

-API version: 0.4.22
+API version: 0.5.1
 */

 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -3,7 +3,7 @@ Hindsight HTTP API

 HTTP API for Hindsight

-API version: 0.4.22
+API version: 0.5.1
 */

 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -3,7 +3,7 @@ Hindsight HTTP API

 HTTP API for Hindsight

-API version: 0.4.22
+API version: 0.5.1
 */

 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
--- a/Show More
+++ b/Show More