ci: trigger CI run

feat: reject tags+tag_groups together, add tag_groups integration tests
- Add model_validator to RecallRequest and ReflectRequest that returns 422 when both `tags` and `tag_groups` are set (mutually exclusive) - Add 5 integration tests for tag_groups compound filtering: * validation: 422 when both fields are set * AND filter: two leaf groups (step scope AND user scope) * OR compound: user:alice OR user:bob * NOT compound: user:alice AND NOT archived * Nested: user:alice AND (step:5 OR step:8)
2026-03-13 14:13:43 +01:00 · 2026-03-13 14:13:43 +01:00 · 2026-03-13 14:13:43 +01:00 · 2026-03-13 14:13:43 +01:00 · 2026-03-13 14:13:43 +01:00 · 2026-03-13 14:12:42 +01:00
619 changed files with 17429 additions and 2934 deletions
@@ -2,7 +2,7 @@
 # Copy this file to .env and fill in your values

 # LLM Configuration (Required)
-# Supported providers: openai, groq, ollama, gemini, anthropic, lmstudio, vertexai
+# Supported providers: openai, groq, ollama, gemini, anthropic, lmstudio, vertexai, minimax
 HINDSIGHT_API_LLM_PROVIDER=openai
 HINDSIGHT_API_LLM_API_KEY=your-api-key-here
 HINDSIGHT_API_LLM_MODEL=gpt-4o-mini
@@ -20,6 +20,11 @@ HINDSIGHT_API_LLM_BASE_URL=https://api.openai.com/v1
 # HINDSIGHT_API_LLM_VERTEXAI_REGION=us-central1
 # HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY=/path/to/service-account-key.json  # Optional, uses ADC if not set

+# Example: MiniMax configuration (204K context window)
+# HINDSIGHT_API_LLM_PROVIDER=minimax
+# HINDSIGHT_API_LLM_API_KEY=your-minimax-api-key
+# HINDSIGHT_API_LLM_MODEL=MiniMax-M2.5
+
 # Example: LM Studio local configuration (Qwen 2.5 32B recommended)
 # HINDSIGHT_API_LLM_PROVIDER=lmstudio
 # HINDSIGHT_API_LLM_API_KEY=lmstudio
@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
@@ -21,20 +21,20 @@ jobs:
  build:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
+      - uses: actions/checkout@v6
+      - uses: actions/setup-node@v6
        with:
          node-version: 20
          cache: npm
          cache-dependency-path: package-lock.json
-      - uses: astral-sh/setup-uv@v4
+      - uses: astral-sh/setup-uv@v7
      - run: npm ci --workspace=hindsight-docs
      - run: uv run generate-llms-full
      - run: npm run build --workspace=hindsight-docs
        env:
          UMAMI_URL: https://analytics.hindsight.vectorize.io
          UMAMI_WEBSITE_ID: ${{ secrets.UMAMI_WEBSITE_ID }}
-      - uses: actions/upload-pages-artifact@v3
+      - uses: actions/upload-pages-artifact@v4
        with:
          path: hindsight-docs/build
  deploy:
@@ -13,10 +13,10 @@ jobs:
      id-token: write

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true

@@ -30,12 +30,20 @@ jobs:
      working-directory: ./hindsight-clients/python
      run: uv build --out-dir dist

+    - name: Build hindsight-api-slim
+      working-directory: ./hindsight-api-slim
+      run: uv build --out-dir dist
+
    - name: Build hindsight-api
      working-directory: ./hindsight-api
      run: uv build --out-dir dist

    - name: Build hindsight-all
-      working-directory: ./hindsight
+      working-directory: ./hindsight-all
+      run: uv build --out-dir dist
+
+    - name: Build hindsight-all-slim
+      working-directory: ./hindsight-all-slim
      run: uv build --out-dir dist

    - name: Build hindsight-litellm
@@ -54,13 +62,19 @@ jobs:
      working-directory: ./hindsight-integrations/pydantic-ai
      run: uv build --out-dir dist

-    # Publish in order (client and api first, then hindsight-all which depends on them)
+    # Publish in order (client and api-slim first, then api/all wrappers which depend on them)
    - name: Publish hindsight-client to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        packages-dir: ./hindsight-clients/python/dist
        skip-existing: true

+    - name: Publish hindsight-api-slim to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: ./hindsight-api-slim/dist
+        skip-existing: true
+
    - name: Publish hindsight-api to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
@@ -70,7 +84,13 @@ jobs:
    - name: Publish hindsight-all to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
-        packages-dir: ./hindsight/dist
+        packages-dir: ./hindsight-all/dist
+        skip-existing: true
+
+    - name: Publish hindsight-all-slim to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: ./hindsight-all-slim/dist
        skip-existing: true

    - name: Publish hindsight-litellm to PyPI
@@ -99,13 +119,15 @@ jobs:

    # Upload artifacts for GitHub release
    - name: Upload artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
      with:
        name: python-packages
        path: |
          hindsight-clients/python/dist/*
+          hindsight-api-slim/dist/*
          hindsight-api/dist/*
-          hindsight/dist/*
+          hindsight-all/dist/*
+          hindsight-all-slim/dist/*
          hindsight-integrations/litellm/dist/*
          hindsight-embed/dist/*
          hindsight-integrations/crewai/dist/*
@@ -117,10 +139,10 @@ jobs:
    environment: npm

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '20'
        registry-url: 'https://registry.npmjs.org'
@@ -155,7 +177,7 @@ jobs:
      run: npm pack

    - name: Upload artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
      with:
        name: typescript-client
        path: hindsight-clients/typescript/*.tgz
@@ -166,10 +188,10 @@ jobs:
    environment: npm

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '22'
        registry-url: 'https://registry.npmjs.org'
@@ -204,7 +226,7 @@ jobs:
      run: npm pack

    - name: Upload artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
      with:
        name: openclaw-integration
        path: hindsight-integrations/openclaw/*.tgz
@@ -215,10 +237,10 @@ jobs:
    environment: npm

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '22'
        registry-url: 'https://registry.npmjs.org'
@@ -253,7 +275,7 @@ jobs:
      run: npm pack

    - name: Upload artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
      with:
        name: ai-sdk-integration
        path: hindsight-integrations/ai-sdk/*.tgz
@@ -264,10 +286,10 @@ jobs:
    environment: npm

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '22'
        registry-url: 'https://registry.npmjs.org'
@@ -302,7 +324,7 @@ jobs:
      run: npm pack

    - name: Upload artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
      with:
        name: chat-integration
        path: hindsight-integrations/chat/*.tgz
@@ -313,10 +335,10 @@ jobs:
    environment: npm

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '20'
        registry-url: 'https://registry.npmjs.org'
@@ -364,7 +386,7 @@ jobs:
      run: npm pack

    - name: Upload artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
      with:
        name: control-plane
        path: hindsight-control-plane/*.tgz
@@ -393,7 +415,7 @@ jobs:
            asset_name: hindsight-linux-arm64

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Install Rust
      uses: dtolnay/rust-toolchain@stable
@@ -411,7 +433,7 @@ jobs:
        chmod +x artifacts/${{ matrix.asset_name }}

    - name: Upload artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
      with:
        name: rust-cli-${{ matrix.asset_name }}
        path: artifacts/${{ matrix.asset_name }}
@@ -452,7 +474,7 @@ jobs:
              PRELOAD_ML_MODELS=false

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Free Disk Space
      uses: jlumbroso/free-disk-space@main
@@ -466,13 +488,13 @@ jobs:
        swap-storage: true

    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
+      uses: docker/setup-qemu-action@v4

    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@v4

    - name: Log in to GitHub Container Registry
-      uses: docker/login-action@v3
+      uses: docker/login-action@v4
      with:
        registry: ghcr.io
        username: ${{ github.actor }}
@@ -484,7 +506,7 @@ jobs:

    - name: Extract metadata for release tags
      id: meta
-      uses: docker/metadata-action@v5
+      uses: docker/metadata-action@v6
      with:
        images: ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}
        flavor: |
@@ -500,7 +522,7 @@ jobs:
    # # Step 1: Build for local testing (single platform, no push)
    # # This creates an identical image to what will be released, just for one platform
    # - name: Build image for testing
-    #   uses: docker/build-push-action@v6
+    #   uses: docker/build-push-action@v7
    #   with:
    #     context: .
    #     file: docker/standalone/Dockerfile
@@ -519,7 +541,7 @@ jobs:

    # Build multi-platform and push to release tags
    - name: Build and push release images
-      uses: docker/build-push-action@v6
+      uses: docker/build-push-action@v7
      with:
        context: .
        file: docker/standalone/Dockerfile
@@ -537,7 +559,7 @@ jobs:
      packages: write

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Install Helm
      uses: azure/setup-helm@v4
@@ -557,7 +579,7 @@ jobs:
      run: helm push helm-packages/*.tgz oci://ghcr.io/${{ github.repository_owner }}/charts

    - name: Upload artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
      with:
        name: helm-chart
        path: helm-packages/*.tgz
@@ -570,7 +592,7 @@ jobs:
      contents: write

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Extract version from tag
      id: get_version
@@ -641,8 +663,10 @@ jobs:
        mkdir -p release-assets
        # Python packages
        cp artifacts/python-packages/hindsight-clients/python/dist/* release-assets/ || true
+        cp artifacts/python-packages/hindsight-api-slim/dist/* release-assets/ || true
        cp artifacts/python-packages/hindsight-api/dist/* release-assets/ || true
-        cp artifacts/python-packages/hindsight/dist/* release-assets/ || true
+        cp artifacts/python-packages/hindsight-all/dist/* release-assets/ || true
+        cp artifacts/python-packages/hindsight-all-slim/dist/* release-assets/ || true
        cp artifacts/python-packages/hindsight-integrations/litellm/dist/* release-assets/ || true
        cp artifacts/python-packages/hindsight-integrations/pydantic-ai/dist/* release-assets/ || true
        cp artifacts/python-packages/hindsight-embed/dist/* release-assets/ || true
@@ -16,10 +16,10 @@ jobs:
        python-version: ['3.11', '3.12', '3.13', '3.14']

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true

@@ -29,17 +29,17 @@ jobs:
        python-version: ${{ matrix.python-version }}

    - name: Build hindsight-api
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: uv build

  build-typescript-client:
    runs-on: ubuntu-latest

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '20'
        cache: 'npm'
@@ -55,10 +55,10 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '22'

@@ -78,10 +78,10 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '22'

@@ -101,10 +101,10 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '22'

@@ -124,10 +124,10 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '20'
        cache: 'npm'
@@ -176,10 +176,10 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '20'
        cache: 'npm'
@@ -202,7 +202,7 @@ jobs:
      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Setup GCP credentials
      run: |
@@ -214,7 +214,7 @@ jobs:
      uses: dtolnay/rust-toolchain@stable

    - name: Cache cargo
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: |
          ~/.cargo/registry
@@ -231,14 +231,14 @@ jobs:
      run: cargo build --release

    - name: Upload CLI artifact
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
      with:
        name: hindsight-cli
        path: hindsight-cli/target/release/hindsight
        retention-days: 1

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -249,23 +249,23 @@ jobs:
        python-version-file: ".python-version"

    - name: Build API
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: uv build

    - name: Install API dependencies
-      working-directory: ./hindsight-api
-      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
+      working-directory: ./hindsight-api-slim
+      run: uv sync --frozen --all-extras --index-strategy unsafe-best-match

    - name: Cache HuggingFace models
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: ~/.cache/huggingface
-        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-huggingface-

    - name: Pre-download models
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: |
        uv run python -c "
        from sentence_transformers import SentenceTransformer, CrossEncoder
@@ -316,7 +316,7 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Install Helm
      uses: azure/setup-helm@v4
@@ -358,7 +358,7 @@ jobs:
              PRELOAD_ML_MODELS=false

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Free Disk Space
      uses: jlumbroso/free-disk-space@main
@@ -372,10 +372,10 @@ jobs:
        swap-storage: true

    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@v4

    - name: Build ${{ matrix.name }} image (${{ matrix.variant }})
-      uses: docker/build-push-action@v6
+      uses: docker/build-push-action@v7
      with:
        context: .
        file: docker/standalone/Dockerfile
@@ -424,7 +424,7 @@ jobs:
      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Setup GCP credentials
      run: |
@@ -433,7 +433,7 @@ jobs:
        echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -444,23 +444,23 @@ jobs:
        python-version-file: ".python-version"

    - name: Build API
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: uv build

    - name: Install dependencies
-      working-directory: ./hindsight-api
-      run: uv sync --frozen --extra test --no-install-project --index-strategy unsafe-best-match
+      working-directory: ./hindsight-api-slim
+      run: uv sync --frozen --all-extras --index-strategy unsafe-best-match

    - name: Cache HuggingFace models
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: ~/.cache/huggingface
-        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-huggingface-

    - name: Pre-download models
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: |
        uv run python -c "
        from sentence_transformers import SentenceTransformer, CrossEncoder
@@ -472,7 +472,7 @@ jobs:
        "

    - name: Run tests
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: uv run pytest tests -v

  test-python-client:
@@ -487,7 +487,7 @@ jobs:
      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Setup GCP credentials
      run: |
@@ -496,7 +496,7 @@ jobs:
        echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -507,7 +507,7 @@ jobs:
        python-version-file: ".python-version"

    - name: Build API
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: uv build

    - name: Build Python client
@@ -519,19 +519,19 @@ jobs:
      run: uv sync --frozen --extra test --index-strategy unsafe-best-match

    - name: Install API dependencies
-      working-directory: ./hindsight-api
-      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
+      working-directory: ./hindsight-api-slim
+      run: uv sync --frozen --all-extras --index-strategy unsafe-best-match

    - name: Cache HuggingFace models
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: ~/.cache/huggingface
-        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-huggingface-

    - name: Pre-download models
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: |
        uv run python -c "
        from sentence_transformers import SentenceTransformer, CrossEncoder
@@ -590,7 +590,7 @@ jobs:
      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Setup GCP credentials
      run: |
@@ -599,7 +599,7 @@ jobs:
        echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -610,17 +610,17 @@ jobs:
        python-version-file: ".python-version"

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '20'

    - name: Build API
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: uv build

    - name: Install API dependencies
-      working-directory: ./hindsight-api
-      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
+      working-directory: ./hindsight-api-slim
+      run: uv sync --frozen --all-extras --index-strategy unsafe-best-match

    - name: Install TypeScript client dependencies
      working-directory: ./hindsight-clients/typescript
@@ -631,15 +631,15 @@ jobs:
      run: npm run build

    - name: Cache HuggingFace models
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: ~/.cache/huggingface
-        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-huggingface-

    - name: Pre-download models
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: |
        uv run python -c "
        from sentence_transformers import SentenceTransformer, CrossEncoder
@@ -690,7 +690,7 @@ jobs:
    runs-on: ubuntu-24.04-arm

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Install Rust
      uses: dtolnay/rust-toolchain@stable
@@ -698,7 +698,7 @@ jobs:
        targets: aarch64-unknown-linux-gnu

    - name: Cache cargo
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: |
          ~/.cargo/registry
@@ -722,7 +722,7 @@ jobs:
      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Setup GCP credentials
      run: |
@@ -731,7 +731,7 @@ jobs:
        echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -745,7 +745,7 @@ jobs:
      uses: dtolnay/rust-toolchain@stable

    - name: Cache cargo
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: |
          ~/.cargo/registry
@@ -754,23 +754,23 @@ jobs:
        key: ${{ runner.os }}-cargo-client-${{ hashFiles('hindsight-clients/rust/Cargo.lock') }}

    - name: Build API
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: uv build

    - name: Install API dependencies
-      working-directory: ./hindsight-api
-      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
+      working-directory: ./hindsight-api-slim
+      run: uv sync --frozen --all-extras --index-strategy unsafe-best-match

    - name: Cache HuggingFace models
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: ~/.cache/huggingface
-        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-huggingface-

    - name: Pre-download models
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: |
        uv run python -c "
        from sentence_transformers import SentenceTransformer, CrossEncoder
@@ -829,7 +829,7 @@ jobs:
      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Setup GCP credentials
      run: |
@@ -838,7 +838,7 @@ jobs:
        echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -849,29 +849,29 @@ jobs:
        python-version-file: ".python-version"

    - name: Set up Go
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@v6
      with:
        go-version: '1.23'
        cache-dependency-path: hindsight-clients/go/go.sum

    - name: Build API
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: uv build

    - name: Install API dependencies
-      working-directory: ./hindsight-api
-      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
+      working-directory: ./hindsight-api-slim
+      run: uv sync --frozen --all-extras --index-strategy unsafe-best-match

    - name: Cache HuggingFace models
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: ~/.cache/huggingface
-        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-huggingface-

    - name: Pre-download models
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: |
        uv run python -c "
        from sentence_transformers import SentenceTransformer, CrossEncoder
@@ -934,7 +934,7 @@ jobs:
      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Setup GCP credentials
      run: |
@@ -943,7 +943,7 @@ jobs:
        echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -954,32 +954,32 @@ jobs:
        python-version-file: ".python-version"

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '22'

    - name: Build API
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: uv build

-    - name: Install API dependencies
-      working-directory: ./hindsight-api
-      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
-
    - name: Install embed dependencies
      working-directory: ./hindsight-embed
      run: uv sync --frozen --index-strategy unsafe-best-match

+    - name: Install API dependencies
+      working-directory: ./hindsight-api-slim
+      run: uv sync --frozen --all-extras --index-strategy unsafe-best-match
+
    - name: Cache HuggingFace models
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: ~/.cache/huggingface
-        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-huggingface-

    - name: Pre-download models
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: |
        uv run python -c "
        from sentence_transformers import SentenceTransformer, CrossEncoder
@@ -1041,7 +1041,7 @@ jobs:
      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Setup GCP credentials
      run: |
@@ -1050,7 +1050,7 @@ jobs:
        echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -1061,27 +1061,27 @@ jobs:
        python-version-file: ".python-version"

    - name: Build API
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: uv build

-    - name: Install API dependencies
-      working-directory: ./hindsight-api
-      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
-
    - name: Install integration test dependencies
      working-directory: ./hindsight-integration-tests
      run: uv sync --frozen

+    - name: Install API dependencies
+      working-directory: ./hindsight-api-slim
+      run: uv sync --frozen --all-extras --index-strategy unsafe-best-match
+
    - name: Cache HuggingFace models
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: ~/.cache/huggingface
-        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-huggingface-

    - name: Pre-download models
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: |
        uv run python -c "
        from sentence_transformers import SentenceTransformer, CrossEncoder
@@ -1132,10 +1132,10 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -1161,10 +1161,10 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -1190,10 +1190,10 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -1215,14 +1215,15 @@ jobs:
      working-directory: ./hindsight-integrations/pydantic-ai
      run: uv run pytest tests -v

-  test-embed:
+  test-pip-slim:
    runs-on: ubuntu-latest
    env:
      HINDSIGHT_API_LLM_PROVIDER: vertexai
      HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY: /tmp/gcp-credentials.json
      HINDSIGHT_API_LLM_MODEL: google/gemini-2.5-flash-lite
-      # Prefer CPU-only PyTorch in CI
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
+      HINDSIGHT_API_EMBEDDINGS_PROVIDER: cohere
+      HINDSIGHT_API_RERANKER_PROVIDER: cohere
+      HINDSIGHT_API_COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}

    steps:
    - uses: actions/checkout@v4
@@ -1244,12 +1245,75 @@ jobs:
      with:
        python-version-file: ".python-version"

+    - name: Install hindsight-api-slim (embedded-db only, no local ML)
+      working-directory: ./hindsight-api-slim
+      run: uv sync --frozen --extra embedded-db --index-strategy unsafe-best-match
+
+    - name: Start API server
+      working-directory: ./hindsight-api-slim
+      run: |
+        uv run hindsight-api --port 8888 > /tmp/slim-api-server.log 2>&1 &
+        for i in $(seq 1 60); do
+          if curl -s http://localhost:8888/health | grep -q "healthy"; then
+            echo "API server is ready after ${i}s"
+            break
+          fi
+          if [ $i -eq 60 ]; then
+            echo "API server failed to start after 60s"
+            cat /tmp/slim-api-server.log
+            exit 1
+          fi
+          sleep 1
+        done
+
+    - name: Smoke test - retain and recall
+      run: ./scripts/smoke-test-slim.sh http://localhost:8888
+
+    - name: Show API server logs
+      if: always()
+      run: |
+        echo "=== API Server Logs ==="
+        cat /tmp/slim-api-server.log 2>/dev/null || true
+
+  test-embed:
+    runs-on: ubuntu-latest
+    env:
+      HINDSIGHT_API_LLM_PROVIDER: vertexai
+      HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY: /tmp/gcp-credentials.json
+      HINDSIGHT_API_LLM_MODEL: google/gemini-2.5-flash-lite
+      # Prefer CPU-only PyTorch in CI
+      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Setup GCP credentials
+      run: |
+        printf '%s' '${{ secrets.GCP_VERTEXAI_CREDENTIALS }}' > /tmp/gcp-credentials.json
+        PROJECT_ID=$(jq -r '.project_id' /tmp/gcp-credentials.json)
+        echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version-file: ".python-version"
+
    - name: Install dependencies
      working-directory: ./hindsight-embed
      run: uv sync --frozen --index-strategy unsafe-best-match

+    - name: Install API dependencies (with local-ml and embedded-db for smoke test)
+      working-directory: ./hindsight-api-slim
+      run: uv sync --frozen --all-extras --index-strategy unsafe-best-match
+
    - name: Cache HuggingFace models
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: ~/.cache/huggingface
        key: ${{ runner.os }}-huggingface-embed-${{ hashFiles('hindsight-embed/pyproject.toml') }}
@@ -1279,7 +1343,7 @@ jobs:
      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Setup GCP credentials
      run: |
@@ -1288,7 +1352,7 @@ jobs:
        echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -1299,24 +1363,24 @@ jobs:
        python-version-file: ".python-version"

    - name: Build hindsight-all
-      working-directory: ./hindsight
+      working-directory: ./hindsight-all
      run: uv build

    - name: Install dependencies
-      working-directory: ./hindsight
+      working-directory: ./hindsight-all
      run: uv sync --frozen --extra test --index-strategy unsafe-best-match

    - name: Cache HuggingFace models
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: ~/.cache/huggingface
-        key: ${{ runner.os }}-huggingface-all-${{ hashFiles('hindsight/pyproject.toml') }}
+        key: ${{ runner.os }}-huggingface-all-${{ hashFiles('hindsight-all/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-huggingface-all-
          ${{ runner.os }}-huggingface-

    - name: Run unit tests
-      working-directory: ./hindsight
+      working-directory: ./hindsight-all
      run: uv run pytest tests/ -v

  test-doc-examples:
@@ -1335,7 +1399,7 @@ jobs:
      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Setup GCP credentials
      run: |
@@ -1349,7 +1413,7 @@ jobs:

    - name: Cache cargo
      if: matrix.language == 'cli'
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: |
          ~/.cargo/registry
@@ -1365,7 +1429,7 @@ jobs:
        cp target/release/hindsight /usr/local/bin/hindsight

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -1377,23 +1441,23 @@ jobs:

    - name: Set up Node.js
      if: matrix.language == 'node'
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '20'
        cache: 'npm'
        cache-dependency-path: package-lock.json

-    - name: Build and install API
-      working-directory: ./hindsight-api
-      run: |
-        uv build
-        uv sync --frozen --no-install-project --index-strategy unsafe-best-match
-
    - name: Install Python client dependencies
      if: matrix.language == 'python'
      working-directory: ./hindsight-clients/python
      run: uv sync --frozen --extra test --index-strategy unsafe-best-match

+    - name: Build and install API
+      working-directory: ./hindsight-api-slim
+      run: |
+        uv build
+        uv sync --frozen --all-extras --index-strategy unsafe-best-match
+
    - name: Install TypeScript client
      if: matrix.language == 'node'
      run: |
@@ -1401,15 +1465,15 @@ jobs:
        npm run build --workspace=hindsight-clients/typescript

    - name: Cache HuggingFace models
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: ~/.cache/huggingface
-        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-huggingface-

    - name: Pre-download models
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: |
        uv run python -c "
        from sentence_transformers import SentenceTransformer, CrossEncoder
@@ -1469,7 +1533,7 @@ jobs:
      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
      with:
        fetch-depth: 0  # Full history needed for git clone of tags

@@ -1483,7 +1547,7 @@ jobs:
      run: git fetch --tags

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        prune-cache: false
@@ -1494,10 +1558,10 @@ jobs:
        python-version-file: ".python-version"

    - name: Cache HuggingFace models
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: ~/.cache/huggingface
-        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-huggingface-

@@ -1506,11 +1570,11 @@ jobs:
      run: uv sync --frozen --extra test --index-strategy unsafe-best-match

    - name: Install current hindsight-api
-      working-directory: ./hindsight-api
-      run: uv sync --frozen --index-strategy unsafe-best-match
+      working-directory: ./hindsight-api-slim
+      run: uv sync --frozen --all-extras --index-strategy unsafe-best-match

    - name: Pre-download models
-      working-directory: ./hindsight-api
+      working-directory: ./hindsight-api-slim
      run: |
        uv run python -c "
        from sentence_transformers import SentenceTransformer, CrossEncoder
@@ -1543,10 +1607,10 @@ jobs:
      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true

@@ -1556,7 +1620,7 @@ jobs:
        python-version-file: ".python-version"

    - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
      with:
        node-version: '20'
        cache: 'npm'
@@ -1566,7 +1630,7 @@ jobs:
      uses: dtolnay/rust-toolchain@stable

    - name: Cache cargo
-      uses: actions/cache@v4
+      uses: actions/cache@v5
      with:
        path: |
          ~/.cargo/registry
@@ -1615,12 +1679,12 @@ jobs:
      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
      with:
        fetch-depth: 0  # Fetch full git history to access base branch

    - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true

@@ -17,20 +17,20 @@ Hindsight is an agent memory system that provides long-term memory for AI agents
 ./scripts/dev/start-api.sh

 # Run all tests (parallelized with pytest-xdist)
-cd hindsight-api && uv run pytest tests/
+cd hindsight-api-slim && uv run pytest tests/

 # Run specific test file
-cd hindsight-api && uv run pytest tests/test_http_api_integration.py -v
+cd hindsight-api-slim && uv run pytest tests/test_http_api_integration.py -v

 # Run single test function
-cd hindsight-api && uv run pytest tests/test_retain.py::test_retain_simple -v
+cd hindsight-api-slim && uv run pytest tests/test_retain.py::test_retain_simple -v

 # Lint and format
-cd hindsight-api && uv run ruff check .
-cd hindsight-api && uv run ruff format .
+cd hindsight-api-slim && uv run ruff check .
+cd hindsight-api-slim && uv run ruff format .

 # Type checking (uses ty - extremely fast type checker from Astral)
-cd hindsight-api && uv run ty check hindsight_api/
+cd hindsight-api-slim && uv run ty check hindsight_api/
 ```

 ### Control Plane (Next.js)
@@ -72,7 +72,7 @@ cd hindsight-control-plane && npm run dev
 ## Architecture

 ### Monorepo Structure
- **hindsight-api/**: Core FastAPI server with memory engine (Python, uv)
+- **hindsight-api-slim/**: Core FastAPI server with memory engine (Python, uv)
 - **hindsight/**: Embedded Python bundle (hindsight-all package)
 - **hindsight-control-plane/**: Admin UI (Next.js, npm)
 - **hindsight-cli/**: CLI tool (Rust, cargo, uses progenitor for API client)
@@ -81,9 +81,9 @@ cd hindsight-control-plane && npm run dev
 - **hindsight-integrations/**: Framework integrations (LiteLLM, OpenAI)
 - **hindsight-dev/**: Development tools and benchmarks

-### Core Engine (hindsight-api/hindsight_api/engine/)
+### Core Engine (hindsight-api-slim/hindsight_api/engine/)
 - `memory_engine.py`: Main orchestrator (~170KB) for retain/recall/reflect operations
- `llm_wrapper.py`: LLM abstraction supporting OpenAI, Anthropic, Gemini, Groq, Ollama, LM Studio
+- `llm_wrapper.py`: LLM abstraction supporting OpenAI, Anthropic, Gemini, Groq, MiniMax, Ollama, LM Studio
 - `embeddings.py`: Embedding generation (local sentence-transformers or TEI)
 - `cross_encoder.py`: Reranking (local or TEI)
 - `entity_resolver.py`: Entity extraction and normalization
@@ -101,7 +101,7 @@ cd hindsight-control-plane && npm run dev
 - `fusion.py`: Reciprocal rank fusion for combining results
 - `reranking.py`: Cross-encoder reranking

-### API Layer (hindsight-api/hindsight_api/api/)
+### API Layer (hindsight-api-slim/hindsight_api/api/)
 - `http.py`: FastAPI HTTP routers (~80KB) for all REST endpoints
 - `mcp.py`: Model Context Protocol server implementation

@@ -111,13 +111,13 @@ Main operations:
 - **Reflect**: Disposition-aware reasoning using memories and mental models.

 ### Database
-PostgreSQL with pgvector. Schema managed via Alembic migrations in `hindsight-api/hindsight_api/alembic/`. Migrations run automatically on API startup.
+PostgreSQL with pgvector. Schema managed via Alembic migrations in `hindsight-api-slim/hindsight_api/alembic/`. Migrations run automatically on API startup.

 Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`

 ### Adding Database Migrations

-1. **Create a new migration file** in `hindsight-api/hindsight_api/alembic/versions/`:
+1. **Create a new migration file** in `hindsight-api-slim/hindsight_api/alembic/versions/`:
   - File name format: `<revision_id>_<description>.py` (e.g., `f1a2b3c4d5e6_add_new_index.py`)
   - Use a unique hex revision ID (12 chars)
   - Set `down_revision` to the previous migration's revision ID
@@ -154,7 +154,7 @@ Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`

 3. **Run migrations locally**:
   ```bash
-   # Set database URL and run migrations
+   # Set database URL and run migrations for the base schema plus all tenants
   uv run hindsight-admin run-db-migration

   # Run on a specific tenant schema
@@ -251,7 +251,7 @@ Fields must be categorized as either **hierarchical** (can be overridden per-ten

 #### Adding a New Configuration Field

-1. **config.py** (`hindsight-api/hindsight_api/config.py`):
+1. **config.py** (`hindsight-api-slim/hindsight_api/config.py`):
   - Add `ENV_*` constant for the environment variable name (e.g., `ENV_MY_SETTING = "HINDSIGHT_API_MY_SETTING"`)
   - Add `DEFAULT_*` constant for the default value
   - Add field to `HindsightConfig` dataclass with type annotation
@@ -268,7 +268,7 @@ Fields must be categorized as either **hierarchical** (can be overridden per-ten
   # Static field - just don't add to _HIERARCHICAL_FIELDS
   ```

-2. **main.py** (`hindsight-api/hindsight_api/main.py`):
+2. **main.py** (`hindsight-api-slim/hindsight_api/main.py`):
   - Add field to the manual `HindsightConfig()` constructor call (search for "CLI override")

 3. **Use hierarchical config in MemoryEngine**:
@@ -308,14 +308,14 @@ cp .env.example .env
 # Edit .env with LLM API key

 # Python deps
-uv sync --directory hindsight-api/
+uv sync --directory hindsight-api-slim/

 # Node deps (uses npm workspaces)
 npm install
 ```

 Required env vars:
- `HINDSIGHT_API_LLM_PROVIDER`: openai, anthropic, gemini, groq, ollama, lmstudio
+- `HINDSIGHT_API_LLM_PROVIDER`: openai, anthropic, gemini, groq, minimax, ollama, lmstudio
 - `HINDSIGHT_API_LLM_API_KEY`: Your API key
 - `HINDSIGHT_API_LLM_MODEL`: Model name (e.g., gpt-4o-mini, claude-sonnet-4-20250514)

@@ -9,8 +9,9 @@
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 ![PyPI - Downloads](https://img.shields.io/pypi/dm/hindsight-api?label=PyPI)
 ![NPM Downloads](https://img.shields.io/npm/dm/%40vectorize-io%2Fhindsight-client?logoColor=orange&label=NPM&color=blue&link=https%3A%2F%2Fwww.npmjs.com%2Fpackage%2F%40vectorize-io%2Fhindsight-client)
+<br/>

-
+<a href="https://trendshift.io/repositories/15603" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15603" alt="vectorize-io%2Fhindsight | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
 </div>

 ---
@@ -69,7 +70,7 @@ docker run --rm -it --pull always -p 8888:8888 -p 9999:9999 \
 >API: http://localhost:8888
 >UI: http://localhost:9999

-You can modify the LLM provider by setting `HINDSIGHT_API_LLM_PROVIDER`. Valid options are `openai`, `anthropic`, `gemini`, `groq`, `ollama`, and `lmstudio`. The documentation provides more details on [supported models](https://hindsight.vectorize.io/developer/models).
+You can modify the LLM provider by setting `HINDSIGHT_API_LLM_PROVIDER`. Valid options are `openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`, and `minimax`. The documentation provides more details on [supported models](https://hindsight.vectorize.io/developer/models).



@@ -42,25 +42,22 @@ RUN apt-get update && apt-get install -y \
    && pip install --no-cache-dir uv

 # Copy dependency files and README (required by pyproject.toml)
-COPY hindsight-api/pyproject.toml ./api/
-COPY hindsight-api/README.md ./api/
+COPY hindsight-api-slim/pyproject.toml ./api/
+COPY hindsight-api-slim/README.md ./api/

 WORKDIR /app/api

-# Remove local ML model dependencies if INCLUDE_LOCAL_MODELS=false
-# This creates a smaller image when using external providers (TEI, OpenAI, Cohere)
-RUN if [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then \
-        echo "Removing local-models dependencies (sentence-transformers, torch, transformers)..." && \
-        sed -i '/"sentence-transformers/d' pyproject.toml && \
-        sed -i '/"transformers/d' pyproject.toml && \
-        sed -i '/"torch/d' pyproject.toml; \
+# Sync dependencies using appropriate extras based on INCLUDE_LOCAL_MODELS
+# local-ml: torch, sentence-transformers, transformers, einops, flashrank, mlx (optional)
+# embedded-db: pg0-embedded (always included for embedded PostgreSQL support)
+RUN if [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
+        uv sync --extra local-ml --extra embedded-db; \
+    else \
+        uv sync --extra embedded-db; \
    fi

-# Sync dependencies (will create lock file if needed)
-RUN uv sync
-
 # Copy source code (alembic migrations are inside hindsight_api/)
-COPY hindsight-api/hindsight_api ./hindsight_api
+COPY hindsight-api-slim/hindsight_api ./hindsight_api

 # Install the local package (uv sync only installed dependencies, not the package itself)
 RUN uv pip install -e .
@@ -77,18 +77,32 @@ PIDS=()
 # Start API if enabled
 if [ "$ENABLE_API" = "true" ]; then
    cd /app/api
+    API_HEALTH_URL="${HINDSIGHT_API_HEALTH_URL:-http://localhost:8888/health}"
+    API_STARTUP_WAIT_SECONDS="${HINDSIGHT_API_STARTUP_WAIT_SECONDS:-300}"
+
    # Run API directly - Python's PYTHONUNBUFFERED=1 handles output buffering
    hindsight-api &
    API_PID=$!
    PIDS+=($API_PID)

    # Wait for API to be ready
-    for i in {1..60}; do
-        if curl -sf http://localhost:8888/health &>/dev/null; then
+    api_ready=false
+    for ((i=1; i<=API_STARTUP_WAIT_SECONDS; i++)); do
+        if ! kill -0 "$API_PID" 2>/dev/null; then
+            wait "$API_PID"
+            exit $?
+        fi
+        if curl -sf "$API_HEALTH_URL" &>/dev/null; then
+            api_ready=true
            break
        fi
        sleep 1
    done
+
+    if [ "$api_ready" != "true" ]; then
+        echo "❌ API did not become healthy within ${API_STARTUP_WAIT_SECONDS}s"
+        exit 1
+    fi
 else
    echo "API disabled (HINDSIGHT_ENABLE_API=false)"
 fi
@@ -49,6 +49,9 @@

 set -euo pipefail

+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(dirname "$SCRIPT_DIR")"
+
 # Colors for output
 RED='\033[0;31m'
 GREEN='\033[0;32m'
@@ -178,6 +181,21 @@ for i in $(seq 1 "$TIMEOUT"); do
        echo "=== Health Response ==="
        curl -s "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" | python3 -m json.tool 2>/dev/null || curl -s "http://localhost:${HEALTH_PORT}${HEALTH_PATH}"
        echo ""
+
+        # Run retain/recall smoke test for API targets
+        if [ "$TARGET" != "cp-only" ]; then
+            echo ""
+            echo "=== Retain/Recall Smoke Test ==="
+            if ! "$REPO_ROOT/scripts/smoke-test-slim.sh" "http://localhost:${HEALTH_PORT}"; then
+                echo ""
+                echo "=== Container Logs (last 50 lines) ==="
+                docker logs "$CONTAINER_NAME" 2>&1 | tail -50
+                echo ""
+                echo -e "${RED}Smoke test FAILED${NC}"
+                exit 1
+            fi
+        fi
+
        echo ""
        echo "=== Container Logs (last 50 lines) ==="
        docker logs "$CONTAINER_NAME" 2>&1 | tail -50
@@ -2,8 +2,8 @@ apiVersion: v2
 name: hindsight
 description: Hindsight helm chart
 type: application
-version: 0.4.16
-appVersion: "0.4.16"
+version: 0.4.17
+appVersion: "0.4.17"
 keywords:
  - ai
  - memory
@@ -0,0 +1,33 @@
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hindsight-all-slim"
+version = "0.4.17"
+description = "Hindsight: Agent Memory That Works Like Human Memory - Slim All-in-One Bundle"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "hindsight-api-slim>=0.4.17",
+    "hindsight-client>=0.0.7",
+    "hindsight-embed>=0.1.0",
+]
+
+[tool.uv.sources]
+hindsight-api-slim = { workspace = true }
+hindsight-client = { workspace = true }
+hindsight-embed = { workspace = true }
+
+[project.optional-dependencies]
+test = [
+    "pytest>=7.0.0",
+    "pytest-asyncio>=0.21.0",
+]
+
+[tool.setuptools]
+packages = []
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
@@ -0,0 +1,48 @@
+# hindsight-all
+
+All-in-one package for Hindsight - Agent Memory That Works Like Human Memory
+
+## Quick Start
+
+```python
+from hindsight import start_server, HindsightClient
+
+# Start server with embedded PostgreSQL
+server = start_server(
+    llm_provider="groq",
+    llm_api_key="your-api-key",
+    llm_model="openai/gpt-oss-120b"
+)
+
+# Create client
+client = HindsightClient(base_url=server.url)
+
+# Store memories
+client.put(agent_id="assistant", content="User prefers Python for data analysis")
+
+# Search memories
+results = client.search(agent_id="assistant", query="programming preferences")
+
+# Generate contextual response
+response = client.think(agent_id="assistant", query="What languages should I recommend?")
+
+# Stop server when done
+server.stop()
+```
+
+## Using Context Manager
+
+```python
+from hindsight import HindsightServer, HindsightClient
+
+with HindsightServer(llm_provider="groq", llm_api_key="...") as server:
+    client = HindsightClient(base_url=server.url)
+    # ... use client ...
+# Server automatically stops
+```
+
+## Installation
+
+```bash
+pip install hindsight-all
+```
@@ -4,18 +4,18 @@ build-backend = "hatchling.build"

 [project]
 name = "hindsight-all"
-version = "0.4.16"
+version = "0.4.17"
 description = "Hindsight: Agent Memory That Works Like Human Memory - All-in-One Bundle"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "hindsight-api>=0.0.7",
+    "hindsight-api-slim[all]>=0.4.17",
    "hindsight-client>=0.0.7",
    "hindsight-embed>=0.1.0",
 ]

 [tool.uv.sources]
-hindsight-api = { workspace = true }
+hindsight-api-slim = { workspace = true }
 hindsight-client = { workspace = true }
 hindsight-embed = { workspace = true }

@@ -0,0 +1,137 @@
+# Hindsight API
+
+**Memory System for AI Agents** — Temporal + Semantic + Entity Memory Architecture using PostgreSQL with pgvector.
+
+Hindsight gives AI agents persistent memory that works like human memory: it stores facts, tracks entities and relationships, handles temporal reasoning ("what happened last spring?"), and forms opinions based on configurable disposition traits.
+
+## Installation
+
+```bash
+pip install hindsight-api
+```
+
+## Quick Start
+
+### Run the Server
+
+```bash
+# Set your LLM provider
+export HINDSIGHT_API_LLM_PROVIDER=openai
+export HINDSIGHT_API_LLM_API_KEY=sk-xxxxxxxxxxxx
+
+# Start the server (uses embedded PostgreSQL by default)
+hindsight-api
+```
+
+The server starts at http://localhost:8888 with:
+- REST API for memory operations
+- MCP server at `/mcp` for tool-use integration
+
+### Use the Python API
+
+```python
+from hindsight_api import MemoryEngine
+
+# Create and initialize the memory engine
+memory = MemoryEngine()
+await memory.initialize()
+
+# Create a memory bank for your agent
+bank = await memory.create_memory_bank(
+    name="my-assistant",
+    background="A helpful coding assistant"
+)
+
+# Store a memory
+await memory.retain(
+    memory_bank_id=bank.id,
+    content="The user prefers Python for data science projects"
+)
+
+# Recall memories
+results = await memory.recall(
+    memory_bank_id=bank.id,
+    query="What programming language does the user prefer?"
+)
+
+# Reflect with reasoning
+response = await memory.reflect(
+    memory_bank_id=bank.id,
+    query="Should I recommend Python or R for this ML project?"
+)
+```
+
+## CLI Options
+
+```bash
+hindsight-api --help
+
+# Common options
+hindsight-api --port 9000          # Custom port (default: 8888)
+hindsight-api --host 127.0.0.1     # Bind to localhost only
+hindsight-api --workers 4          # Multiple worker processes
+hindsight-api --log-level debug    # Verbose logging
+```
+
+## Configuration
+
+Configure via environment variables:
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `HINDSIGHT_API_DATABASE_URL` | PostgreSQL connection string | `pg0` (embedded) |
+| `HINDSIGHT_API_LLM_PROVIDER` | `openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio` | `openai` |
+| `HINDSIGHT_API_LLM_API_KEY` | API key for LLM provider | - |
+| `HINDSIGHT_API_LLM_MODEL` | Model name | `gpt-4o-mini` |
+| `HINDSIGHT_API_HOST` | Server bind address | `0.0.0.0` |
+| `HINDSIGHT_API_PORT` | Server port | `8888` |
+
+### Example with External PostgreSQL
+
+```bash
+export HINDSIGHT_API_DATABASE_URL=postgresql://user:pass@localhost:5432/hindsight
+export HINDSIGHT_API_LLM_PROVIDER=groq
+export HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx
+
+hindsight-api
+```
+
+## Docker
+
+```bash
+docker run --rm -it -p 8888:8888 \
+  -e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \
+  -v $HOME/.hindsight-docker:/home/hindsight/.pg0 \
+  ghcr.io/vectorize-io/hindsight:latest
+```
+
+## MCP Server
+
+For local MCP integration without running the full API server:
+
+```bash
+hindsight-local-mcp
+```
+
+This runs a stdio-based MCP server that can be used directly with MCP-compatible clients.
+
+## Key Features
+
+- **Multi-Strategy Retrieval (TEMPR)** — Semantic, keyword, graph, and temporal search combined with RRF fusion
+- **Entity Graph** — Automatic entity extraction and relationship tracking
+- **Temporal Reasoning** — Native support for time-based queries
+- **Disposition Traits** — Configurable skepticism, literalism, and empathy influence opinion formation
+- **Three Memory Types** — World facts, bank actions, and formed opinions with confidence scores
+
+## Documentation
+
+Full documentation: [https://hindsight.vectorize.io](https://hindsight.vectorize.io)
+
+- [Installation Guide](https://hindsight.vectorize.io/developer/installation)
+- [Configuration Reference](https://hindsight.vectorize.io/developer/configuration)
+- [API Reference](https://hindsight.vectorize.io/api-reference)
+- [Python SDK](https://hindsight.vectorize.io/sdks/python)
+
+## License
+
+Apache 2.0
@@ -46,4 +46,4 @@ __all__ = [
    "RemoteTEICrossEncoder",
    "LLMConfig",
 ]
-__version__ = "0.4.16"
+__version__ = "0.4.17"
@@ -14,7 +14,8 @@ from typing import Any
 import asyncpg
 import typer

-from ..config import HindsightConfig
+from ..config import DEFAULT_DATABASE_SCHEMA, HindsightConfig
+from ..extensions import TenantExtension, load_extension
 from ..pg0 import parse_pg0_url, resolve_database_url


@@ -214,20 +215,81 @@ def restore(
    typer.echo("Restore complete")


-async def _run_migration(db_url: str, schema: str = "public") -> None:
-    """Resolve database URL and run migrations."""
-    from ..migrations import run_migrations
+async def _run_migration(
+    db_url: str,
+    schema: str | None = None,
+    base_schema: str = DEFAULT_DATABASE_SCHEMA,
+    embedding_dimension: int | None = None,
+) -> list[str]:
+    """Resolve database URL and run migrations for one schema or all discovered schemas."""
+    from ..migrations import (
+        ensure_embedding_dimension,
+        ensure_text_search_extension,
+        ensure_vector_extension,
+        run_migrations,
+    )

    is_pg0, instance_name, _ = parse_pg0_url(db_url)
    if is_pg0:
        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
    resolved_url = await resolve_database_url(db_url)
-    run_migrations(resolved_url, schema=schema)
+
+    config = HindsightConfig.from_env()
+    if schema:
+        schemas = [schema]
+    else:
+        tenant_extension = load_extension("TENANT", TenantExtension)
+
+        schemas = [base_schema or DEFAULT_DATABASE_SCHEMA]
+        if tenant_extension:
+            tenants = await tenant_extension.list_tenants()
+            schemas.extend(tenant.schema for tenant in tenants if tenant.schema)
+
+        # Preserve order while removing duplicates.
+        schemas = list(dict.fromkeys(schemas))
+
+    for schema in schemas:
+        run_migrations(resolved_url, schema=schema)
+
+    if embedding_dimension is not None:
+        for schema in schemas:
+            ensure_embedding_dimension(
+                resolved_url,
+                embedding_dimension,
+                schema=schema,
+                vector_extension=config.vector_extension,
+            )
+
+    for schema in schemas:
+        ensure_vector_extension(
+            resolved_url,
+            vector_extension=config.vector_extension,
+            schema=schema,
+        )
+
+    for schema in schemas:
+        ensure_text_search_extension(
+            resolved_url,
+            text_search_extension=config.text_search_extension,
+            schema=schema,
+        )
+
+    return schemas


@app.command(name="run-db-migration")
 def run_db_migration(
-    schema: str = typer.Option("public", "--schema", "-s", help="Database schema to run migrations on"),
+    schema: str | None = typer.Option(
+        None,
+        "--schema",
+        "-s",
+        help="Database schema to run migrations on. If omitted, migrate the base schema and all discovered tenant schemas.",
+    ),
+    embedding_dimension: int | None = typer.Option(
+        None,
+        "--embedding-dimension",
+        help="Expected embedding dimension to enforce after migrations. Omit to skip dimension sync.",
+    ),
 ):
    """Run database migrations to the latest version."""
    config = HindsightConfig.from_env()
@@ -237,11 +299,21 @@ def run_db_migration(
        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
        raise typer.Exit(1)

-    typer.echo(f"Running database migrations (schema: {schema})...")
+    if schema:
+        typer.echo(f"Running database migrations for schema: {schema}...")
+    else:
+        typer.echo("Running database migrations for base schema and all discovered tenant schemas...")

-    asyncio.run(_run_migration(config.database_url, schema))
+    schemas = asyncio.run(
+        _run_migration(
+            config.database_url,
+            schema=schema,
+            base_schema=config.database_schema,
+            embedding_dimension=embedding_dimension,
+        )
+    )

-    typer.echo("Database migrations completed successfully")
+    typer.echo(f"Database migrations completed successfully for {len(schemas)} schema(s)")


 async def _decommission_worker(db_url: str, worker_id: str, schema: str = "public") -> int:
@@ -34,7 +34,7 @@ def upgrade() -> None:
    # Create file_storage table (minimal: just key + data)
    op.execute(
        f"""
-        CREATE TABLE {schema}file_storage (
+        CREATE TABLE IF NOT EXISTS {schema}file_storage (
            storage_key TEXT PRIMARY KEY,
            data BYTEA NOT NULL
        )
@@ -0,0 +1,30 @@
+"""Add history column to mental_models
+
+Revision ID: c3d4e5f6g7h8
+Revises: a2b3c4d5e6f7, a2b3c4d5e6f8
+Create Date: 2026-03-06
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "c3d4e5f6g7h8"
+down_revision: str | Sequence[str] | None = ("a2b3c4d5e6f7", "a2b3c4d5e6f8")
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}mental_models ADD COLUMN IF NOT EXISTS history JSONB DEFAULT '[]'::jsonb")
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS history")
@@ -0,0 +1,53 @@
+"""Recreate idx_memory_units_source_memory_ids GIN index with fastupdate=off
+
+GIN indexes use a "fastupdate" pending list by default: small writes are
+buffered there and flushed to the main GIN tree in bulk. Flushing requires
+AccessExclusiveLock on the index. Under high insert concurrency (e.g. 8
+parallel pytest-xdist workers all calling retain_async) two transactions can
+each trigger a flush simultaneously and deadlock.
+
+Disabling fastupdate makes every insert write directly to the GIN tree
+(slightly slower per insert, but no pending-list lock cycles).
+
+Revision ID: d4e5f6g7h8i9
+Revises: d5e6f7a8b9c0
+Create Date: 2026-03-11
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "d4e5f6g7h8i9"
+down_revision: str | Sequence[str] | None = "d5e6f7a8b9c0"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    # DROP + CREATE CONCURRENTLY must run outside a transaction block.
+    op.execute("COMMIT")
+    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
+    op.execute(
+        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
+        f"ON {schema}memory_units USING GIN (source_memory_ids) "
+        f"WITH (fastupdate=off) "
+        f"WHERE source_memory_ids IS NOT NULL"
+    )
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute("COMMIT")
+    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
+    op.execute(
+        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
+        f"ON {schema}memory_units USING GIN (source_memory_ids) "
+        f"WHERE source_memory_ids IS NOT NULL"
+    )
@@ -0,0 +1,131 @@
+"""Add internal_id to banks and per-(bank, fact_type) partial HNSW indexes
+
+Revision ID: d5e6f7a8b9c0
+Revises: a3b4c5d6e7f8
+Create Date: 2026-03-11
+
+This migration:
+1. Adds internal_id UUID column to banks (stable identifier for index naming)
+2. Drops the global HNSW index (competes with per-bank partial indexes)
+3. Creates per-(bank_id, fact_type) partial HNSW indexes for all existing banks
+   (new banks get indexes created at bank-creation time via bank_utils.create_bank_hnsw_indexes)
+
+Why per-(bank, fact_type) indexes:
+- fact_type-only partial indexes are never chosen by the planner when bank_id is in the WHERE
+  clause, because the idx_memory_units_bank_id B-tree index always wins at planning time.
+- Per-(bank, fact_type) partial indexes have both predicates matching → planner selects them.
+- The global HNSW index competes for larger partitions (world, observation) and must be dropped.
+
+For large deployments, create indexes CONCURRENTLY before running this migration:
+    SELECT internal_id, bank_id FROM banks;
+    -- for each bank and each fact_type in (world, experience, observation):
+    CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mu_emb_{ft}_{uid16}
+        ON memory_units USING hnsw (embedding vector_cosine_ops)
+        WHERE fact_type = '{ft}' AND bank_id = '{bank_id}';
+    DROP INDEX CONCURRENTLY IF EXISTS idx_memory_units_embedding;
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+from sqlalchemy import text
+
+revision: str = "d5e6f7a8b9c0"
+down_revision: str | Sequence[str] | None = "c3d4e5f6g7h8"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_HNSW_FACT_TYPES: dict[str, str] = {
+    "world": "worl",
+    "experience": "expr",
+    "observation": "obsv",
+}
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+
+    # 1. Add internal_id column to banks
+    op.execute(
+        f"ALTER TABLE {schema}banks ADD COLUMN IF NOT EXISTS internal_id UUID DEFAULT gen_random_uuid() NOT NULL"
+    )
+    op.execute(f"ALTER TABLE {schema}banks ADD CONSTRAINT banks_internal_id_unique UNIQUE (internal_id)")
+
+    # 2. Drop any fact_type-only partial HNSW indexes that may exist from prior migrations
+    #    (bank_id B-tree always wins over them when bank_id is in the WHERE clause)
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_mu_emb_world")
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_mu_emb_observation")
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_mu_emb_experience")
+
+    # 4. Drop global HNSW index (competes with per-bank partial indexes)
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_embedding")
+
+    # 5. Create per-(bank, fact_type) partial HNSW indexes for all existing banks
+    bind = op.get_bind()
+    schema_name = context.config.get_main_option("target_schema")
+    table_ref = f'"{schema_name}".memory_units' if schema_name else "memory_units"
+    banks_ref = f'"{schema_name}".banks' if schema_name else "banks"
+
+    rows = bind.execute(text(f"SELECT bank_id, internal_id FROM {banks_ref}")).fetchall()  # noqa: S608
+    for row in rows:
+        bank_id = row[0]
+        internal_id = str(row[1]).replace("-", "")[:16]
+        escaped_bank_id = bank_id.replace("'", "''")
+        for ft, ft_short in _HNSW_FACT_TYPES.items():
+            idx_name = f"idx_mu_emb_{ft_short}_{internal_id}"
+            # Index name is schema-unqualified (indexes live in the schema of their table)
+            bind.execute(
+                text(
+                    f"CREATE INDEX IF NOT EXISTS {idx_name} "
+                    f"ON {table_ref} USING hnsw (embedding vector_cosine_ops) "
+                    f"WHERE fact_type = '{ft}' AND bank_id = '{escaped_bank_id}'"
+                )
+            )
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+
+    # Drop per-bank HNSW indexes (iterate existing banks)
+    bind = op.get_bind()
+    schema_name = context.config.get_main_option("target_schema")
+    banks_ref = f'"{schema_name}".banks' if schema_name else "banks"
+
+    rows = bind.execute(text(f"SELECT internal_id FROM {banks_ref}")).fetchall()  # noqa: S608
+    for row in rows:
+        internal_id = str(row[0]).replace("-", "")[:16]
+        for ft_short in _HNSW_FACT_TYPES.values():
+            idx_name = f"idx_mu_emb_{ft_short}_{internal_id}"
+            bind.execute(text(f"DROP INDEX IF EXISTS {schema}{idx_name}"))
+
+    # Restore the global HNSW index
+    table_ref = f'"{schema_name}".memory_units' if schema_name else "memory_units"
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS idx_memory_units_embedding ON {table_ref} USING hnsw (embedding vector_cosine_ops)"
+    )
+
+    # Restore old fact_type-only partial indexes
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS idx_mu_emb_world "
+        f"ON {table_ref} USING hnsw (embedding vector_cosine_ops) "
+        f"WHERE fact_type = 'world'"
+    )
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS idx_mu_emb_observation "
+        f"ON {table_ref} USING hnsw (embedding vector_cosine_ops) "
+        f"WHERE fact_type = 'observation'"
+    )
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS idx_mu_emb_experience "
+        f"ON {table_ref} USING hnsw (embedding vector_cosine_ops) "
+        f"WHERE fact_type = 'experience'"
+    )
+
+    # Drop internal_id column
+    op.execute(f"ALTER TABLE {schema}banks DROP CONSTRAINT IF EXISTS banks_internal_id_unique")
+    op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS internal_id")
@@ -0,0 +1,73 @@
+"""Add CASCADE DELETE FK from async_operations and webhooks to banks.
+
+When a bank is deleted, all its async_operations and webhooks rows are
+automatically deleted by the database. This ensures that any in-flight
+worker tasks detect the deletion via _check_op_alive() and abort early.
+
+Revision ID: e5f6g7h8i9j0
+Revises: d4e5f6g7h8i9
+Create Date: 2026-03-11
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "e5f6g7h8i9j0"
+down_revision: str | Sequence[str] | None = "d4e5f6g7h8i9"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+
+    # Remove orphaned async_operations rows whose bank no longer exists
+    # (can happen because there was no FK before this migration).
+    op.execute(
+        f"""
+        DELETE FROM {schema}async_operations
+        WHERE bank_id IS NOT NULL
+          AND bank_id NOT IN (SELECT bank_id FROM {schema}banks)
+        """
+    )
+
+    # Remove orphaned webhooks rows whose bank no longer exists.
+    op.execute(
+        f"""
+        DELETE FROM {schema}webhooks
+        WHERE bank_id IS NOT NULL
+          AND bank_id NOT IN (SELECT bank_id FROM {schema}banks)
+        """
+    )
+
+    # Add FK with ON DELETE CASCADE so that deleting a bank automatically
+    # cleans up all its pending/processing operations and webhook configs.
+    op.execute(
+        f"""
+        ALTER TABLE {schema}async_operations
+            ADD CONSTRAINT fk_async_operations_bank_id
+            FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id)
+            ON DELETE CASCADE
+        """
+    )
+
+    op.execute(
+        f"""
+        ALTER TABLE {schema}webhooks
+            ADD CONSTRAINT fk_webhooks_bank_id
+            FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id)
+            ON DELETE CASCADE
+        """
+    )
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}async_operations DROP CONSTRAINT IF EXISTS fk_async_operations_bank_id")
+    op.execute(f"ALTER TABLE {schema}webhooks DROP CONSTRAINT IF EXISTS fk_webhooks_bank_id")
@@ -35,7 +35,7 @@ def upgrade() -> None:

    # Add GIN index for JSONB containment queries (@> operator)
    op.execute(f"""
-        CREATE INDEX idx_async_operations_result_metadata
+        CREATE INDEX IF NOT EXISTS idx_async_operations_result_metadata
        ON {schema}async_operations
        USING gin(result_metadata)
    """)
@@ -34,7 +34,7 @@ def _parse_metadata(metadata: Any) -> dict[str, Any]:

 from typing import Callable

-from pydantic import BaseModel, ConfigDict, Field, field_validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator

 from hindsight_api import MemoryEngine

@@ -73,15 +73,13 @@ def FieldWithDefault(default_factory: Callable, **kwargs) -> Any:
 from hindsight_api.config import get_config
 from hindsight_api.engine.memory_engine import Budget, _current_schema, _get_tiktoken_encoding, fq_table
 from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES, MemoryFact, TokenUsage
-from hindsight_api.engine.search.tags import TagsMatch
+from hindsight_api.engine.search.tags import TagGroup, TagsMatch
 from hindsight_api.extensions import HttpExtension, OperationValidationError, load_extension
 from hindsight_api.metrics import create_metrics_collector, get_metrics_collector, initialize_metrics
 from hindsight_api.models import RequestContext

 logger = logging.getLogger(__name__)

-MAX_QUERY_TOKENS = 500  # Maximum tokens allowed in recall query
-

 class EntityIncludeOptions(BaseModel):
    """Options for including entity observations in recall results."""
@@ -98,7 +96,12 @@ class ChunkIncludeOptions(BaseModel):
 class SourceFactsIncludeOptions(BaseModel):
    """Options for including source facts for observation-type results."""

-    max_tokens: int = Field(default=4096, description="Maximum tokens for source facts")
+    max_tokens: int = Field(
+        default=4096, description="Maximum total tokens for source facts across all observations (-1 = unlimited)"
+    )
+    max_tokens_per_observation: int = Field(
+        default=-1, description="Maximum tokens of source facts per observation (-1 = unlimited)"
+    )


 class IncludeOptions(BaseModel):
@@ -160,6 +163,17 @@ class RecallRequest(BaseModel):
        description="How to match tags: 'any' (OR, includes untagged), 'all' (AND, includes untagged), "
        "'any_strict' (OR, excludes untagged), 'all_strict' (AND, excludes untagged).",
    )
+    tag_groups: list[TagGroup] | None = Field(
+        default=None,
+        description="Compound tag filter using boolean groups. Groups in the list are AND-ed. "
+        "Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}.",
+    )
+
+    @model_validator(mode="after")
+    def validate_tags_exclusive(self) -> "RecallRequest":
+        if self.tags is not None and self.tag_groups is not None:
+            raise ValueError("'tags' and 'tag_groups' are mutually exclusive. Use 'tag_groups' for compound filtering.")
+        return self


 class RecallResult(BaseModel):
@@ -472,6 +486,11 @@ class FileRetainMetadata(BaseModel):
    metadata: dict[str, Any] | None = Field(default=None, description="Additional metadata")
    tags: list[str] | None = Field(default=None, description="Tags for this file")
    timestamp: str | None = Field(default=None, description="ISO timestamp")
+    parser: str | list[str] | None = Field(
+        default=None,
+        description="Parser or ordered fallback chain for this file (overrides request-level parser). "
+        "E.g. 'iris' or ['iris', 'markitdown'].",
+    )


 class FileRetainRequest(BaseModel):
@@ -480,14 +499,21 @@ class FileRetainRequest(BaseModel):
    model_config = ConfigDict(
        json_schema_extra={
            "example": {
+                "parser": "iris",
                "files_metadata": [
                    {"document_id": "report_2024", "tags": ["quarterly"]},
-                    {"context": "meeting notes"},
+                    {"context": "meeting notes", "parser": ["iris", "markitdown"]},
                ],
            }
        }
    )

+    parser: str | list[str] | None = Field(
+        default=None,
+        description="Default parser or ordered fallback chain for all files in this request. "
+        "E.g. 'markitdown' or ['iris', 'markitdown']. Falls back to server default if not set. "
+        "Per-file 'parser' in files_metadata takes precedence over this value.",
+    )
    files_metadata: list[FileRetainMetadata] | None = Field(
        default=None,
        description="Metadata for each file (optional, must match number of files if provided)",
@@ -624,6 +650,17 @@ class ReflectRequest(BaseModel):
        description="How to match tags: 'any' (OR, includes untagged), 'all' (AND, includes untagged), "
        "'any_strict' (OR, excludes untagged), 'all_strict' (AND, excludes untagged).",
    )
+    tag_groups: list[TagGroup] | None = Field(
+        default=None,
+        description="Compound tag filter using boolean groups. Groups in the list are AND-ed. "
+        "Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}.",
+    )
+
+    @model_validator(mode="after")
+    def validate_tags_exclusive(self) -> "ReflectRequest":
+        if self.tags is not None and self.tag_groups is not None:
+            raise ValueError("'tags' and 'tag_groups' are mutually exclusive. Use 'tag_groups' for compound filtering.")
+        return self


 class ReflectFact(BaseModel):
@@ -1189,6 +1226,30 @@ class DocumentResponse(BaseModel):
    tags: list[str] = FieldWithDefault(list, description="Tags associated with this document")


+class UpdateDocumentRequest(BaseModel):
+    """Request model for updating a document's mutable fields."""
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "tags": ["team-a", "team-b"],
+            }
+        }
+    )
+
+    tags: list[str] | None = Field(
+        default=None,
+        description="New tags for the document and its memory units. "
+        "Triggers observation invalidation and re-consolidation.",
+    )
+
+
+class UpdateDocumentResponse(BaseModel):
+    """Response model for update document endpoint."""
+
+    success: bool = True
+
+
 class DeleteDocumentResponse(BaseModel):
    """Response model for delete document endpoint."""

@@ -1517,6 +1578,24 @@ class CancelOperationResponse(BaseModel):
    operation_id: str


+class RetryOperationResponse(BaseModel):
+    """Response model for retry operation endpoint."""
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "success": True,
+                "message": "Operation 550e8400-e29b-41d4-a716-446655440000 queued for retry",
+                "operation_id": "550e8400-e29b-41d4-a716-446655440000",
+            }
+        }
+    )
+
+    success: bool
+    message: str
+    operation_id: str
+
+
 class ChildOperationStatus(BaseModel):
    """Status of a child operation (for batch operations)."""

@@ -2120,7 +2199,7 @@ def _register_routes(app: FastAPI):
    @app.get(
        "/v1/default/banks/{bank_id}/memories/{memory_id}",
        summary="Get memory unit",
-        description="Get a single memory unit by ID with all its metadata including entities and tags.",
+        description="Get a single memory unit by ID with all its metadata including entities and tags. Note: the 'history' field is deprecated and always returns an empty list - use GET /memories/{memory_id}/history instead.",
        operation_id="get_memory",
        tags=["Memory"],
    )
@@ -2150,6 +2229,39 @@ def _register_routes(app: FastAPI):
            logger.error(f"Error in /v1/default/banks/{bank_id}/memories/{memory_id}: {error_detail}")
            raise HTTPException(status_code=500, detail=str(e))

+    @app.get(
+        "/v1/default/banks/{bank_id}/memories/{memory_id}/history",
+        summary="Get observation history",
+        description="Get the full history of an observation, with each change's source facts resolved to their text.",
+        operation_id="get_observation_history",
+        tags=["Memory"],
+    )
+    async def api_get_observation_history(
+        bank_id: str,
+        memory_id: str,
+        request_context: RequestContext = Depends(get_request_context),
+    ):
+        """Get the history of a single observation by ID."""
+        try:
+            data = await app.state.memory.get_observation_history(
+                bank_id=bank_id,
+                memory_id=memory_id,
+                request_context=request_context,
+            )
+            if data is None:
+                raise HTTPException(status_code=404, detail=f"Memory unit '{memory_id}' not found")
+            return data
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in /v1/default/banks/{bank_id}/memories/{memory_id}/history: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
    @app.post(
        "/v1/default/banks/{bank_id}/memories/recall",
        response_model=RecallResponse,
@@ -2171,12 +2283,13 @@ def _register_routes(app: FastAPI):
        metrics = get_metrics_collector()

        # Validate query length to prevent expensive operations on oversized queries
+        max_query_tokens = get_config().recall_max_query_tokens
        encoding = _get_tiktoken_encoding()
        query_tokens = len(encoding.encode(request.query))
-        if query_tokens > MAX_QUERY_TOKENS:
+        if query_tokens > max_query_tokens:
            raise HTTPException(
                status_code=400,
-                detail=f"Query too long: {query_tokens} tokens exceeds maximum of {MAX_QUERY_TOKENS}. Please shorten your query.",
+                detail=f"Query too long: {query_tokens} tokens exceeds maximum of {max_query_tokens}. Please shorten your query.",
            )

        try:
@@ -2205,6 +2318,9 @@ def _register_routes(app: FastAPI):
            # Determine source facts inclusion settings
            include_source_facts = request.include.source_facts is not None
            max_source_facts_tokens = request.include.source_facts.max_tokens if include_source_facts else 4096
+            max_source_facts_tokens_per_observation = (
+                request.include.source_facts.max_tokens_per_observation if include_source_facts else -1
+            )

            pre_recall = time.time() - handler_start
            # Run recall with tracing (record metrics)
@@ -2226,9 +2342,11 @@ def _register_routes(app: FastAPI):
                    max_chunk_tokens=max_chunk_tokens,
                    include_source_facts=include_source_facts,
                    max_source_facts_tokens=max_source_facts_tokens,
+                    max_source_facts_tokens_per_observation=max_source_facts_tokens_per_observation,
                    request_context=request_context,
                    tags=request.tags,
                    tags_match=request.tags_match,
+                    tag_groups=request.tag_groups,
                )

            # Convert core MemoryFact objects to API RecallResult objects (excluding internal metrics)
@@ -2364,6 +2482,7 @@ def _register_routes(app: FastAPI):
                    request_context=request_context,
                    tags=request.tags,
                    tags_match=request.tags_match,
+                    tag_groups=request.tag_groups,
                )

            # Build based_on (memories + mental_models + directives) if facts are requested
@@ -2694,6 +2813,41 @@ def _register_routes(app: FastAPI):
            logger.error(f"Error in GET /v1/default/banks/{bank_id}/mental-models/{mental_model_id}: {error_detail}")
            raise HTTPException(status_code=500, detail=str(e))

+    @app.get(
+        "/v1/default/banks/{bank_id}/mental-models/{mental_model_id}/history",
+        summary="Get mental model history",
+        description="Get the refresh history of a mental model, showing content changes over time.",
+        operation_id="get_mental_model_history",
+        tags=["Mental Models"],
+    )
+    async def api_get_mental_model_history(
+        bank_id: str,
+        mental_model_id: str,
+        request_context: RequestContext = Depends(get_request_context),
+    ):
+        """Get the refresh history of a mental model."""
+        try:
+            data = await app.state.memory.get_mental_model_history(
+                bank_id=bank_id,
+                mental_model_id=mental_model_id,
+                request_context=request_context,
+            )
+            if data is None:
+                raise HTTPException(status_code=404, detail=f"Mental model '{mental_model_id}' not found")
+            return data
+        except (AuthenticationError, HTTPException):
+            raise
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(
+                f"Error in GET /v1/default/banks/{bank_id}/mental-models/{mental_model_id}/history: {error_detail}"
+            )
+            raise HTTPException(status_code=500, detail=str(e))
+
    @app.post(
        "/v1/default/banks/{bank_id}/mental-models",
        response_model=CreateMentalModelResponse,
@@ -3215,6 +3369,55 @@ def _register_routes(app: FastAPI):
            logger.error(f"Error in /v1/default/chunks/{chunk_id}: {error_detail}")
            raise HTTPException(status_code=500, detail=str(e))

+    @app.patch(
+        "/v1/default/banks/{bank_id}/documents/{document_id:path}",
+        response_model=UpdateDocumentResponse,
+        summary="Update document",
+        description="Update mutable fields on a document without re-processing its content.\n\n"
+        "**Tags** (`tags`): Propagated to all associated memory units. Observations derived from "
+        "those units are invalidated and queued for re-consolidation under the new tags. "
+        "Co-source memories from other documents that shared those observations are also reset.\n\n"
+        "At least one field must be provided.",
+        operation_id="update_document",
+        tags=["Documents"],
+    )
+    async def api_update_document(
+        bank_id: str,
+        document_id: str,
+        body: UpdateDocumentRequest,
+        request_context: RequestContext = Depends(get_request_context),
+    ):
+        """
+        Update mutable fields on a document without re-processing its content.
+
+        Args:
+            bank_id: Memory Bank ID (from path)
+            document_id: Document ID (from path)
+            body: Fields to update (tags, metadata, context)
+        """
+        if body.tags is None:
+            raise HTTPException(status_code=422, detail="At least one field (tags) must be provided")
+        try:
+            result = await app.state.memory.update_document(
+                document_id,
+                bank_id,
+                tags=body.tags,
+                request_context=request_context,
+            )
+            if not result:
+                raise HTTPException(status_code=404, detail="Document not found")
+            return UpdateDocumentResponse(success=True)
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in PATCH /v1/default/banks/{bank_id}/documents/{document_id}: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
    @app.delete(
        "/v1/default/banks/{bank_id}/documents/{document_id:path}",
        response_model=DeleteDocumentResponse,
@@ -3265,13 +3468,17 @@ def _register_routes(app: FastAPI):
        "/v1/default/banks/{bank_id}/operations",
        response_model=OperationsListResponse,
        summary="List async operations",
-        description="Get a list of async operations for a specific agent, with optional filtering by status. Results are sorted by most recent first.",
+        description="Get a list of async operations for a specific agent, with optional filtering by status and operation type. Results are sorted by most recent first.",
        operation_id="list_operations",
        tags=["Operations"],
    )
    async def api_list_operations(
        bank_id: str,
        status: str | None = Query(default=None, description="Filter by status: pending, completed, or failed"),
+        type: str | None = Query(
+            default=None,
+            description="Filter by operation type: retain, consolidation, refresh_mental_model, file_convert_retain, webhook_delivery",
+        ),
        limit: int = Query(default=20, ge=1, le=100, description="Maximum number of operations to return"),
        offset: int = Query(default=0, ge=0, description="Number of operations to skip"),
        request_context: RequestContext = Depends(get_request_context),
@@ -3279,7 +3486,7 @@ def _register_routes(app: FastAPI):
        """List async operations for a memory bank with optional filtering and pagination."""
        try:
            result = await app.state.memory.list_operations(
-                bank_id, status=status, limit=limit, offset=offset, request_context=request_context
+                bank_id, status=status, task_type=type, limit=limit, offset=offset, request_context=request_context
            )
            return OperationsListResponse(
                bank_id=bank_id,
@@ -3366,6 +3573,39 @@ def _register_routes(app: FastAPI):
            logger.error(f"Error in /v1/default/banks/{bank_id}/operations/{operation_id}: {error_detail}")
            raise HTTPException(status_code=500, detail=str(e))

+    @app.post(
+        "/v1/default/banks/{bank_id}/operations/{operation_id}/retry",
+        response_model=RetryOperationResponse,
+        summary="Retry a failed async operation",
+        description="Re-queue a failed async operation so the worker picks it up again",
+        operation_id="retry_operation",
+        tags=["Operations"],
+    )
+    async def api_retry_operation(
+        bank_id: str, operation_id: str, request_context: RequestContext = Depends(get_request_context)
+    ):
+        """Retry a failed async operation."""
+        try:
+            try:
+                uuid.UUID(operation_id)
+            except ValueError:
+                raise HTTPException(status_code=400, detail=f"Invalid operation_id format: {operation_id}")
+
+            result = await app.state.memory.retry_operation(bank_id, operation_id, request_context=request_context)
+            return RetryOperationResponse(**result)
+        except ValueError as e:
+            raise HTTPException(status_code=404, detail=str(e))
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in POST /v1/default/banks/{bank_id}/operations/{operation_id}/retry: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
    @app.get(
        "/v1/default/banks/{bank_id}/profile",
        response_model=BankProfileResponse,
@@ -3875,6 +4115,10 @@ def _register_routes(app: FastAPI):
        try:
            pool = await app.state.memory._get_pool()
            from hindsight_api.engine.memory_engine import fq_table
+            from hindsight_api.engine.retain import bank_utils
+
+            # Ensure the bank row exists before inserting into webhooks (FK constraint).
+            await bank_utils.get_bank_profile(pool, bank_id)

            webhook_id = uuid.uuid4()
            now = datetime.utcnow().isoformat() + "Z"
@@ -4312,8 +4556,14 @@ def _register_routes(app: FastAPI):
        "Use the operations endpoint to monitor progress.\n\n"
        "**Request format:** multipart/form-data with:\n"
        "- `files`: One or more files to upload\n"
-        "- `request`: JSON string with FileRetainRequest model (files_metadata)\n\n"
-        "**Note:** File parser is configured server-side via `HINDSIGHT_API_FILE_PARSER` (default: markitdown).",
+        "- `request`: JSON string with FileRetainRequest model\n\n"
+        "**Parser selection:**\n"
+        "- Set `parser` in the request body to override the server default for all files.\n"
+        "- Set `parser` inside a `files_metadata` entry for per-file control.\n"
+        "- Pass a list (e.g. `['iris', 'markitdown']`) to define an ordered fallback chain — "
+        "each parser is tried in sequence until one succeeds.\n"
+        "- Falls back to the server default (`HINDSIGHT_API_FILE_PARSER`) if not specified.\n"
+        "- Only parsers enabled on the server may be requested; others return HTTP 400.",
        operation_id="file_retain",
        tags=["Files"],
    )
@@ -4359,20 +4609,39 @@ def _register_routes(app: FastAPI):
                    detail=f"files_metadata count ({len(request_data.files_metadata)}) must match files count ({len(files)})",
                )

+            # Resolve the registered parser names for allowlist validation
+            registered_parsers = app.state.memory._parser_registry.list_parsers()
+            allowlist = config.file_parser_allowlist if config.file_parser_allowlist is not None else registered_parsers
+
+            def _resolve_parser(raw: str | list[str] | None) -> list[str]:
+                """Normalize parser value to a non-empty list of names."""
+                if raw is None:
+                    return config.file_parser
+                return [raw] if isinstance(raw, str) else list(raw)
+
+            def _validate_parsers(parsers: list[str], context: str) -> None:
+                """Raise HTTP 400 if any parser name is not in the allowlist."""
+                disallowed = [p for p in parsers if p not in allowlist]
+                if disallowed:
+                    raise HTTPException(
+                        status_code=400,
+                        detail=f"Parser(s) not available ({context}): {disallowed}. Available: {allowlist}",
+                    )
+
+            # Validate request-level parser early (before reading files)
+            if request_data.parser is not None:
+                _validate_parsers(_resolve_parser(request_data.parser), "request-level parser")
+
            # Prepare file items and calculate total batch size
+            import io
+
            file_items = []
            total_batch_size = 0

            for i, file in enumerate(files):
                # Read file content to check size
                file_content = await file.read()
-                size = len(file_content)
-                total_batch_size += size
-
-                # Create a temporary file-like object from the bytes
-                import io
-
-                file_obj = io.BytesIO(file_content)
+                total_batch_size += len(file_content)

                # Create a mock UploadFile with the necessary attributes
                class FileWrapper:
@@ -4380,7 +4649,6 @@ def _register_routes(app: FastAPI):
                        self._content = content
                        self.filename = filename
                        self.content_type = content_type
-                        self._buffer = io.BytesIO(content)

                    async def read(self):
                        return self._content
@@ -4391,6 +4659,12 @@ def _register_routes(app: FastAPI):
                file_meta = request_data.files_metadata[i] if request_data.files_metadata else FileRetainMetadata()
                doc_id = file_meta.document_id or f"file_{uuid.uuid4()}"

+                # Resolve and validate per-file parser chain
+                # Priority: per-file > request-level > server default
+                raw_parser = file_meta.parser if file_meta.parser is not None else request_data.parser
+                parser_chain = _resolve_parser(raw_parser)
+                _validate_parsers(parser_chain, f"file '{file.filename}'")
+
                item = {
                    "file": wrapped_file,
                    "document_id": doc_id,
@@ -4398,6 +4672,7 @@ def _register_routes(app: FastAPI):
                    "metadata": file_meta.metadata or {},
                    "tags": file_meta.tags or [],
                    "timestamp": file_meta.timestamp,
+                    "parser": parser_chain,
                }
                file_items.append(item)

@@ -4412,7 +4687,6 @@ def _register_routes(app: FastAPI):
            result = await app.state.memory.submit_async_file_retain(
                bank_id=bank_id,
                file_items=file_items,
-                parser=config.file_parser,
                document_tags=None,
                request_context=request_context,
            )
@@ -193,6 +193,7 @@ ENV_EMBEDDINGS_LITELLM_MODEL = "HINDSIGHT_API_EMBEDDINGS_LITELLM_MODEL"
 ENV_RERANKER_LITELLM_API_BASE = "HINDSIGHT_API_RERANKER_LITELLM_API_BASE"
 ENV_RERANKER_LITELLM_API_KEY = "HINDSIGHT_API_RERANKER_LITELLM_API_KEY"
 ENV_RERANKER_LITELLM_MODEL = "HINDSIGHT_API_RERANKER_LITELLM_MODEL"
+ENV_RERANKER_LITELLM_MAX_TOKENS_PER_DOC = "HINDSIGHT_API_RERANKER_LITELLM_MAX_TOKENS_PER_DOC"

 # LiteLLM SDK configuration (direct API access, no proxy needed)
 ENV_EMBEDDINGS_LITELLM_SDK_API_KEY = "HINDSIGHT_API_EMBEDDINGS_LITELLM_SDK_API_KEY"
@@ -238,6 +239,7 @@ ENV_GRAPH_RETRIEVER = "HINDSIGHT_API_GRAPH_RETRIEVER"
 ENV_MPFP_TOP_K_NEIGHBORS = "HINDSIGHT_API_MPFP_TOP_K_NEIGHBORS"
 ENV_RECALL_MAX_CONCURRENT = "HINDSIGHT_API_RECALL_MAX_CONCURRENT"
 ENV_RECALL_CONNECTION_BUDGET = "HINDSIGHT_API_RECALL_CONNECTION_BUDGET"
+ENV_RECALL_MAX_QUERY_TOKENS = "HINDSIGHT_API_RECALL_MAX_QUERY_TOKENS"
 ENV_MENTAL_MODEL_REFRESH_CONCURRENCY = "HINDSIGHT_API_MENTAL_MODEL_REFRESH_CONCURRENCY"

 # OpenTelemetry tracing configuration
@@ -280,6 +282,7 @@ ENV_FILE_STORAGE_AZURE_CONTAINER = "HINDSIGHT_API_FILE_STORAGE_AZURE_CONTAINER"
 ENV_FILE_STORAGE_AZURE_ACCOUNT_NAME = "HINDSIGHT_API_FILE_STORAGE_AZURE_ACCOUNT_NAME"
 ENV_FILE_STORAGE_AZURE_ACCOUNT_KEY = "HINDSIGHT_API_FILE_STORAGE_AZURE_ACCOUNT_KEY"
 ENV_FILE_PARSER = "HINDSIGHT_API_FILE_PARSER"
+ENV_FILE_PARSER_ALLOWLIST = "HINDSIGHT_API_FILE_PARSER_ALLOWLIST"
 ENV_FILE_PARSER_IRIS_TOKEN = "HINDSIGHT_API_FILE_PARSER_IRIS_TOKEN"
 ENV_FILE_PARSER_IRIS_ORG_ID = "HINDSIGHT_API_FILE_PARSER_IRIS_ORG_ID"
 ENV_FILE_CONVERSION_MAX_BATCH_SIZE_MB = "HINDSIGHT_API_FILE_CONVERSION_MAX_BATCH_SIZE_MB"
@@ -292,7 +295,13 @@ ENV_ENABLE_OBSERVATIONS = "HINDSIGHT_API_ENABLE_OBSERVATIONS"
 ENV_CONSOLIDATION_BATCH_SIZE = "HINDSIGHT_API_CONSOLIDATION_BATCH_SIZE"
 ENV_CONSOLIDATION_LLM_BATCH_SIZE = "HINDSIGHT_API_CONSOLIDATION_LLM_BATCH_SIZE"
 ENV_CONSOLIDATION_MAX_TOKENS = "HINDSIGHT_API_CONSOLIDATION_MAX_TOKENS"
+ENV_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS = "HINDSIGHT_API_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS"
+ENV_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION = (
+    "HINDSIGHT_API_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION"
+)
 ENV_OBSERVATIONS_MISSION = "HINDSIGHT_API_OBSERVATIONS_MISSION"
+ENV_ENABLE_OBSERVATION_HISTORY = "HINDSIGHT_API_ENABLE_OBSERVATION_HISTORY"
+ENV_ENABLE_MENTAL_MODEL_HISTORY = "HINDSIGHT_API_ENABLE_MENTAL_MODEL_HISTORY"

 # Webhook configuration (global, static - server-level only)
 ENV_WEBHOOK_URL = "HINDSIGHT_API_WEBHOOK_URL"
@@ -343,6 +352,7 @@ PROVIDER_DEFAULT_MODELS = {
    "anthropic": "claude-haiku-4-5-20251001",
    "gemini": "gemini-2.5-flash",
    "groq": "openai/gpt-oss-120b",
+    "minimax": "MiniMax-M2.5",
    "ollama": "gemma3:12b",
    "lmstudio": "local-model",
    "vertexai": "google/gemini-2.5-flash-lite",
@@ -400,6 +410,7 @@ DEFAULT_TEXT_SEARCH_EXTENSION = "native"  # Options: "native", "vchord", "pg_tex
 DEFAULT_LITELLM_API_BASE = "http://localhost:4000"
 DEFAULT_EMBEDDINGS_LITELLM_MODEL = "text-embedding-3-small"
 DEFAULT_RERANKER_LITELLM_MODEL = "cohere/rerank-english-v3.0"
+DEFAULT_RERANKER_LITELLM_MAX_TOKENS_PER_DOC: int | None = None

 # LiteLLM SDK defaults
 DEFAULT_EMBEDDINGS_LITELLM_SDK_MODEL = "cohere/embed-english-v3.0"
@@ -418,6 +429,7 @@ DEFAULT_GRAPH_RETRIEVER = "link_expansion"  # Options: "link_expansion", "mpfp",
 DEFAULT_MPFP_TOP_K_NEIGHBORS = 20  # Fan-out limit per node in MPFP graph traversal
 DEFAULT_RECALL_MAX_CONCURRENT = 32  # Max concurrent recall operations per worker
 DEFAULT_RECALL_CONNECTION_BUDGET = 4  # Max concurrent DB connections per recall operation
+DEFAULT_RECALL_MAX_QUERY_TOKENS = 500  # Maximum tokens allowed in recall query
 DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY = 8  # Max concurrent mental model refreshes

 # Retain settings
@@ -435,7 +447,8 @@ DEFAULT_RETAIN_BATCH_POLL_INTERVAL_SECONDS = 60  # Batch API polling interval in

 # File storage defaults
 DEFAULT_FILE_STORAGE_TYPE = "native"  # PostgreSQL BYTEA storage
-DEFAULT_FILE_PARSER = "markitdown"  # File parser to use (markitdown is the only supported parser)
+DEFAULT_FILE_PARSER = "markitdown"  # Default parser fallback chain (comma-separated, e.g. "iris,markitdown")
+DEFAULT_FILE_PARSER_ALLOWLIST = None  # Allowlist of parsers clients may request (None = all registered parsers)
 DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE_MB = 100  # Max total batch size in MB (all files combined)
 DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE = 10  # Max files per batch upload
 DEFAULT_ENABLE_FILE_UPLOAD_API = True  # Enable file upload endpoint
@@ -443,9 +456,17 @@ DEFAULT_FILE_DELETE_AFTER_RETAIN = True  # Delete file bytes after retain (saves

 # Observations defaults (consolidated knowledge from facts)
 DEFAULT_ENABLE_OBSERVATIONS = True  # Observations enabled by default
+DEFAULT_ENABLE_OBSERVATION_HISTORY = True  # Observation history tracking enabled by default
+DEFAULT_ENABLE_MENTAL_MODEL_HISTORY = True  # Mental model history tracking enabled by default
 DEFAULT_CONSOLIDATION_BATCH_SIZE = 50  # Memories to load per batch (internal memory optimization)
 DEFAULT_CONSOLIDATION_LLM_BATCH_SIZE = 8  # Facts per LLM call (1 = no batching; >1 = batch mode)
 DEFAULT_CONSOLIDATION_MAX_TOKENS = 512  # Max tokens for recall when finding related observations
+DEFAULT_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS = (
+    -1
+)  # Total token budget for source facts in consolidation recall (-1 = unlimited)
+DEFAULT_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION = (
+    256  # Max tokens of source facts per observation in consolidation prompt (-1 = unlimited)
+)
 DEFAULT_OBSERVATIONS_MISSION = None  # Declarative spec of what observations are for this bank

 # Database migrations
@@ -540,6 +561,11 @@ class JsonFormatter(logging.Formatter):
        return json.dumps(log_entry)


+def _parse_str_list(value: str) -> list[str]:
+    """Parse a comma-separated string into a non-empty list of stripped tokens."""
+    return [v.strip() for v in value.split(",") if v.strip()]
+
+
 def _validate_extraction_mode(mode: str) -> str:
    """Validate and normalize extraction mode."""
    mode_lower = mode.lower()
@@ -652,6 +678,7 @@ class HindsightConfig:
    reranker_litellm_api_base: str
    reranker_litellm_api_key: str | None
    reranker_litellm_model: str
+    reranker_litellm_max_tokens_per_doc: int | None
    reranker_litellm_sdk_api_key: str | None
    reranker_litellm_sdk_model: str
    reranker_litellm_sdk_api_base: str | None
@@ -673,6 +700,7 @@ class HindsightConfig:
    mpfp_top_k_neighbors: int
    recall_max_concurrent: int
    recall_connection_budget: int
+    recall_max_query_tokens: int
    mental_model_refresh_concurrency: int

    # Retain settings
@@ -699,7 +727,8 @@ class HindsightConfig:
    file_storage_azure_container: str | None  # Azure container name (required for azure storage)
    file_storage_azure_account_name: str | None  # Azure storage account name
    file_storage_azure_account_key: str | None  # Azure storage account key
-    file_parser: str  # File parser to use (e.g., "markitdown", "iris")
+    file_parser: list[str]  # Ordered fallback chain of parsers (e.g. ["iris", "markitdown"])
+    file_parser_allowlist: list[str] | None  # Parsers clients may request (None = all registered)
    file_parser_iris_token: str | None  # Vectorize API token for iris parser (VECTORIZE_TOKEN)
    file_parser_iris_org_id: str | None  # Vectorize org ID for iris parser (VECTORIZE_ORG_ID)
    file_conversion_max_batch_size_mb: int  # Max total batch size in MB (all files combined)
@@ -709,9 +738,13 @@ class HindsightConfig:

    # Observations settings (consolidated knowledge from facts)
    enable_observations: bool
+    enable_observation_history: bool
+    enable_mental_model_history: bool
    consolidation_batch_size: int
    consolidation_llm_batch_size: int
    consolidation_max_tokens: int
+    consolidation_source_facts_max_tokens: int
+    consolidation_source_facts_max_tokens_per_observation: int
    observations_mission: str | None

    # Entity labels (controlled vocabulary of key:value classification labels extracted at retain time)
@@ -812,6 +845,9 @@ class HindsightConfig:
        "entities_allow_free_form",
        # Consolidation settings
        "enable_observations",
+        "consolidation_llm_batch_size",
+        "consolidation_source_facts_max_tokens",
+        "consolidation_source_facts_max_tokens_per_observation",
        "observations_mission",
        # Reflect settings
        "reflect_mission",
@@ -1071,6 +1107,9 @@ class HindsightConfig:
            or os.getenv(ENV_LITELLM_API_BASE, DEFAULT_LITELLM_API_BASE),
            reranker_litellm_api_key=os.getenv(ENV_RERANKER_LITELLM_API_KEY) or os.getenv(ENV_LITELLM_API_KEY),
            reranker_litellm_model=os.getenv(ENV_RERANKER_LITELLM_MODEL, DEFAULT_RERANKER_LITELLM_MODEL),
+            reranker_litellm_max_tokens_per_doc=int(v)
+            if (v := os.getenv(ENV_RERANKER_LITELLM_MAX_TOKENS_PER_DOC))
+            else DEFAULT_RERANKER_LITELLM_MAX_TOKENS_PER_DOC,
            # LiteLLM SDK reranker (direct API access)
            reranker_litellm_sdk_api_key=os.getenv(ENV_RERANKER_LITELLM_SDK_API_KEY),
            reranker_litellm_sdk_model=os.getenv(ENV_RERANKER_LITELLM_SDK_MODEL, DEFAULT_RERANKER_LITELLM_SDK_MODEL),
@@ -1097,6 +1136,7 @@ class HindsightConfig:
            recall_connection_budget=int(
                os.getenv(ENV_RECALL_CONNECTION_BUDGET, str(DEFAULT_RECALL_CONNECTION_BUDGET))
            ),
+            recall_max_query_tokens=int(os.getenv(ENV_RECALL_MAX_QUERY_TOKENS, str(DEFAULT_RECALL_MAX_QUERY_TOKENS))),
            mental_model_refresh_concurrency=int(
                os.getenv(ENV_MENTAL_MODEL_REFRESH_CONCURRENCY, str(DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY))
            ),
@@ -1136,7 +1176,10 @@ class HindsightConfig:
            file_storage_azure_container=os.getenv(ENV_FILE_STORAGE_AZURE_CONTAINER) or None,
            file_storage_azure_account_name=os.getenv(ENV_FILE_STORAGE_AZURE_ACCOUNT_NAME) or None,
            file_storage_azure_account_key=os.getenv(ENV_FILE_STORAGE_AZURE_ACCOUNT_KEY) or None,
-            file_parser=os.getenv(ENV_FILE_PARSER, DEFAULT_FILE_PARSER),
+            file_parser=_parse_str_list(os.getenv(ENV_FILE_PARSER, DEFAULT_FILE_PARSER)),
+            file_parser_allowlist=_parse_str_list(os.getenv(ENV_FILE_PARSER_ALLOWLIST))
+            if os.getenv(ENV_FILE_PARSER_ALLOWLIST)
+            else None,
            file_parser_iris_token=os.getenv(ENV_FILE_PARSER_IRIS_TOKEN) or None,
            file_parser_iris_org_id=os.getenv(ENV_FILE_PARSER_IRIS_ORG_ID) or None,
            file_conversion_max_batch_size_mb=int(
@@ -1153,6 +1196,14 @@ class HindsightConfig:
            == "true",
            # Observations settings (consolidated knowledge from facts)
            enable_observations=os.getenv(ENV_ENABLE_OBSERVATIONS, str(DEFAULT_ENABLE_OBSERVATIONS)).lower() == "true",
+            enable_observation_history=os.getenv(
+                ENV_ENABLE_OBSERVATION_HISTORY, str(DEFAULT_ENABLE_OBSERVATION_HISTORY)
+            ).lower()
+            == "true",
+            enable_mental_model_history=os.getenv(
+                ENV_ENABLE_MENTAL_MODEL_HISTORY, str(DEFAULT_ENABLE_MENTAL_MODEL_HISTORY)
+            ).lower()
+            == "true",
            consolidation_batch_size=int(
                os.getenv(ENV_CONSOLIDATION_BATCH_SIZE, str(DEFAULT_CONSOLIDATION_BATCH_SIZE))
            ),
@@ -1162,6 +1213,15 @@ class HindsightConfig:
            consolidation_max_tokens=int(
                os.getenv(ENV_CONSOLIDATION_MAX_TOKENS, str(DEFAULT_CONSOLIDATION_MAX_TOKENS))
            ),
+            consolidation_source_facts_max_tokens=int(
+                os.getenv(ENV_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS, str(DEFAULT_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS))
+            ),
+            consolidation_source_facts_max_tokens_per_observation=int(
+                os.getenv(
+                    ENV_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION,
+                    str(DEFAULT_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION),
+                )
+            ),
            observations_mission=os.getenv(ENV_OBSERVATIONS_MISSION) or DEFAULT_OBSERVATIONS_MISSION,
            entity_labels=None,
            entities_allow_free_form=True,
@@ -24,9 +24,10 @@ from datetime import datetime, timezone
 from itertools import combinations
 from typing import TYPE_CHECKING, Any

-from pydantic import BaseModel
+from pydantic import BaseModel, field_validator

 from ...config import get_config
+from ..llm_wrapper import sanitize_llm_output
 from ..memory_engine import fq_table
 from ..retain import embedding_utils
 from .prompts import build_batch_consolidation_prompt
@@ -45,12 +46,22 @@ class _CreateAction(BaseModel):
    text: str
    source_fact_ids: list[str]  # memory UUIDs from the NEW FACTS list

+    @field_validator("text", mode="before")
+    @classmethod
+    def sanitize_text(cls, v: str) -> str:
+        return sanitize_llm_output(v) or ""
+

 class _UpdateAction(BaseModel):
    text: str
    observation_id: str  # UUID of the existing observation to update
    source_fact_ids: list[str]  # memory UUIDs from the NEW FACTS list

+    @field_validator("text", mode="before")
+    @classmethod
+    def sanitize_text(cls, v: str) -> str:
+        return sanitize_llm_output(v) or ""
+

 class _DeleteAction(BaseModel):
    observation_id: str  # UUID of the observation to remove
@@ -150,6 +161,7 @@ async def run_consolidation_job(
    memory_engine: "MemoryEngine",
    bank_id: str,
    request_context: "RequestContext",
+    operation_id: str | None = None,
 ) -> dict[str, Any]:
    """
    Run consolidation job for a bank.
@@ -375,6 +387,13 @@ async def run_consolidation_job(
                    [(m["id"],) for m in llm_batch],
                )

+            # Checkpoint: abort if the operation (and thus the bank) was deleted mid-run.
+            if operation_id and not await memory_engine._check_op_alive(operation_id):
+                logger.info(
+                    f"[CONSOLIDATION] bank={bank_id} operation {operation_id} cancelled (bank deleted), stopping early"
+                )
+                return {"status": "cancelled", "bank_id": bank_id, **stats}
+
            for result in results:
                stats["memories_processed"] += 1
                action = result.get("action")
@@ -766,13 +785,17 @@ async def _execute_update_action(
        logger.debug(f"Update skipped: observation {observation_id} not found in recall results")
        return

-    history = [
-        {
-            "previous_text": model.text,
-            "changed_at": datetime.now(timezone.utc).isoformat(),
-            "source_memory_ids": [str(mid) for mid in source_memory_ids],
-        }
-    ]
+    from ...config import get_config
+
+    history_entry = {
+        "previous_text": model.text,
+        "previous_tags": list(model.tags or []),
+        "previous_occurred_start": model.occurred_start,
+        "previous_occurred_end": model.occurred_end,
+        "previous_mentioned_at": model.mentioned_at,
+        "changed_at": datetime.now(timezone.utc).isoformat(),
+        "new_source_memory_ids": [str(mid) for mid in source_memory_ids],
+    }

    source_ids = list(model.source_fact_ids or []) + source_memory_ids

@@ -787,13 +810,18 @@ async def _execute_update_action(
    if perf:
        perf.record_timing("embedding", time.time() - t0)

+    config = get_config()
+    history_clause = (
+        "history = COALESCE(history, '[]'::jsonb) || $3::jsonb," if config.enable_observation_history else ""
+    )
+
    t0 = time.time()
    await conn.execute(
        f"""
        UPDATE {fq_table("memory_units")}
        SET text = $1,
            embedding = $2::vector,
-            history = $3,
+            {history_clause}
            source_memory_ids = $4,
            proof_count = $5,
            tags = $10,
@@ -805,7 +833,7 @@ async def _execute_update_action(
        """,
        new_text,
        embedding_str,
-        json.dumps(history),
+        json.dumps([history_entry]),
        source_ids,
        len(source_ids),
        uuid.UUID(observation_id),
@@ -917,10 +945,9 @@ async def _find_related_observations(
    """
    # Use recall to find related observations with token budget
    # max_tokens naturally limits how many observations are returned
-    from ...config import get_config
    from ...tracing import get_tracer, is_tracing_enabled

-    config = get_config()
+    config = await memory_engine._config_resolver.resolve_full_config(bank_id, request_context)

    # SECURITY: Use all_strict matching if tags provided to prevent cross-scope consolidation
    tags_match = "all_strict" if tags else "any"
@@ -945,7 +972,8 @@ async def _find_related_observations(
            tags=tags,  # Filter by source memory's tags
            tags_match=tags_match,  # Use strict matching for security
            include_source_facts=True,  # Embed source facts so we avoid a separate DB fetch
-            max_source_facts_tokens=-1,  # No token limit — we need all source facts for consolidation
+            max_source_facts_tokens=config.consolidation_source_facts_max_tokens,
+            max_source_facts_tokens_per_observation=config.consolidation_source_facts_max_tokens_per_observation,
            _quiet=True,  # Suppress logging
        )
    finally:
@@ -1009,14 +1037,17 @@ async def _consolidate_batch_with_llm(
        observations_text = "[]"

    def _fact_line(m: dict[str, Any]) -> str:
-        parts = [f"[{m['id']}] {m['text']}"]
+        text = f"[{m['id']}] {m['text']}"
+        temporal_parts = []
        if m.get("occurred_start"):
-            parts.append(f"occurred_start={m['occurred_start']}")
+            temporal_parts.append(f"occurred_start={m['occurred_start']}")
        if m.get("occurred_end"):
-            parts.append(f"occurred_end={m['occurred_end']}")
+            temporal_parts.append(f"occurred_end={m['occurred_end']}")
        if m.get("mentioned_at"):
-            parts.append(f"mentioned_at={m['mentioned_at']}")
-        return " | ".join(parts)
+            temporal_parts.append(f"mentioned_at={m['mentioned_at']}")
+        if temporal_parts:
+            text += f" ({', '.join(temporal_parts)})"
+        return text

    facts_lines = "\n".join(_fact_line(m) for m in memories)

@@ -29,14 +29,31 @@ Compare the facts against existing observations:
 - Same topic as an existing observation → UPDATE it (observation_id + source_fact_ids)
 - New topic with durable knowledge → CREATE a new observation (source_fact_ids)
 - Cross-reference facts within the batch: a later fact may resolve a vague reference in an earlier one
- Purely ephemeral facts → omit them (no create/update needed)"""
+- Purely ephemeral facts → omit them unless the MISSION above explicitly targets such data (e.g. timestamped events, session state, screen content)"""

 # Output format — JSON braces escaped as {{ }} so .format() leaves them literal
 _BATCH_OUTPUT_FORMAT = """
 Output a JSON object with three arrays.

-Example (showing the required UUID format for all IDs):
-{{"creates": [{{"text": "Alice lives in Berlin", "source_fact_ids": ["a1b2c3d4-e5f6-7890-abcd-ef1234567890", "b2c3d4e5-f6a7-8901-bcde-f12345678901"]}}],
+## EXAMPLE
+
+Input facts:
+[a1b2c3d4-e5f6-7890-abcd-ef1234567890] Alice mentioned she works long hours, often past midnight | Involving: Alice (occurred_start=2024-01-15, mentioned_at=2024-01-15)
+[b2c3d4e5-f6a7-8901-bcde-f12345678901] Alice said she's exhausted from the project deadlines | Involving: Alice (occurred_start=2024-01-20, mentioned_at=2024-01-20)
+
+Good observation text — clean prose, no metadata, each fact tracked distinctly:
+  "Alice works long hours, often past midnight."
+  "Alice feels exhausted from project deadlines."
+
+Bad observation text — NEVER do this (verbatim copy of fact text with metadata):
+  "Alice mentioned she works long hours, often past midnight | Involving: Alice (occurred_start=2024-01-15, mentioned_at=2024-01-15)"
+
+Observation text rules:
+- Write clean prose — NEVER copy raw fact lines or their metadata (temporal fields, "Involving:", "When:" labels, UUIDs).
+- Parenthesized metadata like (occurred_start=...) and pipe-separated labels like "| Involving: ..." are fact formatting — strip them entirely from observation text.
+- How many observations to create and how much to aggregate is driven by the MISSION above.
+
+{{"creates": [{{"text": "Alice works long hours, often past midnight.", "source_fact_ids": ["a1b2c3d4-e5f6-7890-abcd-ef1234567890"]}}, {{"text": "Alice feels exhausted from project deadlines.", "source_fact_ids": ["b2c3d4e5-f6a7-8901-bcde-f12345678901"]}}],
  "updates": [{{"text": "Alice works at Acme Corp as a senior engineer", "observation_id": "c3d4e5f6-a7b8-9012-cdef-123456789012", "source_fact_ids": ["d4e5f6a7-b8c9-0123-defa-234567890123"]}}],
  "deletes": [{{"observation_id": "e5f6a7b8-c9d0-1234-efab-345678901234"}}]}}

@@ -20,6 +20,7 @@ from ..config import (
    DEFAULT_RERANKER_COHERE_MODEL,
    DEFAULT_RERANKER_FLASHRANK_CACHE_DIR,
    DEFAULT_RERANKER_FLASHRANK_MODEL,
+    DEFAULT_RERANKER_LITELLM_MAX_TOKENS_PER_DOC,
    DEFAULT_RERANKER_LITELLM_MODEL,
    DEFAULT_RERANKER_LITELLM_SDK_MODEL,
    DEFAULT_RERANKER_LOCAL_FORCE_CPU,
@@ -820,6 +821,17 @@ class FlashRankCrossEncoder(CrossEncoderModel):
        return await loop.run_in_executor(FlashRankCrossEncoder._executor, self._predict_sync, pairs)


+def _truncate_to_tokens(text: str, max_tokens: int) -> str:
+    """Truncate text to at most max_tokens using the shared tiktoken encoder."""
+    from .memory_engine import _get_tiktoken_encoding
+
+    enc = _get_tiktoken_encoding()
+    tokens = enc.encode(text)
+    if len(tokens) <= max_tokens:
+        return text
+    return enc.decode(tokens[:max_tokens])
+
+
 class LiteLLMCrossEncoder(CrossEncoderModel):
    """
    LiteLLM cross-encoder implementation using LiteLLM proxy's /rerank endpoint.
@@ -843,6 +855,7 @@ class LiteLLMCrossEncoder(CrossEncoderModel):
        api_key: str | None = None,
        model: str = DEFAULT_RERANKER_LITELLM_MODEL,
        timeout: float = 60.0,
+        max_tokens_per_doc: int | None = DEFAULT_RERANKER_LITELLM_MAX_TOKENS_PER_DOC,
    ):
        """
        Initialize LiteLLM cross-encoder client.
@@ -853,11 +866,15 @@ class LiteLLMCrossEncoder(CrossEncoderModel):
            model: Reranking model name (default: cohere/rerank-english-v3.0)
                   Use provider prefix (e.g., cohere/, together_ai/, voyage/)
            timeout: Request timeout in seconds (default: 60.0)
+            max_tokens_per_doc: If set, truncate each document to this many tokens before
+                                sending to the reranker (uses tiktoken cl100k_base encoding).
+                                Useful for models with small context windows (e.g. 1024 tokens).
        """
        self.api_base = api_base.rstrip("/")
        self.api_key = api_key
        self.model = model
        self.timeout = timeout
+        self.max_tokens_per_doc = max_tokens_per_doc
        self._async_client: httpx.AsyncClient | None = None

    @property
@@ -905,6 +922,8 @@ class LiteLLMCrossEncoder(CrossEncoderModel):

        for query, indexed_texts in query_groups.items():
            texts = [text for _, text in indexed_texts]
+            if self.max_tokens_per_doc is not None:
+                texts = [_truncate_to_tokens(t, self.max_tokens_per_doc) for t in texts]
            indices = [idx for idx, _ in indexed_texts]

            # LiteLLM /rerank follows Cohere API format
@@ -950,6 +969,7 @@ class LiteLLMSDKCrossEncoder(CrossEncoderModel):
        model: str = DEFAULT_RERANKER_LITELLM_SDK_MODEL,
        api_base: str | None = None,
        timeout: float = 60.0,
+        max_tokens_per_doc: int | None = DEFAULT_RERANKER_LITELLM_MAX_TOKENS_PER_DOC,
    ):
        """
        Initialize LiteLLM SDK cross-encoder client.
@@ -959,11 +979,15 @@ class LiteLLMSDKCrossEncoder(CrossEncoderModel):
            model: Model name with provider prefix (e.g., "deepinfra/Qwen3-reranker-8B")
            api_base: Custom base URL for API (optional)
            timeout: Request timeout in seconds (default: 60.0)
+            max_tokens_per_doc: If set, truncate each document to this many tokens before
+                                sending to the reranker (uses tiktoken cl100k_base encoding).
+                                Useful for models with small context windows (e.g. 1024 tokens).
        """
        self.api_key = api_key
        self.model = model
        self.api_base = api_base
        self.timeout = timeout
+        self.max_tokens_per_doc = max_tokens_per_doc
        self._initialized = False
        self._litellm = None  # Will be set during initialization

@@ -1017,6 +1041,8 @@ class LiteLLMSDKCrossEncoder(CrossEncoderModel):

        for query, indexed_texts in query_groups.items():
            texts = [text for _, text in indexed_texts]
+            if self.max_tokens_per_doc is not None:
+                texts = [_truncate_to_tokens(t, self.max_tokens_per_doc) for t in texts]
            indices = [idx for idx, _ in indexed_texts]

            # Build kwargs for rerank call
@@ -1050,6 +1076,97 @@ class LiteLLMSDKCrossEncoder(CrossEncoderModel):
        return all_scores


+class JinaMLXCrossEncoder(CrossEncoderModel):
+    """
+    Jina Reranker v3 MLX implementation for Apple Silicon.
+
+    Uses jinaai/jina-reranker-v3-mlx — a 0.6B parameter multilingual listwise reranker
+    optimized for Apple Silicon via the MLX framework. No transformers/PyTorch dependency.
+
+    The model is downloaded automatically from HuggingFace Hub on first use.
+    Requires: mlx>=0.31.0, mlx-lm>=0.31.1, safetensors>=0.6.2
+    """
+
+    HF_REPO_ID = "jinaai/jina-reranker-v3-mlx"
+
+    def __init__(self, model_path: str | None = None):
+        """
+        Args:
+            model_path: Local path to the downloaded model directory.
+                        If None, the model is downloaded from HuggingFace Hub.
+        """
+        self.model_path = model_path
+        self._reranker = None
+
+    @property
+    def provider_name(self) -> str:
+        return "jina-mlx"
+
+    async def initialize(self) -> None:
+        if self._reranker is not None:
+            return
+
+        try:
+            import mlx.core  # noqa: F401
+            import mlx_lm  # noqa: F401
+        except ImportError:
+            raise ImportError(
+                "mlx and mlx-lm are required for JinaMLXCrossEncoder. "
+                "Install with: pip install mlx>=0.31.0 mlx-lm>=0.31.1 safetensors>=0.6.2"
+            )
+
+        loop = asyncio.get_event_loop()
+        await loop.run_in_executor(None, self._load_model)
+
+    def _load_model(self) -> None:
+        """Download (if needed) and load the MLX reranker. Runs in a thread."""
+        import os
+
+        from huggingface_hub import snapshot_download
+
+        from .jina_mlx_reranker import MLXReranker
+
+        model_path = self.model_path
+        if model_path is None:
+            logger.info(f"Reranker: downloading {self.HF_REPO_ID} from HuggingFace Hub...")
+            model_path = snapshot_download(repo_id=self.HF_REPO_ID)
+
+        logger.info(f"Reranker: loading jina-reranker-v3-mlx from {model_path}")
+        self._reranker = MLXReranker(
+            model_path=model_path,
+            projector_path=os.path.join(model_path, "projector.safetensors"),
+        )
+        logger.info("Reranker: jina-mlx provider initialized")
+
+    def _predict_sync(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """Score pairs grouped by query. Runs in a thread."""
+        if not pairs:
+            return []
+
+        query_groups: dict[str, list[tuple[int, str]]] = {}
+        for idx, (query, doc) in enumerate(pairs):
+            query_groups.setdefault(query, []).append((idx, doc))
+
+        all_scores = [0.0] * len(pairs)
+
+        for query, indexed_docs in query_groups.items():
+            docs = [doc for _, doc in indexed_docs]
+            indices = [idx for idx, _ in indexed_docs]
+            results = self._reranker.rerank(query, docs)
+            for result in results:
+                original_idx = result["index"]
+                all_scores[indices[original_idx]] = result["relevance_score"]
+
+        return all_scores
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+        if self._reranker is None:
+            raise RuntimeError("Reranker not initialized. Call initialize() first.")
+
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, self._predict_sync, pairs)
+
+
 def create_cross_encoder_from_env() -> CrossEncoderModel:
    """
    Create a CrossEncoderModel instance based on configuration.
@@ -1098,6 +1215,7 @@ def create_cross_encoder_from_env() -> CrossEncoderModel:
            api_base=config.reranker_litellm_api_base,
            api_key=config.reranker_litellm_api_key,
            model=config.reranker_litellm_model,
+            max_tokens_per_doc=config.reranker_litellm_max_tokens_per_doc,
        )
    elif provider == "litellm-sdk":
        api_key = config.reranker_litellm_sdk_api_key
@@ -1109,6 +1227,7 @@ def create_cross_encoder_from_env() -> CrossEncoderModel:
            api_key=api_key,
            model=config.reranker_litellm_sdk_model,
            api_base=config.reranker_litellm_sdk_api_base,
+            max_tokens_per_doc=config.reranker_litellm_max_tokens_per_doc,
        )
    elif provider == "zeroentropy":
        api_key = config.reranker_zeroentropy_api_key
@@ -1122,7 +1241,9 @@ def create_cross_encoder_from_env() -> CrossEncoderModel:
        )
    elif provider == "rrf":
        return RRFPassthroughCrossEncoder()
+    elif provider == "jina-mlx":
+        return JinaMLXCrossEncoder()
    else:
        raise ValueError(
-            f"Unknown reranker provider: {provider}. Supported: 'local', 'tei', 'cohere', 'zeroentropy', 'flashrank', 'litellm', 'litellm-sdk', 'rrf'"
+            f"Unknown reranker provider: {provider}. Supported: 'local', 'tei', 'cohere', 'zeroentropy', 'flashrank', 'litellm', 'litellm-sdk', 'rrf', 'jina-mlx'"
        )
@@ -58,10 +58,16 @@ async def retry_with_backoff(
            last_exception = e
            if attempt < max_retries:
                delay = min(base_delay * (2**attempt), max_delay)
-                logger.warning(
-                    f"Database operation failed (attempt {attempt + 1}/{max_retries + 1}): {e}. "
-                    f"Retrying in {delay:.1f}s..."
-                )
+                if isinstance(e, asyncpg.exceptions.DeadlockDetectedError):
+                    logger.warning(
+                        f"Deadlock detected during parallel document processing — this is expected and will resolve automatically "
+                        f"(attempt {attempt + 1}/{max_retries + 1}, retrying in {delay:.1f}s)"
+                    )
+                else:
+                    logger.warning(
+                        f"Database operation failed (attempt {attempt + 1}/{max_retries + 1}): {e}. "
+                        f"Retrying in {delay:.1f}s..."
+                    )
                await asyncio.sleep(delay)
            else:
                logger.error(f"Database operation failed after {max_retries + 1} attempts: {e}")
@@ -459,10 +459,12 @@ class EntityResolver:
            entity_dates = [g.event_date for _, g in sorted_groups]

            # INSERT ... ON CONFLICT DO NOTHING — no row lock on already-existing entities.
+            # mention_count starts at 0 here; flush_pending_stats() is the sole source of
+            # truth for mention counting (one stat per original mention in the batch).
            inserted_rows = await conn.fetch(
                f"""
                INSERT INTO {fq_table("entities")} (bank_id, canonical_name, first_seen, last_seen, mention_count)
-                SELECT $1, name, COALESCE(event_date, now()), COALESCE(event_date, now()), 1
+                SELECT $1, name, COALESCE(event_date, now()), COALESCE(event_date, now()), 0
                FROM unnest($2::text[], $3::timestamptz[]) AS t(name, event_date)
                ON CONFLICT (bank_id, LOWER(canonical_name))
                DO NOTHING
@@ -489,13 +491,15 @@ class EntityResolver:
                for row in existing_rows:
                    id_by_name[row["name_lower"]] = row["id"]

-            # Assign entity IDs back and queue for post-txn stats flush.
+            # Assign entity IDs back and queue one stat per original mention so that
+            # flush_pending_stats() increments mention_count by the true mention count,
+            # not just 1 per unique name.
            for name_lower, g in sorted_groups:
                entity_id = id_by_name.get(name_lower)
                if entity_id:
                    for original_idx in g.indices:
                        entity_ids[original_idx] = entity_id
-                    pending.append(_EntityStat(entity_id=entity_id, event_date=g.event_date))
+                        pending.append(_EntityStat(entity_id=entity_id, event_date=g.event_date))

        # Accumulate into the resolver's pending list; the orchestrator flushes
        # these with await entity_resolver.flush_pending_stats() after the txn.
@@ -0,0 +1,144 @@
+"""
+MLX implementation of jina-reranker-v3 for Apple Silicon.
+
+This file is adapted from the official model repository:
+  https://huggingface.co/jinaai/jina-reranker-v3-mlx/blob/main/rerank.py
+
+License: CC BY-NC 4.0 (contact Jina AI for commercial usage)
+
+Changes from upstream:
+- Removed the __main__ example block
+- Type annotations added to public methods
+- top_n parameter added to rerank() (upstream only exposed it implicitly)
+"""
+
+import numpy as np
+
+
+class _MLPProjector:
+    def __init__(self):
+        import mlx.nn as nn
+
+        self.linear1 = nn.Linear(1024, 512, bias=False)
+        self.linear2 = nn.Linear(512, 512, bias=False)
+
+    def __call__(self, x):
+        import mlx.nn as nn
+
+        x = self.linear1(x)
+        x = nn.relu(x)
+        x = self.linear2(x)
+        return x
+
+
+def _load_projector(projector_path: str) -> _MLPProjector:
+    import mlx.core as mx
+    from safetensors import safe_open
+
+    projector = _MLPProjector()
+    with safe_open(projector_path, framework="numpy") as f:
+        projector.linear1.weight = mx.array(f.get_tensor("linear1.weight"))
+        projector.linear2.weight = mx.array(f.get_tensor("linear2.weight"))
+    return projector
+
+
+def _sanitize(text: str, special_tokens: dict[str, str]) -> str:
+    for token in special_tokens.values():
+        text = text.replace(token, "")
+    return text
+
+
+def _format_prompt(query: str, docs: list[str], special_tokens: dict[str, str]) -> str:
+    query = _sanitize(query, special_tokens)
+    docs = [_sanitize(d, special_tokens) for d in docs]
+
+    doc_token = special_tokens["doc_embed_token"]
+    query_token = special_tokens["query_embed_token"]
+
+    prefix = (
+        "<|im_start|>system\n"
+        "You are a search relevance expert who can determine a ranking of the passages based on how relevant they are to the query. "
+        "If the query is a question, how relevant a passage is depends on how well it answers the question. "
+        "If not, try to analyze the intent of the query and assess how well each passage satisfies the intent. "
+        "If an instruction is provided, you should follow the instruction when determining the ranking."
+        "<|im_end|>\n<|im_start|>user\n"
+    )
+    suffix = "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
+
+    body = (
+        f"I will provide you with {len(docs)} passages, each indicated by a numerical identifier. "
+        f"Rank the passages based on their relevance to query: {query}\n"
+    )
+    body += "\n".join(f'<passage id="{i}">\n{doc}{doc_token}\n</passage>' for i, doc in enumerate(docs))
+    body += f"\n<query>\n{query}{query_token}\n</query>"
+    return prefix + body + suffix
+
+
+class MLXReranker:
+    """
+    MLX-accelerated jina-reranker-v3 for Apple Silicon.
+
+    Loads the model from a local directory (use huggingface_hub.snapshot_download
+    to fetch jinaai/jina-reranker-v3-mlx if you don't have it already).
+    """
+
+    _SPECIAL_TOKENS = {
+        "query_embed_token": "<|rerank_token|>",
+        "doc_embed_token": "<|embed_token|>",
+    }
+    _DOC_TOKEN_ID = 151670
+    _QUERY_TOKEN_ID = 151671
+
+    def __init__(self, model_path: str, projector_path: str):
+        from mlx_lm import load
+
+        self.model, self.tokenizer = load(model_path)
+        self.model.eval()
+        self.projector = _load_projector(projector_path)
+
+    def rerank(self, query: str, documents: list[str], top_n: int | None = None) -> list[dict]:
+        """
+        Rank documents by relevance to a query.
+
+        Returns a list of dicts with keys: document, relevance_score, index.
+        Sorted by descending relevance_score.
+        """
+        import mlx.core as mx
+
+        prompt = _format_prompt(query, documents, self._SPECIAL_TOKENS)
+        input_ids = self.tokenizer.encode(prompt)
+        hidden_states = self.model.model([input_ids])[0]  # [seq_len, hidden_size]
+
+        input_ids_np = np.array(input_ids)
+        query_positions = np.where(input_ids_np == self._QUERY_TOKEN_ID)[0]
+        doc_positions = np.where(input_ids_np == self._DOC_TOKEN_ID)[0]
+
+        if len(query_positions) == 0:
+            raise ValueError("Query embed token not found in prompt")
+        if len(doc_positions) == 0:
+            raise ValueError("Document embed tokens not found in prompt")
+
+        query_hidden = mx.expand_dims(hidden_states[int(query_positions[0])], axis=0)
+        doc_hidden = mx.stack([hidden_states[int(p)] for p in doc_positions])
+
+        query_emb = self.projector(query_hidden)  # [1, 512]
+        doc_emb = self.projector(doc_hidden)  # [num_docs, 512]
+
+        query_exp = mx.broadcast_to(mx.expand_dims(query_emb, 0), (1, len(documents), 512))
+        doc_exp = mx.expand_dims(doc_emb, 0)
+
+        scores = mx.sum(doc_exp * query_exp, axis=-1) / (
+            mx.sqrt(mx.sum(doc_exp * doc_exp, axis=-1)) * mx.sqrt(mx.sum(query_exp * query_exp, axis=-1))
+        )  # [1, num_docs]
+        scores_np = np.array(scores[0])
+
+        order = np.argsort(scores_np)[::-1]
+        n = min(top_n, len(documents)) if top_n is not None else len(documents)
+        return [
+            {
+                "document": documents[order[i]],
+                "relevance_score": float(scores_np[order[i]]),
+                "index": int(order[i]),
+            }
+            for i in range(n)
+        ]
@@ -48,6 +48,28 @@ _llm_max_concurrent = int(os.getenv(ENV_LLM_MAX_CONCURRENT, str(DEFAULT_LLM_MAX_
 _global_llm_semaphore = asyncio.Semaphore(_llm_max_concurrent)


+def sanitize_llm_output(text: str | None) -> str | None:
+    """
+    Sanitize text by removing characters that break downstream systems.
+
+    Removes:
+    - ASCII control characters (0x00-0x08, 0x0B-0x0C, 0x0E-0x1F, 0x7F): break
+      json.loads and PostgreSQL UTF-8 encoding; tab (0x09), newline (0x0A), and
+      carriage return (0x0D) are preserved as they are valid in text and JSON.
+    - Unicode surrogates (U+D800-U+DFFF): Invalid in UTF-8, break LLM APIs
+
+    Surrogate characters are used in UTF-16 encoding but cannot be encoded
+    in UTF-8. They can appear in Python strings from improperly decoded data
+    (e.g., from JavaScript or broken files). Control characters commonly appear
+    in LLM output embedded inside JSON string values.
+    """
+    if text is None:
+        return None
+    if not text:
+        return text
+    return re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f\ud800-\udfff]", "", text)
+
+
 class OutputTooLongError(Exception):
    """
    Bridge exception raised when LLM output exceeds token limits.
@@ -205,7 +227,7 @@ def create_llm_provider(
            reasoning_effort=reasoning_effort,
        )

-    elif provider_lower in ("openai", "groq", "ollama", "lmstudio"):
+    elif provider_lower in ("openai", "groq", "ollama", "lmstudio", "minimax"):
        return OpenAICompatibleLLM(
            provider=provider,
            api_key=api_key,
@@ -274,6 +296,7 @@ class LLMProvider:
            "openai-codex",
            "claude-code",
            "mock",
+            "minimax",
        ]
        if self.provider not in valid_providers:
            raise ValueError(f"Invalid LLM provider: {self.provider}. Must be one of: {', '.join(valid_providers)}")
@@ -286,6 +309,8 @@ class LLMProvider:
                self.base_url = "http://localhost:11434/v1"
            elif self.provider == "lmstudio":
                self.base_url = "http://localhost:1234/v1"
+            elif self.provider == "minimax":
+                self.base_url = "https://api.minimax.io/v1"

        # Prepare Vertex AI config (if applicable)
        vertexai_project_id = None
@@ -16,7 +16,7 @@ import logging
 import time
 import uuid
 from collections.abc import Awaitable, Callable
-from datetime import UTC, datetime, timedelta
+from datetime import UTC, datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Any

 import asyncpg
@@ -51,13 +51,9 @@ def get_current_schema() -> str:
    return schema


-# Initialize tiktoken encoder once at module level for efficiency
-_tiktoken_encoder = tiktoken.get_encoding("cl100k_base")  # GPT-4/GPT-3.5-turbo encoding
-
-
 def count_tokens(text: str) -> int:
    """Count tokens in text using tiktoken (cl100k_base encoding for GPT-4/3.5)."""
-    return len(_tiktoken_encoder.encode(text))
+    return len(_get_tiktoken_encoding().encode(text))


 def fq_table(table_name: str) -> str:
@@ -168,7 +164,7 @@ from enum import Enum
 from ..metrics import get_metrics_collector
 from ..pg0 import EmbeddedPostgres, parse_pg0_url
 from .entity_resolver import EntityResolver
-from .llm_wrapper import LLMConfig, requires_api_key
+from .llm_wrapper import LLMConfig, requires_api_key, sanitize_llm_output
 from .query_analyzer import QueryAnalyzer
 from .reflect import run_reflect_agent
 from .reflect.tools import tool_expand, tool_recall, tool_search_mental_models, tool_search_observations
@@ -188,7 +184,7 @@ from .retain import bank_utils, embedding_utils
 from .retain.types import RetainContentDict
 from .search import think_utils
 from .search.reranking import CrossEncoderReranker, apply_combined_scoring
-from .search.tags import TagsMatch, build_tags_where_clause
+from .search.tags import TagGroup, TagsMatch, build_tags_where_clause
 from .task_backend import BrokerTaskBackend, SyncTaskBackend, TaskBackend


@@ -208,8 +204,6 @@ def utcnow():
 # Logger for memory system
 logger = logging.getLogger(__name__)

-import tiktoken
-
 from .db_utils import acquire_with_retry

 # Cache tiktoken encoding for token budget filtering (module-level singleton)
@@ -653,13 +647,19 @@ class MemoryEngine(MemoryEngineInterface):
            # Retrieve file from storage
            file_data = await self._file_storage.retrieve(storage_key)

-            # Convert to markdown
-            parser = self._parser_registry.get_parser(
-                name=task_dict.get("parser"),
+            # Convert to markdown using the ordered fallback chain stored in the task payload.
+            # task_dict["parser"] is always a list[str] set at submission time.
+            parser_chain: list[str] = task_dict.get("parser") or []
+            if not parser_chain:
+                raise ValueError("No parser chain defined for file_convert_retain task")
+            convert_result = await self._parser_registry.convert_with_fallback(
+                parsers=parser_chain,
+                file_data=file_data,
                filename=filename,
                content_type=task_dict.get("content_type"),
            )
-            markdown_content = await parser.convert(file_data, filename)
+            markdown_content = sanitize_llm_output(convert_result.content) or ""
+            winning_parser = convert_result.parser_name
        except Exception as e:
            # Re-raise with filename context for better error reporting
            error_msg = f"Failed to parse file '{filename}': {str(e)}"
@@ -671,6 +671,31 @@ class MemoryEngine(MemoryEngineInterface):
            f"document_id={document_id}, {len(markdown_content)} chars. Submitting retain task."
        )

+        # Fire file conversion hook (e.g., for Iris billing)
+        if self._operation_validator:
+            try:
+                from hindsight_api.extensions.operation_validator import FileConvertResult
+                from hindsight_api.models import RequestContext
+
+                convert_context = RequestContext(
+                    internal=True,
+                    user_initiated=True,
+                    tenant_id=task_dict.get("_tenant_id"),
+                    api_key_id=task_dict.get("_api_key_id"),
+                )
+                await self._operation_validator.on_file_convert_complete(
+                    FileConvertResult(
+                        bank_id=bank_id,
+                        parser_name=winning_parser,
+                        filename=filename,
+                        output_chars=len(markdown_content),
+                        output_text=markdown_content,
+                        request_context=convert_context,
+                    )
+                )
+            except Exception as e:
+                logger.warning(f"[FILE_CONVERT_RETAIN] on_file_convert_complete hook failed: {e}")
+
        # Build retain task payload
        retain_contents = [
            {
@@ -789,6 +814,7 @@ class MemoryEngine(MemoryEngineInterface):
            memory_engine=self,
            bank_id=bank_id,
            request_context=internal_context,
+            operation_id=task_dict.get("operation_id"),
        )

        logger.info(f"[CONSOLIDATION] bank={bank_id} completed: {result.get('memories_processed', 0)} processed")
@@ -1108,8 +1134,13 @@ class MemoryEngine(MemoryEngineInterface):
            )

        async def _callback(conn: asyncpg.Connection) -> None:
+            # Resolve schema at call time (not at callback creation time) because
+            # _current_schema contextvar may not yet be set when the callback is built
+            # from the HTTP path (http.py calls _build_retain_outbox_callback before
+            # retain_batch_async which is where _authenticate_tenant sets the schema).
+            resolved_schema = schema or _current_schema.get()
            for event in events:
-                await webhook_manager.fire_event_with_conn(event, conn, schema=schema)
+                await webhook_manager.fire_event_with_conn(event, conn, schema=resolved_schema)

        return _callback

@@ -1213,6 +1244,24 @@ class MemoryEngine(MemoryEngineInterface):
        except Exception as e:
            logger.error(f"Failed to delete async operation record {operation_id}: {e}")

+    async def _check_op_alive(self, operation_id: str) -> bool:
+        """Return False if the operation row no longer exists (e.g. bank was deleted via CASCADE).
+
+        Long-running operations should call this at natural checkpoints (e.g. after each
+        committed batch) to detect bank deletion early and abort cleanly.
+        """
+        try:
+            pool = await self._get_pool()
+            async with acquire_with_retry(pool) as conn:
+                row = await conn.fetchrow(
+                    f"SELECT operation_id FROM {fq_table('async_operations')} WHERE operation_id = $1",
+                    uuid.UUID(operation_id),
+                )
+                return row is not None
+        except Exception as e:
+            logger.error(f"Failed to check operation liveness {operation_id}: {e}")
+            return True  # Assume alive on DB error to avoid false-positive aborts
+
    async def _mark_operation_failed(self, operation_id: str, error_message: str, error_traceback: str):
        """Helper to mark an operation as failed in the database.

@@ -1228,15 +1277,19 @@ class MemoryEngine(MemoryEngineInterface):
            async with acquire_with_retry(pool) as conn:
                async with conn.transaction():
                    # Mark this operation as failed
-                    await conn.execute(
+                    row = await conn.fetchrow(
                        f"""
                        UPDATE {fq_table("async_operations")}
                        SET status = 'failed', error_message = $2, updated_at = NOW()
                        WHERE operation_id = $1
+                        RETURNING operation_id
                        """,
                        uuid.UUID(operation_id),
                        truncated_error,
                    )
+                    if row is None:
+                        logger.info(f"Operation {operation_id} no longer exists (bank deleted), skipping mark-failed")
+                        return
                    logger.info(f"Marked async operation as failed: {operation_id}")

                    # Check if this is a child operation and update parent if all siblings are done
@@ -1256,14 +1309,20 @@ class MemoryEngine(MemoryEngineInterface):
            async with acquire_with_retry(pool) as conn:
                async with conn.transaction():
                    # Mark this operation as completed
-                    await conn.execute(
+                    row = await conn.fetchrow(
                        f"""
                        UPDATE {fq_table("async_operations")}
                        SET status = 'completed', updated_at = NOW(), completed_at = NOW()
                        WHERE operation_id = $1
+                        RETURNING operation_id
                        """,
                        uuid.UUID(operation_id),
                    )
+                    if row is None:
+                        logger.info(
+                            f"Operation {operation_id} no longer exists (bank deleted), skipping mark-completed"
+                        )
+                        return
                    logger.info(f"Marked async operation as completed: {operation_id}")

                    # Check if this is a child operation and update parent if all siblings are done
@@ -1293,14 +1352,20 @@ class MemoryEngine(MemoryEngineInterface):
            pool = await self._get_pool()
            async with acquire_with_retry(pool) as conn:
                async with conn.transaction():
-                    await conn.execute(
+                    row = await conn.fetchrow(
                        f"""
                        UPDATE {fq_table("async_operations")}
                        SET status = 'completed', updated_at = NOW(), completed_at = NOW()
                        WHERE operation_id = $1
+                        RETURNING operation_id
                        """,
                        uuid.UUID(operation_id),
                    )
+                    if row is None:
+                        logger.info(
+                            f"Operation {operation_id} no longer exists (bank deleted), skipping mark-completed"
+                        )
+                        return
                    logger.info(f"Marked async operation as completed: {operation_id}")
                    await self._maybe_update_parent_operation(operation_id, conn)

@@ -1598,6 +1663,15 @@ class MemoryEngine(MemoryEngineInterface):
        # Create connection pool
        # For read-heavy workloads with many parallel think/search operations,
        # we need a larger pool. Read operations don't need strong isolation.
+        async def _init_connection(conn: asyncpg.Connection) -> None:
+            # SET (not SET LOCAL) so it persists for the connection lifetime.
+            # ef_search=200 improves HNSW recall quality for the per-fact_type
+            # semantic queries in retrieve_semantic_bm25_combined().
+            try:
+                await conn.execute("SET hnsw.ef_search = 200")
+            except Exception:
+                logger.debug("Could not set hnsw.ef_search — extension may not support it")
+
        self._pool = await asyncpg.create_pool(
            self.db_url,
            min_size=self._pool_min_size,
@@ -1605,6 +1679,7 @@ class MemoryEngine(MemoryEngineInterface):
            command_timeout=self._db_command_timeout,
            statement_cache_size=0,  # Disable prepared statement cache
            timeout=self._db_acquire_timeout,  # Connection acquisition timeout (seconds)
+            init=_init_connection,
        )

        # Initialize entity resolver with pool and configured lookup strategy
@@ -2011,6 +2086,15 @@ class MemoryEngine(MemoryEngineInterface):
            # Process each sub-batch
            all_results = []
            for i, sub_batch in enumerate(sub_batches, 1):
+                # Checkpoint: abort if the operation was deleted (bank was deleted) between sub-batches.
+                if operation_id and not await self._check_op_alive(operation_id):
+                    logger.info(
+                        f"[BATCH_RETAIN] bank={bank_id} operation {operation_id} cancelled (bank deleted), stopping after {i - 1}/{len(sub_batches)} sub-batches"
+                    )
+                    if return_usage:
+                        return all_results, total_usage
+                    return all_results
+
                sub_batch_tokens = sum(count_tokens(item.get("content", "")) for item in sub_batch)
                logger.info(
                    f"Processing sub-batch {i}/{len(sub_batches)}: {len(sub_batch)} items, {sub_batch_tokens:,} tokens"
@@ -2151,7 +2235,7 @@ class MemoryEngine(MemoryEngineInterface):
                    document_tags=document_tags,
                    config=resolved_config,
                    operation_id=operation_id,
-                    schema=request_context.tenant_id if request_context else None,
+                    schema=_current_schema.get(),
                    outbox_callback=outbox_callback,
                )

@@ -2212,9 +2296,11 @@ class MemoryEngine(MemoryEngineInterface):
        max_chunk_tokens: int = 8192,
        include_source_facts: bool = False,
        max_source_facts_tokens: int = 4096,
+        max_source_facts_tokens_per_observation: int = -1,
        request_context: "RequestContext",
        tags: list[str] | None = None,
        tags_match: TagsMatch = "any",
+        tag_groups: list[TagGroup] | None = None,
        _connection_budget: int | None = None,
        _quiet: bool = False,
    ) -> RecallResultModel:
@@ -2349,10 +2435,12 @@ class MemoryEngine(MemoryEngineInterface):
                            semaphore_wait=semaphore_wait,
                            tags=tags,
                            tags_match=tags_match,
+                            tag_groups=tag_groups,
                            connection_budget=_connection_budget,
                            quiet=_quiet,
                            include_source_facts=include_source_facts,
                            max_source_facts_tokens=max_source_facts_tokens,
+                            max_source_facts_tokens_per_observation=max_source_facts_tokens_per_observation,
                        )
                        break  # Success - exit retry loop
                    except Exception as e:
@@ -2475,10 +2563,12 @@ class MemoryEngine(MemoryEngineInterface):
        semaphore_wait: float = 0.0,
        tags: list[str] | None = None,
        tags_match: TagsMatch = "any",
+        tag_groups: list[TagGroup] | None = None,
        connection_budget: int | None = None,
        quiet: bool = False,
        include_source_facts: bool = False,
        max_source_facts_tokens: int = 4096,
+        max_source_facts_tokens_per_observation: int = -1,
    ) -> RecallResultModel:
        """
        Search implementation with modular retrieval and reranking.
@@ -2593,6 +2683,7 @@ class MemoryEngine(MemoryEngineInterface):
                        self.query_analyzer,
                        tags=tags,
                        tags_match=tags_match,
+                        tag_groups=tag_groups,
                    )
                    parallel_duration = time.time() - parallel_start
            finally:
@@ -3100,18 +3191,9 @@ class MemoryEngine(MemoryEngineInterface):

                            encoding = _get_tiktoken_encoding()
                            source_facts_dict = {}
-                            total_source_tokens = 0
-                            for sid in source_ids_ordered:
-                                if sid not in source_row_by_id:
-                                    continue
-                                r = source_row_by_id[sid]
-                                fact_tokens = len(encoding.encode(r["text"]))
-                                if (
-                                    max_source_facts_tokens >= 0
-                                    and total_source_tokens + fact_tokens > max_source_facts_tokens
-                                ):
-                                    break
-                                source_facts_dict[sid] = MemoryFact(
+
+                            def _make_source_fact(sid: str, r: Any) -> MemoryFact:
+                                return MemoryFact(
                                    id=sid,
                                    text=r["text"],
                                    fact_type=r["fact_type"],
@@ -3123,7 +3205,37 @@ class MemoryEngine(MemoryEngineInterface):
                                    chunk_id=str(r["chunk_id"]) if r["chunk_id"] else None,
                                    tags=r["tags"] or None,
                                )
-                                total_source_tokens += fact_tokens
+
+                            if max_source_facts_tokens_per_observation >= 0:
+                                # Per-observation capping: each observation independently selects
+                                # source facts up to its token budget.
+                                for obs_id, sids in source_fact_ids_by_obs.items():
+                                    obs_tokens = 0
+                                    for sid in sids:
+                                        if sid not in source_row_by_id:
+                                            continue
+                                        r = source_row_by_id[sid]
+                                        fact_tokens = len(encoding.encode(r["text"]))
+                                        if obs_tokens + fact_tokens > max_source_facts_tokens_per_observation:
+                                            break
+                                        obs_tokens += fact_tokens
+                                        if sid not in source_facts_dict:
+                                            source_facts_dict[sid] = _make_source_fact(sid, r)
+                            else:
+                                # Global budget: fill in order of first appearance until exhausted.
+                                total_source_tokens = 0
+                                for sid in source_ids_ordered:
+                                    if sid not in source_row_by_id:
+                                        continue
+                                    r = source_row_by_id[sid]
+                                    fact_tokens = len(encoding.encode(r["text"]))
+                                    if (
+                                        max_source_facts_tokens >= 0
+                                        and total_source_tokens + fact_tokens > max_source_facts_tokens
+                                    ):
+                                        break
+                                    source_facts_dict[sid] = _make_source_fact(sid, r)
+                                    total_source_tokens += fact_tokens

            # Get entities for each fact if include_entities is requested
            fact_entity_map = {}  # unit_id -> list of (entity_id, entity_name)
@@ -3385,6 +3497,140 @@ class MemoryEngine(MemoryEngineInterface):

        return result

+    async def update_document(
+        self,
+        document_id: str,
+        bank_id: str,
+        *,
+        tags: list[str] | None = None,
+        request_context: "RequestContext",
+    ) -> bool:
+        """
+        Update mutable fields on a document without re-processing its content.
+
+        Tag changes propagate to all associated memory units and trigger observation
+        invalidation + re-consolidation (same semantics as delete_document):
+        - Observations referencing the document's memory units are deleted.
+        - The document's own units and any co-source memories from other documents
+          have consolidated_at reset so they are re-consolidated under the new tags.
+
+        Args:
+            document_id: Document ID to update
+            bank_id: Bank ID that owns the document
+            tags: New tags to apply to the document and all its memory units (optional)
+            request_context: Request context for authentication.
+
+        Returns:
+            True if the document was found and updated, False if not found
+        """
+        await self._authenticate_tenant(request_context)
+        if self._operation_validator:
+            from hindsight_api.extensions import BankWriteContext
+
+            ctx = BankWriteContext(bank_id=bank_id, operation="update_document", request_context=request_context)
+            await self._validate_operation(self._operation_validator.validate_bank_write(ctx))
+        pool = await self._get_pool()
+        invalidated_obs = 0
+        async with acquire_with_retry(pool) as conn:
+            async with conn.transaction():
+                set_parts: list[str] = ["updated_at = now()"]
+                params: list[Any] = []
+                p = 1
+
+                if tags is not None:
+                    set_parts.append(f"tags = ${p}")
+                    params.append(tags)
+                    p += 1
+
+                params.extend([document_id, bank_id])
+                doc_id_found = await conn.fetchval(
+                    f"""
+                    UPDATE {fq_table("documents")}
+                    SET {", ".join(set_parts)}
+                    WHERE id = ${p} AND bank_id = ${p + 1}
+                    RETURNING id
+                    """,
+                    *params,
+                )
+                if not doc_id_found:
+                    return False
+
+                if tags is not None:
+                    unit_rows = await conn.fetch(
+                        f"SELECT id FROM {fq_table('memory_units')} WHERE document_id = $1 AND fact_type IN ('experience', 'world')",
+                        document_id,
+                    )
+                    unit_ids = [str(row["id"]) for row in unit_rows]
+
+                    await conn.execute(
+                        f"UPDATE {fq_table('memory_units')} SET tags = $1 WHERE document_id = $2",
+                        tags,
+                        document_id,
+                    )
+
+                    if unit_ids:
+                        import uuid as uuid_module
+
+                        unit_uuids = [uuid_module.UUID(uid) for uid in unit_ids]
+                        unit_uuid_set = {str(u) for u in unit_uuids}
+                        affected_obs = await conn.fetch(
+                            f"""
+                            SELECT id, source_memory_ids FROM {fq_table("memory_units")}
+                            WHERE bank_id = $1
+                              AND fact_type = 'observation'
+                              AND source_memory_ids && $2::uuid[]
+                            """,
+                            bank_id,
+                            unit_uuids,
+                        )
+                        if affected_obs:
+                            obs_ids = [obs["id"] for obs in affected_obs]
+
+                            seen: set[str] = set()
+                            other_source_uuids: list[uuid_module.UUID] = []
+                            for obs in affected_obs:
+                                for src_id in obs["source_memory_ids"] or []:
+                                    src_str = str(src_id)
+                                    if src_str not in unit_uuid_set and src_str not in seen:
+                                        other_source_uuids.append(src_id)
+                                        seen.add(src_str)
+
+                            await conn.execute(
+                                f"DELETE FROM {fq_table('memory_units')} WHERE id = ANY($1::uuid[])",
+                                obs_ids,
+                            )
+                            await conn.execute(
+                                f"""
+                                UPDATE {fq_table("memory_units")}
+                                SET consolidated_at = NULL
+                                WHERE id = ANY($1::uuid[])
+                                  AND fact_type IN ('experience', 'world')
+                                """,
+                                unit_uuids,
+                            )
+                            if other_source_uuids:
+                                await conn.execute(
+                                    f"""
+                                    UPDATE {fq_table("memory_units")}
+                                    SET consolidated_at = NULL
+                                    WHERE id = ANY($1::uuid[])
+                                      AND fact_type IN ('experience', 'world')
+                                    """,
+                                    other_source_uuids,
+                                )
+                            invalidated_obs = len(obs_ids)
+                            logger.info(
+                                f"[OBSERVATIONS] Deleted {invalidated_obs} observations, reset "
+                                f"{len(unit_ids)} document source memories and "
+                                f"{len(other_source_uuids)} co-source memories for re-consolidation "
+                                f"after document update on '{document_id}' in bank {bank_id}"
+                            )
+
+        if invalidated_obs > 0:
+            await self.submit_async_consolidation(bank_id=bank_id, request_context=request_context)
+
+        return True
+
    async def delete_memory_unit(
        self,
        unit_id: str,
@@ -3482,6 +3728,7 @@ class MemoryEngine(MemoryEngineInterface):
        pool = await self._get_pool()
        invalidated_obs = 0
        result: dict[str, int] = {}
+        bank_internal_id: str | None = None
        async with acquire_with_retry(pool) as conn:
            # Ensure connection is not in read-only mode (can happen with connection poolers)
            await conn.execute("SET SESSION CHARACTERISTICS AS TRANSACTION READ WRITE")
@@ -3537,8 +3784,12 @@ class MemoryEngine(MemoryEngineInterface):
                        # Delete entities (cascades to unit_entities, entity_cooccurrences, memory_links with entity_id)
                        await conn.execute(f"DELETE FROM {fq_table('entities')} WHERE bank_id = $1", bank_id)

-                        # Delete the bank profile itself
-                        await conn.execute(f"DELETE FROM {fq_table('banks')} WHERE bank_id = $1", bank_id)
+                        # Delete the bank profile and retrieve internal_id for HNSW index cleanup
+                        internal_id = await conn.fetchval(
+                            f"DELETE FROM {fq_table('banks')} WHERE bank_id = $1 RETURNING internal_id", bank_id
+                        )
+                        if internal_id:
+                            bank_internal_id = str(internal_id)

                        result = {
                            "memory_units_deleted": units_count,
@@ -3550,6 +3801,12 @@ class MemoryEngine(MemoryEngineInterface):
                except Exception as e:
                    raise Exception(f"Failed to delete agent data: {str(e)}")

+            # Drop per-bank HNSW indexes AFTER the transaction commits to avoid
+            # AccessExclusiveLock deadlocks with concurrent bank deletions.
+            # (DROP INDEX on memory_units conflicts with RowExclusiveLock from DELETE inside tx)
+            if bank_internal_id:
+                await bank_utils.drop_bank_hnsw_indexes(conn, bank_internal_id)
+
        if invalidated_obs > 0:
            await self.submit_async_consolidation(bank_id=bank_id, request_context=request_context)

@@ -4290,7 +4547,11 @@ class MemoryEngine(MemoryEngineInterface):
                "observation_scopes": row["observation_scopes"] if row["observation_scopes"] else None,
            }

-            # For observations, include source_memory_ids and fetch source_memories
+            # For observations, include source_memory_ids
+            # history is deprecated here - use GET /memories/{id}/history instead
+            if row["fact_type"] == "observation":
+                result["history"] = []
+
            if row["fact_type"] == "observation" and row["source_memory_ids"]:
                source_ids = row["source_memory_ids"]
                result["source_memory_ids"] = [str(sid) for sid in source_ids]
@@ -4319,6 +4580,95 @@ class MemoryEngine(MemoryEngineInterface):

            return result

+    async def get_observation_history(
+        self,
+        bank_id: str,
+        memory_id: str,
+        request_context: "RequestContext",
+    ) -> list[dict] | None:
+        """
+        Get the history of an observation, with source facts resolved to their text.
+
+        Returns None if the memory is not found or is not an observation.
+        Returns a list of history entries (most recent first), each with source_facts resolved.
+        """
+        await self._authenticate_tenant(request_context)
+        if self._operation_validator:
+            from hindsight_api.extensions import BankReadContext
+
+            ctx = BankReadContext(bank_id=bank_id, operation="get_observation_history", request_context=request_context)
+            await self._validate_operation(self._operation_validator.validate_bank_read(ctx))
+        pool = await self._get_pool()
+        async with acquire_with_retry(pool) as conn:
+            row = await conn.fetchrow(
+                f"""
+                SELECT fact_type, history, source_memory_ids
+                FROM {fq_table("memory_units")}
+                WHERE id = $1 AND bank_id = $2
+                """,
+                uuid.UUID(memory_id),
+                bank_id,
+            )
+            if not row:
+                return None
+            if row["fact_type"] != "observation":
+                return []
+
+            raw_history = row["history"]
+            if isinstance(raw_history, str):
+                raw_history = json.loads(raw_history)
+            if not raw_history:
+                return []
+
+            # Collect all source memory IDs (current full set + all historical new ones)
+            current_source_ids: list[str] = [str(sid) for sid in (row["source_memory_ids"] or [])]
+            all_source_ids: set[uuid.UUID] = set(uuid.UUID(sid) for sid in current_source_ids)
+            for entry in raw_history:
+                for sid in entry.get("new_source_memory_ids", []):
+                    try:
+                        all_source_ids.add(uuid.UUID(sid))
+                    except (ValueError, AttributeError):
+                        pass
+
+            # Resolve all source memories in one query
+            source_map: dict[str, dict] = {}
+            if all_source_ids:
+                source_rows = await conn.fetch(
+                    f"""
+                    SELECT id, text, fact_type, context
+                    FROM {fq_table("memory_units")}
+                    WHERE id = ANY($1::uuid[])
+                    """,
+                    list(all_source_ids),
+                )
+                for r in source_rows:
+                    source_map[str(r["id"])] = {
+                        "id": str(r["id"]),
+                        "text": r["text"],
+                        "type": r["fact_type"],
+                        "context": r["context"] or None,
+                    }
+
+            # Reconstruct cumulative source IDs per change by working backwards from current state.
+            # Source IDs are only ever accumulated (never removed), so:
+            #   after_change_N = before_change_N + new_source_memory_ids_N
+            cumulative_ids: list[str] = list(current_source_ids)
+            enriched: list[dict] = []
+            for entry in reversed(raw_history):
+                new_ids_in_entry: set[str] = set(entry.get("new_source_memory_ids", []))
+                source_facts = []
+                for sid in cumulative_ids:
+                    fact = source_map.get(sid, {"id": sid, "text": None, "type": None, "context": None})
+                    source_facts.append({**fact, "is_new": sid in new_ids_in_entry})
+                enriched_entry = dict(entry)
+                enriched_entry["source_facts"] = source_facts
+                enriched.append(enriched_entry)
+                # Step back: remove the new IDs added by this change to get the prior state
+                cumulative_ids = [sid for sid in cumulative_ids if sid not in new_ids_in_entry]
+
+            enriched.reverse()
+            return enriched
+
    async def list_documents(
        self,
        bank_id: str,
@@ -4694,6 +5044,7 @@ class MemoryEngine(MemoryEngineInterface):
        request_context: "RequestContext",
        tags: list[str] | None = None,
        tags_match: TagsMatch = "any",
+        tag_groups: list[TagGroup] | None = None,
        exclude_mental_model_ids: list[str] | None = None,
        _skip_span: bool = False,
    ) -> ReflectResult:
@@ -4796,6 +5147,7 @@ class MemoryEngine(MemoryEngineInterface):
                    max_results=max_results,
                    tags=tags,
                    tags_match=tags_match,
+                    tag_groups=tag_groups,
                    exclude_ids=exclude_mental_model_ids,
                    pending_consolidation=pending_consolidation,
                )
@@ -4809,6 +5161,7 @@ class MemoryEngine(MemoryEngineInterface):
                max_tokens=max_tokens,
                tags=tags,
                tags_match=tags_match,
+                tag_groups=tag_groups,
                last_consolidated_at=last_consolidated_at,
                pending_consolidation=pending_consolidation,
            )
@@ -4822,6 +5175,7 @@ class MemoryEngine(MemoryEngineInterface):
                max_tokens=max_tokens,
                tags=tags,
                tags_match=tags_match,
+                tag_groups=tag_groups,
                max_chunk_tokens=max_chunk_tokens,
            )

@@ -5851,6 +6205,39 @@ class MemoryEngine(MemoryEngineInterface):

        return result

+    async def get_mental_model_history(
+        self,
+        bank_id: str,
+        mental_model_id: str,
+        *,
+        request_context: "RequestContext",
+    ) -> list[dict] | None:
+        """Get the refresh history of a mental model.
+
+        Returns None if the mental model is not found.
+        Returns a list of history entries (most recent first), each with previous_content and changed_at.
+        """
+        await self._authenticate_tenant(request_context)
+        pool = await self._get_pool()
+        async with acquire_with_retry(pool) as conn:
+            row = await conn.fetchrow(
+                f"""
+                SELECT history
+                FROM {fq_table("mental_models")}
+                WHERE bank_id = $1 AND id = $2
+                """,
+                bank_id,
+                mental_model_id,
+            )
+            if row is None:
+                return None
+            raw_history = row["history"]
+            if isinstance(raw_history, str):
+                raw_history = json.loads(raw_history)
+            if not raw_history:
+                return []
+            return list(reversed(raw_history))
+
    async def create_mental_model(
        self,
        bank_id: str,
@@ -6071,6 +6458,17 @@ class MemoryEngine(MemoryEngineInterface):
        pool = await self._get_pool()

        async with acquire_with_retry(pool) as conn:
+            # If content is changing, fetch current content first to record history
+            previous_content: str | None = None
+            if content is not None:
+                current_row = await conn.fetchrow(
+                    f"SELECT content FROM {fq_table('mental_models')} WHERE bank_id = $1 AND id = $2",
+                    bank_id,
+                    mental_model_id,
+                )
+                if current_row:
+                    previous_content = current_row["content"]
+
            # Build dynamic update
            updates = []
            params: list[Any] = [bank_id, mental_model_id]
@@ -6086,6 +6484,14 @@ class MemoryEngine(MemoryEngineInterface):
                params.append(content)
                param_idx += 1
                updates.append("last_refreshed_at = NOW()")
+                # Record history entry with the previous content
+                if get_config().enable_mental_model_history:
+                    history_entry = json.dumps(
+                        [{"previous_content": previous_content, "changed_at": datetime.now(timezone.utc).isoformat()}]
+                    )
+                    updates.append(f"history = COALESCE(history, '[]'::jsonb) || ${param_idx}::jsonb")
+                    params.append(history_entry)
+                    param_idx += 1
                # Also update embedding (convert to string for asyncpg vector type)
                embedding_text = f"{name or ''} {content}"
                embedding = await embedding_utils.generate_embeddings_batch(self.embeddings, [embedding_text])
@@ -6506,6 +6912,7 @@ class MemoryEngine(MemoryEngineInterface):
        bank_id: str,
        *,
        status: str | None = None,
+        task_type: str | None = None,
        limit: int = 20,
        offset: int = 0,
        request_context: "RequestContext",
@@ -6515,6 +6922,7 @@ class MemoryEngine(MemoryEngineInterface):
        Args:
            bank_id: Bank identifier
            status: Optional status filter (pending, completed, failed)
+            task_type: Optional operation type filter (retain, consolidation, etc.)
            limit: Maximum number of operations to return (default 20)
            offset: Number of operations to skip (default 0)
            request_context: Request context for authentication
@@ -6543,6 +6951,10 @@ class MemoryEngine(MemoryEngineInterface):
                    where_conditions.append(f"status = ${len(params) + 1}")
                    params.append(status)

+            if task_type:
+                where_conditions.append(f"operation_type = ${len(params) + 1}")
+                params.append(task_type)
+
            where_clause = " AND ".join(where_conditions)

            # Get total count (with filter)
@@ -6773,6 +7185,64 @@ class MemoryEngine(MemoryEngineInterface):
                "bank_id": bank_id,
            }

+    async def retry_operation(
+        self,
+        bank_id: str,
+        operation_id: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """Re-queue a failed async operation."""
+        await self._authenticate_tenant(request_context)
+        from hindsight_api.extensions import OperationValidationError
+
+        if self._operation_validator:
+            from hindsight_api.extensions import BankWriteContext
+
+            ctx = BankWriteContext(bank_id=bank_id, operation="retry_operation", request_context=request_context)
+            await self._validate_operation(self._operation_validator.validate_bank_write(ctx))
+        pool = await self._get_pool()
+
+        op_uuid = uuid.UUID(operation_id)
+
+        async with acquire_with_retry(pool) as conn:
+            row = await conn.fetchrow(
+                f"SELECT bank_id, status FROM {fq_table('async_operations')} WHERE operation_id = $1 AND bank_id = $2",
+                op_uuid,
+                bank_id,
+            )
+
+            if not row:
+                raise ValueError(f"Operation {operation_id} not found for bank {bank_id}")
+
+            if row["status"] != "failed":
+                raise OperationValidationError(
+                    f"Operation {operation_id} cannot be retried: status is '{row['status']}', expected 'failed'",
+                    409,
+                )
+
+            await conn.execute(
+                f"""
+                UPDATE {fq_table("async_operations")}
+                SET status = 'pending',
+                    error_message = NULL,
+                    completed_at = NULL,
+                    next_retry_at = NULL,
+                    worker_id = NULL,
+                    claimed_at = NULL,
+                    retry_count = 0,
+                    updated_at = NOW()
+                WHERE operation_id = $1
+                """,
+                op_uuid,
+            )
+
+            return {
+                "success": True,
+                "message": f"Operation {operation_id} queued for retry",
+                "operation_id": operation_id,
+            }
+
    async def update_bank(
        self,
        bank_id: str,
@@ -6978,6 +7448,10 @@ class MemoryEngine(MemoryEngineInterface):
        parent_operation_id = uuid.uuid4()
        pool = await self._get_pool()

+        # Ensure the bank row exists before inserting async_operations (which now has a FK).
+        # Banks are created lazily on first retain, but the FK requires the row to exist first.
+        await bank_utils.get_bank_profile(pool, bank_id)
+
        # Create typed metadata for parent operation
        parent_metadata = BatchRetainParentMetadata(
            items_count=len(contents),
@@ -7044,7 +7518,6 @@ class MemoryEngine(MemoryEngineInterface):
        self,
        bank_id: str,
        file_items: list[dict[str, Any]],
-        parser: str,
        document_tags: list[str] | None,
        request_context: "RequestContext",
    ) -> dict[str, Any]:
@@ -7063,7 +7536,7 @@ class MemoryEngine(MemoryEngineInterface):
                - metadata: Optional metadata dict
                - tags: Optional tags list
                - timestamp: Optional timestamp
-            parser: Parser name (e.g., "markitdown")
+                - parser: Ordered list of parser names to try (fallback chain)
            document_tags: Tags applied to all documents
            request_context: Request context for authentication

@@ -7119,7 +7592,7 @@ class MemoryEngine(MemoryEngineInterface):
                "storage_key": storage_key,
                "original_filename": file.filename,
                "content_type": file.content_type or "application/octet-stream",
-                "parser": parser,
+                "parser": item["parser"],
                "context": item.get("context"),
                "metadata": item.get("metadata", {}),
                "tags": item.get("tags", []),
@@ -0,0 +1,128 @@
+"""File parser implementations."""
+
+import logging
+from dataclasses import dataclass
+
+from .base import FileParser, UnsupportedFileTypeError
+from .iris import IrisParser
+from .markitdown import MarkitdownParser
+
+__all__ = [
+    "FileParser",
+    "UnsupportedFileTypeError",
+    "IrisParser",
+    "MarkitdownParser",
+    "FileParserRegistry",
+    "ConvertResult",
+]
+
+
+@dataclass
+class ConvertResult:
+    """Result of a successful file conversion."""
+
+    content: str
+    parser_name: str
+
+
+logger = logging.getLogger(__name__)
+
+
+class FileParserRegistry:
+    """Registry for file parsers with auto-detection."""
+
+    def __init__(self):
+        """Initialize empty parser registry."""
+        self._parsers: dict[str, FileParser] = {}
+
+    def register(self, parser: FileParser):
+        """
+        Register a parser.
+
+        Args:
+            parser: FileParser instance
+        """
+        self._parsers[parser.name()] = parser
+
+    def get_parser(
+        self,
+        name: str | None,
+        filename: str,
+        content_type: str | None = None,
+    ) -> FileParser:
+        """
+        Get parser by name or auto-detect.
+
+        Args:
+            name: Parser name (e.g., "markitdown") or None for auto-detect
+            filename: File name for auto-detection
+            content_type: MIME type (optional)
+
+        Returns:
+            FileParser instance
+
+        Raises:
+            ValueError: If no suitable parser found
+        """
+        if name:
+            # Explicit parser requested — return it directly, let the parser
+            # raise UnsupportedFileTypeError from convert() if needed
+            if name not in self._parsers:
+                raise ValueError(f"Parser '{name}' not found. Available: {list(self._parsers.keys())}")
+            return self._parsers[name]
+
+        # Auto-detect parser
+        for parser in self._parsers.values():
+            if parser.supports(filename, content_type):
+                return parser
+
+        raise ValueError(f"No parser found for {filename}. Available parsers: {list(self._parsers.keys())}")
+
+    async def convert_with_fallback(
+        self,
+        parsers: list[str],
+        file_data: bytes,
+        filename: str,
+        content_type: str | None = None,
+    ) -> ConvertResult:
+        """
+        Try each parser in order, falling back on failure or empty content.
+
+        Moves to the next parser if the current one raises UnsupportedFileTypeError
+        or returns empty content. Any other exception (RuntimeError, network error,
+        etc.) also triggers a fallback so the chain is exhausted before failing.
+
+        Args:
+            parsers: Ordered list of parser names to try
+            file_data: Raw file bytes
+            filename: Original filename
+            content_type: MIME type (optional)
+
+        Returns:
+            ConvertResult with the parsed content and the name of the parser that succeeded
+
+        Raises:
+            ValueError: If a parser name is not registered
+            RuntimeError: If all parsers fail or return empty content
+        """
+        last_error: Exception | None = None
+        for name in parsers:
+            parser = self.get_parser(name, filename, content_type)
+            try:
+                content = await parser.convert(file_data, filename)
+                if content and content.strip():
+                    return ConvertResult(content=content, parser_name=name)
+                logger.warning(f"Parser '{name}' returned empty content for '{filename}', trying next")
+                last_error = RuntimeError(f"Parser '{name}' returned no content for '{filename}'")
+            except UnsupportedFileTypeError as e:
+                logger.warning(f"Parser '{name}' does not support '{filename}', trying next: {e}")
+                last_error = e
+            except Exception as e:
+                logger.warning(f"Parser '{name}' failed for '{filename}', trying next: {e}")
+                last_error = e
+
+        raise last_error or RuntimeError(f"No parsers available for '{filename}'")
+
+    def list_parsers(self) -> list[str]:
+        """Get list of registered parser names."""
+        return list(self._parsers.keys())
@@ -62,7 +62,7 @@ class IrisParser(FileParser):
        """
        content_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"

-        async with httpx.AsyncClient() as client:
+        async with httpx.AsyncClient(timeout=httpx.Timeout(30.0, read=120.0)) as client:
            # Step 1: Request a presigned upload URL
            init_resp = await client.post(
                f"{_IRIS_BASE_URL}/org/{self._org_id}/files",
@@ -75,9 +75,10 @@ class IrisParser(FileParser):
            upload_url: str = init_data["uploadUrl"]

            # Step 2: Upload the file bytes to the presigned URL (no auth header)
+            # Ensure file_data is plain bytes (GCS storage may return obstore.Bytes)
            upload_resp = await client.put(
                upload_url,
-                content=file_data,
+                content=bytes(file_data),
                headers={"Content-Type": content_type},
            )
            _raise_for_status(upload_resp, filename, "file upload")
--- a/Show More
+++ b/Show More