fix(docs): update Go quickstart example to use NullableTimestamp for timestamp field

When a LabelGroup has tag=True, extracted key:value entities for that group
are automatically written to the memory unit's tags array. This lets entity
labels double as tags, enabling immediate filtering via the existing
tags/tags_match API params with no extra infrastructure.

- Add tag: bool = False to LabelGroup
- _inject_label_tags() helper called in both sync and batch extraction paths
- UI: add Tag checkbox per label group row
- Docs: document the new tag field
- Tests: 4 new unit tests covering all tag injection paths

.env.example

@@ -2,7 +2,7 @@
 # Copy this file to .env and fill in your values
 
 # LLM Configuration (Required)
-# Supported providers: openai, groq, ollama, gemini, anthropic, lmstudio, vertexai, minimax
+# Supported providers: openai, groq, ollama, gemini, anthropic, lmstudio, vertexai
 HINDSIGHT_API_LLM_PROVIDER=openai
 HINDSIGHT_API_LLM_API_KEY=your-api-key-here
 HINDSIGHT_API_LLM_MODEL=gpt-4o-mini
@@ -20,11 +20,6 @@ HINDSIGHT_API_LLM_BASE_URL=https://api.openai.com/v1
 # HINDSIGHT_API_LLM_VERTEXAI_REGION=us-central1
 # HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY=/path/to/service-account-key.json  # Optional, uses ADC if not set
 
-# Example: MiniMax configuration (1M context window)
-# HINDSIGHT_API_LLM_PROVIDER=minimax
-# HINDSIGHT_API_LLM_API_KEY=your-minimax-api-key
-# HINDSIGHT_API_LLM_MODEL=MiniMax-M2.7
-
 # Example: LM Studio local configuration (Qwen 2.5 32B recommended)
 # HINDSIGHT_API_LLM_PROVIDER=lmstudio
 # HINDSIGHT_API_LLM_API_KEY=lmstudio

.github/dependabot.yml

@@ -1,6 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "weekly"

.github/workflows/deploy-docs.yml

@@ -21,20 +21,20 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
-      - uses: actions/setup-node@v6
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: npm
           cache-dependency-path: package-lock.json
-      - uses: astral-sh/setup-uv@v7
+      - uses: astral-sh/setup-uv@v4
       - run: npm ci --workspace=hindsight-docs
       - run: uv run generate-llms-full
       - run: npm run build --workspace=hindsight-docs
         env:
           UMAMI_URL: https://analytics.hindsight.vectorize.io
           UMAMI_WEBSITE_ID: ${{ secrets.UMAMI_WEBSITE_ID }}
-      - uses: actions/upload-pages-artifact@v4
+      - uses: actions/upload-pages-artifact@v3
         with:
           path: hindsight-docs/build
   deploy:

.github/workflows/release-integration.yml

@@ -1,99 +0,0 @@
-name: Release Integration
-
-on:
-  push:
-    tags:
-      - 'integrations/**'
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    permissions:
-      id-token: write  # for PyPI trusted publishing
-
-    steps:
-    - uses: actions/checkout@v6
-
-    - name: Extract integration info
-      id: info
-      run: |
-        # refs/tags/integrations/litellm/v0.1.0 → integration=litellm, version=0.1.0
-        TAG="${GITHUB_REF#refs/tags/}"
-        INTEGRATION=$(echo "$TAG" | cut -d'/' -f2)
-        VERSION=$(echo "$TAG" | cut -d'/' -f3 | sed 's/^v//')
-        echo "integration=$INTEGRATION" >> $GITHUB_OUTPUT
-        echo "version=$VERSION" >> $GITHUB_OUTPUT
-        echo "tag=$TAG" >> $GITHUB_OUTPUT
-        echo "Integration: $INTEGRATION, Version: $VERSION"
-
-    - name: Detect integration type
-      id: type
-      run: |
-        if [ -f "hindsight-integrations/${{ steps.info.outputs.integration }}/pyproject.toml" ]; then
-          echo "type=python" >> $GITHUB_OUTPUT
-        else
-          echo "type=typescript" >> $GITHUB_OUTPUT
-        fi
-
-    # ── Python integrations (litellm, pydantic-ai, crewai) ──────────────────
-
-    - name: Install uv
-      if: steps.type.outputs.type == 'python'
-      uses: astral-sh/setup-uv@v7
-      with:
-        enable-cache: true
-
-    - name: Set up Python
-      if: steps.type.outputs.type == 'python'
-      uses: actions/setup-python@v5
-      with:
-        python-version-file: ".python-version"
-
-    - name: Build Python package
-      if: steps.type.outputs.type == 'python'
-      working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
-      run: uv build --out-dir dist
-
-    - name: Publish Python package to PyPI
-      if: steps.type.outputs.type == 'python'
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        packages-dir: ./hindsight-integrations/${{ steps.info.outputs.integration }}/dist
-        skip-existing: true
-
-    # ── TypeScript integrations (ai-sdk, chat, openclaw) ────────────────────
-
-    - name: Set up Node.js
-      if: steps.type.outputs.type == 'typescript'
-      uses: actions/setup-node@v6
-      with:
-        node-version: '22'
-        registry-url: 'https://registry.npmjs.org'
-
-    - name: Install dependencies
-      if: steps.type.outputs.type == 'typescript'
-      working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
-      run: npm ci
-
-    - name: Build TypeScript package
-      if: steps.type.outputs.type == 'typescript'
-      working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
-      run: npm run build
-
-    - name: Publish TypeScript package to npm
-      if: steps.type.outputs.type == 'typescript'
-      working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
-      run: |
-        set +e
-        OUTPUT=$(npm publish --access public 2>&1)
-        EXIT_CODE=$?
-        echo "$OUTPUT"
-        if [ $EXIT_CODE -ne 0 ]; then
-          if echo "$OUTPUT" | grep -q "cannot publish over"; then
-            echo "Package version already published, skipping..."
-            exit 0
-          fi
-          exit $EXIT_CODE
-        fi
-      env:
-        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/release.yml

@@ -13,15 +13,15 @@ jobs:
       id-token: write
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Install uv
-      uses: astral-sh/setup-uv@v7
+      uses: astral-sh/setup-uv@v5
       with:
         enable-cache: true
 
     - name: Set up Python
-      uses: actions/setup-python@v6
+      uses: actions/setup-python@v5
       with:
         python-version-file: ".python-version"
 
@@ -30,39 +30,33 @@ jobs:
       working-directory: ./hindsight-clients/python
       run: uv build --out-dir dist
 
-    - name: Build hindsight-api-slim
-      working-directory: ./hindsight-api-slim
-      run: uv build --out-dir dist
-
     - name: Build hindsight-api
       working-directory: ./hindsight-api
       run: uv build --out-dir dist
 
     - name: Build hindsight-all
-      working-directory: ./hindsight-all
+      working-directory: ./hindsight
       run: uv build --out-dir dist
 
-    - name: Build hindsight-all-slim
-      working-directory: ./hindsight-all-slim
+    - name: Build hindsight-litellm
+      working-directory: ./hindsight-integrations/litellm
       run: uv build --out-dir dist
 
     - name: Build hindsight-embed
       working-directory: ./hindsight-embed
       run: uv build --out-dir dist
 
-    # Publish in order (client and api-slim first, then api/all wrappers which depend on them)
+    - name: Build hindsight-crewai
+      working-directory: ./hindsight-integrations/crewai
+      run: uv build --out-dir dist
+
+    # Publish in order (client and api first, then hindsight-all which depends on them)
     - name: Publish hindsight-client to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         packages-dir: ./hindsight-clients/python/dist
         skip-existing: true
 
-    - name: Publish hindsight-api-slim to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        packages-dir: ./hindsight-api-slim/dist
-        skip-existing: true
-
     - name: Publish hindsight-api to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
@@ -72,13 +66,13 @@ jobs:
     - name: Publish hindsight-all to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
-        packages-dir: ./hindsight-all/dist
+        packages-dir: ./hindsight/dist
         skip-existing: true
 
-    - name: Publish hindsight-all-slim to PyPI
+    - name: Publish hindsight-litellm to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
-        packages-dir: ./hindsight-all-slim/dist
+        packages-dir: ./hindsight-integrations/litellm/dist
         skip-existing: true
 
     - name: Publish hindsight-embed to PyPI
@@ -87,18 +81,24 @@ jobs:
         packages-dir: ./hindsight-embed/dist
         skip-existing: true
 
+    - name: Publish hindsight-crewai to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: ./hindsight-integrations/crewai/dist
+        skip-existing: true
+
     # Upload artifacts for GitHub release
     - name: Upload artifacts
-      uses: actions/upload-artifact@v7
+      uses: actions/upload-artifact@v4
       with:
         name: python-packages
         path: |
           hindsight-clients/python/dist/*
-          hindsight-api-slim/dist/*
           hindsight-api/dist/*
-          hindsight-all/dist/*
-          hindsight-all-slim/dist/*
+          hindsight/dist/*
+          hindsight-integrations/litellm/dist/*
           hindsight-embed/dist/*
+          hindsight-integrations/crewai/dist/*
         retention-days: 1
 
   release-typescript-client:
@@ -106,10 +106,10 @@ jobs:
     environment: npm
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Set up Node.js
-      uses: actions/setup-node@v6
+      uses: actions/setup-node@v4
       with:
         node-version: '20'
         registry-url: 'https://registry.npmjs.org'
@@ -144,21 +144,168 @@ jobs:
       run: npm pack
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@v7
+      uses: actions/upload-artifact@v4
       with:
         name: typescript-client
         path: hindsight-clients/typescript/*.tgz
         retention-days: 1
 
+  release-openclaw-integration:
+    runs-on: ubuntu-latest
+    environment: npm
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '22'
+        registry-url: 'https://registry.npmjs.org'
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/openclaw
+      run: npm ci
+
+    - name: Build
+      working-directory: ./hindsight-integrations/openclaw
+      run: npm run build
+
+    - name: Publish to npm
+      working-directory: ./hindsight-integrations/openclaw
+      run: |
+        set +e
+        OUTPUT=$(npm publish --access public 2>&1)
+        EXIT_CODE=$?
+        echo "$OUTPUT"
+        if [ $EXIT_CODE -ne 0 ]; then
+          if echo "$OUTPUT" | grep -q "cannot publish over"; then
+            echo "Package version already published, skipping..."
+            exit 0
+          fi
+          exit $EXIT_CODE
+        fi
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+    - name: Pack for GitHub release
+      working-directory: ./hindsight-integrations/openclaw
+      run: npm pack
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: openclaw-integration
+        path: hindsight-integrations/openclaw/*.tgz
+        retention-days: 1
+
+  release-ai-sdk-integration:
+    runs-on: ubuntu-latest
+    environment: npm
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '22'
+        registry-url: 'https://registry.npmjs.org'
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/ai-sdk
+      run: npm ci
+
+    - name: Build
+      working-directory: ./hindsight-integrations/ai-sdk
+      run: npm run build
+
+    - name: Publish to npm
+      working-directory: ./hindsight-integrations/ai-sdk
+      run: |
+        set +e
+        OUTPUT=$(npm publish --access public 2>&1)
+        EXIT_CODE=$?
+        echo "$OUTPUT"
+        if [ $EXIT_CODE -ne 0 ]; then
+          if echo "$OUTPUT" | grep -q "cannot publish over"; then
+            echo "Package version already published, skipping..."
+            exit 0
+          fi
+          exit $EXIT_CODE
+        fi
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+    - name: Pack for GitHub release
+      working-directory: ./hindsight-integrations/ai-sdk
+      run: npm pack
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: ai-sdk-integration
+        path: hindsight-integrations/ai-sdk/*.tgz
+        retention-days: 1
+
+  release-chat-integration:
+    runs-on: ubuntu-latest
+    environment: npm
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '22'
+        registry-url: 'https://registry.npmjs.org'
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/chat
+      run: npm ci
+
+    - name: Build
+      working-directory: ./hindsight-integrations/chat
+      run: npm run build
+
+    - name: Publish to npm
+      working-directory: ./hindsight-integrations/chat
+      run: |
+        set +e
+        OUTPUT=$(npm publish --access public 2>&1)
+        EXIT_CODE=$?
+        echo "$OUTPUT"
+        if [ $EXIT_CODE -ne 0 ]; then
+          if echo "$OUTPUT" | grep -q "cannot publish over"; then
+            echo "Package version already published, skipping..."
+            exit 0
+          fi
+          exit $EXIT_CODE
+        fi
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+    - name: Pack for GitHub release
+      working-directory: ./hindsight-integrations/chat
+      run: npm pack
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: chat-integration
+        path: hindsight-integrations/chat/*.tgz
+        retention-days: 1
+
   release-control-plane:
     runs-on: ubuntu-latest
     environment: npm
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Set up Node.js
-      uses: actions/setup-node@v6
+      uses: actions/setup-node@v4
       with:
         node-version: '20'
         registry-url: 'https://registry.npmjs.org'
@@ -206,7 +353,7 @@ jobs:
       run: npm pack
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@v7
+      uses: actions/upload-artifact@v4
       with:
         name: control-plane
         path: hindsight-control-plane/*.tgz
@@ -229,13 +376,9 @@ jobs:
             target: aarch64-apple-darwin
             artifact_name: hindsight
             asset_name: hindsight-darwin-arm64
-          - os: ubuntu-24.04-arm
-            target: aarch64-unknown-linux-gnu
-            artifact_name: hindsight
-            asset_name: hindsight-linux-arm64
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Install Rust
       uses: dtolnay/rust-toolchain@stable
@@ -253,7 +396,7 @@ jobs:
         chmod +x artifacts/${{ matrix.asset_name }}
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@v7
+      uses: actions/upload-artifact@v4
       with:
         name: rust-cli-${{ matrix.asset_name }}
         path: artifacts/${{ matrix.asset_name }}
@@ -294,7 +437,7 @@ jobs:
               PRELOAD_ML_MODELS=false
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Free Disk Space
       uses: jlumbroso/free-disk-space@main
@@ -308,13 +451,13 @@ jobs:
         swap-storage: true
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v4
+      uses: docker/setup-qemu-action@v3
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v4
+      uses: docker/setup-buildx-action@v3
 
     - name: Log in to GitHub Container Registry
-      uses: docker/login-action@v4
+      uses: docker/login-action@v3
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
@@ -326,7 +469,7 @@ jobs:
 
     - name: Extract metadata for release tags
       id: meta
-      uses: docker/metadata-action@v6
+      uses: docker/metadata-action@v5
       with:
         images: ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}
         flavor: |
@@ -342,7 +485,7 @@ jobs:
     # # Step 1: Build for local testing (single platform, no push)
     # # This creates an identical image to what will be released, just for one platform
     # - name: Build image for testing
-    #   uses: docker/build-push-action@v7
+    #   uses: docker/build-push-action@v6
     #   with:
     #     context: .
     #     file: docker/standalone/Dockerfile
@@ -361,7 +504,7 @@ jobs:
 
     # Build multi-platform and push to release tags
     - name: Build and push release images
-      uses: docker/build-push-action@v7
+      uses: docker/build-push-action@v6
       with:
         context: .
         file: docker/standalone/Dockerfile
@@ -379,7 +522,7 @@ jobs:
       packages: write
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Install Helm
       uses: azure/setup-helm@v4
@@ -399,7 +542,7 @@ jobs:
       run: helm push helm-packages/*.tgz oci://ghcr.io/${{ github.repository_owner }}/charts
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@v7
+      uses: actions/upload-artifact@v4
       with:
         name: helm-chart
         path: helm-packages/*.tgz
@@ -407,55 +550,73 @@ jobs:
 
   create-github-release:
     runs-on: ubuntu-latest
-    needs: [release-python-packages, release-typescript-client, release-control-plane, release-rust-cli, release-docker-images, release-helm-chart]
+    needs: [release-python-packages, release-typescript-client, release-openclaw-integration, release-ai-sdk-integration, release-chat-integration, release-control-plane, release-rust-cli, release-docker-images, release-helm-chart]
     permissions:
       contents: write
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Extract version from tag
       id: get_version
       run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
 
     - name: Download Python packages
-      uses: actions/download-artifact@v8
+      uses: actions/download-artifact@v4
       with:
         name: python-packages
         path: ./artifacts/python-packages
 
     - name: Download TypeScript client
-      uses: actions/download-artifact@v8
+      uses: actions/download-artifact@v4
       with:
         name: typescript-client
         path: ./artifacts/typescript-client
 
+    - name: Download OpenClaw Integration
+      uses: actions/download-artifact@v4
+      with:
+        name: openclaw-integration
+        path: ./artifacts/openclaw-integration
+
+    - name: Download AI SDK Integration
+      uses: actions/download-artifact@v4
+      with:
+        name: ai-sdk-integration
+        path: ./artifacts/ai-sdk-integration
+
+    - name: Download Chat Integration
+      uses: actions/download-artifact@v4
+      with:
+        name: chat-integration
+        path: ./artifacts/chat-integration
+
     - name: Download Control Plane
-      uses: actions/download-artifact@v8
+      uses: actions/download-artifact@v4
       with:
         name: control-plane
         path: ./artifacts/control-plane
 
     - name: Download Rust CLI (Linux)
-      uses: actions/download-artifact@v8
+      uses: actions/download-artifact@v4
       with:
         name: rust-cli-hindsight-linux-amd64
         path: ./artifacts/rust-cli-linux
 
     - name: Download Rust CLI (macOS Intel)
-      uses: actions/download-artifact@v8
+      uses: actions/download-artifact@v4
       with:
         name: rust-cli-hindsight-darwin-amd64
         path: ./artifacts/rust-cli-darwin-amd64
 
     - name: Download Rust CLI (macOS ARM)
-      uses: actions/download-artifact@v8
+      uses: actions/download-artifact@v4
       with:
         name: rust-cli-hindsight-darwin-arm64
         path: ./artifacts/rust-cli-darwin-arm64
 
     - name: Download Helm chart
-      uses: actions/download-artifact@v8
+      uses: actions/download-artifact@v4
       with:
         name: helm-chart
         path: ./artifacts/helm-chart
@@ -465,13 +626,18 @@ jobs:
         mkdir -p release-assets
         # Python packages
         cp artifacts/python-packages/hindsight-clients/python/dist/* release-assets/ || true
-        cp artifacts/python-packages/hindsight-api-slim/dist/* release-assets/ || true
         cp artifacts/python-packages/hindsight-api/dist/* release-assets/ || true
-        cp artifacts/python-packages/hindsight-all/dist/* release-assets/ || true
-        cp artifacts/python-packages/hindsight-all-slim/dist/* release-assets/ || true
+        cp artifacts/python-packages/hindsight/dist/* release-assets/ || true
+        cp artifacts/python-packages/hindsight-integrations/litellm/dist/* release-assets/ || true
         cp artifacts/python-packages/hindsight-embed/dist/* release-assets/ || true
         # TypeScript client
         cp artifacts/typescript-client/*.tgz release-assets/ || true
+        # OpenClaw Integration
+        cp artifacts/openclaw-integration/*.tgz release-assets/ || true
+        # AI SDK Integration
+        cp artifacts/ai-sdk-integration/*.tgz release-assets/ || true
+        # Chat Integration
+        cp artifacts/chat-integration/*.tgz release-assets/ || true
         # Control Plane
         cp artifacts/control-plane/*.tgz release-assets/ || true
         # Rust CLI binaries

CLAUDE.md

@@ -17,20 +17,20 @@ Hindsight is an agent memory system that provides long-term memory for AI agents
 ./scripts/dev/start-api.sh
 
 # Run all tests (parallelized with pytest-xdist)
-cd hindsight-api-slim && uv run pytest tests/
+cd hindsight-api && uv run pytest tests/
 
 # Run specific test file
-cd hindsight-api-slim && uv run pytest tests/test_http_api_integration.py -v
+cd hindsight-api && uv run pytest tests/test_http_api_integration.py -v
 
 # Run single test function
-cd hindsight-api-slim && uv run pytest tests/test_retain.py::test_retain_simple -v
+cd hindsight-api && uv run pytest tests/test_retain.py::test_retain_simple -v
 
 # Lint and format
-cd hindsight-api-slim && uv run ruff check .
-cd hindsight-api-slim && uv run ruff format .
+cd hindsight-api && uv run ruff check .
+cd hindsight-api && uv run ruff format .
 
 # Type checking (uses ty - extremely fast type checker from Astral)
-cd hindsight-api-slim && uv run ty check hindsight_api/
+cd hindsight-api && uv run ty check hindsight_api/
 ```
 
 ### Control Plane (Next.js)
@@ -72,7 +72,7 @@ cd hindsight-control-plane && npm run dev
 ## Architecture
 
 ### Monorepo Structure
-- **hindsight-api-slim/**: Core FastAPI server with memory engine (Python, uv)
+- **hindsight-api/**: Core FastAPI server with memory engine (Python, uv)
 - **hindsight/**: Embedded Python bundle (hindsight-all package)
 - **hindsight-control-plane/**: Admin UI (Next.js, npm)
 - **hindsight-cli/**: CLI tool (Rust, cargo, uses progenitor for API client)
@@ -81,9 +81,9 @@ cd hindsight-control-plane && npm run dev
 - **hindsight-integrations/**: Framework integrations (LiteLLM, OpenAI)
 - **hindsight-dev/**: Development tools and benchmarks
 
-### Core Engine (hindsight-api-slim/hindsight_api/engine/)
+### Core Engine (hindsight-api/hindsight_api/engine/)
 - `memory_engine.py`: Main orchestrator (~170KB) for retain/recall/reflect operations
-- `llm_wrapper.py`: LLM abstraction supporting OpenAI, Anthropic, Gemini, Groq, MiniMax, Ollama, LM Studio
+- `llm_wrapper.py`: LLM abstraction supporting OpenAI, Anthropic, Gemini, Groq, Ollama, LM Studio
 - `embeddings.py`: Embedding generation (local sentence-transformers or TEI)
 - `cross_encoder.py`: Reranking (local or TEI)
 - `entity_resolver.py`: Entity extraction and normalization
@@ -101,7 +101,7 @@ cd hindsight-control-plane && npm run dev
 - `fusion.py`: Reciprocal rank fusion for combining results
 - `reranking.py`: Cross-encoder reranking
 
-### API Layer (hindsight-api-slim/hindsight_api/api/)
+### API Layer (hindsight-api/hindsight_api/api/)
 - `http.py`: FastAPI HTTP routers (~80KB) for all REST endpoints
 - `mcp.py`: Model Context Protocol server implementation
 
@@ -111,13 +111,13 @@ Main operations:
 - **Reflect**: Disposition-aware reasoning using memories and mental models.
 
 ### Database
-PostgreSQL with pgvector. Schema managed via Alembic migrations in `hindsight-api-slim/hindsight_api/alembic/`. Migrations run automatically on API startup.
+PostgreSQL with pgvector. Schema managed via Alembic migrations in `hindsight-api/hindsight_api/alembic/`. Migrations run automatically on API startup.
 
 Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
 
 ### Adding Database Migrations
 
-1. **Create a new migration file** in `hindsight-api-slim/hindsight_api/alembic/versions/`:
+1. **Create a new migration file** in `hindsight-api/hindsight_api/alembic/versions/`:
    - File name format: `<revision_id>_<description>.py` (e.g., `f1a2b3c4d5e6_add_new_index.py`)
    - Use a unique hex revision ID (12 chars)
    - Set `down_revision` to the previous migration's revision ID
@@ -154,7 +154,7 @@ Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
 
 3. **Run migrations locally**:
    ```bash
-   # Set database URL and run migrations for the base schema plus all tenants
+   # Set database URL and run migrations
    uv run hindsight-admin run-db-migration
 
    # Run on a specific tenant schema
@@ -251,7 +251,7 @@ Fields must be categorized as either **hierarchical** (can be overridden per-ten
 
 #### Adding a New Configuration Field
 
-1. **config.py** (`hindsight-api-slim/hindsight_api/config.py`):
+1. **config.py** (`hindsight-api/hindsight_api/config.py`):
    - Add `ENV_*` constant for the environment variable name (e.g., `ENV_MY_SETTING = "HINDSIGHT_API_MY_SETTING"`)
    - Add `DEFAULT_*` constant for the default value
    - Add field to `HindsightConfig` dataclass with type annotation
@@ -268,7 +268,7 @@ Fields must be categorized as either **hierarchical** (can be overridden per-ten
    # Static field - just don't add to _HIERARCHICAL_FIELDS
    ```
 
-2. **main.py** (`hindsight-api-slim/hindsight_api/main.py`):
+2. **main.py** (`hindsight-api/hindsight_api/main.py`):
    - Add field to the manual `HindsightConfig()` constructor call (search for "CLI override")
 
 3. **Use hierarchical config in MemoryEngine**:
@@ -308,14 +308,14 @@ cp .env.example .env
 # Edit .env with LLM API key
 
 # Python deps
-uv sync --directory hindsight-api-slim/
+uv sync --directory hindsight-api/
 
 # Node deps (uses npm workspaces)
 npm install
 ```
 
 Required env vars:
-- `HINDSIGHT_API_LLM_PROVIDER`: openai, anthropic, gemini, groq, minimax, ollama, lmstudio
+- `HINDSIGHT_API_LLM_PROVIDER`: openai, anthropic, gemini, groq, ollama, lmstudio
 - `HINDSIGHT_API_LLM_API_KEY`: Your API key
 - `HINDSIGHT_API_LLM_MODEL`: Model name (e.g., gpt-4o-mini, claude-sonnet-4-20250514)
 

README.md

@@ -9,9 +9,8 @@
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 ![PyPI - Downloads](https://img.shields.io/pypi/dm/hindsight-api?label=PyPI)
 ![NPM Downloads](https://img.shields.io/npm/dm/%40vectorize-io%2Fhindsight-client?logoColor=orange&label=NPM&color=blue&link=https%3A%2F%2Fwww.npmjs.com%2Fpackage%2F%40vectorize-io%2Fhindsight-client)
-<br/>
 
-<a href="https://trendshift.io/repositories/15603" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15603" alt="vectorize-io%2Fhindsight | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+
 </div>
 
 ---
@@ -70,7 +69,7 @@ docker run --rm -it --pull always -p 8888:8888 -p 9999:9999 \
 >API: http://localhost:8888
 >UI: http://localhost:9999
 
-You can modify the LLM provider by setting `HINDSIGHT_API_LLM_PROVIDER`. Valid options are `openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`, and `minimax`. The documentation provides more details on [supported models](https://hindsight.vectorize.io/developer/models).
+You can modify the LLM provider by setting `HINDSIGHT_API_LLM_PROVIDER`. Valid options are `openai`, `anthropic`, `gemini`, `groq`, `ollama`, and `lmstudio`. The documentation provides more details on [supported models](https://hindsight.vectorize.io/developer/models).
 
 
 

deno.lock

@@ -1,139 +0,0 @@
-{
-  "version": "5",
-  "specifiers": {
-    "jsr:@std/assert@^1.0.17": "1.0.19",
-    "jsr:@std/assert@^1.0.19": "1.0.19",
-    "jsr:@std/expect@*": "1.0.18",
-    "jsr:@std/internal@^1.0.12": "1.0.12",
-    "jsr:@std/path@^1.1.4": "1.1.4",
-    "jsr:@std/testing@*": "1.0.17"
-  },
-  "jsr": {
-    "@std/[email protected]": {
-      "integrity": "eaada96ee120cb980bc47e040f82814d786fe8162ecc53c91d8df60b8755991e",
-      "dependencies": [
-        "jsr:@std/internal"
-      ]
-    },
-    "@std/[email protected]": {
-      "integrity": "8566eab35200466f8609eb7e7aed062ed0db314e9a258d5d201b1b8997ce801a",
-      "dependencies": [
-        "jsr:@std/assert@^1.0.19",
-        "jsr:@std/internal",
-        "jsr:@std/path"
-      ]
-    },
-    "@std/[email protected]": {
-      "integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027"
-    },
-    "@std/[email protected]": {
-      "integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5",
-      "dependencies": [
-        "jsr:@std/internal"
-      ]
-    },
-    "@std/[email protected]": {
-      "integrity": "87bdc2700fa98249d48a17cd72413352d3d3680dcfbdb64947fd0982d6bbf681",
-      "dependencies": [
-        "jsr:@std/assert@^1.0.17",
-        "jsr:@std/internal"
-      ]
-    }
-  },
-  "workspace": {
-    "members": {
-      "hindsight-clients/typescript": {
-        "packageJson": {
-          "dependencies": [
-            "npm:@hey-api/[email protected]",
-            "npm:@types/jest@29",
-            "npm:@types/node@20",
-            "npm:jest@29",
-            "npm:ts-jest@29",
-            "npm:tsup@^8.5.1",
-            "npm:typescript@5"
-          ]
-        }
-      },
-      "hindsight-control-plane": {
-        "packageJson": {
-          "dependencies": [
-            "npm:@eslint/eslintrc@^3.3.3",
-            "npm:@eslint/js@^9.39.2",
-            "npm:@radix-ui/react-alert-dialog@^1.1.15",
-            "npm:@radix-ui/react-checkbox@^1.3.3",
-            "npm:@radix-ui/react-dialog@^1.1.15",
-            "npm:@radix-ui/react-dropdown-menu@^2.1.16",
-            "npm:@radix-ui/react-label@^2.1.8",
-            "npm:@radix-ui/react-popover@^1.1.15",
-            "npm:@radix-ui/react-radio-group@^1.3.8",
-            "npm:@radix-ui/react-select@^2.2.6",
-            "npm:@radix-ui/react-slider@^1.3.6",
-            "npm:@radix-ui/react-slot@^1.2.4",
-            "npm:@radix-ui/react-switch@^1.2.6",
-            "npm:@radix-ui/react-tabs@^1.1.13",
-            "npm:@radix-ui/react-tooltip@^1.2.8",
-            "npm:@tailwindcss/postcss@^4.1.17",
-            "npm:@tailwindcss/typography@~0.5.19",
-            "npm:@types/cytoscape@^3.21.9",
-            "npm:@types/node@^24.10.0",
-            "npm:@types/react-dom@^19.2.2",
-            "npm:@types/react@^19.2.2",
-            "npm:autoprefixer@^10.4.21",
-            "npm:class-variance-authority@~0.7.1",
-            "npm:clsx@^2.1.1",
-            "npm:cmdk@^1.1.1",
-            "npm:cytoscape-fcose@^2.2.0",
-            "npm:cytoscape@^3.33.1",
-            "npm:eslint-config-next@^16.0.1",
-            "npm:eslint-plugin-react-hooks@^7.0.1",
-            "npm:eslint-plugin-react@^7.37.5",
-            "npm:eslint@^9.39.1",
-            "npm:[email protected]",
-            "npm:next-themes@~0.4.6",
-            "npm:next@^16.1.6",
-            "npm:postcss@^8.5.6",
-            "npm:prettier@^3.7.4",
-            "npm:react-chrono@^2.9.1",
-            "npm:react-dom@^19.2.0",
-            "npm:react-markdown@^10.1.0",
-            "npm:react18-json-view@~0.2.9",
-            "npm:react@^19.2.0",
-            "npm:recharts@^3.5.1",
-            "npm:remark-gfm@^4.0.1",
-            "npm:sonner@^2.0.7",
-            "npm:tailwind-merge@^3.4.0",
-            "npm:tailwindcss-animate@^1.0.7",
-            "npm:tailwindcss@^4.1.17",
-            "npm:[email protected]",
-            "npm:typescript-eslint@^8.50.0",
-            "npm:typescript@^5.9.3"
-          ]
-        }
-      },
-      "hindsight-docs": {
-        "packageJson": {
-          "dependencies": [
-            "npm:@docusaurus/[email protected]",
-            "npm:@docusaurus/[email protected]",
-            "npm:@docusaurus/[email protected]",
-            "npm:@docusaurus/theme-common@^3.9.2",
-            "npm:@docusaurus/theme-mermaid@^3.9.2",
-            "npm:@docusaurus/[email protected]",
-            "npm:@docusaurus/[email protected]",
-            "npm:@easyops-cn/docusaurus-search-local@~0.52.2",
-            "npm:@mdx-js/react@3",
-            "npm:clsx@2",
-            "npm:prism-react-renderer@^2.3.0",
-            "npm:raw-loader@^4.0.2",
-            "npm:react-dom@19",
-            "npm:react-icons@^5.6.0",
-            "npm:react@19",
-            "npm:redocusaurus@^2.5.0",
-            "npm:typescript@~5.6.2"
-          ]
-        }
-      }
-    }
-  }
-}

docker/standalone/Dockerfile

@@ -42,22 +42,25 @@ RUN apt-get update && apt-get install -y \
     && pip install --no-cache-dir uv
 
 # Copy dependency files and README (required by pyproject.toml)
-COPY hindsight-api-slim/pyproject.toml ./api/
-COPY hindsight-api-slim/README.md ./api/
+COPY hindsight-api/pyproject.toml ./api/
+COPY hindsight-api/README.md ./api/
 
 WORKDIR /app/api
 
-# Sync dependencies using appropriate extras based on INCLUDE_LOCAL_MODELS
-# local-ml: torch, sentence-transformers, transformers, einops, flashrank, mlx (optional)
-# embedded-db: pg0-embedded (always included for embedded PostgreSQL support)
-RUN if [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
-        uv sync --extra local-ml --extra embedded-db; \
-    else \
-        uv sync --extra embedded-db; \
+# Remove local ML model dependencies if INCLUDE_LOCAL_MODELS=false
+# This creates a smaller image when using external providers (TEI, OpenAI, Cohere)
+RUN if [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then \
+        echo "Removing local-models dependencies (sentence-transformers, torch, transformers)..." && \
+        sed -i '/"sentence-transformers/d' pyproject.toml && \
+        sed -i '/"transformers/d' pyproject.toml && \
+        sed -i '/"torch/d' pyproject.toml; \
     fi
 
+# Sync dependencies (will create lock file if needed)
+RUN uv sync
+
 # Copy source code (alembic migrations are inside hindsight_api/)
-COPY hindsight-api-slim/hindsight_api ./hindsight_api
+COPY hindsight-api/hindsight_api ./hindsight_api
 
 # Install the local package (uv sync only installed dependencies, not the package itself)
 RUN uv pip install -e .

docker/standalone/start-all.sh

@@ -77,32 +77,18 @@ PIDS=()
 # Start API if enabled
 if [ "$ENABLE_API" = "true" ]; then
     cd /app/api
-    API_HEALTH_URL="${HINDSIGHT_API_HEALTH_URL:-http://localhost:8888/health}"
-    API_STARTUP_WAIT_SECONDS="${HINDSIGHT_API_STARTUP_WAIT_SECONDS:-300}"
-
     # Run API directly - Python's PYTHONUNBUFFERED=1 handles output buffering
     hindsight-api &
     API_PID=$!
     PIDS+=($API_PID)
 
     # Wait for API to be ready
-    api_ready=false
-    for ((i=1; i<=API_STARTUP_WAIT_SECONDS; i++)); do
-        if ! kill -0 "$API_PID" 2>/dev/null; then
-            wait "$API_PID"
-            exit $?
-        fi
-        if curl -sf "$API_HEALTH_URL" &>/dev/null; then
-            api_ready=true
+    for i in {1..60}; do
+        if curl -sf http://localhost:8888/health &>/dev/null; then
             break
         fi
         sleep 1
     done
-
-    if [ "$api_ready" != "true" ]; then
-        echo "❌ API did not become healthy within ${API_STARTUP_WAIT_SECONDS}s"
-        exit 1
-    fi
 else
     echo "API disabled (HINDSIGHT_ENABLE_API=false)"
 fi
@@ -111,7 +97,6 @@ fi
 if [ "$ENABLE_CP" = "true" ]; then
     echo "🎛️  Starting Control Plane..."
     cd /app/control-plane
-    export HOSTNAME="${HINDSIGHT_CP_HOSTNAME:-0.0.0.0}"
     PORT="${HINDSIGHT_CP_PORT:-9999}" node server.js &
     CP_PID=$!
     PIDS+=($CP_PID)

docker/test-image.sh

@@ -49,9 +49,6 @@
 
 set -euo pipefail
 
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-REPO_ROOT="$(dirname "$SCRIPT_DIR")"
-
 # Colors for output
 RED='\033[0;31m'
 GREEN='\033[0;32m'
@@ -181,21 +178,6 @@ for i in $(seq 1 "$TIMEOUT"); do
         echo "=== Health Response ==="
         curl -s "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" | python3 -m json.tool 2>/dev/null || curl -s "http://localhost:${HEALTH_PORT}${HEALTH_PATH}"
         echo ""
-
-        # Run retain/recall smoke test for API targets
-        if [ "$TARGET" != "cp-only" ]; then
-            echo ""
-            echo "=== Retain/Recall Smoke Test ==="
-            if ! "$REPO_ROOT/scripts/smoke-test-slim.sh" "http://localhost:${HEALTH_PORT}"; then
-                echo ""
-                echo "=== Container Logs (last 50 lines) ==="
-                docker logs "$CONTAINER_NAME" 2>&1 | tail -50
-                echo ""
-                echo -e "${RED}Smoke test FAILED${NC}"
-                exit 1
-            fi
-        fi
-
         echo ""
         echo "=== Container Logs (last 50 lines) ==="
         docker logs "$CONTAINER_NAME" 2>&1 | tail -50

helm/hindsight/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: hindsight
 description: Hindsight helm chart
 type: application
-version: 0.4.19
-appVersion: "0.4.19"
+version: 0.4.14
+appVersion: "0.4.14"
 keywords:
   - ai
   - memory

hindsight-all-slim/pyproject.toml

@@ -1,33 +0,0 @@
-[build-system]
-requires = ["setuptools>=61"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "hindsight-all-slim"
-version = "0.4.19"
-description = "Hindsight: Agent Memory That Works Like Human Memory - Slim All-in-One Bundle"
-readme = "README.md"
-requires-python = ">=3.11"
-dependencies = [
-    "hindsight-api-slim>=0.4.17",
-    "hindsight-client>=0.0.7",
-    "hindsight-embed>=0.1.0",
-]
-
-[tool.uv.sources]
-hindsight-api-slim = { workspace = true }
-hindsight-client = { workspace = true }
-hindsight-embed = { workspace = true }
-
-[project.optional-dependencies]
-test = [
-    "pytest>=7.0.0",
-    "pytest-asyncio>=0.21.0",
-]
-
-[tool.setuptools]
-packages = []
-
-[tool.pytest.ini_options]
-asyncio_mode = "auto"
-asyncio_default_fixture_loop_scope = "function"

hindsight-all/README.md

@@ -1,48 +0,0 @@
-# hindsight-all
-
-All-in-one package for Hindsight - Agent Memory That Works Like Human Memory
-
-## Quick Start
-
-```python
-from hindsight import start_server, HindsightClient
-
-# Start server with embedded PostgreSQL
-server = start_server(
-    llm_provider="groq",
-    llm_api_key="your-api-key",
-    llm_model="openai/gpt-oss-120b"
-)
-
-# Create client
-client = HindsightClient(base_url=server.url)
-
-# Store memories
-client.put(agent_id="assistant", content="User prefers Python for data analysis")
-
-# Search memories
-results = client.search(agent_id="assistant", query="programming preferences")
-
-# Generate contextual response
-response = client.think(agent_id="assistant", query="What languages should I recommend?")
-
-# Stop server when done
-server.stop()
-```
-
-## Using Context Manager
-
-```python
-from hindsight import HindsightServer, HindsightClient
-
-with HindsightServer(llm_provider="groq", llm_api_key="...") as server:
-    client = HindsightClient(base_url=server.url)
-    # ... use client ...
-# Server automatically stops
-```
-
-## Installation
-
-```bash
-pip install hindsight-all
-```

hindsight-all/hindsight/client_wrapper.py

@@ -1,423 +0,0 @@
-"""
-Wrapper for Hindsight client that adds API namespaces.
-
-Provides organized access to different parts of the Hindsight API through
-namespaces like .banks, .mental_models, etc.
-"""
-
-from __future__ import annotations
-
-from typing import Any
-
-from hindsight_client import Hindsight
-
-
-class BanksAPI:
-    """Namespace for bank-related operations.
-    
-    Provides methods to create, delete, and manage memory banks.
-    """
-
-    def __init__(self, client: Hindsight):
-        self._client = client
-
-    def create(
-        self,
-        bank_id: str,
-        name: str | None = None,
-        mission: str | None = None,
-        disposition: dict[str, Any] | None = None,
-    ) -> Any:
-        """Create a new bank.
-        
-        Args:
-            bank_id: Unique identifier for the bank.
-            name: Optional display name for the bank.
-            mission: Optional mission statement for the bank.
-            disposition: Optional disposition configuration dict.
-            
-        Returns:
-            Bank creation response from the API.
-        """
-        return self._client.create_bank(
-            bank_id=bank_id,
-            name=name,
-            mission=mission,
-            disposition=disposition,
-        )
-
-    def delete(self, bank_id: str) -> Any:
-        """Delete a bank.
-        
-        Args:
-            bank_id: The ID of the bank to delete.
-            
-        Returns:
-            Deletion response from the API.
-        """
-        return self._client.delete_bank(bank_id=bank_id)
-
-    def set_mission(self, bank_id: str, mission: str) -> Any:
-        """Set or update the mission for a bank.
-        
-        Args:
-            bank_id: The ID of the bank.
-            mission: The mission statement to set.
-            
-        Returns:
-            API response confirming the update.
-        """
-        return self._client.set_mission(bank_id=bank_id, mission=mission)
-
-    def set_disposition(self, bank_id: str, disposition: dict[str, Any]) -> Any:
-        """Set or update the disposition for a bank.
-        
-        Args:
-            bank_id: The ID of the bank.
-            disposition: The disposition configuration dict.
-            
-        Returns:
-            API response confirming the update.
-        """
-        return self._client.set_disposition(bank_id=bank_id, disposition=disposition)
-
-    def list(self) -> Any:
-        """List all banks.
-        
-        Returns:
-            List of banks from the API.
-        """
-        from hindsight_client.hindsight_client import _run_async
-
-        return _run_async(self._client._banks_api.list_banks())
-
-
-class MentalModelsAPI:
-    """Namespace for mental model operations.
-    
-    Mental models are reusable knowledge structures that guide agent behavior.
-    """
-
-    def __init__(self, client: Hindsight):
-        self._client = client
-
-    def create(
-        self,
-        bank_id: str,
-        name: str,
-        content: str,
-        tags: list[str] | None = None,
-    ) -> Any:
-        """Create a new mental model.
-        
-        Args:
-            bank_id: The ID of the bank to add the model to.
-            name: Name for the mental model.
-            content: The content/instructions for the mental model.
-            tags: Optional list of tags for categorization.
-            
-        Returns:
-            Creation response from the API.
-        """
-        return self._client.create_mental_model(
-            bank_id=bank_id,
-            name=name,
-            content=content,
-            tags=tags,
-        )
-
-    def list(self, bank_id: str, tags: list[str] | None = None) -> Any:
-        """List all mental models for a bank.
-        
-        Args:
-            bank_id: The ID of the bank.
-            tags: Optional filter by tags.
-            
-        Returns:
-            List of mental models.
-        """
-        return self._client.list_mental_models(bank_id=bank_id, tags=tags)
-
-    def get(self, bank_id: str, mental_model_id: str) -> Any:
-        """Get a specific mental model.
-        
-        Args:
-            bank_id: The ID of the bank.
-            mental_model_id: The ID of the mental model.
-            
-        Returns:
-            The mental model details.
-        """
-        return self._client.get_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
-
-    def refresh(self, bank_id: str, mental_model_id: str) -> Any:
-        """Refresh a mental model.
-        
-        Args:
-            bank_id: The ID of the bank.
-            mental_model_id: The ID of the mental model to refresh.
-            
-        Returns:
-            Refresh response from the API.
-        """
-        return self._client.refresh_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
-
-    def update(
-        self,
-        bank_id: str,
-        mental_model_id: str,
-        name: str | None = None,
-        content: str | None = None,
-        tags: list[str] | None = None,
-    ) -> Any:
-        """Update a mental model.
-        
-        Args:
-            bank_id: The ID of the bank.
-            mental_model_id: The ID of the mental model to update.
-            name: Optional new name.
-            content: Optional new content.
-            tags: Optional new tags list.
-            
-        Returns:
-            Update response from the API.
-        """
-        return self._client.update_mental_model(
-            bank_id=bank_id,
-            mental_model_id=mental_model_id,
-            name=name,
-            content=content,
-            tags=tags,
-        )
-
-    def delete(self, bank_id: str, mental_model_id: str) -> Any:
-        """Delete a mental model.
-        
-        Args:
-            bank_id: The ID of the bank.
-            mental_model_id: The ID of the mental model to delete.
-            
-        Returns:
-            Deletion response from the API.
-        """
-        return self._client.delete_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
-
-
-class DirectivesAPI:
-    """Namespace for directive operations.
-    
-    Directives are explicit instructions that guide agent behavior.
-    """
-
-    def __init__(self, client: Hindsight):
-        self._client = client
-
-    def create(
-        self,
-        bank_id: str,
-        name: str,
-        content: str,
-        tags: list[str] | None = None,
-    ) -> Any:
-        """Create a new directive.
-        
-        Args:
-            bank_id: The ID of the bank to add the directive to.
-            name: Name for the directive.
-            content: The directive content/instructions.
-            tags: Optional list of tags for categorization.
-            
-        Returns:
-            Creation response from the API.
-        """
-        return self._client.create_directive(
-            bank_id=bank_id,
-            name=name,
-            content=content,
-            tags=tags,
-        )
-
-    def list(self, bank_id: str, tags: list[str] | None = None) -> Any:
-        """List all directives for a bank.
-        
-        Args:
-            bank_id: The ID of the bank.
-            tags: Optional filter by tags.
-            
-        Returns:
-            List of directives.
-        """
-        return self._client.list_directives(bank_id=bank_id, tags=tags)
-
-    def get(self, bank_id: str, directive_id: str) -> Any:
-        """Get a specific directive.
-        
-        Args:
-            bank_id: The ID of the bank.
-            directive_id: The ID of the directive.
-            
-        Returns:
-            The directive details.
-        """
-        return self._client.get_directive(bank_id=bank_id, directive_id=directive_id)
-
-    def update(
-        self,
-        bank_id: str,
-        directive_id: str,
-        name: str | None = None,
-        content: str | None = None,
-        tags: list[str] | None = None,
-    ) -> Any:
-        """Update a directive.
-        
-        Args:
-            bank_id: The ID of the bank.
-            directive_id: The ID of the directive to update.
-            name: Optional new name.
-            content: Optional new content.
-            tags: Optional new tags list.
-            
-        Returns:
-            Update response from the API.
-        """
-        return self._client.update_directive(
-            bank_id=bank_id,
-            directive_id=directive_id,
-            name=name,
-            content=content,
-            tags=tags,
-        )
-
-    def delete(self, bank_id: str, directive_id: str) -> Any:
-        """Delete a directive.
-        
-        Args:
-            bank_id: The ID of the bank.
-            directive_id: The ID of the directive to delete.
-            
-        Returns:
-            Deletion response from the API.
-        """
-        return self._client.delete_directive(bank_id=bank_id, directive_id=directive_id)
-
-
-class MemoriesAPI:
-    """Namespace for memory operations.
-    
-    Provides methods to query and retrieve stored memories.
-    """
-
-    def __init__(self, client: Hindsight):
-        self._client = client
-
-    def list(
-        self,
-        bank_id: str,
-        type: str | None = None,
-        search_query: str | None = None,
-        limit: int = 100,
-        offset: int = 0,
-    ) -> Any:
-        """List memories in a bank.
-        
-        Args:
-            bank_id: The ID of the bank to query.
-            type: Optional filter by memory type.
-            search_query: Optional search query for filtering.
-            limit: Maximum number of results to return (default: 100).
-            offset: Number of results to skip for pagination (default: 0).
-            
-        Returns:
-            List of memories matching the criteria.
-        """
-        return self._client.list_memories(
-            bank_id=bank_id,
-            type=type,
-            search_query=search_query,
-            limit=limit,
-            offset=offset,
-        )
-
-
-class HindsightClient(Hindsight):
-    """
-    Enhanced Hindsight client with organized API namespaces.
-
-    This wrapper extends the auto-generated Hindsight client with organized
-    access to different parts of the API through namespaces.
-
-    Example:
-        ```python
-        from hindsight import HindsightClient
-
-        client = HindsightClient(base_url="http://localhost:8888")
-
-        # Core operations (inherited from Hindsight)
-        client.retain(bank_id="test", content="Hello")
-        results = client.recall(bank_id="test", query="Hello")
-
-        # Organized API access through namespaces
-        client.banks.create(bank_id="test", name="Test Bank")
-        models = client.mental_models.list(bank_id="test")
-        directives = client.directives.list(bank_id="test")
-        memories = client.memories.list(bank_id="test")
-        ```
-        
-    Attributes:
-        banks: Namespace for bank management operations.
-        mental_models: Namespace for mental model operations.
-        directives: Namespace for directive operations.
-        memories: Namespace for memory listing operations.
-    """
-
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        super().__init__(*args, **kwargs)
-        self._banks_namespace: BanksAPI | None = None
-        self._mental_models_namespace: MentalModelsAPI | None = None
-        self._directives_namespace: DirectivesAPI | None = None
-        self._memories_namespace: MemoriesAPI | None = None
-
-    @property
-    def banks(self) -> BanksAPI:
-        """Access bank management operations.
-        
-        Returns:
-            BanksAPI instance for bank operations.
-        """
-        if self._banks_namespace is None:
-            self._banks_namespace = BanksAPI(self)
-        return self._banks_namespace
-
-    @property
-    def mental_models(self) -> MentalModelsAPI:
-        """Access mental model operations.
-        
-        Returns:
-            MentalModelsAPI instance for mental model operations.
-        """
-        if self._mental_models_namespace is None:
-            self._mental_models_namespace = MentalModelsAPI(self)
-        return self._mental_models_namespace
-
-    @property
-    def directives(self) -> DirectivesAPI:
-        """Access directive operations.
-        
-        Returns:
-            DirectivesAPI instance for directive operations.
-        """
-        if self._directives_namespace is None:
-            self._directives_namespace = DirectivesAPI(self)
-        return self._directives_namespace
-
-    @property
-    def memories(self) -> MemoriesAPI:
-        """Access memory listing operations.
-        
-        Returns:
-            MemoriesAPI instance for memory operations.
-        """
-        if self._memories_namespace is None:
-            self._memories_namespace = MemoriesAPI(self)
-        return self._memories_namespace

hindsight-api-slim/README.md

@@ -1,137 +0,0 @@
-# Hindsight API
-
-**Memory System for AI Agents** — Temporal + Semantic + Entity Memory Architecture using PostgreSQL with pgvector.
-
-Hindsight gives AI agents persistent memory that works like human memory: it stores facts, tracks entities and relationships, handles temporal reasoning ("what happened last spring?"), and forms opinions based on configurable disposition traits.
-
-## Installation
-
-```bash
-pip install hindsight-api
-```
-
-## Quick Start
-
-### Run the Server
-
-```bash
-# Set your LLM provider
-export HINDSIGHT_API_LLM_PROVIDER=openai
-export HINDSIGHT_API_LLM_API_KEY=sk-xxxxxxxxxxxx
-
-# Start the server (uses embedded PostgreSQL by default)
-hindsight-api
-```
-
-The server starts at http://localhost:8888 with:
-- REST API for memory operations
-- MCP server at `/mcp` for tool-use integration
-
-### Use the Python API
-
-```python
-from hindsight_api import MemoryEngine
-
-# Create and initialize the memory engine
-memory = MemoryEngine()
-await memory.initialize()
-
-# Create a memory bank for your agent
-bank = await memory.create_memory_bank(
-    name="my-assistant",
-    background="A helpful coding assistant"
-)
-
-# Store a memory
-await memory.retain(
-    memory_bank_id=bank.id,
-    content="The user prefers Python for data science projects"
-)
-
-# Recall memories
-results = await memory.recall(
-    memory_bank_id=bank.id,
-    query="What programming language does the user prefer?"
-)
-
-# Reflect with reasoning
-response = await memory.reflect(
-    memory_bank_id=bank.id,
-    query="Should I recommend Python or R for this ML project?"
-)
-```
-
-## CLI Options
-
-```bash
-hindsight-api --help
-
-# Common options
-hindsight-api --port 9000          # Custom port (default: 8888)
-hindsight-api --host 127.0.0.1     # Bind to localhost only
-hindsight-api --workers 4          # Multiple worker processes
-hindsight-api --log-level debug    # Verbose logging
-```
-
-## Configuration
-
-Configure via environment variables:
-
-| Variable | Description | Default |
-|----------|-------------|---------|
-| `HINDSIGHT_API_DATABASE_URL` | PostgreSQL connection string | `pg0` (embedded) |
-| `HINDSIGHT_API_LLM_PROVIDER` | `openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio` | `openai` |
-| `HINDSIGHT_API_LLM_API_KEY` | API key for LLM provider | - |
-| `HINDSIGHT_API_LLM_MODEL` | Model name | `gpt-4o-mini` |
-| `HINDSIGHT_API_HOST` | Server bind address | `0.0.0.0` |
-| `HINDSIGHT_API_PORT` | Server port | `8888` |
-
-### Example with External PostgreSQL
-
-```bash
-export HINDSIGHT_API_DATABASE_URL=postgresql://user:pass@localhost:5432/hindsight
-export HINDSIGHT_API_LLM_PROVIDER=groq
-export HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx
-
-hindsight-api
-```
-
-## Docker
-
-```bash
-docker run --rm -it -p 8888:8888 \
-  -e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \
-  -v $HOME/.hindsight-docker:/home/hindsight/.pg0 \
-  ghcr.io/vectorize-io/hindsight:latest
-```
-
-## MCP Server
-
-For local MCP integration without running the full API server:
-
-```bash
-hindsight-local-mcp
-```
-
-This runs a stdio-based MCP server that can be used directly with MCP-compatible clients.
-
-## Key Features
-
-- **Multi-Strategy Retrieval (TEMPR)** — Semantic, keyword, graph, and temporal search combined with RRF fusion
-- **Entity Graph** — Automatic entity extraction and relationship tracking
-- **Temporal Reasoning** — Native support for time-based queries
-- **Disposition Traits** — Configurable skepticism, literalism, and empathy influence opinion formation
-- **Three Memory Types** — World facts, bank actions, and formed opinions with confidence scores
-
-## Documentation
-
-Full documentation: [https://hindsight.vectorize.io](https://hindsight.vectorize.io)
-
-- [Installation Guide](https://hindsight.vectorize.io/developer/installation)
-- [Configuration Reference](https://hindsight.vectorize.io/developer/configuration)
-- [API Reference](https://hindsight.vectorize.io/api-reference)
-- [Python SDK](https://hindsight.vectorize.io/sdks/python)
-
-## License
-
-Apache 2.0

hindsight-api-slim/hindsight_api/alembic/versions/a2b3c4d5e6f8_add_gin_index_source_memory_ids.py

@@ -1,54 +0,0 @@
-"""Add GIN index on source_memory_ids for observation lookup performance
-
-Without this index, queries using the array overlap operator (&&) or array
-containment (@>) on source_memory_ids require a full sequential scan over all
-observation memory_units. At ~77k observations this was measured at 45ms per
-query, becoming a bottleneck during consolidation recall (57-64s timeouts) and
-user recall (18-27s average).
-
-The GIN index reduces these queries to index scans: 45ms → 0.049ms (927x
-speedup). Recall dropped from 18-27s to ~6s, and consolidation recall
-stabilised from timeout to ~15s.
-
-Created with CONCURRENTLY so the migration does not block reads or writes.
-CONCURRENTLY requires running outside a transaction block, so the migration
-emits an explicit COMMIT before the statement and uses IF NOT EXISTS for
-idempotency.
-
-Revision ID: a2b3c4d5e6f8
-Revises: f7g8h9i0j1k2
-Create Date: 2026-03-04
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-revision: str = "a2b3c4d5e6f8"
-down_revision: str | Sequence[str] | None = "f7g8h9i0j1k2"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    schema = _get_schema_prefix()
-
-    # CREATE INDEX CONCURRENTLY cannot run inside a transaction block.
-    # Commit the current Alembic transaction first.
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
-        f"ON {schema}memory_units USING GIN (source_memory_ids) "
-        f"WHERE source_memory_ids IS NOT NULL"
-    )
-
-
-def downgrade() -> None:
-    schema = _get_schema_prefix()
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")

hindsight-api-slim/hindsight_api/alembic/versions/a3b4c5d6e7f8_add_consolidation_failed_at_to_memory_units.py

@@ -1,52 +0,0 @@
-"""Add consolidation_failed_at column to memory_units for tracking persistent LLM failures.
-
-When all LLM retries are exhausted on a single-memory batch, the memory is marked
-with consolidation_failed_at instead of consolidated_at, so it is not silently lost
-and can be retried later via the API.
-
-Revision ID: a3b4c5d6e7f8
-Revises: g7h8i9j0k1l2
-Create Date: 2026-03-17
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-revision: str = "a3b4c5d6e7f8"
-down_revision: str | Sequence[str] | None = "g7h8i9j0k1l2"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    """Get schema prefix for table names (required for multi-tenant support)."""
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    schema = _get_schema_prefix()
-
-    op.execute(
-        f"""
-        ALTER TABLE {schema}memory_units
-        ADD COLUMN IF NOT EXISTS consolidation_failed_at TIMESTAMPTZ DEFAULT NULL
-        """
-    )
-
-    # Index to efficiently query memories that failed consolidation for a given bank
-    op.execute(
-        f"""
-        CREATE INDEX IF NOT EXISTS idx_memory_units_consolidation_failed
-        ON {schema}memory_units (bank_id, consolidation_failed_at)
-        WHERE consolidation_failed_at IS NOT NULL AND fact_type IN ('experience', 'world')
-        """
-    )
-
-
-def downgrade() -> None:
-    schema = _get_schema_prefix()
-
-    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_consolidation_failed")
-    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS consolidation_failed_at")

hindsight-api-slim/hindsight_api/alembic/versions/b3c4d5e6f7g8_add_temporal_date_indexes.py

@@ -1,68 +0,0 @@
-"""Add partial indexes on memory_units temporal date fields for fast temporal retrieval
-
-Revision ID: b3c4d5e6f7g8
-Revises: c1a2b3d4e5f6
-Create Date: 2026-03-02
-
-The temporal retrieval entry-point query filters memory_units by occurred_start,
-occurred_end, and mentioned_at using OR conditions. Without dedicated indexes the
-planner falls back to a sequential scan of all bank rows after applying the
-(bank_id, fact_type) index, then re-checks each date field.
-
-These three partial indexes give the planner bitmap-index scan options for the
-three most common date predicates, dramatically reducing the row set before any
-embedding computation is required.
-
-All indexes are created CONCURRENTLY so the migration does not block writes on
-memory_units during production deployments. CONCURRENTLY requires running outside
-a transaction block; see migrations.py for how this is handled safely.
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-revision: str = "b3c4d5e6f7g8"
-down_revision: str | Sequence[str] | None = "c1a2b3d4e5f6"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    schema = _get_schema_prefix()
-    # Partial index on occurred_start (covers "occurred_start BETWEEN $4 AND $5")
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_occurred_start "
-        f"ON {schema}memory_units(bank_id, fact_type, occurred_start) "
-        f"WHERE occurred_start IS NOT NULL"
-    )
-    # Partial index on occurred_end (covers "occurred_end BETWEEN $4 AND $5")
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_occurred_end "
-        f"ON {schema}memory_units(bank_id, fact_type, occurred_end) "
-        f"WHERE occurred_end IS NOT NULL"
-    )
-    # Partial index on mentioned_at (covers "mentioned_at BETWEEN $4 AND $5")
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_mentioned_at "
-        f"ON {schema}memory_units(bank_id, fact_type, mentioned_at) "
-        f"WHERE mentioned_at IS NOT NULL"
-    )
-
-
-def downgrade() -> None:
-    schema = _get_schema_prefix()
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_mentioned_at")
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_occurred_end")
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_occurred_start")

hindsight-api-slim/hindsight_api/alembic/versions/c1a2b3d4e5f6_enable_pg_trgm_and_entities_trgm_index.py

@@ -1,46 +0,0 @@
-"""Enable pg_trgm extension and add GIN trigram index on entities.canonical_name
-
-Revision ID: c1a2b3d4e5f6
-Revises: b4c5d6e7f8a9
-Create Date: 2026-03-02
-
-Index is created CONCURRENTLY so the migration does not block writes on entities
-during production deployments. CONCURRENTLY requires running outside a transaction
-block; see migrations.py for how this is handled safely.
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-revision: str = "c1a2b3d4e5f6"
-down_revision: str | Sequence[str] | None = "b4c5d6e7f8a9"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    # pg_trgm ships with every standard PostgreSQL installation as a contrib module.
-    # It enables fast similarity lookups via GIN indexes, used for entity name matching.
-    op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
-
-    schema = _get_schema_prefix()
-    # GIN index on canonical_name enables sub-millisecond trigram similarity queries
-    # (% operator, similarity()) instead of full-table scans across all bank entities.
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS entities_canonical_name_trgm_idx "
-        f"ON {schema}entities USING GIN (canonical_name gin_trgm_ops)"
-    )
-
-
-def downgrade() -> None:
-    schema = _get_schema_prefix()
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}entities_canonical_name_trgm_idx")
-    # Note: not dropping pg_trgm extension as other indexes may depend on it

hindsight-api-slim/hindsight_api/alembic/versions/c3d4e5f6g7h8_add_history_to_mental_models.py

@@ -1,30 +0,0 @@
-"""Add history column to mental_models
-
-Revision ID: c3d4e5f6g7h8
-Revises: a2b3c4d5e6f7, a2b3c4d5e6f8
-Create Date: 2026-03-06
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-revision: str = "c3d4e5f6g7h8"
-down_revision: str | Sequence[str] | None = ("a2b3c4d5e6f7", "a2b3c4d5e6f8")
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    schema = _get_schema_prefix()
-    op.execute(f"ALTER TABLE {schema}mental_models ADD COLUMN IF NOT EXISTS history JSONB DEFAULT '[]'::jsonb")
-
-
-def downgrade() -> None:
-    schema = _get_schema_prefix()
-    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS history")

hindsight-api-slim/hindsight_api/alembic/versions/d2e3f4a5b6c7_add_memory_links_expansion_indexes.py

@@ -1,83 +0,0 @@
-"""Add covering and composite indexes to speed up link expansion graph retrieval.
-
-Two indexes target the two bottlenecks identified by EXPLAIN ANALYZE on a 17M-row
-memory_links table:
-
-1. idx_memory_links_to_type_weight  (to_unit_id, link_type, weight DESC)
-   The semantic incoming direction — finding facts that consider seeds as their
-   nearest neighbour — currently hits an expensive BitmapAnd of two separate
-   bitmap scans (to_unit_id bitmap ∩ link_type bitmap).  A composite index
-   on (to_unit_id, link_type) turns this into a single index scan and reduces
-   latency from ~36 ms to < 5 ms per query.
-
-2. idx_memory_links_entity_covering  (from_unit_id) INCLUDE (to_unit_id, entity_id)
-   WHERE link_type = 'entity'
-   The entity co-occurrence expansion uses COUNT(DISTINCT ml.entity_id) and
-   joins on ml.to_unit_id.  Without a covering index the planner must read
-   ~2 500 heap pages to fetch entity_id and to_unit_id after the bitmap index
-   scan, adding ~230 ms of random I/O.  INCLUDE adds those two columns to the
-   index leaf pages so the entire query can be served from the index (index-only
-   scan), eliminating the heap reads entirely.
-   Partial index (WHERE link_type = 'entity') keeps index size ~40 % smaller.
-
-Both indexes are created with CONCURRENTLY so the migration does not block
-concurrent reads or writes on memory_links.  CONCURRENTLY requires running
-outside a transaction block, so the migration emits an explicit COMMIT before
-each statement and uses IF NOT EXISTS for idempotency.
-
-Revision ID: d2e3f4a5b6c7
-Revises: b3c4d5e6f7g8
-Create Date: 2026-03-02
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-revision: str = "d2e3f4a5b6c7"
-down_revision: str | Sequence[str] | None = "b3c4d5e6f7g8"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    schema = _get_schema_prefix()
-
-    # CREATE INDEX CONCURRENTLY cannot run inside a transaction block.
-    # Commit the current Alembic transaction, then issue each CONCURRENTLY
-    # statement in its own implicit autocommit transaction.
-    # IF NOT EXISTS makes each statement idempotent if the migration is retried.
-
-    # Index for the semantic *incoming* direction in link_expansion_retrieval.py.
-    # Replaces the BitmapAnd of idx_memory_links_to_unit ∩ idx_memory_links_link_type
-    # with a single composite index scan.
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_to_type_weight "
-        f"ON {schema}memory_links(to_unit_id, link_type, weight DESC)"
-    )
-
-    # Covering index for entity co-occurrence expansion.
-    # Enables an index-only scan: entity_id and to_unit_id are read from the
-    # index leaf pages instead of the heap, eliminating ~2 500 random heap-page
-    # reads per expansion query.
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_entity_covering "
-        f"ON {schema}memory_links(from_unit_id) "
-        f"INCLUDE (to_unit_id, entity_id) "
-        f"WHERE link_type = 'entity'"
-    )
-
-
-def downgrade() -> None:
-    schema = _get_schema_prefix()
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_entity_covering")
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_to_type_weight")

hindsight-api-slim/hindsight_api/alembic/versions/d4e5f6g7h8i9_gin_source_memory_ids_fastupdate_off.py

@@ -1,53 +0,0 @@
-"""Recreate idx_memory_units_source_memory_ids GIN index with fastupdate=off
-
-GIN indexes use a "fastupdate" pending list by default: small writes are
-buffered there and flushed to the main GIN tree in bulk. Flushing requires
-AccessExclusiveLock on the index. Under high insert concurrency (e.g. 8
-parallel pytest-xdist workers all calling retain_async) two transactions can
-each trigger a flush simultaneously and deadlock.
-
-Disabling fastupdate makes every insert write directly to the GIN tree
-(slightly slower per insert, but no pending-list lock cycles).
-
-Revision ID: d4e5f6g7h8i9
-Revises: d5e6f7a8b9c0
-Create Date: 2026-03-11
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-revision: str = "d4e5f6g7h8i9"
-down_revision: str | Sequence[str] | None = "d5e6f7a8b9c0"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    schema = _get_schema_prefix()
-    # DROP + CREATE CONCURRENTLY must run outside a transaction block.
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
-        f"ON {schema}memory_units USING GIN (source_memory_ids) "
-        f"WITH (fastupdate=off) "
-        f"WHERE source_memory_ids IS NOT NULL"
-    )
-
-
-def downgrade() -> None:
-    schema = _get_schema_prefix()
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
-        f"ON {schema}memory_units USING GIN (source_memory_ids) "
-        f"WHERE source_memory_ids IS NOT NULL"
-    )

hindsight-api-slim/hindsight_api/alembic/versions/d5e6f7a8b9c0_add_bank_internal_id_and_per_bank_hnsw.py

@@ -1,131 +0,0 @@
-"""Add internal_id to banks and per-(bank, fact_type) partial HNSW indexes
-
-Revision ID: d5e6f7a8b9c0
-Revises: a3b4c5d6e7f8
-Create Date: 2026-03-11
-
-This migration:
-1. Adds internal_id UUID column to banks (stable identifier for index naming)
-2. Drops the global HNSW index (competes with per-bank partial indexes)
-3. Creates per-(bank_id, fact_type) partial HNSW indexes for all existing banks
-   (new banks get indexes created at bank-creation time via bank_utils.create_bank_hnsw_indexes)
-
-Why per-(bank, fact_type) indexes:
-- fact_type-only partial indexes are never chosen by the planner when bank_id is in the WHERE
-  clause, because the idx_memory_units_bank_id B-tree index always wins at planning time.
-- Per-(bank, fact_type) partial indexes have both predicates matching → planner selects them.
-- The global HNSW index competes for larger partitions (world, observation) and must be dropped.
-
-For large deployments, create indexes CONCURRENTLY before running this migration:
-    SELECT internal_id, bank_id FROM banks;
-    -- for each bank and each fact_type in (world, experience, observation):
-    CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mu_emb_{ft}_{uid16}
-        ON memory_units USING hnsw (embedding vector_cosine_ops)
-        WHERE fact_type = '{ft}' AND bank_id = '{bank_id}';
-    DROP INDEX CONCURRENTLY IF EXISTS idx_memory_units_embedding;
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-from sqlalchemy import text
-
-revision: str = "d5e6f7a8b9c0"
-down_revision: str | Sequence[str] | None = "c3d4e5f6g7h8"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-_HNSW_FACT_TYPES: dict[str, str] = {
-    "world": "worl",
-    "experience": "expr",
-    "observation": "obsv",
-}
-
-
-def _get_schema_prefix() -> str:
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    schema = _get_schema_prefix()
-
-    # 1. Add internal_id column to banks
-    op.execute(
-        f"ALTER TABLE {schema}banks ADD COLUMN IF NOT EXISTS internal_id UUID DEFAULT gen_random_uuid() NOT NULL"
-    )
-    op.execute(f"ALTER TABLE {schema}banks ADD CONSTRAINT banks_internal_id_unique UNIQUE (internal_id)")
-
-    # 2. Drop any fact_type-only partial HNSW indexes that may exist from prior migrations
-    #    (bank_id B-tree always wins over them when bank_id is in the WHERE clause)
-    op.execute(f"DROP INDEX IF EXISTS {schema}idx_mu_emb_world")
-    op.execute(f"DROP INDEX IF EXISTS {schema}idx_mu_emb_observation")
-    op.execute(f"DROP INDEX IF EXISTS {schema}idx_mu_emb_experience")
-
-    # 4. Drop global HNSW index (competes with per-bank partial indexes)
-    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_embedding")
-
-    # 5. Create per-(bank, fact_type) partial HNSW indexes for all existing banks
-    bind = op.get_bind()
-    schema_name = context.config.get_main_option("target_schema")
-    table_ref = f'"{schema_name}".memory_units' if schema_name else "memory_units"
-    banks_ref = f'"{schema_name}".banks' if schema_name else "banks"
-
-    rows = bind.execute(text(f"SELECT bank_id, internal_id FROM {banks_ref}")).fetchall()  # noqa: S608
-    for row in rows:
-        bank_id = row[0]
-        internal_id = str(row[1]).replace("-", "")[:16]
-        escaped_bank_id = bank_id.replace("'", "''")
-        for ft, ft_short in _HNSW_FACT_TYPES.items():
-            idx_name = f"idx_mu_emb_{ft_short}_{internal_id}"
-            # Index name is schema-unqualified (indexes live in the schema of their table)
-            bind.execute(
-                text(
-                    f"CREATE INDEX IF NOT EXISTS {idx_name} "
-                    f"ON {table_ref} USING hnsw (embedding vector_cosine_ops) "
-                    f"WHERE fact_type = '{ft}' AND bank_id = '{escaped_bank_id}'"
-                )
-            )
-
-
-def downgrade() -> None:
-    schema = _get_schema_prefix()
-
-    # Drop per-bank HNSW indexes (iterate existing banks)
-    bind = op.get_bind()
-    schema_name = context.config.get_main_option("target_schema")
-    banks_ref = f'"{schema_name}".banks' if schema_name else "banks"
-
-    rows = bind.execute(text(f"SELECT internal_id FROM {banks_ref}")).fetchall()  # noqa: S608
-    for row in rows:
-        internal_id = str(row[0]).replace("-", "")[:16]
-        for ft_short in _HNSW_FACT_TYPES.values():
-            idx_name = f"idx_mu_emb_{ft_short}_{internal_id}"
-            bind.execute(text(f"DROP INDEX IF EXISTS {schema}{idx_name}"))
-
-    # Restore the global HNSW index
-    table_ref = f'"{schema_name}".memory_units' if schema_name else "memory_units"
-    op.execute(
-        f"CREATE INDEX IF NOT EXISTS idx_memory_units_embedding ON {table_ref} USING hnsw (embedding vector_cosine_ops)"
-    )
-
-    # Restore old fact_type-only partial indexes
-    op.execute(
-        f"CREATE INDEX IF NOT EXISTS idx_mu_emb_world "
-        f"ON {table_ref} USING hnsw (embedding vector_cosine_ops) "
-        f"WHERE fact_type = 'world'"
-    )
-    op.execute(
-        f"CREATE INDEX IF NOT EXISTS idx_mu_emb_observation "
-        f"ON {table_ref} USING hnsw (embedding vector_cosine_ops) "
-        f"WHERE fact_type = 'observation'"
-    )
-    op.execute(
-        f"CREATE INDEX IF NOT EXISTS idx_mu_emb_experience "
-        f"ON {table_ref} USING hnsw (embedding vector_cosine_ops) "
-        f"WHERE fact_type = 'experience'"
-    )
-
-    # Drop internal_id column
-    op.execute(f"ALTER TABLE {schema}banks DROP CONSTRAINT IF EXISTS banks_internal_id_unique")
-    op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS internal_id")

hindsight-api-slim/hindsight_api/alembic/versions/e4f5a6b7c8d9_add_webhooks_tables.py

@@ -1,62 +0,0 @@
-"""Add webhooks table and next_retry_at to async_operations.
-
-Webhook deliveries are handled as async_operations tasks (operation_type='webhook_delivery')
-rather than a dedicated webhook_deliveries table.
-
-Revision ID: e4f5a6b7c8d9
-Revises: d2e3f4a5b6c7
-Create Date: 2026-03-04
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-revision: str = "e4f5a6b7c8d9"
-down_revision: str | Sequence[str] | None = "d2e3f4a5b6c7"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    schema = _get_schema_prefix()
-
-    op.execute(
-        f"""
-        CREATE TABLE IF NOT EXISTS {schema}webhooks (
-            id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-            bank_id TEXT,
-            url TEXT NOT NULL,
-            secret TEXT,
-            event_types TEXT[] NOT NULL DEFAULT '{{}}',
-            enabled BOOLEAN NOT NULL DEFAULT TRUE,
-            created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-            updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
-        )
-        """
-    )
-
-    # Index for bank-scoped webhook lookup
-    op.execute(f"CREATE INDEX IF NOT EXISTS idx_webhooks_bank_id ON {schema}webhooks(bank_id)")
-
-    # Add next_retry_at to async_operations for task-owned retry scheduling
-    op.execute(f"ALTER TABLE {schema}async_operations ADD COLUMN IF NOT EXISTS next_retry_at TIMESTAMPTZ NULL")
-
-    # Index for polling: status + next_retry_at
-    op.execute(
-        f"CREATE INDEX IF NOT EXISTS idx_async_operations_status_retry "
-        f"ON {schema}async_operations(status, next_retry_at)"
-    )
-
-
-def downgrade() -> None:
-    schema = _get_schema_prefix()
-    op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_status_retry")
-    op.execute(f"ALTER TABLE {schema}async_operations DROP COLUMN IF EXISTS next_retry_at")
-    op.execute(f"DROP INDEX IF EXISTS {schema}idx_webhooks_bank_id")
-    op.execute(f"DROP TABLE IF EXISTS {schema}webhooks")

hindsight-api-slim/hindsight_api/alembic/versions/e5f6g7h8i9j0_cascade_delete_ops_on_bank_delete.py

@@ -1,73 +0,0 @@
-"""Add CASCADE DELETE FK from async_operations and webhooks to banks.
-
-When a bank is deleted, all its async_operations and webhooks rows are
-automatically deleted by the database. This ensures that any in-flight
-worker tasks detect the deletion via _check_op_alive() and abort early.
-
-Revision ID: e5f6g7h8i9j0
-Revises: d4e5f6g7h8i9
-Create Date: 2026-03-11
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-revision: str = "e5f6g7h8i9j0"
-down_revision: str | Sequence[str] | None = "d4e5f6g7h8i9"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    schema = _get_schema_prefix()
-
-    # Remove orphaned async_operations rows whose bank no longer exists
-    # (can happen because there was no FK before this migration).
-    op.execute(
-        f"""
-        DELETE FROM {schema}async_operations
-        WHERE bank_id IS NOT NULL
-          AND bank_id NOT IN (SELECT bank_id FROM {schema}banks)
-        """
-    )
-
-    # Remove orphaned webhooks rows whose bank no longer exists.
-    op.execute(
-        f"""
-        DELETE FROM {schema}webhooks
-        WHERE bank_id IS NOT NULL
-          AND bank_id NOT IN (SELECT bank_id FROM {schema}banks)
-        """
-    )
-
-    # Add FK with ON DELETE CASCADE so that deleting a bank automatically
-    # cleans up all its pending/processing operations and webhook configs.
-    op.execute(
-        f"""
-        ALTER TABLE {schema}async_operations
-            ADD CONSTRAINT fk_async_operations_bank_id
-            FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id)
-            ON DELETE CASCADE
-        """
-    )
-
-    op.execute(
-        f"""
-        ALTER TABLE {schema}webhooks
-            ADD CONSTRAINT fk_webhooks_bank_id
-            FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id)
-            ON DELETE CASCADE
-        """
-    )
-
-
-def downgrade() -> None:
-    schema = _get_schema_prefix()
-    op.execute(f"ALTER TABLE {schema}async_operations DROP CONSTRAINT IF EXISTS fk_async_operations_bank_id")
-    op.execute(f"ALTER TABLE {schema}webhooks DROP CONSTRAINT IF EXISTS fk_webhooks_bank_id")

hindsight-api-slim/hindsight_api/alembic/versions/f6g7h8i9j0k1_chunk_fk_cascade_delete.py

@@ -1,38 +0,0 @@
-"""chunk_fk_cascade_delete
-
-Revision ID: f6g7h8i9j0k1
-Revises: e5f6g7h8i9j0
-Create Date: 2026-03-16 00:00:00.000000
-
-"""
-
-from collections.abc import Sequence
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision: str = "f6g7h8i9j0k1"
-down_revision: str | Sequence[str] | None = "e5f6g7h8i9j0"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def upgrade() -> None:
-    """Change memory_units.chunk_id FK from SET NULL to CASCADE.
-
-    When a document is deleted the CASCADE reaches chunks first; with SET NULL
-    the memory_units rows survived with chunk_id = NULL, leaving ghost records.
-    Switching to CASCADE ensures they are removed together with their chunk.
-    """
-    op.drop_constraint("memory_units_chunk_fkey", "memory_units", type_="foreignkey")
-    op.create_foreign_key(
-        "memory_units_chunk_fkey", "memory_units", "chunks", ["chunk_id"], ["chunk_id"], ondelete="CASCADE"
-    )
-
-
-def downgrade() -> None:
-    """Revert to SET NULL behaviour."""
-    op.drop_constraint("memory_units_chunk_fkey", "memory_units", type_="foreignkey")
-    op.create_foreign_key(
-        "memory_units_chunk_fkey", "memory_units", "chunks", ["chunk_id"], ["chunk_id"], ondelete="SET NULL"
-    )

hindsight-api-slim/hindsight_api/alembic/versions/f7g8h9i0j1k2_add_webhook_http_config.py

@@ -1,33 +0,0 @@
-"""Add http_config JSONB column to webhooks table.
-
-Stores HTTP delivery configuration (method, timeout, headers, params) as a
-single JSONB column rather than separate columns.
-
-Revision ID: f7g8h9i0j1k2
-Revises: e4f5a6b7c8d9
-Create Date: 2026-03-04
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-revision: str = "f7g8h9i0j1k2"
-down_revision: str | Sequence[str] | None = "e4f5a6b7c8d9"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    schema = _get_schema_prefix()
-    op.execute(f"ALTER TABLE {schema}webhooks ADD COLUMN IF NOT EXISTS http_config JSONB NOT NULL DEFAULT '{{}}'")
-
-
-def downgrade() -> None:
-    schema = _get_schema_prefix()
-    op.execute(f"ALTER TABLE {schema}webhooks DROP COLUMN IF EXISTS http_config")

hindsight-api-slim/hindsight_api/alembic/versions/g7h8i9j0k1l2_backsweep_orphan_observations.py

@@ -1,71 +0,0 @@
-"""backsweep_orphan_memory_units
-
-Two-pass cleanup of memory_units rows that were never removed by earlier bugs:
-
-Pass 1 — any fact_type, bank gone:
-  memory_units whose bank_id no longer exists in banks.  These accumulate when
-  a bank is deleted without a proper cascade (no FK from memory_units to banks
-  exists in the schema).
-
-Pass 2 — observations only, all sources gone:
-  observation rows whose bank still exists but every source_memory_id points
-  to a deleted memory unit.  These were left behind before PR #580 fixed the
-  chunk FK cascade and before delete_document() called
-  _delete_stale_observations_for_memories.
-
-Revision ID: g7h8i9j0k1l2
-Revises: f6g7h8i9j0k1
-Create Date: 2026-03-16
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-revision: str = "g7h8i9j0k1l2"
-down_revision: str | Sequence[str] | None = "f6g7h8i9j0k1"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    schema = _get_schema_prefix()
-    mu = f"{schema}memory_units"
-    banks = f"{schema}banks"
-
-    # Pass 1: delete all memory_units (any fact_type) whose bank no longer exists.
-    # There is no FK from memory_units to banks, so these never cascade away.
-    op.execute(
-        f"""
-        DELETE FROM {mu}
-        WHERE NOT EXISTS (
-            SELECT 1 FROM {banks} b WHERE b.bank_id = {mu}.bank_id
-        )
-        """
-    )
-
-    # Pass 2: delete orphaned observations whose bank still exists but every
-    # source_memory_id refers to a now-deleted memory unit (or the array is
-    # empty).  Observations with at least one surviving source are left alone.
-    op.execute(
-        f"""
-        DELETE FROM {mu} orphan
-        WHERE orphan.fact_type = 'observation'
-          AND NOT EXISTS (
-              SELECT 1
-              FROM {mu} src
-              WHERE src.id = ANY(orphan.source_memory_ids)
-                AND src.bank_id = orphan.bank_id
-          )
-        """
-    )
-
-
-def downgrade() -> None:
-    # Deleted rows cannot be restored.
-    pass

hindsight-api-slim/hindsight_api/engine/jina_mlx_reranker.py

@@ -1,144 +0,0 @@
-"""
-MLX implementation of jina-reranker-v3 for Apple Silicon.
-
-This file is adapted from the official model repository:
-  https://huggingface.co/jinaai/jina-reranker-v3-mlx/blob/main/rerank.py
-
-License: CC BY-NC 4.0 (contact Jina AI for commercial usage)
-
-Changes from upstream:
-- Removed the __main__ example block
-- Type annotations added to public methods
-- top_n parameter added to rerank() (upstream only exposed it implicitly)
-"""
-
-import numpy as np
-
-
-class _MLPProjector:
-    def __init__(self):
-        import mlx.nn as nn
-
-        self.linear1 = nn.Linear(1024, 512, bias=False)
-        self.linear2 = nn.Linear(512, 512, bias=False)
-
-    def __call__(self, x):
-        import mlx.nn as nn
-
-        x = self.linear1(x)
-        x = nn.relu(x)
-        x = self.linear2(x)
-        return x
-
-
-def _load_projector(projector_path: str) -> _MLPProjector:
-    import mlx.core as mx
-    from safetensors import safe_open
-
-    projector = _MLPProjector()
-    with safe_open(projector_path, framework="numpy") as f:
-        projector.linear1.weight = mx.array(f.get_tensor("linear1.weight"))
-        projector.linear2.weight = mx.array(f.get_tensor("linear2.weight"))
-    return projector
-
-
-def _sanitize(text: str, special_tokens: dict[str, str]) -> str:
-    for token in special_tokens.values():
-        text = text.replace(token, "")
-    return text
-
-
-def _format_prompt(query: str, docs: list[str], special_tokens: dict[str, str]) -> str:
-    query = _sanitize(query, special_tokens)
-    docs = [_sanitize(d, special_tokens) for d in docs]
-
-    doc_token = special_tokens["doc_embed_token"]
-    query_token = special_tokens["query_embed_token"]
-
-    prefix = (
-        "<|im_start|>system\n"
-        "You are a search relevance expert who can determine a ranking of the passages based on how relevant they are to the query. "
-        "If the query is a question, how relevant a passage is depends on how well it answers the question. "
-        "If not, try to analyze the intent of the query and assess how well each passage satisfies the intent. "
-        "If an instruction is provided, you should follow the instruction when determining the ranking."
-        "<|im_end|>\n<|im_start|>user\n"
-    )
-    suffix = "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
-
-    body = (
-        f"I will provide you with {len(docs)} passages, each indicated by a numerical identifier. "
-        f"Rank the passages based on their relevance to query: {query}\n"
-    )
-    body += "\n".join(f'<passage id="{i}">\n{doc}{doc_token}\n</passage>' for i, doc in enumerate(docs))
-    body += f"\n<query>\n{query}{query_token}\n</query>"
-    return prefix + body + suffix
-
-
-class MLXReranker:
-    """
-    MLX-accelerated jina-reranker-v3 for Apple Silicon.
-
-    Loads the model from a local directory (use huggingface_hub.snapshot_download
-    to fetch jinaai/jina-reranker-v3-mlx if you don't have it already).
-    """
-
-    _SPECIAL_TOKENS = {
-        "query_embed_token": "<|rerank_token|>",
-        "doc_embed_token": "<|embed_token|>",
-    }
-    _DOC_TOKEN_ID = 151670
-    _QUERY_TOKEN_ID = 151671
-
-    def __init__(self, model_path: str, projector_path: str):
-        from mlx_lm import load
-
-        self.model, self.tokenizer = load(model_path)
-        self.model.eval()
-        self.projector = _load_projector(projector_path)
-
-    def rerank(self, query: str, documents: list[str], top_n: int | None = None) -> list[dict]:
-        """
-        Rank documents by relevance to a query.
-
-        Returns a list of dicts with keys: document, relevance_score, index.
-        Sorted by descending relevance_score.
-        """
-        import mlx.core as mx
-
-        prompt = _format_prompt(query, documents, self._SPECIAL_TOKENS)
-        input_ids = self.tokenizer.encode(prompt)
-        hidden_states = self.model.model([input_ids])[0]  # [seq_len, hidden_size]
-
-        input_ids_np = np.array(input_ids)
-        query_positions = np.where(input_ids_np == self._QUERY_TOKEN_ID)[0]
-        doc_positions = np.where(input_ids_np == self._DOC_TOKEN_ID)[0]
-
-        if len(query_positions) == 0:
-            raise ValueError("Query embed token not found in prompt")
-        if len(doc_positions) == 0:
-            raise ValueError("Document embed tokens not found in prompt")
-
-        query_hidden = mx.expand_dims(hidden_states[int(query_positions[0])], axis=0)
-        doc_hidden = mx.stack([hidden_states[int(p)] for p in doc_positions])
-
-        query_emb = self.projector(query_hidden)  # [1, 512]
-        doc_emb = self.projector(doc_hidden)  # [num_docs, 512]
-
-        query_exp = mx.broadcast_to(mx.expand_dims(query_emb, 0), (1, len(documents), 512))
-        doc_exp = mx.expand_dims(doc_emb, 0)
-
-        scores = mx.sum(doc_exp * query_exp, axis=-1) / (
-            mx.sqrt(mx.sum(doc_exp * doc_exp, axis=-1)) * mx.sqrt(mx.sum(query_exp * query_exp, axis=-1))
-        )  # [1, num_docs]
-        scores_np = np.array(scores[0])
-
-        order = np.argsort(scores_np)[::-1]
-        n = min(top_n, len(documents)) if top_n is not None else len(documents)
-        return [
-            {
-                "document": documents[order[i]],
-                "relevance_score": float(scores_np[order[i]]),
-                "index": int(order[i]),
-            }
-            for i in range(n)
-        ]

hindsight-api-slim/hindsight_api/engine/parsers/__init__.py

@@ -1,128 +0,0 @@
-"""File parser implementations."""
-
-import logging
-from dataclasses import dataclass
-
-from .base import FileParser, UnsupportedFileTypeError
-from .iris import IrisParser
-from .markitdown import MarkitdownParser
-
-__all__ = [
-    "FileParser",
-    "UnsupportedFileTypeError",
-    "IrisParser",
-    "MarkitdownParser",
-    "FileParserRegistry",
-    "ConvertResult",
-]
-
-
-@dataclass
-class ConvertResult:
-    """Result of a successful file conversion."""
-
-    content: str
-    parser_name: str
-
-
-logger = logging.getLogger(__name__)
-
-
-class FileParserRegistry:
-    """Registry for file parsers with auto-detection."""
-
-    def __init__(self):
-        """Initialize empty parser registry."""
-        self._parsers: dict[str, FileParser] = {}
-
-    def register(self, parser: FileParser):
-        """
-        Register a parser.
-
-        Args:
-            parser: FileParser instance
-        """
-        self._parsers[parser.name()] = parser
-
-    def get_parser(
-        self,
-        name: str | None,
-        filename: str,
-        content_type: str | None = None,
-    ) -> FileParser:
-        """
-        Get parser by name or auto-detect.
-
-        Args:
-            name: Parser name (e.g., "markitdown") or None for auto-detect
-            filename: File name for auto-detection
-            content_type: MIME type (optional)
-
-        Returns:
-            FileParser instance
-
-        Raises:
-            ValueError: If no suitable parser found
-        """
-        if name:
-            # Explicit parser requested — return it directly, let the parser
-            # raise UnsupportedFileTypeError from convert() if needed
-            if name not in self._parsers:
-                raise ValueError(f"Parser '{name}' not found. Available: {list(self._parsers.keys())}")
-            return self._parsers[name]
-
-        # Auto-detect parser
-        for parser in self._parsers.values():
-            if parser.supports(filename, content_type):
-                return parser
-
-        raise ValueError(f"No parser found for {filename}. Available parsers: {list(self._parsers.keys())}")
-
-    async def convert_with_fallback(
-        self,
-        parsers: list[str],
-        file_data: bytes,
-        filename: str,
-        content_type: str | None = None,
-    ) -> ConvertResult:
-        """
-        Try each parser in order, falling back on failure or empty content.
-
-        Moves to the next parser if the current one raises UnsupportedFileTypeError
-        or returns empty content. Any other exception (RuntimeError, network error,
-        etc.) also triggers a fallback so the chain is exhausted before failing.
-
-        Args:
-            parsers: Ordered list of parser names to try
-            file_data: Raw file bytes
-            filename: Original filename
-            content_type: MIME type (optional)
-
-        Returns:
-            ConvertResult with the parsed content and the name of the parser that succeeded
-
-        Raises:
-            ValueError: If a parser name is not registered
-            RuntimeError: If all parsers fail or return empty content
-        """
-        last_error: Exception | None = None
-        for name in parsers:
-            parser = self.get_parser(name, filename, content_type)
-            try:
-                content = await parser.convert(file_data, filename)
-                if content and content.strip():
-                    return ConvertResult(content=content, parser_name=name)
-                logger.warning(f"Parser '{name}' returned empty content for '{filename}', trying next")
-                last_error = RuntimeError(f"Parser '{name}' returned no content for '{filename}'")
-            except UnsupportedFileTypeError as e:
-                logger.warning(f"Parser '{name}' does not support '{filename}', trying next: {e}")
-                last_error = e
-            except Exception as e:
-                logger.warning(f"Parser '{name}' failed for '{filename}', trying next: {e}")
-                last_error = e
-
-        raise last_error or RuntimeError(f"No parsers available for '{filename}'")
-
-    def list_parsers(self) -> list[str]:
-        """Get list of registered parser names."""
-        return list(self._parsers.keys())

hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py

@@ -1,544 +0,0 @@
-"""
-Main orchestrator for the retain pipeline.
-
-Coordinates all retain pipeline modules to store memories efficiently.
-"""
-
-import logging
-import time
-import uuid
-from collections.abc import Awaitable, Callable
-from datetime import UTC, datetime
-from typing import Any
-
-from ..db_utils import acquire_with_retry, retry_with_backoff
-from . import bank_utils
-
-
-def utcnow():
-    """Get current UTC time."""
-    return datetime.now(UTC)
-
-
-def parse_datetime_flexible(value: Any) -> datetime:
-    """
-    Parse a datetime value that could be either a datetime object or an ISO string.
-
-    This handles datetime values from both direct Python calls and deserialized JSON
-    (where datetime objects are serialized as ISO strings).
-
-    Args:
-        value: Either a datetime object or an ISO format string
-
-    Returns:
-        datetime object (timezone-aware)
-
-    Raises:
-        TypeError: If value is neither datetime nor string
-        ValueError: If string is not a valid ISO datetime
-    """
-    if isinstance(value, datetime):
-        # Ensure timezone-aware
-        if value.tzinfo is None:
-            return value.replace(tzinfo=UTC)
-        return value
-    elif isinstance(value, str):
-        # Parse ISO format string (handles both 'Z' and '+00:00' timezone formats)
-        dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
-        # Ensure timezone-aware
-        if dt.tzinfo is None:
-            return dt.replace(tzinfo=UTC)
-        return dt
-    else:
-        raise TypeError(f"Expected datetime or string, got {type(value).__name__}")
-
-
-import asyncpg
-
-from ..response_models import TokenUsage
-from . import (
-    chunk_storage,
-    embedding_processing,
-    entity_processing,
-    fact_extraction,
-    fact_storage,
-    link_creation,
-)
-from .types import EntityLink, ExtractedFact, ProcessedFact, RetainContent, RetainContentDict
-
-logger = logging.getLogger(__name__)
-
-
-async def retain_batch(
-    pool,
-    embeddings_model,
-    llm_config,
-    entity_resolver,
-    format_date_fn,
-    bank_id: str,
-    contents_dicts: list[RetainContentDict],
-    config,
-    document_id: str | None = None,
-    is_first_batch: bool = True,
-    fact_type_override: str | None = None,
-    confidence_score: float | None = None,
-    document_tags: list[str] | None = None,
-    operation_id: str | None = None,
-    schema: str | None = None,
-    outbox_callback: Callable[["asyncpg.Connection"], Awaitable[None]] | None = None,
-) -> tuple[list[list[str]], TokenUsage]:
-    """
-    Process a batch of content through the retain pipeline.
-
-    Args:
-        pool: Database connection pool
-        embeddings_model: Embeddings model for generating embeddings
-        llm_config: LLM configuration for fact extraction
-        entity_resolver: Entity resolver for entity processing
-        format_date_fn: Function to format datetime to readable string
-        bank_id: Bank identifier
-        contents_dicts: List of content dictionaries
-        config: Resolved HindsightConfig for this bank
-        document_id: Optional document ID
-        is_first_batch: Whether this is the first batch
-        fact_type_override: Override fact type for all facts
-        confidence_score: Confidence score for opinions
-        document_tags: Tags applied to all items in this batch
-
-    Returns:
-        Tuple of (unit ID lists, token usage for fact extraction)
-    """
-    start_time = time.time()
-    total_chars = sum(len(item.get("content", "")) for item in contents_dicts)
-
-    # Buffer all logs
-    log_buffer = []
-    log_buffer.append(f"{'=' * 60}")
-    log_buffer.append(f"RETAIN_BATCH START: {bank_id}")
-    log_buffer.append(f"Batch size: {len(contents_dicts)} content items, {total_chars:,} chars")
-    log_buffer.append(f"{'=' * 60}")
-
-    # Get bank profile
-    profile = await bank_utils.get_bank_profile(pool, bank_id)
-    agent_name = profile["name"]
-
-    # Convert dicts to RetainContent objects
-    contents = []
-    for item in contents_dicts:
-        # Merge item-level tags with document-level tags
-        item_tags = item.get("tags", []) or []
-        merged_tags = list(set(item_tags + (document_tags or [])))
-
-        # Handle event_date: distinguish "not provided" (default to now) from
-        # "explicitly None" (caller opted into no timestamp).
-        if "event_date" in item and item["event_date"] is None:
-            event_date_value = None  # Caller explicitly signalled "unknown date"
-        elif item.get("event_date"):
-            event_date_value = parse_datetime_flexible(item["event_date"])
-        else:
-            event_date_value = utcnow()  # Backward-compatible default
-
-        content = RetainContent(
-            content=item["content"],
-            context=item.get("context", ""),
-            event_date=event_date_value,
-            metadata=item.get("metadata", {}),
-            entities=item.get("entities", []),
-            tags=merged_tags,
-            observation_scopes=item.get("observation_scopes"),
-        )
-        contents.append(content)
-
-    # Step 1: Extract facts from all contents
-    step_start = time.time()
-
-    extracted_facts, chunks, usage = await fact_extraction.extract_facts_from_contents(
-        contents, llm_config, agent_name, config, pool, operation_id, schema
-    )
-    log_buffer.append(
-        f"[1] Extract facts: {len(extracted_facts)} facts, {len(chunks)} chunks from {len(contents)} contents in {time.time() - step_start:.3f}s"
-    )
-
-    if not extracted_facts:
-        # Still need to create document if document_id was provided or chunks exist
-        from collections import defaultdict
-
-        docs_tracked = 0
-        async with acquire_with_retry(pool) as conn:
-            async with conn.transaction():
-                # Group contents by document_id (consistent with normal path)
-                contents_by_doc_early = defaultdict(list)
-                for idx, content_dict in enumerate(contents_dicts):
-                    doc_id = content_dict.get("document_id")
-                    contents_by_doc_early[doc_id].append((idx, content_dict))
-
-                if document_id:
-                    # Legacy: single document_id parameter
-                    combined_content = "\n".join([c.get("content", "") for c in contents_dicts])
-                    # Collect tags from all content items and merge with document_tags
-                    all_tags = set(document_tags or [])
-                    for item in contents_dicts:
-                        item_tags = item.get("tags", []) or []
-                        all_tags.update(item_tags)
-                    merged_tags = list(all_tags)
-
-                    retain_params = {}
-                    if contents_dicts:
-                        first_item = contents_dicts[0]
-                        if first_item.get("context"):
-                            retain_params["context"] = first_item["context"]
-                        if first_item.get("event_date"):
-                            retain_params["event_date"] = (
-                                first_item["event_date"].isoformat()
-                                if hasattr(first_item["event_date"], "isoformat")
-                                else str(first_item["event_date"])
-                            )
-                        if first_item.get("metadata"):
-                            retain_params["metadata"] = first_item["metadata"]
-                    await fact_storage.handle_document_tracking(
-                        conn, bank_id, document_id, combined_content, is_first_batch, retain_params, merged_tags
-                    )
-                    docs_tracked += 1
-                else:
-                    # Handle per-item document_ids and/or chunks (mirrors normal path logic)
-                    has_any_doc_ids = any(item.get("document_id") for item in contents_dicts)
-
-                    if has_any_doc_ids or chunks:
-                        for original_doc_id, doc_contents in contents_by_doc_early.items():
-                            should_create_doc = (original_doc_id is not None) or chunks
-                            if not should_create_doc:
-                                continue
-
-                            actual_doc_id = original_doc_id
-                            if actual_doc_id is None:
-                                # No document_id but have chunks - generate one
-                                actual_doc_id = str(uuid.uuid4())
-
-                            combined_content = "\n".join([c.get("content", "") for _, c in doc_contents])
-                            all_tags = set(document_tags or [])
-                            for _, item in doc_contents:
-                                item_tags = item.get("tags", []) or []
-                                all_tags.update(item_tags)
-                            merged_tags = list(all_tags)
-
-                            retain_params = {}
-                            if doc_contents:
-                                first_item = doc_contents[0][1]
-                                if first_item.get("context"):
-                                    retain_params["context"] = first_item["context"]
-                                if first_item.get("event_date"):
-                                    retain_params["event_date"] = (
-                                        first_item["event_date"].isoformat()
-                                        if hasattr(first_item["event_date"], "isoformat")
-                                        else str(first_item["event_date"])
-                                    )
-                                if first_item.get("metadata"):
-                                    retain_params["metadata"] = first_item["metadata"]
-                            await fact_storage.handle_document_tracking(
-                                conn,
-                                bank_id,
-                                actual_doc_id,
-                                combined_content,
-                                is_first_batch,
-                                retain_params,
-                                merged_tags,
-                            )
-                            docs_tracked += 1
-
-        total_time = time.time() - start_time
-        doc_status = f"{docs_tracked} document(s) tracked" if docs_tracked > 0 else "no document tracked"
-        logger.info(
-            f"RETAIN_BATCH COMPLETE: 0 facts extracted from {len(contents)} contents in {total_time:.3f}s ({doc_status}, no facts)"
-        )
-        return [[] for _ in contents], usage
-
-    # Apply fact_type_override if provided
-    if fact_type_override:
-        for fact in extracted_facts:
-            fact.fact_type = fact_type_override
-
-    # Step 2: Augment texts and generate embeddings
-    step_start = time.time()
-    augmented_texts = embedding_processing.augment_texts_with_dates(extracted_facts, format_date_fn)
-    embeddings = await embedding_processing.generate_embeddings_batch(embeddings_model, augmented_texts)
-    log_buffer.append(f"[2] Generate embeddings: {len(embeddings)} embeddings in {time.time() - step_start:.3f}s")
-
-    # Step 3: Convert to ProcessedFact objects (without chunk_ids yet)
-    processed_facts = [
-        ProcessedFact.from_extracted_fact(extracted_fact, embedding)
-        for extracted_fact, embedding in zip(extracted_facts, embeddings)
-    ]
-
-    # Group contents by document_id for document tracking and chunk storage
-    from collections import defaultdict
-
-    contents_by_doc = defaultdict(list)
-    for idx, content_dict in enumerate(contents_dicts):
-        doc_id = content_dict.get("document_id")
-        contents_by_doc[doc_id].append((idx, content_dict))
-
-    # Step 4: Database transaction (retried on deadlock)
-    result_unit_ids: list[list[str]] = []
-
-    log_buffer_pre_db = len(log_buffer)
-
-    async def _run_db_work() -> None:
-        nonlocal result_unit_ids
-
-        # Reset per-fact mutations and log buffer so each retry attempt starts clean
-        del log_buffer[log_buffer_pre_db:]
-        document_ids_added: list[str] = []
-        for pf in processed_facts:
-            pf.document_id = None
-            pf.chunk_id = None
-
-        async with acquire_with_retry(pool) as conn:
-            async with conn.transaction():
-                # Handle document tracking for all documents
-                step_start = time.time()
-                # Map None document_id to generated UUIDs
-                doc_id_mapping = {}  # Maps original doc_id (including None) to actual doc_id used
-
-                if document_id:
-                    # Legacy: single document_id parameter
-                    combined_content = "\n".join([c.get("content", "") for c in contents_dicts])
-                    retain_params = {}
-                    # Collect tags from all content items and merge with document_tags
-                    all_tags = set(document_tags or [])
-                    for item in contents_dicts:
-                        item_tags = item.get("tags", []) or []
-                        all_tags.update(item_tags)
-                    merged_tags = list(all_tags)
-
-                    if contents_dicts:
-                        first_item = contents_dicts[0]
-                        if first_item.get("context"):
-                            retain_params["context"] = first_item["context"]
-                        if first_item.get("event_date"):
-                            retain_params["event_date"] = (
-                                first_item["event_date"].isoformat()
-                                if hasattr(first_item["event_date"], "isoformat")
-                                else str(first_item["event_date"])
-                            )
-                        if first_item.get("metadata"):
-                            retain_params["metadata"] = first_item["metadata"]
-
-                    await fact_storage.handle_document_tracking(
-                        conn, bank_id, document_id, combined_content, is_first_batch, retain_params, merged_tags
-                    )
-                    document_ids_added.append(document_id)
-                    doc_id_mapping[None] = document_id  # For backwards compatibility
-                else:
-                    # Handle per-item document_ids (create documents if any item has document_id or if chunks exist)
-                    has_any_doc_ids = any(item.get("document_id") for item in contents_dicts)
-
-                    if has_any_doc_ids or chunks:
-                        for original_doc_id, doc_contents in contents_by_doc.items():
-                            actual_doc_id = original_doc_id
-
-                            # Only create document record if:
-                            # 1. Item has explicit document_id, OR
-                            # 2. There are chunks (need document for chunk storage)
-                            should_create_doc = (original_doc_id is not None) or chunks
-
-                            if should_create_doc:
-                                if actual_doc_id is None:
-                                    # No document_id but have chunks - generate one
-                                    actual_doc_id = str(uuid.uuid4())
-
-                                # Store mapping for later use
-                                doc_id_mapping[original_doc_id] = actual_doc_id
-
-                                # Combine content for this document
-                                combined_content = "\n".join([c.get("content", "") for _, c in doc_contents])
-
-                                # Collect tags from all content items for this document and merge with document_tags
-                                all_tags = set(document_tags or [])
-                                for _, item in doc_contents:
-                                    item_tags = item.get("tags", []) or []
-                                    all_tags.update(item_tags)
-                                merged_tags = list(all_tags)
-
-                                # Extract retain params from first content item
-                                retain_params = {}
-                                if doc_contents:
-                                    first_item = doc_contents[0][1]
-                                    if first_item.get("context"):
-                                        retain_params["context"] = first_item["context"]
-                                    if first_item.get("event_date"):
-                                        retain_params["event_date"] = (
-                                            first_item["event_date"].isoformat()
-                                            if hasattr(first_item["event_date"], "isoformat")
-                                            else str(first_item["event_date"])
-                                        )
-                                    if first_item.get("metadata"):
-                                        retain_params["metadata"] = first_item["metadata"]
-
-                                await fact_storage.handle_document_tracking(
-                                    conn,
-                                    bank_id,
-                                    actual_doc_id,
-                                    combined_content,
-                                    is_first_batch,
-                                    retain_params,
-                                    merged_tags,
-                                )
-                                document_ids_added.append(actual_doc_id)
-
-                if document_ids_added:
-                    log_buffer.append(
-                        f"[2.5] Document tracking: {len(document_ids_added)} documents in {time.time() - step_start:.3f}s"
-                    )
-
-                # Store chunks and map to facts for all documents
-                step_start = time.time()
-                chunk_id_map_by_doc = {}  # Maps (doc_id, chunk_index) -> chunk_id
-
-                if chunks:
-                    # Group chunks by their source document
-                    chunks_by_doc = defaultdict(list)
-                    for chunk in chunks:
-                        # chunk.content_index tells us which content this chunk came from
-                        original_doc_id = contents_dicts[chunk.content_index].get("document_id")
-                        # Map to actual document_id (handles None -> generated UUID mapping)
-                        actual_doc_id = doc_id_mapping.get(original_doc_id, original_doc_id)
-                        if actual_doc_id is None and document_id:
-                            actual_doc_id = document_id
-                        chunks_by_doc[actual_doc_id].append(chunk)
-
-                    # Store chunks for each document
-                    for doc_id, doc_chunks in chunks_by_doc.items():
-                        chunk_id_map = await chunk_storage.store_chunks_batch(conn, bank_id, doc_id, doc_chunks)
-                        # Store mapping with document context
-                        for chunk_idx, chunk_id in chunk_id_map.items():
-                            chunk_id_map_by_doc[(doc_id, chunk_idx)] = chunk_id
-
-                    log_buffer.append(
-                        f"[3] Store chunks: {len(chunks)} chunks for {len(chunks_by_doc)} documents in {time.time() - step_start:.3f}s"
-                    )
-
-                    # Map chunk_ids and document_ids to facts
-                    for fact, processed_fact in zip(extracted_facts, processed_facts):
-                        # Get the original document_id for this fact's source content
-                        original_doc_id = contents_dicts[fact.content_index].get("document_id")
-                        # Map to actual document_id (handles None -> generated UUID mapping)
-                        actual_doc_id = doc_id_mapping.get(original_doc_id, original_doc_id)
-                        if actual_doc_id is None and document_id:
-                            actual_doc_id = document_id
-
-                        # Set document_id on the fact
-                        processed_fact.document_id = actual_doc_id
-
-                        # Map chunk_id if this fact came from a chunk
-                        if fact.chunk_index is not None:
-                            # Look up chunk_id using (doc_id, chunk_index)
-                            chunk_id = chunk_id_map_by_doc.get((actual_doc_id, fact.chunk_index))
-                            if chunk_id:
-                                processed_fact.chunk_id = chunk_id
-                else:
-                    # No chunks - still need to set document_id on facts
-                    for fact, processed_fact in zip(extracted_facts, processed_facts):
-                        original_doc_id = contents_dicts[fact.content_index].get("document_id")
-                        # Map to actual document_id (handles None -> generated UUID mapping)
-                        actual_doc_id = doc_id_mapping.get(original_doc_id, original_doc_id)
-                        if actual_doc_id is None and document_id:
-                            actual_doc_id = document_id
-                        processed_fact.document_id = actual_doc_id
-
-                non_duplicate_facts = processed_facts
-
-                # Insert facts (document_id is now stored per-fact)
-                step_start = time.time()
-                unit_ids = await fact_storage.insert_facts_batch(conn, bank_id, non_duplicate_facts)
-                log_buffer.append(f"[5] Insert facts: {len(unit_ids)} units in {time.time() - step_start:.3f}s")
-
-                # Process entities
-                step_start = time.time()
-                # Build map of content_index -> user entities for merging
-                user_entities_per_content = {
-                    idx: content.entities for idx, content in enumerate(contents) if content.entities
-                }
-                entity_links = await entity_processing.process_entities_batch(
-                    entity_resolver,
-                    conn,
-                    bank_id,
-                    unit_ids,
-                    non_duplicate_facts,
-                    log_buffer,
-                    user_entities_per_content=user_entities_per_content,
-                    entity_labels=getattr(config, "entity_labels", None),
-                )
-                log_buffer.append(f"[6] Process entities: {len(entity_links)} links in {time.time() - step_start:.3f}s")
-
-                # Create temporal links
-                step_start = time.time()
-                temporal_link_count = await link_creation.create_temporal_links_batch(conn, bank_id, unit_ids)
-                log_buffer.append(f"[7] Temporal links: {temporal_link_count} links in {time.time() - step_start:.3f}s")
-
-                # Create semantic links
-                step_start = time.time()
-                embeddings_for_links = [fact.embedding for fact in non_duplicate_facts]
-                semantic_link_count = await link_creation.create_semantic_links_batch(
-                    conn, bank_id, unit_ids, embeddings_for_links
-                )
-                log_buffer.append(f"[8] Semantic links: {semantic_link_count} links in {time.time() - step_start:.3f}s")
-
-                # Insert entity links
-                step_start = time.time()
-                if entity_links:
-                    await entity_processing.insert_entity_links_batch(conn, entity_links)
-                log_buffer.append(
-                    f"[9] Entity links: {len(entity_links) if entity_links else 0} links in {time.time() - step_start:.3f}s"
-                )
-
-                # Create causal links
-                step_start = time.time()
-                causal_link_count = await link_creation.create_causal_links_batch(conn, unit_ids, non_duplicate_facts)
-                log_buffer.append(f"[10] Causal links: {causal_link_count} links in {time.time() - step_start:.3f}s")
-
-                # Map results back to original content items
-                result_unit_ids = _map_results_to_contents(contents, extracted_facts, unit_ids)
-
-                # Transactional outbox: queue any side-effect tasks (e.g. webhook deliveries)
-                # inside the same transaction so they are atomically committed with the retain data.
-                if outbox_callback:
-                    await outbox_callback(conn)
-
-            # Flush entity stats (mention_count / last_seen) now that the transaction
-            # has committed.  Uses a fresh pool connection — no locks held.
-            await entity_resolver.flush_pending_stats()
-
-            # Log final summary
-            total_time = time.time() - start_time
-            log_buffer.append(f"{'=' * 60}")
-            log_buffer.append(f"RETAIN_BATCH COMPLETE: {len(unit_ids)} units in {total_time:.3f}s")
-            if document_ids_added:
-                log_buffer.append(f"Documents: {', '.join(document_ids_added)}")
-            log_buffer.append(f"{'=' * 60}")
-
-            logger.info("\n" + "\n".join(log_buffer) + "\n")
-
-    await retry_with_backoff(_run_db_work)
-    return result_unit_ids, usage
-
-
-def _map_results_to_contents(
-    contents: list[RetainContent],
-    extracted_facts: list[ExtractedFact],
-    unit_ids: list[str],
-) -> list[list[str]]:
-    """Map created unit IDs back to original content items."""
-    facts_by_content: dict[int, list[int]] = {i: [] for i in range(len(contents))}
-    for i, fact in enumerate(extracted_facts):
-        facts_by_content[fact.content_index].append(i)
-
-    result_unit_ids = []
-    unit_idx = 0
-    for content_index in range(len(contents)):
-        content_unit_ids = []
-        for _ in facts_by_content[content_index]:
-            content_unit_ids.append(unit_ids[unit_idx])
-            unit_idx += 1
-        result_unit_ids.append(content_unit_ids)
-
-    return result_unit_ids

hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py

@@ -1,493 +0,0 @@
-"""
-Link Expansion graph retrieval.
-
-Expands from semantic/temporal seeds through three parallel, first-class signals
-stored in memory_links:
-
-1. Entity links  — precomputed co-occurrence graph (created at retain time, bounded to
-                   MAX_LINKS_PER_ENTITY per entity). Score = number of distinct shared
-                   entities between the seed set and each candidate.
-2. Semantic links — precomputed kNN graph (each new fact linked to its top-5 most
-                    similar existing facts at insert time, similarity >= 0.7). Checked
-                    in both directions since the graph is not symmetric. Score = weight.
-3. Causal links  — explicit causal chains (causes/caused_by/enables/prevents).
-                   Score = weight + 1.0 (boosted as highest-quality signal).
-
-All three signals are bounded at retain time, so no LATERAL fan-out caps are needed
-at query time. Each expansion is a simple aggregation over a small result set.
-
-For non-observation fact types the three expansions are issued as a single CTE query
-(one roundtrip, one connection) with a `source` discriminator column so the Python
-merge step can apply per-signal score transformations.
-"""
-
-import logging
-import math
-import time
-
-from ..db_utils import acquire_with_retry
-from ..memory_engine import fq_table
-from .graph_retrieval import GraphRetriever
-from .tags import TagGroup, TagsMatch, filter_results_by_tag_groups, filter_results_by_tags
-from .types import MPFPTimings, RetrievalResult
-
-logger = logging.getLogger(__name__)
-
-
-async def _find_semantic_seeds(
-    conn,
-    query_embedding_str: str,
-    bank_id: str,
-    fact_type: str,
-    limit: int = 20,
-    threshold: float = 0.3,
-    tags: list[str] | None = None,
-    tags_match: TagsMatch = "any",
-    tag_groups: list[TagGroup] | None = None,
-) -> list[RetrievalResult]:
-    """Find semantic seeds via embedding search."""
-    from .tags import build_tag_groups_where_clause, build_tags_where_clause_simple
-
-    tags_clause = build_tags_where_clause_simple(tags, 6, match=tags_match)
-    tag_groups_param_start = 6 + (1 if tags else 0)
-    groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start)
-    params = [query_embedding_str, bank_id, fact_type, threshold, limit]
-    if tags:
-        params.append(tags)
-    params.extend(groups_params)
-
-    rows = await conn.fetch(
-        f"""
-        SELECT id, text, context, event_date, occurred_start, occurred_end,
-               mentioned_at, fact_type, document_id, chunk_id, tags,
-               1 - (embedding <=> $1::vector) AS similarity
-        FROM {fq_table("memory_units")}
-        WHERE bank_id = $2
-          AND embedding IS NOT NULL
-          AND fact_type = $3
-          AND (1 - (embedding <=> $1::vector)) >= $4
-          {tags_clause}
-          {groups_clause}
-        ORDER BY embedding <=> $1::vector
-        LIMIT $5
-        """,
-        *params,
-    )
-    return [RetrievalResult.from_db_row(dict(r)) for r in rows]
-
-
-class LinkExpansionRetriever(GraphRetriever):
-    """
-    Graph retrieval via direct link expansion from seeds.
-
-    Runs three expansions through precomputed memory_links: entity co-occurrence,
-    semantic kNN, and causal chains, all bounded at retain time.
-
-    For non-observation fact types the three expansions are issued as a single CTE
-    query (one roundtrip, one connection slot) with a `source` discriminator column.
-    The Python merge step applies per-signal score transformations.
-    """
-
-    def __init__(
-        self,
-        causal_weight_threshold: float = 0.3,
-    ):
-        """
-        Args:
-            causal_weight_threshold: Minimum weight for causal links to follow.
-        """
-        self.causal_weight_threshold = causal_weight_threshold
-
-    @property
-    def name(self) -> str:
-        return "link_expansion"
-
-    async def retrieve(
-        self,
-        pool,
-        query_embedding_str: str,
-        bank_id: str,
-        fact_type: str,
-        budget: int,
-        query_text: str | None = None,
-        semantic_seeds: list[RetrievalResult] | None = None,
-        temporal_seeds: list[RetrievalResult] | None = None,
-        adjacency=None,
-        tags: list[str] | None = None,
-        tags_match: TagsMatch = "any",
-        tag_groups: list[TagGroup] | None = None,
-    ) -> tuple[list[RetrievalResult], MPFPTimings | None]:
-        """
-        Retrieve facts by expanding links from seeds.
-
-        Args:
-            pool: Database connection pool
-            query_embedding_str: Query embedding as string
-            bank_id: Memory bank ID
-            fact_type: Fact type to filter
-            budget: Maximum results to return
-            query_text: Original query text (unused)
-            semantic_seeds: Pre-computed semantic entry points
-            temporal_seeds: Pre-computed temporal entry points
-            adjacency: Unused, kept for interface compatibility
-            tags: Optional list of tags for visibility filtering
-
-        Returns:
-            Tuple of (results, timings)
-        """
-        start_time = time.time()
-        timings = MPFPTimings(fact_type=fact_type)
-
-        async with acquire_with_retry(pool) as conn:
-            # Find seeds if not provided
-            if semantic_seeds:
-                all_seeds = list(semantic_seeds)
-            else:
-                seeds_start = time.time()
-                all_seeds = await _find_semantic_seeds(
-                    conn,
-                    query_embedding_str,
-                    bank_id,
-                    fact_type,
-                    limit=20,
-                    threshold=0.3,
-                    tags=tags,
-                    tags_match=tags_match,
-                    tag_groups=tag_groups,
-                )
-                timings.seeds_time = time.time() - seeds_start
-                logger.debug(
-                    f"[LinkExpansion] Found {len(all_seeds)} semantic seeds for fact_type={fact_type} "
-                    f"(tags={tags}, tags_match={tags_match})"
-                )
-
-            if temporal_seeds:
-                all_seeds.extend(temporal_seeds)
-
-            if not all_seeds:
-                return [], timings
-
-            seed_ids = list({s.id for s in all_seeds})
-            timings.pattern_count = len(seed_ids)
-
-            query_start = time.time()
-
-            if fact_type == "observation":
-                entity_rows, semantic_rows, causal_rows = await self._expand_observations(conn, seed_ids, budget)
-            else:
-                entity_rows, semantic_rows, causal_rows = await self._expand_combined(conn, seed_ids, fact_type, budget)
-
-            timings.edge_load_time = time.time() - query_start
-            timings.db_queries = 1
-            timings.edge_count = len(entity_rows) + len(semantic_rows) + len(causal_rows)
-
-        # Merge results with additive intra-score: entity + semantic + causal ∈ [0, 3].
-        #
-        # Entity score: tanh(count × 0.5) maps shared-entity count to [0, 1]:
-        #   1 entity → 0.46,  2 → 0.76,  3 → 0.91,  4 → 0.96  (saturates naturally)
-        # Semantic score: similarity weight, already ∈ [0.7, 1.0].
-        # Causal score:   link weight, already ∈ [0, 1].
-        #
-        # Facts appearing in multiple signals accumulate higher scores, rewarding
-        # convergent evidence. The outer RRF uses rank position from this sorted list.
-        entity_scores: dict[str, float] = {}
-        semantic_scores: dict[str, float] = {}
-        causal_scores: dict[str, float] = {}
-        row_map: dict[str, dict] = {}
-
-        for row in entity_rows:
-            fact_id = str(row["id"])
-            entity_scores[fact_id] = math.tanh(row["score"] * 0.5)
-            row_map[fact_id] = dict(row)
-
-        for row in semantic_rows:
-            fact_id = str(row["id"])
-            semantic_scores[fact_id] = max(semantic_scores.get(fact_id, 0.0), row["score"])
-            row_map.setdefault(fact_id, dict(row))
-
-        for row in causal_rows:
-            fact_id = str(row["id"])
-            causal_scores[fact_id] = max(causal_scores.get(fact_id, 0.0), row["score"])
-            row_map.setdefault(fact_id, dict(row))
-
-        all_ids = set(entity_scores) | set(semantic_scores) | set(causal_scores)
-        score_map = {
-            fid: entity_scores.get(fid, 0.0) + semantic_scores.get(fid, 0.0) + causal_scores.get(fid, 0.0)
-            for fid in all_ids
-        }
-
-        sorted_ids = sorted(score_map.keys(), key=lambda x: score_map[x], reverse=True)[:budget]
-        rows = [row_map[fact_id] for fact_id in sorted_ids]
-
-        results = []
-        for row in rows:
-            result = RetrievalResult.from_db_row(dict(row))
-            result.activation = row["score"]
-            results.append(result)
-
-        if tags:
-            results = filter_results_by_tags(results, tags, match=tags_match)
-
-        if tag_groups:
-            results = filter_results_by_tag_groups(results, tag_groups)
-
-        timings.result_count = len(results)
-        timings.traverse = time.time() - start_time
-
-        logger.debug(
-            f"LinkExpansion: {len(results)} results from {len(seed_ids)} seeds "
-            f"in {timings.traverse * 1000:.1f}ms (query: {timings.edge_load_time * 1000:.1f}ms)"
-        )
-
-        return results, timings
-
-    async def _expand_combined(
-        self,
-        conn,
-        seed_ids: list,
-        fact_type: str,
-        budget: int,
-    ) -> tuple[list, list, list]:
-        """
-        Single-roundtrip CTE query combining entity, semantic, and causal expansions.
-
-        Uses a `source` discriminator column so the caller can apply per-signal
-        score transformations.  The three CTEs share one connection slot — important
-        for asyncpg which does not allow concurrent queries on the same connection.
-
-        Index coverage (requires migration d2e3f4a5b6c7):
-          entity:   idx_memory_links_entity_covering (from_unit_id) INCLUDE (to_unit_id, entity_id)
-                    WHERE link_type = 'entity'  → index-only scan, no heap reads
-          semantic incoming:
-                    idx_memory_links_to_type_weight (to_unit_id, link_type, weight DESC)
-                    → replaces costly BitmapAnd of two separate scans
-        """
-        ml = fq_table("memory_links")
-        mu = fq_table("memory_units")
-        all_rows = await conn.fetch(
-            f"""
-            WITH entity_expanded AS (
-                -- Entity co-occurrence: seeds → their precomputed entity-link neighbors.
-                -- Score = distinct shared entities (bounded at retain time to
-                -- MAX_LINKS_PER_ENTITY=50).  GROUP BY mu.id is sufficient because mu.id
-                -- is the primary key and functionally determines all other mu columns.
-                SELECT
-                    mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
-                    mu.occurred_end, mu.mentioned_at,
-                    mu.fact_type, mu.document_id, mu.chunk_id, mu.tags,
-                    COUNT(DISTINCT ml.entity_id)::float AS score,
-                    'entity'::text AS source
-                FROM {ml} ml
-                JOIN {mu} mu ON mu.id = ml.to_unit_id
-                WHERE ml.from_unit_id = ANY($1::uuid[])
-                  AND ml.link_type = 'entity'
-                  AND mu.fact_type = $2
-                  AND mu.id != ALL($1::uuid[])
-                GROUP BY mu.id
-                ORDER BY score DESC
-                LIMIT $3
-            ),
-            semantic_expanded AS (
-                -- Semantic kNN: both outgoing (seeds → their kNN at insert time) and
-                -- incoming (facts inserted after seeds that found seeds as kNN).
-                -- Score = max similarity weight across both directions.
-                SELECT
-                    id, text, context, event_date, occurred_start,
-                    occurred_end, mentioned_at,
-                    fact_type, document_id, chunk_id, tags,
-                    MAX(weight) AS score,
-                    'semantic'::text AS source
-                FROM (
-                    SELECT
-                        mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
-                        mu.occurred_end, mu.mentioned_at,
-                        mu.fact_type, mu.document_id, mu.chunk_id, mu.tags,
-                        ml.weight
-                    FROM {ml} ml
-                    JOIN {mu} mu ON mu.id = ml.to_unit_id
-                    WHERE ml.from_unit_id = ANY($1::uuid[])
-                      AND ml.link_type = 'semantic'
-                      AND mu.fact_type = $2
-                      AND mu.id != ALL($1::uuid[])
-                    UNION ALL
-                    SELECT
-                        mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
-                        mu.occurred_end, mu.mentioned_at,
-                        mu.fact_type, mu.document_id, mu.chunk_id, mu.tags,
-                        ml.weight
-                    FROM {ml} ml
-                    JOIN {mu} mu ON mu.id = ml.from_unit_id
-                    WHERE ml.to_unit_id = ANY($1::uuid[])
-                      AND ml.link_type = 'semantic'
-                      AND mu.fact_type = $2
-                      AND mu.id != ALL($1::uuid[])
-                ) sem_raw
-                GROUP BY id, text, context, event_date, occurred_start,
-                         occurred_end, mentioned_at,
-                         fact_type, document_id, chunk_id, tags
-                ORDER BY score DESC
-                LIMIT $3
-            ),
-            causal_expanded AS (
-                -- Causal chains: explicit causes/enables/prevents links from seeds.
-                -- DISTINCT ON handles the case where a seed has multiple causal links
-                -- to the same target; best weight wins.
-                SELECT DISTINCT ON (mu.id)
-                    mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
-                    mu.occurred_end, mu.mentioned_at,
-                    mu.fact_type, mu.document_id, mu.chunk_id, mu.tags,
-                    ml.weight AS score,
-                    'causal'::text AS source
-                FROM {ml} ml
-                JOIN {mu} mu ON ml.to_unit_id = mu.id
-                WHERE ml.from_unit_id = ANY($1::uuid[])
-                  AND ml.link_type IN ('causes', 'caused_by', 'enables', 'prevents')
-                  AND ml.weight >= $4
-                  AND mu.fact_type = $2
-                ORDER BY mu.id, ml.weight DESC
-                LIMIT $3
-            )
-            SELECT * FROM entity_expanded
-            UNION ALL
-            SELECT * FROM semantic_expanded
-            UNION ALL
-            SELECT * FROM causal_expanded
-            """,
-            seed_ids,
-            fact_type,
-            budget,
-            self.causal_weight_threshold,
-        )
-
-        entity_rows = [r for r in all_rows if r["source"] == "entity"]
-        semantic_rows = [r for r in all_rows if r["source"] == "semantic"]
-        causal_rows = [r for r in all_rows if r["source"] == "causal"]
-        return entity_rows, semantic_rows, causal_rows
-
-    async def _expand_observations(
-        self,
-        conn,
-        seed_ids: list,
-        budget: int,
-    ) -> tuple[list, list, list]:
-        """
-        Observation-specific expansion.
-
-        Observations don't have direct entity links in memory_links (they're created
-        by consolidation, not retain).  Instead, traverse source_memory_ids → world
-        facts → entities → other world facts → their observations.
-
-        Semantic and causal expansions run as a second combined CTE query.
-        """
-        source_ids_found: list = []
-        if logger.isEnabledFor(logging.DEBUG):
-            debug_rows = await conn.fetch(
-                f"""
-                SELECT id, source_memory_ids
-                FROM {fq_table("memory_units")}
-                WHERE id = ANY($1::uuid[])
-                """,
-                seed_ids,
-            )
-            for row in debug_rows:
-                if row["source_memory_ids"]:
-                    source_ids_found.extend(row["source_memory_ids"])
-            logger.debug(
-                f"[LinkExpansion] observation graph: {len(seed_ids)} seeds, "
-                f"{len(source_ids_found)} source_memory_ids found"
-            )
-
-        entity_rows = await conn.fetch(
-            f"""
-            WITH seed_sources AS (
-                SELECT DISTINCT unnest(source_memory_ids) AS source_id
-                FROM {fq_table("memory_units")}
-                WHERE id = ANY($1::uuid[])
-                  AND source_memory_ids IS NOT NULL
-            ),
-            connected_sources AS (
-                -- Mirror the non-observation entity expansion: follow pre-bounded entity
-                -- links in memory_links (capped to MAX_LINKS_PER_ENTITY=50 at retain time).
-                -- Score = number of distinct shared entities, same as the non-obs path.
-                SELECT DISTINCT ml.to_unit_id AS source_id
-                FROM seed_sources ss
-                JOIN {fq_table("memory_links")} ml ON ml.from_unit_id = ss.source_id
-                WHERE ml.link_type = 'entity'
-            ),
-            connected_array AS (
-                SELECT array_agg(source_id) AS source_ids FROM connected_sources
-            )
-            SELECT
-                mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
-                mu.occurred_end, mu.mentioned_at,
-                mu.fact_type, mu.document_id, mu.chunk_id, mu.tags,
-                (SELECT COUNT(DISTINCT s) FROM unnest(mu.source_memory_ids) s WHERE s = ANY(ca.source_ids))::float AS score
-            FROM {fq_table("memory_units")} mu, connected_array ca
-            WHERE mu.fact_type = 'observation'
-              AND mu.id != ALL($1::uuid[])
-              AND ca.source_ids IS NOT NULL
-              AND mu.source_memory_ids && ca.source_ids
-            ORDER BY score DESC
-            LIMIT $2
-            """,
-            seed_ids,
-            budget,
-        )
-        logger.debug(f"[LinkExpansion] observation graph: found {len(entity_rows)} connected observations")
-
-        # Semantic + causal for observations in one query
-        ml = fq_table("memory_links")
-        mu = fq_table("memory_units")
-        sem_causal_rows = await conn.fetch(
-            f"""
-            WITH semantic_expanded AS (
-                SELECT
-                    id, text, context, event_date, occurred_start,
-                    occurred_end, mentioned_at,
-                    fact_type, document_id, chunk_id, tags,
-                    MAX(weight) AS score,
-                    'semantic'::text AS source
-                FROM (
-                    SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
-                           mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id,
-                           mu.chunk_id, mu.tags, ml.weight
-                    FROM {ml} ml JOIN {mu} mu ON mu.id = ml.to_unit_id
-                    WHERE ml.from_unit_id = ANY($1::uuid[])
-                      AND ml.link_type = 'semantic' AND mu.fact_type = 'observation'
-                      AND mu.id != ALL($1::uuid[])
-                    UNION ALL
-                    SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
-                           mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id,
-                           mu.chunk_id, mu.tags, ml.weight
-                    FROM {ml} ml JOIN {mu} mu ON mu.id = ml.from_unit_id
-                    WHERE ml.to_unit_id = ANY($1::uuid[])
-                      AND ml.link_type = 'semantic' AND mu.fact_type = 'observation'
-                      AND mu.id != ALL($1::uuid[])
-                ) sem_raw
-                GROUP BY id, text, context, event_date, occurred_start, occurred_end,
-                         mentioned_at, fact_type, document_id, chunk_id, tags
-                ORDER BY score DESC LIMIT $2
-            ),
-            causal_expanded AS (
-                SELECT DISTINCT ON (mu.id)
-                    mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
-                    mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id,
-                    mu.chunk_id, mu.tags, ml.weight AS score, 'causal'::text AS source
-                FROM {ml} ml JOIN {mu} mu ON ml.to_unit_id = mu.id
-                WHERE ml.from_unit_id = ANY($1::uuid[])
-                  AND ml.link_type IN ('causes', 'caused_by', 'enables', 'prevents')
-                  AND ml.weight >= $3 AND mu.fact_type = 'observation'
-                ORDER BY mu.id, ml.weight DESC LIMIT $2
-            )
-            SELECT * FROM semantic_expanded
-            UNION ALL
-            SELECT * FROM causal_expanded
-            """,
-            seed_ids,
-            budget,
-            self.causal_weight_threshold,
-        )
-
-        semantic_rows = [r for r in sem_causal_rows if r["source"] == "semantic"]
-        causal_rows = [r for r in sem_causal_rows if r["source"] == "causal"]
-        return entity_rows, semantic_rows, causal_rows

hindsight-api-slim/hindsight_api/engine/search/tags.py

@@ -1,390 +0,0 @@
-"""
-Tags filtering utilities for retrieval.
-
-Provides SQL building functions for filtering memories by tags.
-Supports four matching modes via TagsMatch enum:
-- "any": OR matching, includes untagged memories (default, backward compatible)
-- "all": AND matching, includes untagged memories
-- "any_strict": OR matching, excludes untagged memories
-- "all_strict": AND matching, excludes untagged memories
-
-OR matching (any/any_strict): Memory matches if ANY of its tags overlap with request tags
-AND matching (all/all_strict): Memory matches if ALL request tags are present in its tags
-"""
-
-from __future__ import annotations
-
-from typing import Annotated, Literal
-
-from pydantic import BaseModel, ConfigDict, Field
-
-TagsMatch = Literal["any", "all", "any_strict", "all_strict"]
-
-
-def _parse_tags_match(match: TagsMatch) -> tuple[str, bool]:
-    """
-    Parse TagsMatch into operator and include_untagged flag.
-
-    Returns:
-        Tuple of (operator, include_untagged)
-        - operator: "&&" for any/any_strict, "@>" for all/all_strict
-        - include_untagged: True for any/all, False for any_strict/all_strict
-    """
-    if match == "any":
-        return "&&", True
-    elif match == "all":
-        return "@>", True
-    elif match == "any_strict":
-        return "&&", False
-    elif match == "all_strict":
-        return "@>", False
-    else:
-        # Default to "any" behavior
-        return "&&", True
-
-
-def build_tags_where_clause(
-    tags: list[str] | None,
-    param_offset: int = 1,
-    table_alias: str = "",
-    match: TagsMatch = "any",
-) -> tuple[str, list, int]:
-    """
-    Build a SQL WHERE clause for filtering by tags.
-
-    Supports four matching modes:
-    - "any" (default): OR matching, includes untagged memories
-    - "all": AND matching, includes untagged memories
-    - "any_strict": OR matching, excludes untagged memories
-    - "all_strict": AND matching, excludes untagged memories
-
-    Args:
-        tags: List of tags to filter by. If None or empty, returns empty clause (no filtering).
-        param_offset: Starting parameter number for SQL placeholders (default 1).
-        table_alias: Optional table alias prefix (e.g., "mu." for "memory_units mu").
-        match: Matching mode. Defaults to "any".
-
-    Returns:
-        Tuple of (sql_clause, params, next_param_offset):
-        - sql_clause: SQL WHERE clause string
-        - params: List of parameter values to bind
-        - next_param_offset: Next available parameter number
-
-    Example:
-        >>> clause, params, next_offset = build_tags_where_clause(['user_a'], 3, 'mu.', 'any_strict')
-        >>> print(clause)  # "AND mu.tags IS NOT NULL AND mu.tags != '{}' AND mu.tags && $3"
-    """
-    if not tags:
-        return "", [], param_offset
-
-    column = f"{table_alias}tags" if table_alias else "tags"
-    operator, include_untagged = _parse_tags_match(match)
-
-    if include_untagged:
-        # Include untagged memories (NULL or empty array) OR matching tags
-        clause = f"AND ({column} IS NULL OR {column} = '{{}}' OR {column} {operator} ${param_offset})"
-    else:
-        # Strict: only memories with matching tags (exclude NULL and empty)
-        clause = f"AND {column} IS NOT NULL AND {column} != '{{}}' AND {column} {operator} ${param_offset}"
-
-    return clause, [tags], param_offset + 1
-
-
-def build_tags_where_clause_simple(
-    tags: list[str] | None,
-    param_num: int,
-    table_alias: str = "",
-    match: TagsMatch = "any",
-) -> str:
-    """
-    Build a simple SQL WHERE clause for tags filtering.
-
-    This is a convenience version that returns just the clause string,
-    assuming the caller will add the tags array to their params list.
-
-    Args:
-        tags: List of tags to filter by. If None or empty, returns empty string.
-        param_num: Parameter number to use in the clause.
-        table_alias: Optional table alias prefix.
-        match: Matching mode. Defaults to "any".
-
-    Returns:
-        SQL clause string or empty string.
-    """
-    if not tags:
-        return ""
-
-    column = f"{table_alias}tags" if table_alias else "tags"
-    operator, include_untagged = _parse_tags_match(match)
-
-    if include_untagged:
-        # Include untagged memories (NULL or empty array) OR matching tags
-        return f"AND ({column} IS NULL OR {column} = '{{}}' OR {column} {operator} ${param_num})"
-    else:
-        # Strict: only memories with matching tags (exclude NULL and empty)
-        return f"AND {column} IS NOT NULL AND {column} != '{{}}' AND {column} {operator} ${param_num}"
-
-
-def filter_results_by_tags(
-    results: list,
-    tags: list[str] | None,
-    match: TagsMatch = "any",
-) -> list:
-    """
-    Filter retrieval results by tags in Python (for post-processing).
-
-    Used when SQL filtering isn't possible (e.g., graph traversal results).
-
-    Args:
-        results: List of RetrievalResult objects with a 'tags' attribute.
-        tags: List of tags to filter by. If None or empty, returns all results.
-        match: Matching mode. Defaults to "any".
-
-    Returns:
-        Filtered list of results.
-    """
-    if not tags:
-        return results
-
-    _, include_untagged = _parse_tags_match(match)
-    is_any_match = match in ("any", "any_strict")
-
-    tags_set = set(tags)
-    filtered = []
-
-    for result in results:
-        result_tags = getattr(result, "tags", None)
-
-        # Check if untagged
-        is_untagged = result_tags is None or len(result_tags) == 0
-
-        if is_untagged:
-            if include_untagged:
-                filtered.append(result)
-            # else: skip untagged
-        else:
-            result_tags_set = set(result_tags)
-            if is_any_match:
-                # Any overlap
-                if result_tags_set & tags_set:
-                    filtered.append(result)
-            else:
-                # All tags must be present
-                if tags_set <= result_tags_set:
-                    filtered.append(result)
-
-    return filtered
-
-
-# =============================================================================
-# Compound tag group models (recursive boolean expressions)
-# =============================================================================
-
-
-class TagGroupLeaf(BaseModel):
-    """A leaf tag filter: matches memories by tag list and match mode."""
-
-    tags: list[str]
-    match: TagsMatch = "any_strict"
-
-
-class TagGroupAnd(BaseModel):
-    """Compound AND group: all child filters must match."""
-
-    model_config = ConfigDict(populate_by_name=True)
-    filters: list[TagGroup] = Field(alias="and")
-
-
-class TagGroupOr(BaseModel):
-    """Compound OR group: at least one child filter must match."""
-
-    model_config = ConfigDict(populate_by_name=True)
-    filters: list[TagGroup] = Field(alias="or")
-
-
-class TagGroupNot(BaseModel):
-    """Compound NOT group: child filter must NOT match."""
-
-    model_config = ConfigDict(populate_by_name=True)
-    filter: TagGroup = Field(alias="not")
-
-
-# TagGroup is a discriminated union; Pydantic will try left-to-right.
-# TagGroupLeaf is identified by the presence of 'tags'.
-# TagGroupAnd / TagGroupOr / TagGroupNot are compound (no 'tags' key).
-TagGroup = Annotated[
-    TagGroupLeaf | TagGroupAnd | TagGroupOr | TagGroupNot,
-    Field(union_mode="left_to_right"),
-]
-
-# Rebuild forward-reference models so recursive TagGroup is resolved.
-TagGroupAnd.model_rebuild()
-TagGroupOr.model_rebuild()
-TagGroupNot.model_rebuild()
-
-
-# =============================================================================
-# SQL builder for compound tag groups
-# =============================================================================
-
-
-def _build_group_clause(
-    group: TagGroup,
-    param_offset: int,
-    table_alias: str,
-) -> tuple[str, list, int]:
-    """
-    Recursively build an inner SQL clause (no leading AND/OR) for a single TagGroup.
-
-    Returns:
-        (inner_clause, params, next_param_offset)
-    """
-    if isinstance(group, TagGroupLeaf):
-        column = f"{table_alias}tags" if table_alias else "tags"
-        operator, include_untagged = _parse_tags_match(group.match)
-        if include_untagged:
-            clause = f"({column} IS NULL OR {column} = '{{}}' OR {column} {operator} ${param_offset})"
-        else:
-            clause = f"({column} IS NOT NULL AND {column} != '{{}}' AND {column} {operator} ${param_offset})"
-        return clause, [group.tags], param_offset + 1
-
-    elif isinstance(group, TagGroupAnd):
-        parts = []
-        params: list = []
-        offset = param_offset
-        for child in group.filters:
-            child_clause, child_params, offset = _build_group_clause(child, offset, table_alias)
-            parts.append(child_clause)
-            params.extend(child_params)
-        inner = " AND ".join(parts)
-        return f"({inner})", params, offset
-
-    elif isinstance(group, TagGroupOr):
-        parts = []
-        params = []
-        offset = param_offset
-        for child in group.filters:
-            child_clause, child_params, offset = _build_group_clause(child, offset, table_alias)
-            parts.append(child_clause)
-            params.extend(child_params)
-        inner = " OR ".join(parts)
-        return f"({inner})", params, offset
-
-    elif isinstance(group, TagGroupNot):
-        child_clause, child_params, next_offset = _build_group_clause(group.filter, param_offset, table_alias)
-        return f"NOT {child_clause}", child_params, next_offset
-
-    else:
-        # Should never happen with proper Pydantic validation
-        return "", [], param_offset
-
-
-def build_tag_groups_where_clause(
-    tag_groups: list[TagGroup] | None,
-    param_offset: int,
-    table_alias: str = "",
-) -> tuple[str, list, int]:
-    """
-    Build a SQL WHERE clause for compound tag group filtering.
-
-    Top-level groups are AND-ed together. Each group is a recursive boolean
-    expression (leaf, and, or, not).
-
-    Args:
-        tag_groups: List of TagGroup objects. If None or empty, returns empty clause.
-        param_offset: Starting parameter number for SQL placeholders.
-        table_alias: Optional table alias prefix (e.g., "mu." for "memory_units mu").
-
-    Returns:
-        Tuple of (sql_clause, params, next_param_offset):
-        - sql_clause: SQL WHERE clause string starting with "AND" (or empty string)
-        - params: List of parameter values to bind (one per leaf node)
-        - next_param_offset: Next available parameter number
-
-    Example:
-        >>> groups = [TagGroupLeaf(tags=["user:alice"], match="all_strict")]
-        >>> clause, params, next_offset = build_tag_groups_where_clause(groups, 3)
-        >>> print(clause)  # "AND (tags IS NOT NULL AND tags != '{}' AND tags @> $3)"
-    """
-    if not tag_groups:
-        return "", [], param_offset
-
-    all_params: list = []
-    all_clauses: list[str] = []
-    offset = param_offset
-
-    for group in tag_groups:
-        inner_clause, group_params, offset = _build_group_clause(group, offset, table_alias)
-        all_clauses.append(inner_clause)
-        all_params.extend(group_params)
-
-    combined = " AND ".join(all_clauses)
-    return f"AND {combined}", all_params, offset
-
-
-# =============================================================================
-# Python-side filter for compound tag groups (post-retrieval filtering)
-# =============================================================================
-
-
-def _match_group(result: object, group: TagGroup) -> bool:
-    """
-    Recursively evaluate a TagGroup against a retrieval result.
-
-    Args:
-        result: Any object with a 'tags' attribute (list[str] or None).
-        group: The TagGroup to evaluate.
-
-    Returns:
-        True if the result matches the group, False otherwise.
-    """
-    if isinstance(group, TagGroupLeaf):
-        result_tags = getattr(result, "tags", None)
-        is_untagged = result_tags is None or len(result_tags) == 0
-        _, include_untagged = _parse_tags_match(group.match)
-        is_any_match = group.match in ("any", "any_strict")
-        tags_set = set(group.tags)
-
-        if is_untagged:
-            return include_untagged
-        else:
-            result_tags_set = set(result_tags)
-            if is_any_match:
-                return bool(result_tags_set & tags_set)
-            else:
-                return tags_set <= result_tags_set
-
-    elif isinstance(group, TagGroupAnd):
-        return all(_match_group(result, child) for child in group.filters)
-
-    elif isinstance(group, TagGroupOr):
-        return any(_match_group(result, child) for child in group.filters)
-
-    elif isinstance(group, TagGroupNot):
-        return not _match_group(result, group.filter)
-
-    else:
-        return True
-
-
-def filter_results_by_tag_groups(
-    results: list,
-    tag_groups: list[TagGroup] | None,
-) -> list:
-    """
-    Filter retrieval results by compound tag groups in Python (for post-processing).
-
-    Used when SQL filtering isn't possible (e.g., graph traversal results).
-    Top-level groups are AND-ed together.
-
-    Args:
-        results: List of RetrievalResult objects with a 'tags' attribute.
-        tag_groups: List of TagGroup objects. If None or empty, returns all results.
-
-    Returns:
-        Filtered list of results where ALL top-level groups match.
-    """
-    if not tag_groups:
-        return results
-
-    return [r for r in results if all(_match_group(r, group) for group in tag_groups)]

hindsight-api-slim/hindsight_api/engine/storage/gcs.py

@@ -1,105 +0,0 @@
-"""Google Cloud Storage backend using obstore."""
-
-import logging
-import os
-from datetime import datetime, timedelta, timezone
-
-import obstore as obs
-from obstore.store import GCSStore
-
-from .base import FileStorage
-
-logger = logging.getLogger(__name__)
-
-
-def _make_google_auth_credential_provider():
-    """Create a credential provider using google.auth (supports all credential types).
-
-    obstore's built-in credential parsing only supports service_account and
-    authorized_user JSON types.  This provider uses the google-auth library
-    which additionally handles external_account (Workload Identity Federation),
-    impersonated credentials, and metadata-server credentials.
-    """
-    import google.auth
-    import google.auth.transport.requests
-
-    credentials, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/cloud-platform"])
-    request = google.auth.transport.requests.Request()
-
-    def _provide():
-        credentials.refresh(request)
-        expiry = credentials.expiry
-        if expiry and expiry.tzinfo is None:
-            expiry = expiry.replace(tzinfo=timezone.utc)
-        return {"token": credentials.token, "expires_at": expiry}
-
-    return _provide
-
-
-class GCSFileStorage(FileStorage):
-    """
-    Google Cloud Storage backend.
-
-    Uses obstore (Rust-backed) for high-throughput async access to GCS.
-    Supports Application Default Credentials, service account keys, and explicit credentials.
-    """
-
-    def __init__(
-        self,
-        bucket: str,
-        service_account_key: str | None = None,
-    ):
-        kwargs: dict = {}
-        if service_account_key:
-            kwargs["service_account_key"] = service_account_key
-        else:
-            # Use google.auth credential provider for broad credential type support
-            # (service_account, authorized_user, external_account, metadata server, etc.)
-            try:
-                kwargs["credential_provider"] = _make_google_auth_credential_provider()
-                logger.info("Using google.auth credential provider for GCS")
-            except Exception as e:
-                logger.warning(
-                    f"Failed to create google.auth credential provider, falling back to obstore defaults: {e}"
-                )
-
-        # Workaround for https://github.com/developmentseed/obstore/issues/605
-        # obstore's Rust layer doesn't support external_account credentials (Workload
-        # Identity Federation) and eagerly parses GOOGLE_APPLICATION_CREDENTIALS even
-        # when credential_provider is given. Per the obstore maintainer's guidance,
-        # remove env vars so the Rust code doesn't try to authenticate itself.
-        # google.auth (used by credential_provider above) has already loaded credentials.
-        gac = os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None)
-        try:
-            self._store = GCSStore(bucket, **kwargs)
-        finally:
-            if gac is not None:
-                os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = gac
-        logger.info(f"Initialized GCS file storage: bucket={bucket}")
-
-    async def store(self, file_data: bytes, key: str, metadata: dict[str, str] | None = None) -> str:
-        await obs.put_async(self._store, key, file_data)
-        logger.debug(f"Stored file {key} ({len(file_data)} bytes) in GCS")
-        return key
-
-    async def retrieve(self, key: str) -> bytes:
-        try:
-            response = await obs.get_async(self._store, key)
-            return await response.bytes_async()
-        except Exception as e:
-            if "not found" in str(e).lower():
-                raise FileNotFoundError(f"File not found: {key}") from e
-            raise
-
-    async def delete(self, key: str) -> None:
-        await obs.delete_async(self._store, key)
-
-    async def exists(self, key: str) -> bool:
-        try:
-            await obs.head_async(self._store, key)
-            return True
-        except Exception:
-            return False
-
-    async def get_download_url(self, key: str, expires_in: int = 3600) -> str:
-        return await obs.sign_async(self._store, "GET", key, timedelta(seconds=expires_in))

hindsight-api-slim/hindsight_api/webhooks/__init__.py

@@ -1,13 +0,0 @@
-"""Webhook system for Hindsight API event notifications."""
-
-from .manager import WebhookManager
-from .models import ConsolidationEventData, RetainEventData, WebhookConfig, WebhookEvent, WebhookEventType
-
-__all__ = [
-    "WebhookManager",
-    "WebhookConfig",
-    "WebhookEvent",
-    "WebhookEventType",
-    "ConsolidationEventData",
-    "RetainEventData",
-]

hindsight-api-slim/hindsight_api/webhooks/manager.py

@@ -1,242 +0,0 @@
-"""Webhook manager for delivering event notifications."""
-
-import hashlib
-import hmac
-import json
-import logging
-import uuid
-from datetime import datetime, timezone
-from typing import TYPE_CHECKING
-
-import asyncpg
-
-from .models import WebhookConfig, WebhookEvent, WebhookHttpConfig
-
-if TYPE_CHECKING:
-    from hindsight_api.extensions.tenant import TenantExtension
-
-logger = logging.getLogger(__name__)
-
-# Retry delay schedule in seconds: 5 retries after the first attempt.
-# Fast early retries catch transient failures; later retries handle longer outages.
-RETRY_DELAYS = [5, 300, 1800, 7200, 18000]
-MAX_ATTEMPTS = len(RETRY_DELAYS) + 1  # first attempt + len(RETRY_DELAYS) retries
-
-
-def _fq_table(table: str, schema: str | None = None) -> str:
-    """Get fully-qualified table name with optional schema prefix."""
-    if schema:
-        return f'"{schema}".{table}'
-    return table
-
-
-def _parse_http_config(value: str | dict | None) -> WebhookHttpConfig:
-    """Parse http_config column value (JSONB returned as text or dict) into a model."""
-    if value is None:
-        return WebhookHttpConfig()
-    if isinstance(value, str):
-        return WebhookHttpConfig.model_validate_json(value)
-    return WebhookHttpConfig.model_validate(value)
-
-
-class WebhookManager:
-    """
-    Manages webhook registration and event firing.
-
-    Supports both global webhooks (configured via env vars) and per-bank
-    webhooks stored in the database. Deliveries are queued as async_operations
-    tasks (operation_type='webhook_delivery') and picked up by the worker poller.
-    """
-
-    def __init__(
-        self,
-        pool: asyncpg.Pool,
-        global_webhooks: list[WebhookConfig],
-        tenant_extension: "TenantExtension | None" = None,
-    ):
-        self._pool = pool
-        self._global_webhooks = global_webhooks
-        self._tenant_extension = tenant_extension
-
-    def _sign_payload(self, secret: str, payload_bytes: bytes) -> str:
-        """Compute HMAC-SHA256 signature for a payload."""
-        return "sha256=" + hmac.new(secret.encode(), payload_bytes, hashlib.sha256).hexdigest()
-
-    async def fire_event(self, event: WebhookEvent, schema: str | None = None) -> None:
-        """
-        Queue webhook deliveries for an event as async_operations tasks.
-
-        Loads per-bank and global webhooks, inserts pending webhook_delivery tasks for
-        any webhook whose event_types list matches the fired event type. The worker
-        poller picks these up and calls MemoryEngine._handle_webhook_delivery().
-
-        Args:
-            event: The event to deliver.
-            schema: Database schema (for multi-tenant). None = default schema.
-        """
-        webhook_table = _fq_table("webhooks", schema)
-        ops_table = _fq_table("async_operations", schema)
-        now = datetime.now(timezone.utc)
-        payload_str = event.model_dump_json()
-
-        try:
-            # Load per-bank webhooks from DB (bank-specific + global NULL rows)
-            rows = await self._pool.fetch(
-                f"""
-                SELECT id, bank_id, url, secret, event_types, enabled, http_config::text
-                FROM {webhook_table}
-                WHERE (bank_id = $1 OR bank_id IS NULL) AND enabled = true
-                """,
-                event.bank_id,
-            )
-
-            db_webhooks = [
-                WebhookConfig(
-                    id=str(row["id"]),
-                    bank_id=row["bank_id"],
-                    url=row["url"],
-                    secret=row["secret"],
-                    event_types=list(row["event_types"]) if row["event_types"] else [],
-                    enabled=row["enabled"],
-                    http_config=_parse_http_config(row["http_config"]),
-                )
-                for row in rows
-            ]
-
-            # Merge with global webhooks from env config
-            all_webhooks = self._global_webhooks + db_webhooks
-            matched = 0
-
-            for webhook in all_webhooks:
-                if not webhook.enabled:
-                    continue
-                if event.event.value not in webhook.event_types:
-                    continue
-
-                operation_id = uuid.uuid4()
-                webhook_id = webhook.id if webhook.id else None
-
-                task_payload = json.dumps(
-                    {
-                        "type": "webhook_delivery",
-                        "operation_id": str(operation_id),
-                        "bank_id": event.bank_id,
-                        "url": webhook.url,
-                        "secret": webhook.secret,
-                        "event_type": event.event.value,
-                        "payload": payload_str,
-                        "webhook_id": webhook_id,
-                        "http_config": webhook.http_config.model_dump(),
-                    }
-                )
-
-                await self._pool.execute(
-                    f"""
-                    INSERT INTO {ops_table}
-                      (operation_id, bank_id, operation_type, status, task_payload, result_metadata, created_at, updated_at)
-                    VALUES ($1, $2, 'webhook_delivery', 'pending', $3::jsonb, '{{}}'::jsonb, $4, $4)
-                    """,
-                    operation_id,
-                    event.bank_id,
-                    task_payload,
-                    now,
-                )
-                matched += 1
-
-            logger.debug(f"Fired webhook event {event.event} for bank {event.bank_id}: {matched} delivery(ies) queued")
-
-        except Exception as e:
-            logger.error(f"Failed to queue webhook deliveries for event {event.event}: {e}")
-
-    async def fire_event_with_conn(
-        self, event: WebhookEvent, conn: asyncpg.Connection, schema: str | None = None
-    ) -> None:
-        """
-        Queue webhook deliveries within an existing database connection/transaction.
-
-        Identical to fire_event() but uses the provided connection instead of acquiring
-        one from the pool. Use this to atomically insert delivery tasks in the same
-        transaction as the primary operation (transactional outbox pattern).
-
-        Args:
-            event: The event to deliver.
-            conn: Existing asyncpg connection (may be inside an active transaction).
-            schema: Database schema (for multi-tenant). None = default schema.
-        """
-        webhook_table = _fq_table("webhooks", schema)
-        ops_table = _fq_table("async_operations", schema)
-        now = datetime.now(timezone.utc)
-        payload_str = event.model_dump_json()
-
-        try:
-            rows = await conn.fetch(
-                f"""
-                SELECT id, bank_id, url, secret, event_types, enabled, http_config::text
-                FROM {webhook_table}
-                WHERE (bank_id = $1 OR bank_id IS NULL) AND enabled = true
-                """,
-                event.bank_id,
-            )
-
-            db_webhooks = [
-                WebhookConfig(
-                    id=str(row["id"]),
-                    bank_id=row["bank_id"],
-                    url=row["url"],
-                    secret=row["secret"],
-                    event_types=list(row["event_types"]) if row["event_types"] else [],
-                    enabled=row["enabled"],
-                    http_config=_parse_http_config(row["http_config"]),
-                )
-                for row in rows
-            ]
-
-            all_webhooks = self._global_webhooks + db_webhooks
-            matched = 0
-
-            for webhook in all_webhooks:
-                if not webhook.enabled:
-                    continue
-                if event.event.value not in webhook.event_types:
-                    continue
-
-                operation_id = uuid.uuid4()
-                webhook_id = webhook.id if webhook.id else None
-
-                task_payload = json.dumps(
-                    {
-                        "type": "webhook_delivery",
-                        "operation_id": str(operation_id),
-                        "bank_id": event.bank_id,
-                        "url": webhook.url,
-                        "secret": webhook.secret,
-                        "event_type": event.event.value,
-                        "payload": payload_str,
-                        "webhook_id": webhook_id,
-                        "http_config": webhook.http_config.model_dump(),
-                    }
-                )
-
-                await conn.execute(
-                    f"""
-                    INSERT INTO {ops_table}
-                      (operation_id, bank_id, operation_type, status, task_payload, result_metadata, created_at, updated_at)
-                    VALUES ($1, $2, 'webhook_delivery', 'pending', $3::jsonb, '{{}}'::jsonb, $4, $4)
-                    """,
-                    operation_id,
-                    event.bank_id,
-                    task_payload,
-                    now,
-                )
-                matched += 1
-
-            logger.debug(
-                f"Fired webhook event {event.event} for bank {event.bank_id}: {matched} delivery(ies) queued (in-transaction)"
-            )
-
-        except Exception as e:
-            logger.error(
-                f"Failed to queue webhook deliveries (in-transaction) for event {event.event}: {e}. "
-                "CRITICAL: The enclosing database transaction is now aborted and will roll back all changes."
-            )
-            raise

hindsight-api-slim/hindsight_api/webhooks/models.py

@@ -1,51 +0,0 @@
-"""Pydantic models for the webhook system."""
-
-from datetime import datetime
-from enum import StrEnum
-
-from pydantic import BaseModel, Field
-
-
-class WebhookEventType(StrEnum):
-    CONSOLIDATION_COMPLETED = "consolidation.completed"
-    RETAIN_COMPLETED = "retain.completed"
-
-
-class ConsolidationEventData(BaseModel):
-    observations_created: int | None = None
-    observations_updated: int | None = None
-    observations_deleted: int | None = None
-    error_message: str | None = None
-
-
-class RetainEventData(BaseModel):
-    document_id: str | None = None
-    tags: list[str] | None = None
-
-
-class WebhookEvent(BaseModel):
-    event: WebhookEventType
-    bank_id: str
-    operation_id: str
-    status: str  # "completed" or "failed"
-    timestamp: datetime
-    data: ConsolidationEventData | RetainEventData
-
-
-class WebhookHttpConfig(BaseModel):
-    """HTTP delivery configuration for a webhook."""
-
-    method: str = Field(default="POST", description="HTTP method: GET or POST")
-    timeout_seconds: int = Field(default=30, description="HTTP request timeout in seconds")
-    headers: dict[str, str] = Field(default_factory=dict, description="Custom HTTP headers")
-    params: dict[str, str] = Field(default_factory=dict, description="Custom HTTP query parameters")
-
-
-class WebhookConfig(BaseModel):
-    id: str
-    bank_id: str | None
-    url: str
-    secret: str | None
-    event_types: list[str]
-    enabled: bool
-    http_config: WebhookHttpConfig = Field(default_factory=WebhookHttpConfig)

hindsight-api-slim/hindsight_api/worker/exceptions.py

@@ -1,9 +0,0 @@
-from datetime import datetime
-
-
-class RetryTaskAt(Exception):
-    """Raise from a task handler to schedule a retry at a specific time."""
-
-    def __init__(self, retry_at: datetime, message: str = ""):
-        self.retry_at = retry_at
-        super().__init__(message)

hindsight-api-slim/pyproject.toml

@@ -1,205 +0,0 @@
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[project]
-name = "hindsight-api-slim"
-version = "0.4.19"
-description = "Hindsight: Agent Memory That Works Like Human Memory"
-readme = "README.md"
-requires-python = ">=3.11"
-dependencies = [
-    "asyncpg>=0.29.0",
-    "python-dotenv>=1.0.0",
-    "openai>=1.0.0",
-    "pydantic>=2.0.0",
-    "rich>=13.0.0",
-    "langchain-text-splitters>=0.3.0",
-    "fastapi[standard]>=0.120.3",
-    "uvicorn>=0.38.0",
-    "wsproto>=1.0.0",
-    "sqlalchemy>=2.0.44",
-    "alembic>=1.17.1",
-    "pgvector>=0.4.1",
-    "greenlet>=3.2.4",
-    "psycopg2-binary>=2.9.11",
-    "tiktoken>=0.12.0",
-    "httpx>=0.27.0",
-    "PyJWT[crypto]>=2.8.0",
-    "fastmcp>=2.14.0", # CVE-2025-66416
-    "python-dateutil>=2.8.0",
-    "opentelemetry-api>=1.20.0",
-    "opentelemetry-sdk>=1.20.0",
-    "opentelemetry-instrumentation-fastapi>=0.41b0",
-    "opentelemetry-exporter-prometheus>=0.41b0",
-    "opentelemetry-exporter-otlp-proto-http>=1.20.0",
-    "opentelemetry-semantic-conventions>=0.41b0",
-    "dateparser>=1.2.2",
-    "google-genai>=1.0.0",
-    "google-auth>=2.0.0",
-    "anthropic>=0.40.0",
-    "typer>=0.9.0",
-    "cohere>=5.0.0",
-    "litellm>=1.0.0",
-    "markitdown[pdf,docx,pptx,xlsx,xls]>=0.1.4", # File to markdown conversion
-    "obstore>=0.4.0", # S3/GCS/Azure object storage client (Rust-backed)
-    "uvloop>=0.22.1",
-    # Transitive dependency security fixes
-    "pyasn1>=0.6.2", # DoS vulnerability fix
-    "urllib3>=2.6.3", # Decompression-bomb safeguards bypass fix
-    "langchain-core>=1.2.11", # Serialization injection + SSRF vulnerability fix
-    "langsmith>=0.6.3", # SSRF via tracing header injection fix
-    "protobuf>=6.33.5", # JSON recursion depth bypass fix
-    "pillow>=12.1.1", # Out-of-bounds write in PSD image loading fix
-    "cryptography>=46.0.5", # Subgroup attack vulnerability fix
-    "filelock>=3.20.1", # TOCTOU race condition fix
-    "authlib>=1.6.6", # Account takeover vulnerability fix
-    "aiohttp>=3.13.3", # Multiple DoS vulnerabilities
-    "claude-agent-sdk>=0.1.27; sys_platform == 'darwin'",
-]
-
-[project.optional-dependencies]
-local-ml = [
-    # Local ML models for embeddings/reranking
-    "sentence-transformers>=3.3.0",
-    "transformers>=4.53.0", # Security fixes for ReDoS vulnerabilities
-    "torch>=2.6.0", # CVE fix for remote code execution
-    "einops>=0.8.2",
-    "flashrank>=0.2.0",
-    # Apple Silicon local inference
-    "mlx>=0.31.0",
-    "mlx-lm>=0.31.1",
-    "safetensors>=0.6.2",
-]
-embedded-db = [
-    "pg0-embedded>=0.11.0",
-]
-all = [
-    "hindsight-api-slim[local-ml,embedded-db]",
-]
-test = [
-    "pytest>=7.0.0",
-    "pytest-asyncio>=0.21.0",
-    "pytest-timeout>=2.4.0",
-    "pytest-xdist>=3.0.0",
-    "filelock>=3.20.1",  # TOCTOU race condition fix
-    "testcontainers>=4.0.0",
-]
-
-[project.scripts]
-hindsight-api = "hindsight_api.main:main"
-hindsight-worker = "hindsight_api.worker.main:main"
-hindsight-local-mcp = "hindsight_api.mcp_local:main"
-hindsight-admin = "hindsight_api.admin.cli:main"
-
-[tool.hatch.build.targets.wheel]
-packages = ["hindsight_api"]
-
-[tool.hatch.build.targets.wheel.sources]
-"hindsight_api" = "hindsight_api"
-
-[tool.hatch.build.targets.sdist]
-include = [
-    "hindsight_api/**/*",
-]
-
-[tool.hatch.build]
-include = [
-    "hindsight_api/**/*.py",
-    "hindsight_api/alembic/**/*",
-]
-
-[tool.pytest.ini_options]
-log_cli = true
-log_cli_level = "INFO"
-log_cli_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"
-log_cli_date_format = "%Y-%m-%d %H:%M:%S"
-addopts = "--timeout 300 -n 8 --dist loadgroup --durations=10 -v"
-asyncio_mode = "auto"
-asyncio_default_fixture_loop_scope = "function"
-log_auto_indent = true
-filterwarnings = [
-    "ignore:The @wait_container_is_ready decorator is deprecated:DeprecationWarning",
-    "ignore::RuntimeWarning:asyncio",
-]
-
-[dependency-groups]
-dev = [
-    "pytest>=9.0.0",
-    "pytest-asyncio>=1.3.0",
-    "pytest-timeout>=2.4.0",
-    "pytest-xdist>=3.8.0",
-    "python-dotenv>=1.2.1",
-    "filelock>=3.20.1",  # TOCTOU race condition fix
-    "ruff>=0.8.0",
-    "ty>=0.0.1",
-    "testcontainers>=4.0.0",
-]
-
-[tool.ruff]
-line-length = 120
-target-version = "py311"
-exclude = [
-    "tests/",
-    "**/tests/",
-]
-
-[tool.ruff.lint]
-select = [
-    "E",      # pycodestyle errors
-    "W",      # pycodestyle warnings
-    "F",      # Pyflakes
-    "I",      # isort
-]
-ignore = [
-    "E501",   # line too long (handled by formatter)
-    "E402",   # module import not at top of file
-    "F401",   # unused import (too noisy during development)
-    "F841",   # unused variable (too noisy during development)
-    "F811",   # redefined while unused
-    "F821",   # undefined name (forward references in type hints)
-]
-
-[tool.ruff.lint.isort]
-known-third-party = ["alembic"]
-
-[tool.ruff.format]
-quote-style = "double"
-indent-style = "space"
-
-[tool.uv]
-# Use explicit index for PyTorch to prevent the pytorch index from serving
-# non-pytorch packages (e.g. markupsafe) with incompatible wheels
-[[tool.uv.index]]
-name = "pytorch-cpu"
-url = "https://download.pytorch.org/whl/cpu"
-explicit = true
-
-[tool.uv.sources]
-# Route torch to the CPU-only PyTorch index; everything else uses PyPI
-torch = { index = "pytorch-cpu" }
-
-[tool.ty]
-# Type checking configuration
-# ty is an extremely fast Python type checker from Astral (same team as ruff/uv)
-
-[tool.ty.environment]
-python-version = "3.11"
-
-[tool.ty.src]
-exclude = [
-    "tests/",
-    "hindsight_api/alembic/",
-]
-
-[tool.ty.rules]
-# Disable noisy rules while keeping important ones
-invalid-argument-type = "ignore"    # False positives with **kwargs patterns
-invalid-return-type = "ignore"      # Often intentional in async code
-invalid-parameter-default = "ignore" # Optional params with None default
-possibly-missing-attribute = "ignore" # Common with Optional types
-invalid-raise = "ignore"            # False positives with exception tracking
-call-non-callable = "ignore"        # False positives with Optional types
-invalid-key = "ignore"              # Pydantic ConfigDict not understood
-invalid-method-override = "ignore"  # Intentional signature differences
-unresolved-reference = "ignore"     # Forward references not always resolved

hindsight-api-slim/tests/test_combined_scoring.py

@@ -1,171 +0,0 @@
-"""
-Tests for combined scoring (apply_combined_scoring).
-
-The function applies multiplicative recency/temporal boosts to the cross-encoder
-score so that the relative influence of these signals is proportional to the base
-relevance score, independent of the cross-encoder model's score calibration.
-"""
-
-from datetime import datetime, timedelta, timezone
-from unittest.mock import MagicMock
-
-import pytest
-
-from hindsight_api.engine.search.reranking import apply_combined_scoring, _RECENCY_ALPHA, _TEMPORAL_ALPHA
-from hindsight_api.engine.search.types import MergedCandidate, RetrievalResult, ScoredResult
-
-UTC = timezone.utc
-NOW = datetime(2024, 6, 1, tzinfo=UTC)
-
-
-def _make_result(
-    ce_norm: float,
-    occurred_start: datetime | None = None,
-    temporal_proximity: float | None = None,
-) -> ScoredResult:
-    retrieval = MagicMock(spec=RetrievalResult)
-    retrieval.occurred_start = occurred_start
-    retrieval.temporal_proximity = temporal_proximity
-
-    candidate = MagicMock(spec=MergedCandidate)
-    candidate.retrieval = retrieval
-    candidate.rrf_score = 0.05
-
-    return ScoredResult(
-        candidate=candidate,
-        cross_encoder_score=1.0,
-        cross_encoder_score_normalized=ce_norm,
-        weight=ce_norm,
-    )
-
-
-class TestBoostFormula:
-    def test_neutral_signals_leave_score_unchanged(self):
-        """recency=0.5 and temporal=0.5 both produce boost=1.0, so weight == ce."""
-        sr = _make_result(ce_norm=0.6)
-        apply_combined_scoring([sr], now=NOW)
-        assert abs(sr.weight - 0.6) < 1e-9
-
-    def test_max_recency_boost(self):
-        """A memory from today (recency≈1.0) should boost by (1 + alpha*0.5)."""
-        sr = _make_result(ce_norm=0.5, occurred_start=NOW)
-        apply_combined_scoring([sr], now=NOW)
-        expected = 0.5 * (1.0 + _RECENCY_ALPHA * 0.5) * 1.0  # temporal neutral
-        assert abs(sr.weight - expected) < 1e-6
-
-    def test_min_recency_penalty(self):
-        """A memory from >365 days ago (recency=0.1) should penalise score."""
-        old = NOW - timedelta(days=400)
-        sr = _make_result(ce_norm=0.5, occurred_start=old)
-        apply_combined_scoring([sr], now=NOW)
-        expected = 0.5 * (1.0 + _RECENCY_ALPHA * (0.1 - 0.5)) * 1.0
-        assert abs(sr.weight - expected) < 1e-6
-
-    def test_max_temporal_boost(self):
-        """temporal_proximity=1.0 should boost by (1 + alpha*0.5)."""
-        sr = _make_result(ce_norm=0.5, temporal_proximity=1.0)
-        apply_combined_scoring([sr], now=NOW)
-        expected = 0.5 * 1.0 * (1.0 + _TEMPORAL_ALPHA * 0.5)  # recency neutral
-        assert abs(sr.weight - expected) < 1e-6
-
-    def test_temporal_none_is_neutral(self):
-        """temporal_proximity=None must be treated as 0.5 (no boost/penalty)."""
-        sr_none = _make_result(ce_norm=0.5, temporal_proximity=None)
-        sr_half = _make_result(ce_norm=0.5, temporal_proximity=0.5)
-        apply_combined_scoring([sr_none], now=NOW)
-        apply_combined_scoring([sr_half], now=NOW)
-        assert abs(sr_none.weight - sr_half.weight) < 1e-9
-
-    def test_both_signals_combined(self):
-        """Both boosts are applied multiplicatively."""
-        sr = _make_result(ce_norm=0.5, occurred_start=NOW, temporal_proximity=1.0)
-        apply_combined_scoring([sr], now=NOW)
-        recency_boost = 1.0 + _RECENCY_ALPHA * (1.0 - 0.5)
-        temporal_boost = 1.0 + _TEMPORAL_ALPHA * (1.0 - 0.5)
-        expected = 0.5 * recency_boost * temporal_boost
-        assert abs(sr.weight - expected) < 1e-6
-
-    def test_boost_is_proportional_to_ce(self):
-        """The absolute boost from recency scales with the CE score."""
-        sr_high = _make_result(ce_norm=0.9, occurred_start=NOW)
-        sr_low = _make_result(ce_norm=0.3, occurred_start=NOW)
-        apply_combined_scoring([sr_high, sr_low], now=NOW)
-
-        # Both get the same recency boost factor — absolute gain is proportional to CE
-        boost_factor = 1.0 + _RECENCY_ALPHA * 0.5
-        assert abs(sr_high.weight - 0.9 * boost_factor) < 1e-6
-        assert abs(sr_low.weight - 0.3 * boost_factor) < 1e-6
-
-    def test_boost_capped(self):
-        """Max boost: recency=1.0 + temporal=1.0 gives ≤21% uplift on CE."""
-        sr = _make_result(ce_norm=1.0, occurred_start=NOW, temporal_proximity=1.0)
-        apply_combined_scoring([sr], now=NOW)
-        assert sr.weight <= 1.0 * (1 + _RECENCY_ALPHA / 2) * (1 + _TEMPORAL_ALPHA / 2) + 1e-9
-
-    def test_rrf_normalized_always_zero(self):
-        """RRF is excluded from scoring; rrf_normalized is set to 0.0 for trace clarity."""
-        sr = _make_result(ce_norm=0.5)
-        apply_combined_scoring([sr], now=NOW)
-        assert sr.rrf_normalized == 0.0
-
-    def test_combined_score_equals_weight(self):
-        """combined_score and weight must stay in sync."""
-        sr = _make_result(ce_norm=0.7, occurred_start=NOW, temporal_proximity=0.8)
-        apply_combined_scoring([sr], now=NOW)
-        assert sr.combined_score == sr.weight
-
-    def test_model_calibration_independence(self):
-        """
-        A low-calibration model (low CE scores) and a high-calibration model
-        (high CE scores) should produce the same ranking for identical content.
-
-        With additive scoring the recency term would dominate for low-CE models;
-        with multiplicative boosting the relative ranking is stable.
-        """
-        recent = NOW - timedelta(days=10)
-        old = NOW - timedelta(days=300)
-
-        # High-calibration model: clear winner is #1 (more relevant, slightly older)
-        h_relevant = _make_result(ce_norm=0.85, occurred_start=old)
-        h_recent = _make_result(ce_norm=0.60, occurred_start=recent)
-        apply_combined_scoring([h_relevant, h_recent], now=NOW)
-        assert h_relevant.weight > h_recent.weight, "High-CE model: relevance should win"
-
-        # Low-calibration model: same relative difference, just compressed scores
-        l_relevant = _make_result(ce_norm=0.34, occurred_start=old)
-        l_recent = _make_result(ce_norm=0.24, occurred_start=recent)
-        apply_combined_scoring([l_relevant, l_recent], now=NOW)
-        assert l_relevant.weight > l_recent.weight, "Low-CE model: relevance should still win"
-
-    def test_no_occurred_start_defaults_recency_neutral(self):
-        """Missing occurred_start → recency=0.5 → no boost/penalty."""
-        sr = _make_result(ce_norm=0.5, occurred_start=None)
-        apply_combined_scoring([sr], now=NOW)
-        assert sr.recency == 0.5
-        assert abs(sr.weight - 0.5) < 1e-9
-
-    def test_timezone_naive_occurred_start_handled(self):
-        """Naive datetimes in occurred_start should not raise."""
-        naive_date = datetime(2024, 1, 1)  # no tzinfo
-        sr = _make_result(ce_norm=0.5, occurred_start=naive_date)
-        apply_combined_scoring([sr], now=NOW)  # must not raise
-        assert 0.0 < sr.weight < 1.0
-
-    def test_custom_alpha_values(self):
-        """Custom alpha parameters are respected."""
-        sr = _make_result(ce_norm=0.5, occurred_start=NOW)
-        apply_combined_scoring([sr], now=NOW, recency_alpha=0.4, temporal_alpha=0.0)
-        expected = 0.5 * (1.0 + 0.4 * 0.5) * 1.0
-        assert abs(sr.weight - expected) < 1e-6
-
-    def test_future_event_recency_capped_at_one(self):
-        """Events in the future must not produce recency > 1.0, keeping boost within bounds."""
-        future = NOW + timedelta(days=180)
-        sr = _make_result(ce_norm=0.5, occurred_start=future)
-        apply_combined_scoring([sr], now=NOW)
-        assert sr.recency == 1.0
-        expected_max_boost = 1.0 + _RECENCY_ALPHA * 0.5
-        assert sr.weight <= 0.5 * expected_max_boost + 1e-9
-
-    def test_empty_list_is_noop(self):
-        apply_combined_scoring([], now=NOW)  # must not raise

hindsight-api-slim/tests/test_consolidation_failure_recovery.py

@@ -1,476 +0,0 @@
-"""Tests for consolidation failure handling: adaptive batch splitting, consolidation_failed_at,
-and the recovery API.
-
-These tests use a mock LLM to simulate LLM failures deterministically, without making real
-API calls. All tests insert memories directly into the database to bypass retain's LLM calls
-and focus exclusively on the consolidation code paths.
-"""
-
-import uuid
-from unittest.mock import MagicMock
-
-import pytest
-import pytest_asyncio
-
-from hindsight_api.engine.consolidation.consolidator import run_consolidation_job
-from hindsight_api.engine.memory_engine import MemoryEngine
-from hindsight_api.engine.providers.mock_llm import MockLLM
-from hindsight_api.engine.task_backend import SyncTaskBackend
-
-
-@pytest_asyncio.fixture(scope="function")
-async def memory_no_llm_verify(pg0_db_url, embeddings, cross_encoder, query_analyzer):
-    """MemoryEngine with mock LLM.
-
-    Migrations are already applied by the session-scoped pg0_db_url fixture, so
-    run_migrations=False avoids advisory-lock serialization overhead per test.
-    """
-    mem = MemoryEngine(
-        db_url=pg0_db_url,
-        memory_llm_provider="mock",
-        memory_llm_api_key="",
-        memory_llm_model="mock",
-        embeddings=embeddings,
-        cross_encoder=cross_encoder,
-        query_analyzer=query_analyzer,
-        pool_min_size=1,
-        pool_max_size=5,
-        run_migrations=False,
-        task_backend=SyncTaskBackend(),
-        skip_llm_verification=True,
-    )
-    await mem.initialize()
-    yield mem
-    try:
-        if mem._pool and not mem._pool._closing:
-            await mem.close()
-    except Exception:
-        pass
-
-
-@pytest.fixture(autouse=True)
-def enable_observations():
-    """Enable observations for all tests in this module."""
-    from hindsight_api.config import _get_raw_config
-
-    config = _get_raw_config()
-    original = config.enable_observations
-    config.enable_observations = True
-    yield
-    config.enable_observations = original
-
-
-def _make_failing_mock_llm(*, fail_first_n: int = 999) -> MockLLM:
-    """Return a MockLLM that raises ValueError for the first `fail_first_n` consolidation calls."""
-    mock_llm = MockLLM(provider="mock", api_key="", base_url="", model="mock-model")
-    call_count = 0
-
-    def callback(messages, scope):
-        nonlocal call_count
-        if scope == "consolidation":
-            call_count += 1
-            if call_count <= fail_first_n:
-                raise ValueError(f"Simulated LLM failure (call {call_count})")
-        # Return empty response — no creates/updates/deletes
-        from hindsight_api.engine.consolidation.consolidator import _ConsolidationBatchResponse
-
-        return _ConsolidationBatchResponse()
-
-    mock_llm.set_response_callback(callback)
-    return mock_llm
-
-
-def _make_always_success_mock_llm() -> MockLLM:
-    """Return a MockLLM that always succeeds with an empty consolidation response."""
-    mock_llm = MockLLM(provider="mock", api_key="", base_url="", model="mock-model")
-
-    def callback(messages, scope):
-        from hindsight_api.engine.consolidation.consolidator import _ConsolidationBatchResponse
-
-        return _ConsolidationBatchResponse()
-
-    mock_llm.set_response_callback(callback)
-    return mock_llm
-
-
-def _inject_mock_llm(memory: MemoryEngine, mock_llm: MockLLM) -> None:
-    """Replace memory._consolidation_llm_config with a wrapper that returns mock_llm from with_config."""
-    wrapper = MagicMock()
-    wrapper.with_config.return_value = mock_llm
-    memory._consolidation_llm_config = wrapper
-
-
-async def _insert_memories(conn, bank_id: str, texts: list[str]) -> list[uuid.UUID]:
-    """Insert experience memories directly, bypassing LLM-based retain."""
-    ids = []
-    for text in texts:
-        mem_id = uuid.uuid4()
-        await conn.execute(
-            """
-            INSERT INTO memory_units (id, bank_id, text, fact_type, created_at)
-            VALUES ($1, $2, $3, 'experience', now())
-            """,
-            mem_id,
-            bank_id,
-            text,
-        )
-        ids.append(mem_id)
-    return ids
-
-
-class TestAdaptiveBatchSplitting:
-    """Verify that a failing batch is halved and retried until batch_size=1 succeeds."""
-
-    @pytest.mark.asyncio
-    async def test_splitting_recovers_all_memories(self, memory_no_llm_verify: MemoryEngine, request_context):
-        """When a batch of 2 fails, both are retried individually and succeed."""
-        bank_id = f"test-split-recovery-{uuid.uuid4().hex[:8]}"
-        await memory_no_llm_verify.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            mem_ids = await _insert_memories(
-                conn,
-                bank_id,
-                [
-                    "Alice runs marathons every spring.",
-                    "Alice trained for six months for her last race.",
-                ],
-            )
-
-        # Exhaust all 3 retries for batch=2 (calls 1-3 fail), then each batch=1 succeeds (calls 4-5)
-        mock_llm = _make_failing_mock_llm(fail_first_n=3)
-        _inject_mock_llm(memory_no_llm_verify, mock_llm)
-
-        result = await run_consolidation_job(
-            memory_engine=memory_no_llm_verify,
-            bank_id=bank_id,
-            request_context=request_context,
-        )
-
-        assert result["status"] == "completed"
-        assert result["memories_processed"] == 2
-        assert result["memories_failed"] == 0
-
-        # Both memories must have consolidated_at set and consolidation_failed_at NULL
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            rows = await conn.fetch(
-                """
-                SELECT id, consolidated_at, consolidation_failed_at
-                FROM memory_units
-                WHERE bank_id = $1 AND fact_type = 'experience'
-                """,
-                bank_id,
-            )
-        assert len(rows) == 2
-        for row in rows:
-            assert row["consolidated_at"] is not None, f"Memory {row['id']} should have consolidated_at set"
-            assert row["consolidation_failed_at"] is None, (
-                f"Memory {row['id']} should NOT have consolidation_failed_at set"
-            )
-
-        # LLM called 5 times: 3 retries failed (batch=2) + 1 succeeded (batch=1) + 1 succeeded (batch=1)
-        consolidation_calls = [c for c in mock_llm.get_mock_calls() if c["scope"] == "consolidation"]
-        assert len(consolidation_calls) == 5
-
-        await memory_no_llm_verify.delete_bank(bank_id, request_context=request_context)
-
-    @pytest.mark.asyncio
-    async def test_splitting_with_larger_batch(self, memory_no_llm_verify: MemoryEngine, request_context):
-        """A batch of 4 that always fails at size>1 resolves to 4 individual calls."""
-        bank_id = f"test-split-large-{uuid.uuid4().hex[:8]}"
-        await memory_no_llm_verify.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            await _insert_memories(
-                conn,
-                bank_id,
-                [
-                    "Bob plays chess competitively.",
-                    "Bob won a regional chess tournament.",
-                    "Bob practices tactics every morning.",
-                    "Bob coaches youth chess on weekends.",
-                ],
-            )
-
-        # Exhaust all 3 retries for batch=4 (calls 1-3 fail), then both batch=2 halves succeed
-        # (calls 4-5). This verifies that halving once is sufficient when batch=2 works.
-        mock_llm = _make_failing_mock_llm(fail_first_n=3)
-        _inject_mock_llm(memory_no_llm_verify, mock_llm)
-
-        result = await run_consolidation_job(
-            memory_engine=memory_no_llm_verify,
-            bank_id=bank_id,
-            request_context=request_context,
-        )
-
-        assert result["memories_processed"] == 4
-        assert result["memories_failed"] == 0
-
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            rows = await conn.fetch(
-                "SELECT consolidated_at, consolidation_failed_at FROM memory_units "
-                "WHERE bank_id = $1 AND fact_type = 'experience'",
-                bank_id,
-            )
-        assert all(r["consolidated_at"] is not None for r in rows)
-        assert all(r["consolidation_failed_at"] is None for r in rows)
-
-        await memory_no_llm_verify.delete_bank(bank_id, request_context=request_context)
-
-
-class TestConsolidationFailedAt:
-    """Verify that consolidation_failed_at is set — and consolidated_at is NOT — when all retries fail."""
-
-    @pytest.mark.asyncio
-    async def test_single_memory_permanent_failure(self, memory_no_llm_verify: MemoryEngine, request_context):
-        """A single memory that exhausts all LLM retries gets consolidation_failed_at, not consolidated_at."""
-        bank_id = f"test-perm-fail-{uuid.uuid4().hex[:8]}"
-        await memory_no_llm_verify.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            (mem_id,) = await _insert_memories(conn, bank_id, ["Carol enjoys painting watercolors."])
-
-        # Always fail
-        mock_llm = _make_failing_mock_llm(fail_first_n=999)
-        _inject_mock_llm(memory_no_llm_verify, mock_llm)
-
-        result = await run_consolidation_job(
-            memory_engine=memory_no_llm_verify,
-            bank_id=bank_id,
-            request_context=request_context,
-        )
-
-        assert result["memories_failed"] == 1
-        assert result["memories_processed"] == 1
-
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            row = await conn.fetchrow(
-                "SELECT consolidated_at, consolidation_failed_at FROM memory_units WHERE id = $1",
-                mem_id,
-            )
-
-        assert row["consolidated_at"] is None, "consolidated_at must NOT be set for a permanently failed memory"
-        assert row["consolidation_failed_at"] is not None, "consolidation_failed_at must be set"
-
-        await memory_no_llm_verify.delete_bank(bank_id, request_context=request_context)
-
-    @pytest.mark.asyncio
-    async def test_failed_memory_excluded_from_next_run(self, memory_no_llm_verify: MemoryEngine, request_context):
-        """A memory marked consolidation_failed_at is not re-processed on the next consolidation run."""
-        bank_id = f"test-excluded-{uuid.uuid4().hex[:8]}"
-        await memory_no_llm_verify.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            (mem_id,) = await _insert_memories(conn, bank_id, ["Dave collects vinyl records."])
-            # Manually stamp consolidation_failed_at to simulate a prior failed run
-            await conn.execute(
-                "UPDATE memory_units SET consolidation_failed_at = NOW() WHERE id = $1",
-                mem_id,
-            )
-
-        # Even with a healthy LLM, the memory should be skipped
-        mock_llm = _make_always_success_mock_llm()
-        _inject_mock_llm(memory_no_llm_verify, mock_llm)
-
-        result = await run_consolidation_job(
-            memory_engine=memory_no_llm_verify,
-            bank_id=bank_id,
-            request_context=request_context,
-        )
-
-        # No unconsolidated memories to pick up (consolidation_failed_at ≠ NULL, consolidated_at = NULL
-        # but the SELECT filters on consolidated_at IS NULL AND fact_type IN ('experience','world'))
-        assert result["status"] in ("no_new_memories", "completed")
-        if result["status"] == "completed":
-            assert result["memories_processed"] == 0
-
-        # Memory still has consolidation_failed_at set and consolidated_at NULL
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            row = await conn.fetchrow(
-                "SELECT consolidated_at, consolidation_failed_at FROM memory_units WHERE id = $1",
-                mem_id,
-            )
-        assert row["consolidated_at"] is None
-        assert row["consolidation_failed_at"] is not None
-
-        await memory_no_llm_verify.delete_bank(bank_id, request_context=request_context)
-
-    @pytest.mark.asyncio
-    async def test_partial_batch_failure(self, memory_no_llm_verify: MemoryEngine, request_context):
-        """In a batch of 2, if only the first individual retry fails, the second still succeeds."""
-        bank_id = f"test-partial-fail-{uuid.uuid4().hex[:8]}"
-        await memory_no_llm_verify.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            mem_ids = await _insert_memories(
-                conn,
-                bank_id,
-                [
-                    "Eve speaks three languages fluently.",
-                    "Eve learned Japanese in two years.",
-                ],
-            )
-
-        # Exhaust 3 retries for batch=2 (calls 1-3), exhaust 3 retries for first batch=1 (calls 4-6),
-        # second batch=1 succeeds (call 7)
-        mock_llm = _make_failing_mock_llm(fail_first_n=6)
-        _inject_mock_llm(memory_no_llm_verify, mock_llm)
-
-        result = await run_consolidation_job(
-            memory_engine=memory_no_llm_verify,
-            bank_id=bank_id,
-            request_context=request_context,
-        )
-
-        assert result["memories_processed"] == 2
-        assert result["memories_failed"] == 1
-
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            rows = {
-                str(r["id"]): r
-                for r in await conn.fetch(
-                    "SELECT id, consolidated_at, consolidation_failed_at FROM memory_units "
-                    "WHERE bank_id = $1 AND fact_type = 'experience'",
-                    bank_id,
-                )
-            }
-
-        # One should have failed, one should have succeeded
-        failed = [r for r in rows.values() if r["consolidation_failed_at"] is not None]
-        succeeded = [r for r in rows.values() if r["consolidated_at"] is not None]
-        assert len(failed) == 1
-        assert len(succeeded) == 1
-        # They must be different memories
-        assert str(failed[0]["id"]) != str(succeeded[0]["id"])
-
-        await memory_no_llm_verify.delete_bank(bank_id, request_context=request_context)
-
-
-class TestRecoverConsolidation:
-    """Verify the retry_failed_consolidation() method and the /consolidation/recover endpoint."""
-
-    @pytest.mark.asyncio
-    async def test_recover_resets_failed_memories(self, memory_no_llm_verify: MemoryEngine, request_context):
-        """retry_failed_consolidation resets consolidation_failed_at and consolidated_at."""
-        bank_id = f"test-recover-reset-{uuid.uuid4().hex[:8]}"
-        await memory_no_llm_verify.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            ids = await _insert_memories(
-                conn,
-                bank_id,
-                [
-                    "Frank is a competitive cyclist.",
-                    "Frank completed the Tour de France route.",
-                ],
-            )
-            # Mark both as failed
-            for mem_id in ids:
-                await conn.execute(
-                    "UPDATE memory_units SET consolidation_failed_at = NOW() WHERE id = $1",
-                    mem_id,
-                )
-
-        result = await memory_no_llm_verify.retry_failed_consolidation(
-            bank_id, request_context=request_context
-        )
-
-        assert result["retried_count"] == 2
-
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            rows = await conn.fetch(
-                "SELECT consolidated_at, consolidation_failed_at FROM memory_units "
-                "WHERE bank_id = $1 AND fact_type = 'experience'",
-                bank_id,
-            )
-        assert all(r["consolidation_failed_at"] is None for r in rows), "consolidation_failed_at must be cleared"
-        assert all(r["consolidated_at"] is None for r in rows), "consolidated_at must also be cleared"
-
-        await memory_no_llm_verify.delete_bank(bank_id, request_context=request_context)
-
-    @pytest.mark.asyncio
-    async def test_recover_returns_zero_when_none_failed(self, memory_no_llm_verify: MemoryEngine, request_context):
-        """retry_failed_consolidation returns 0 when no memories have failed."""
-        bank_id = f"test-recover-zero-{uuid.uuid4().hex[:8]}"
-        await memory_no_llm_verify.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-        result = await memory_no_llm_verify.retry_failed_consolidation(
-            bank_id, request_context=request_context
-        )
-
-        assert result["retried_count"] == 0
-
-        await memory_no_llm_verify.delete_bank(bank_id, request_context=request_context)
-
-    @pytest.mark.asyncio
-    async def test_recover_then_consolidate_succeeds(self, memory_no_llm_verify: MemoryEngine, request_context):
-        """After recovery, the memory is picked up by the next consolidation run."""
-        bank_id = f"test-recover-consolidate-{uuid.uuid4().hex[:8]}"
-        await memory_no_llm_verify.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            (mem_id,) = await _insert_memories(conn, bank_id, ["Grace is an expert rock climber."])
-            await conn.execute(
-                "UPDATE memory_units SET consolidation_failed_at = NOW() WHERE id = $1", mem_id
-            )
-
-        # Recover
-        recover_result = await memory_no_llm_verify.retry_failed_consolidation(
-            bank_id, request_context=request_context
-        )
-        assert recover_result["retried_count"] == 1
-
-        # Now consolidate with a healthy LLM
-        mock_llm = _make_always_success_mock_llm()
-        _inject_mock_llm(memory_no_llm_verify, mock_llm)
-
-        run_result = await run_consolidation_job(
-            memory_engine=memory_no_llm_verify,
-            bank_id=bank_id,
-            request_context=request_context,
-        )
-
-        assert run_result["memories_processed"] == 1
-        assert run_result["memories_failed"] == 0
-
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            row = await conn.fetchrow(
-                "SELECT consolidated_at, consolidation_failed_at FROM memory_units WHERE id = $1",
-                mem_id,
-            )
-        assert row["consolidated_at"] is not None, "Memory should be consolidated after recovery"
-        assert row["consolidation_failed_at"] is None
-
-        await memory_no_llm_verify.delete_bank(bank_id, request_context=request_context)
-
-    @pytest.mark.asyncio
-    async def test_recover_endpoint_via_http(self, memory_no_llm_verify: MemoryEngine, request_context):
-        """The POST /consolidation/recover endpoint returns the correct retried_count."""
-        import httpx
-
-        from hindsight_api.api.http import create_app
-
-        bank_id = f"test-recover-http-{uuid.uuid4().hex[:8]}"
-        await memory_no_llm_verify.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-        async with memory_no_llm_verify._pool.acquire() as conn:
-            ids = await _insert_memories(
-                conn,
-                bank_id,
-                ["Henry is a professional chef.", "Henry trained at Le Cordon Bleu."],
-            )
-            for mem_id in ids:
-                await conn.execute(
-                    "UPDATE memory_units SET consolidation_failed_at = NOW() WHERE id = $1", mem_id
-                )
-
-        app = create_app(memory_no_llm_verify, initialize_memory=False)
-        transport = httpx.ASGITransport(app=app)
-        async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
-            response = await client.post(f"/v1/default/banks/{bank_id}/consolidation/recover")
-
-        assert response.status_code == 200
-        body = response.json()
-        assert body["retried_count"] == 2
-
-        await memory_no_llm_verify.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_entity_resolver.py

@@ -1,70 +0,0 @@
-"""
-Tests for EntityResolver edge cases.
-"""
-
-import uuid
-from datetime import datetime, timezone
-
-import asyncpg
-import pytest
-
-from hindsight_api.engine.entity_resolver import EntityResolver
-from hindsight_api.pg0 import resolve_database_url
-
-
-@pytest.mark.asyncio
-async def test_resolve_entities_batch_handles_unicode_lower_conflicts(pg0_db_url):
-    """
-    Existing entities with PostgreSQL/Python lowercase mismatches should resolve
-    to the conflicted row instead of leaving a missing entity_id.
-    """
-    resolved_url = await resolve_database_url(pg0_db_url)
-    pool = await asyncpg.create_pool(resolved_url, min_size=1, max_size=2, command_timeout=30)
-    bank_id = f"test-entity-resolver-{uuid.uuid4().hex[:8]}"
-    event_date = datetime(2024, 1, 15, tzinfo=timezone.utc)
-    resolver = EntityResolver(pool=pool, entity_lookup="full")
-
-    try:
-        async with pool.acquire() as conn:
-            existing_entity_id = await conn.fetchval(
-                """
-                INSERT INTO entities (bank_id, canonical_name, first_seen, last_seen, mention_count)
-                VALUES ($1, $2, $3, $3, 1)
-                RETURNING id
-                """,
-                bank_id,
-                "İstanbul",
-                event_date,
-            )
-
-            resolved_ids = await resolver.resolve_entities_batch(
-                bank_id=bank_id,
-                entities_data=[
-                    {
-                        "text": "istanbul",
-                        "nearby_entities": [],
-                        "event_date": event_date,
-                    }
-                ],
-                context="unicode case mismatch",
-                unit_event_date=event_date,
-                conn=conn,
-            )
-
-            entity_rows = await conn.fetch(
-                """
-                SELECT id, canonical_name
-                FROM entities
-                WHERE bank_id = $1
-                ORDER BY canonical_name
-                """,
-                bank_id,
-            )
-
-        assert resolved_ids == [existing_entity_id]
-        assert len(entity_rows) == 1
-        assert entity_rows[0]["id"] == existing_entity_id
-        assert entity_rows[0]["canonical_name"] == "İstanbul"
-    finally:
-        await pool.execute("DELETE FROM entities WHERE bank_id = $1", bank_id)
-        await pool.close()

hindsight-api-slim/tests/test_fact_extraction_retry.py

@@ -1,141 +0,0 @@
-"""
-Unit tests for fact extraction retry logic.
-
-Tests the fix for the TypeError when LLM returns invalid JSON across all retries.
-Previously, `raise last_error` would raise None (TypeError) because last_error was
-only set in the BadRequestError handler, not when the LLM returned non-dict JSON.
-"""
-
-from datetime import datetime, timezone
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-
-def _make_config(llm_max_retries: int = 3, retain_llm_max_retries: int | None = None):
-    """Build a minimal HindsightConfig for fact extraction tests."""
-    from hindsight_api.config import HindsightConfig
-
-    cfg = MagicMock(spec=HindsightConfig)
-    cfg.retain_llm_max_retries = retain_llm_max_retries
-    cfg.llm_max_retries = llm_max_retries
-    cfg.retain_llm_initial_backoff = None
-    cfg.llm_initial_backoff = 0.0
-    cfg.retain_llm_max_backoff = None
-    cfg.llm_max_backoff = 0.0
-    cfg.retain_max_completion_tokens = 8192
-    cfg.retain_extraction_mode = "concise"
-    cfg.retain_extract_causal_links = False
-    cfg.retain_mission = None
-    return cfg
-
-
-def _make_llm_config(mock_response):
-    """Build a mock LLMProvider that returns the given response."""
-    from hindsight_api.engine.llm_wrapper import LLMProvider
-
-    llm = MagicMock(spec=LLMProvider)
-    llm.provider = "mock"
-    token_usage = MagicMock()
-    token_usage.__add__ = lambda self, other: self
-    llm.call = AsyncMock(return_value=(mock_response, token_usage))
-    return llm
-
-
-@pytest.mark.asyncio
-async def test_non_dict_json_all_retries_returns_empty():
-    """
-    When LLM returns non-dict JSON on every attempt, extraction should return []
-    without raising TypeError ('exceptions must derive from BaseException').
-
-    This was the bug: the loop ran range(2) times (hardcoded), but comparisons
-    used config.llm_max_retries (default 10). On the last loop iteration (attempt=1),
-    `attempt < 10 - 1` was True, so the code called `continue`, the loop
-    exhausted, and `raise last_error` raised None → TypeError.
-    """
-    from hindsight_api.engine.retain.fact_extraction import _extract_facts_from_chunk
-
-    # llm_max_retries=3 ensures the bug triggers with the old code (3 != 2 hardcoded)
-    config = _make_config(llm_max_retries=3, retain_llm_max_retries=None)
-
-    # Mock: always returns a list (non-dict), which is invalid
-    llm_config = _make_llm_config(mock_response=[{"invalid": "response"}])
-
-    with patch(
-        "hindsight_api.engine.retain.fact_extraction._build_extraction_prompt_and_schema",
-        return_value=("system prompt", MagicMock()),
-    ):
-        facts, usage = await _extract_facts_from_chunk(
-            chunk="Alice visited Paris in 2023.",
-            chunk_index=0,
-            total_chunks=1,
-            event_date=datetime(2023, 1, 1, tzinfo=timezone.utc),
-            context="travel notes",
-            llm_config=llm_config,
-            config=config,
-            agent_name="test-agent",
-        )
-
-    assert facts == []
-
-
-@pytest.mark.asyncio
-async def test_non_dict_json_with_default_max_retries_returns_empty():
-    """
-    Same scenario with the default llm_max_retries=10 (matching real default config).
-    The old code ran range(2) but checked against 10, always continuing until
-    the loop exhausted, then raised None → TypeError.
-    """
-    from hindsight_api.engine.retain.fact_extraction import _extract_facts_from_chunk
-
-    config = _make_config(llm_max_retries=10, retain_llm_max_retries=None)
-    llm_config = _make_llm_config(mock_response="not a dict at all")
-
-    with patch(
-        "hindsight_api.engine.retain.fact_extraction._build_extraction_prompt_and_schema",
-        return_value=("system prompt", MagicMock()),
-    ):
-        facts, usage = await _extract_facts_from_chunk(
-            chunk="Some text.",
-            chunk_index=0,
-            total_chunks=1,
-            event_date=datetime(2023, 6, 1, tzinfo=timezone.utc),
-            context="",
-            llm_config=llm_config,
-            config=config,
-            agent_name="agent",
-        )
-
-    assert facts == []
-
-
-@pytest.mark.asyncio
-async def test_retain_llm_max_retries_overrides_global():
-    """
-    When retain_llm_max_retries is set, it should be used for the loop range
-    and all comparisons (no shadowing bug).
-    """
-    from hindsight_api.engine.retain.fact_extraction import _extract_facts_from_chunk
-
-    # retain_llm_max_retries=5 should override llm_max_retries=10
-    config = _make_config(llm_max_retries=10, retain_llm_max_retries=5)
-    llm_config = _make_llm_config(mock_response=42)  # non-dict: integer
-
-    with patch(
-        "hindsight_api.engine.retain.fact_extraction._build_extraction_prompt_and_schema",
-        return_value=("system prompt", MagicMock()),
-    ):
-        facts, usage = await _extract_facts_from_chunk(
-            chunk="Bob likes Python.",
-            chunk_index=0,
-            total_chunks=1,
-            event_date=datetime(2024, 1, 1, tzinfo=timezone.utc),
-            context="",
-            llm_config=llm_config,
-            config=config,
-            agent_name="agent",
-        )
-
-    assert facts == []
-    # Verify it retried exactly retain_llm_max_retries times
-    assert llm_config.call.call_count == 5

hindsight-api-slim/tests/test_gemini_safety_settings.py

@@ -1,340 +0,0 @@
-"""
-Tests for Gemini safety settings feature.
-
-Verifies that:
-- Safety settings are read from env var and stored on GeminiLLM instances
-- Settings are applied to GenerateContentConfig in call() and call_with_tools()
-- The context variable override allows per-bank settings at request time
-- None (unset) means Gemini's default safety settings are used (no override)
-"""
-
-import os
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-pytest.importorskip("google.genai")
-
-
-SAMPLE_SAFETY_SETTINGS = [
-    {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"},
-    {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"},
-    {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"},
-    {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"},
-]
-
-
-# ─── Config / env var parsing ─────────────────────────────────────────────────
-
-
-def test_gemini_safety_settings_parsed_from_env():
-    """Safety settings JSON from env var is parsed into HindsightConfig."""
-    import json
-
-    from hindsight_api.config import ENV_LLM_GEMINI_SAFETY_SETTINGS, HindsightConfig, clear_config_cache
-
-    settings_json = json.dumps(SAMPLE_SAFETY_SETTINGS)
-    with patch.dict(os.environ, {ENV_LLM_GEMINI_SAFETY_SETTINGS: settings_json}, clear=False):
-        clear_config_cache()
-        config = HindsightConfig.from_env()
-        assert config.llm_gemini_safety_settings == SAMPLE_SAFETY_SETTINGS
-        clear_config_cache()
-
-
-def test_gemini_safety_settings_default_is_none():
-    """When env var is not set, llm_gemini_safety_settings defaults to None."""
-    from hindsight_api.config import ENV_LLM_GEMINI_SAFETY_SETTINGS, HindsightConfig, clear_config_cache
-
-    env = {k: v for k, v in os.environ.items() if k != ENV_LLM_GEMINI_SAFETY_SETTINGS}
-    with patch.dict(os.environ, env, clear=True):
-        clear_config_cache()
-        config = HindsightConfig.from_env()
-        assert config.llm_gemini_safety_settings is None
-        clear_config_cache()
-
-
-def test_gemini_safety_settings_is_configurable_field():
-    """llm_gemini_safety_settings appears in configurable (per-bank) fields."""
-    from hindsight_api.config import HindsightConfig
-
-    assert "llm_gemini_safety_settings" in HindsightConfig.get_configurable_fields()
-
-
-def test_gemini_safety_settings_not_in_credential_fields():
-    """llm_gemini_safety_settings is NOT a credential — it is safe to expose via API."""
-    from hindsight_api.config import HindsightConfig
-
-    assert "llm_gemini_safety_settings" not in HindsightConfig.get_credential_fields()
-
-
-# ─── GeminiLLM instance ───────────────────────────────────────────────────────
-
-
-def _make_gemini_provider(safety_settings=None):
-    """Return a GeminiLLM instance with a mocked genai.Client."""
-    with patch("google.genai.Client") as mock_client_cls:
-        mock_client_cls.return_value = MagicMock()
-        from hindsight_api.engine.providers.gemini_llm import GeminiLLM
-
-        provider = GeminiLLM(
-            provider="gemini",
-            api_key="fake-api-key",
-            base_url="",
-            model="gemini-2.5-flash",
-            gemini_safety_settings=safety_settings,
-        )
-        # Replace client with a fresh mock so we can inspect calls
-        provider._client = MagicMock()
-        return provider
-
-
-def test_gemini_llm_stores_safety_settings():
-    """GeminiLLM stores safety settings passed at construction."""
-    provider = _make_gemini_provider(safety_settings=SAMPLE_SAFETY_SETTINGS)
-    assert provider._safety_settings == SAMPLE_SAFETY_SETTINGS
-
-
-def test_gemini_llm_no_safety_settings_is_none():
-    """GeminiLLM._safety_settings is None when not provided."""
-    provider = _make_gemini_provider(safety_settings=None)
-    assert provider._safety_settings is None
-
-
-# ─── call() applies safety settings ──────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_call_applies_safety_settings():
-    """call() includes safety_settings in GenerateContentConfig when configured."""
-    from google.genai import types as genai_types
-
-    provider = _make_gemini_provider(safety_settings=SAMPLE_SAFETY_SETTINGS)
-
-    # Build a fake successful response
-    fake_response = MagicMock()
-    fake_response.text = "hello"
-    fake_response.candidates = [MagicMock(finish_reason="STOP")]
-    fake_response.usage_metadata = MagicMock(prompt_token_count=5, candidates_token_count=2)
-
-    provider._client.aio.models.generate_content = AsyncMock(return_value=fake_response)
-
-    await provider.call(
-        messages=[{"role": "user", "content": "hi"}],
-        scope="test",
-    )
-
-    # Inspect the config passed to generate_content
-    call_args = provider._client.aio.models.generate_content.call_args
-    config_arg = call_args.kwargs.get("config") or call_args.args[0] if call_args.args else None
-    # config may be in kwargs or positional; grab from kwargs
-    config_arg = call_args.kwargs.get("config")
-
-    assert config_arg is not None, "GenerateContentConfig should have been passed"
-    assert hasattr(config_arg, "safety_settings"), "Config should have safety_settings"
-    assert config_arg.safety_settings is not None
-
-    categories = [s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings]
-    assert "HARM_CATEGORY_HARASSMENT" in categories
-    assert "HARM_CATEGORY_HATE_SPEECH" in categories
-    assert "HARM_CATEGORY_SEXUALLY_EXPLICIT" in categories
-    assert "HARM_CATEGORY_DANGEROUS_CONTENT" in categories
-
-    thresholds = [s.threshold.value if hasattr(s.threshold, "value") else str(s.threshold) for s in config_arg.safety_settings]
-    assert all(t == "BLOCK_NONE" for t in thresholds)
-
-
-@pytest.mark.asyncio
-async def test_call_no_safety_settings_omits_key():
-    """call() does NOT add safety_settings to GenerateContentConfig when none configured."""
-    provider = _make_gemini_provider(safety_settings=None)
-
-    fake_response = MagicMock()
-    fake_response.text = "hello"
-    fake_response.candidates = [MagicMock(finish_reason="STOP")]
-    fake_response.usage_metadata = MagicMock(prompt_token_count=5, candidates_token_count=2)
-
-    provider._client.aio.models.generate_content = AsyncMock(return_value=fake_response)
-
-    await provider.call(
-        messages=[{"role": "user", "content": "hi"}],
-        scope="test",
-    )
-
-    call_args = provider._client.aio.models.generate_content.call_args
-    config_arg = call_args.kwargs.get("config")
-
-    # When no safety settings, config is either None or lacks safety_settings
-    if config_arg is not None:
-        assert not hasattr(config_arg, "safety_settings") or config_arg.safety_settings is None
-
-
-# ─── call_with_tools() applies safety settings ────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_call_with_tools_applies_safety_settings():
-    """call_with_tools() includes safety_settings in GenerateContentConfig."""
-    provider = _make_gemini_provider(safety_settings=SAMPLE_SAFETY_SETTINGS)
-
-    # Build a fake tool-use response (no tool calls, just text)
-    fake_part = MagicMock()
-    fake_part.text = "answer"
-    fake_part.function_call = None
-
-    fake_candidate = MagicMock()
-    fake_candidate.content = MagicMock(parts=[fake_part])
-
-    fake_response = MagicMock()
-    fake_response.candidates = [fake_candidate]
-    fake_response.usage_metadata = MagicMock(prompt_token_count=5, candidates_token_count=3)
-
-    provider._client.aio.models.generate_content = AsyncMock(return_value=fake_response)
-
-    tools = [
-        {
-            "type": "function",
-            "function": {
-                "name": "test_tool",
-                "description": "A test tool",
-                "parameters": {"type": "object", "properties": {}, "required": []},
-            },
-        }
-    ]
-
-    await provider.call_with_tools(
-        messages=[{"role": "user", "content": "hi"}],
-        tools=tools,
-        scope="test",
-    )
-
-    call_args = provider._client.aio.models.generate_content.call_args
-    config_arg = call_args.kwargs.get("config")
-
-    assert config_arg is not None
-    assert config_arg.safety_settings is not None
-    categories = [s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings]
-    assert "HARM_CATEGORY_HARASSMENT" in categories
-
-
-# ─── with_config() override ───────────────────────────────────────────────────
-
-
-def _make_llm_provider(safety_settings=None):
-    """Return an LLMProvider (wrapping GeminiLLM) with a mocked genai.Client."""
-    with patch("google.genai.Client") as mock_client_cls:
-        mock_client_cls.return_value = MagicMock()
-        from hindsight_api.engine.llm_wrapper import LLMProvider
-
-        provider = LLMProvider(
-            provider="gemini",
-            api_key="fake-api-key",
-            base_url="",
-            model="gemini-2.5-flash",
-            gemini_safety_settings=safety_settings,
-        )
-        # Replace the underlying Gemini client with a fresh mock
-        provider._provider_impl._client = MagicMock()
-        return provider
-
-
-def _fake_response():
-    r = MagicMock()
-    r.text = "hello"
-    r.candidates = [MagicMock(finish_reason="STOP")]
-    r.usage_metadata = MagicMock(prompt_token_count=5, candidates_token_count=2)
-    return r
-
-
-def _make_config(safety_settings):
-    """Return a minimal config-like object with llm_gemini_safety_settings."""
-    cfg = MagicMock()
-    cfg.llm_gemini_safety_settings = safety_settings
-    return cfg
-
-
-@pytest.mark.asyncio
-async def test_with_config_overrides_instance_settings():
-    """with_config() settings take precedence over the provider instance defaults."""
-    instance_settings = [{"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_ONLY_HIGH"}]
-    override_settings = [{"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"}]
-
-    provider = _make_llm_provider(safety_settings=instance_settings)
-    provider._provider_impl._client.aio.models.generate_content = AsyncMock(return_value=_fake_response())
-
-    configured = provider.with_config(_make_config(override_settings))
-    await configured.call(messages=[{"role": "user", "content": "hi"}], scope="test")
-
-    config_arg = provider._provider_impl._client.aio.models.generate_content.call_args.kwargs.get("config")
-    assert config_arg is not None
-    categories = [s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings]
-    # Should use override_settings (HATE_SPEECH), not instance_settings (HARASSMENT)
-    assert "HARM_CATEGORY_HATE_SPEECH" in categories
-    assert "HARM_CATEGORY_HARASSMENT" not in categories
-
-
-@pytest.mark.asyncio
-async def test_with_config_none_falls_back_to_instance():
-    """When with_config() supplies None, the instance default is used."""
-    instance_settings = [{"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"}]
-
-    provider = _make_llm_provider(safety_settings=instance_settings)
-    provider._provider_impl._client.aio.models.generate_content = AsyncMock(return_value=_fake_response())
-
-    configured = provider.with_config(_make_config(None))
-    await configured.call(messages=[{"role": "user", "content": "hi"}], scope="test")
-
-    config_arg = provider._provider_impl._client.aio.models.generate_content.call_args.kwargs.get("config")
-    assert config_arg is not None
-    categories = [s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings]
-    assert "HARM_CATEGORY_HARASSMENT" in categories
-
-
-@pytest.mark.asyncio
-async def test_with_config_resets_after_call():
-    """The ContextVar is properly reset after a with_config() call (no leakage)."""
-    from hindsight_api.engine.providers.gemini_llm import _safety_settings_ctx
-
-    settings = [{"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"}]
-    provider = _make_llm_provider(safety_settings=None)
-    provider._provider_impl._client.aio.models.generate_content = AsyncMock(return_value=_fake_response())
-
-    before = _safety_settings_ctx.get()
-    configured = provider.with_config(_make_config(settings))
-    await configured.call(messages=[{"role": "user", "content": "hi"}], scope="test")
-    after = _safety_settings_ctx.get()
-
-    assert after == before  # ContextVar restored to its original value
-
-
-# ─── LLMProvider reads safety settings from config ────────────────────────────
-
-
-def test_llm_provider_reads_safety_settings_from_config():
-    """LLMProvider reads llm_gemini_safety_settings from global config for Gemini provider."""
-    import json
-
-    from hindsight_api.config import ENV_LLM_GEMINI_SAFETY_SETTINGS, clear_config_cache
-
-    settings_json = json.dumps(SAMPLE_SAFETY_SETTINGS)
-    env_overrides = {
-        "HINDSIGHT_API_LLM_PROVIDER": "gemini",
-        "HINDSIGHT_API_LLM_API_KEY": "fake-key",
-        ENV_LLM_GEMINI_SAFETY_SETTINGS: settings_json,
-    }
-
-    with patch.dict(os.environ, env_overrides, clear=False):
-        clear_config_cache()
-        with patch("google.genai.Client") as mock_client_cls:
-            mock_client_cls.return_value = MagicMock()
-            from hindsight_api.engine.llm_wrapper import LLMProvider
-
-            provider = LLMProvider(
-                provider="gemini",
-                api_key="fake-key",
-                base_url="",
-                model="gemini-2.5-flash",
-            )
-
-            assert provider.gemini_safety_settings == SAMPLE_SAFETY_SETTINGS
-
-        clear_config_cache()

hindsight-api-slim/tests/test_hnsw_indexes.py

@@ -1,193 +0,0 @@
-"""
-Tests for per-bank HNSW index lifecycle and UNION ALL retrieval.
-
-Covers:
-- _hnsw_index_name deterministic naming
-- Per-bank HNSW indexes created on bank creation (retain_async / ensure_bank_exists)
-- Per-bank HNSW indexes dropped on bank deletion
-- retrieve_semantic_bm25_combined groups results correctly by fact_type and source
-"""
-import uuid
-from datetime import datetime, timezone
-
-import pytest
-
-from hindsight_api.engine.retain.bank_utils import _HNSW_FACT_TYPES, _hnsw_index_name
-
-
-# ---------------------------------------------------------------------------
-# Unit tests — no DB required
-# ---------------------------------------------------------------------------
-
-
-class TestHnswIndexName:
-    def test_deterministic(self):
-        uid = "550e8400-e29b-41d4-a716-446655440000"
-        assert _hnsw_index_name("world", uid) == _hnsw_index_name("world", uid)
-
-    def test_strips_dashes(self):
-        uid = "550e8400-e29b-41d4-a716-446655440000"
-        name = _hnsw_index_name("world", uid)
-        # uid16 should be hex chars only
-        assert "-" not in name
-
-    def test_uses_first_16_hex_chars(self):
-        uid = "550e8400-e29b-41d4-a716-446655440000"
-        uid16 = uid.replace("-", "")[:16]  # "550e8400e29b41d4"
-        assert name_ends_with(name=_hnsw_index_name("world", uid), suffix=uid16)
-
-    def test_suffix_per_fact_type(self):
-        uid = "550e8400-e29b-41d4-a716-446655440000"
-        names = {ft: _hnsw_index_name(ft, uid) for ft in _HNSW_FACT_TYPES}
-        # All three names must be distinct
-        assert len(set(names.values())) == 3
-
-    def test_all_fact_types_covered(self):
-        assert set(_HNSW_FACT_TYPES) == {"world", "experience", "observation"}
-
-    def test_fits_pg_identifier_limit(self):
-        # PostgreSQL max identifier length is 63 chars
-        uid = "f" * 32  # simulated UUID without dashes
-        for ft in _HNSW_FACT_TYPES:
-            assert len(_hnsw_index_name(ft, uid)) <= 63
-
-
-def name_ends_with(name: str, suffix: str) -> bool:
-    return name.endswith(suffix)
-
-
-# ---------------------------------------------------------------------------
-# Integration tests — require DB (memory fixture)
-# ---------------------------------------------------------------------------
-
-
-async def _get_bank_hnsw_indexes(pool, bank_id: str) -> list[str]:
-    """Return index names for memory_units that match the per-bank pattern."""
-    async with pool.acquire() as conn:
-        rows = await conn.fetch(
-            """
-            SELECT indexname
-            FROM pg_indexes
-            WHERE tablename = 'memory_units'
-              AND indexname LIKE 'idx_mu_emb_%'
-              AND indexdef LIKE $1
-            ORDER BY indexname
-            """,
-            f"%bank_id = '{bank_id}'%",
-        )
-    return [row["indexname"] for row in rows]
-
-
-@pytest.mark.asyncio
-async def test_retain_creates_per_bank_hnsw_indexes(memory, request_context):
-    """retain_async on a new bank must create 3 per-(bank, fact_type) HNSW indexes."""
-    bank_id = f"test_hnsw_create_{uuid.uuid4().hex[:8]}"
-    try:
-        await memory.retain_async(
-            bank_id=bank_id,
-            content="Alice is a software engineer.",
-            request_context=request_context,
-        )
-        indexes = await _get_bank_hnsw_indexes(memory._pool, bank_id)
-        assert len(indexes) == 3, f"Expected 3 per-bank HNSW indexes, got: {indexes}"
-        for ft_short in _HNSW_FACT_TYPES.values():
-            assert any(ft_short in idx for idx in indexes), (
-                f"Missing index for fact_type short '{ft_short}' in {indexes}"
-            )
-    finally:
-        await memory.delete_bank(bank_id, request_context=request_context)
-
-
-@pytest.mark.asyncio
-async def test_delete_bank_drops_hnsw_indexes(memory, request_context):
-    """delete_bank must drop all per-bank HNSW indexes."""
-    bank_id = f"test_hnsw_drop_{uuid.uuid4().hex[:8]}"
-
-    await memory.retain_async(
-        bank_id=bank_id,
-        content="Bob is a data scientist.",
-        request_context=request_context,
-    )
-    # Verify indexes exist before deletion
-    indexes_before = await _get_bank_hnsw_indexes(memory._pool, bank_id)
-    assert len(indexes_before) == 3
-
-    await memory.delete_bank(bank_id, request_context=request_context)
-
-    indexes_after = await _get_bank_hnsw_indexes(memory._pool, bank_id)
-    assert indexes_after == [], f"Indexes should be dropped after bank deletion, got: {indexes_after}"
-
-
-@pytest.mark.asyncio
-async def test_retain_idempotent_bank_creation(memory, request_context):
-    """Retaining into the same bank twice must not error and still have exactly 3 indexes."""
-    bank_id = f"test_hnsw_idem_{uuid.uuid4().hex[:8]}"
-    try:
-        await memory.retain_async(
-            bank_id=bank_id,
-            content="Carol is a product manager.",
-            request_context=request_context,
-        )
-        await memory.retain_async(
-            bank_id=bank_id,
-            content="Carol joined the company in 2022.",
-            request_context=request_context,
-        )
-        indexes = await _get_bank_hnsw_indexes(memory._pool, bank_id)
-        assert len(indexes) == 3
-    finally:
-        await memory.delete_bank(bank_id, request_context=request_context)
-
-
-@pytest.mark.asyncio
-async def test_retrieve_semantic_bm25_grouped_by_fact_type(memory, request_context):
-    """
-    retrieve_semantic_bm25_combined must return a dict keyed by fact_type with
-    (semantic_list, bm25_list) tuples.  All returned facts must belong to their
-    declared fact_type.
-    """
-    from hindsight_api.engine.search.retrieval import retrieve_semantic_bm25_combined
-
-    bank_id = f"test_retrieval_{uuid.uuid4().hex[:8]}"
-    try:
-        await memory.retain_async(
-            bank_id=bank_id,
-            content=(
-                "Alice is a software engineer at TechCorp. "
-                "She visited Paris in 2023 for a conference."
-            ),
-            context="background",
-            event_date=datetime(2023, 6, 1, tzinfo=timezone.utc),
-            request_context=request_context,
-        )
-
-        query_emb = memory.embeddings.encode(["software engineer Alice"])
-        query_emb_str = str(query_emb[0])
-
-        fact_types = ["world", "experience"]
-        async with memory._pool.acquire() as conn:
-            results = await retrieve_semantic_bm25_combined(
-                conn=conn,
-                query_emb_str=query_emb_str,
-                query_text="software engineer Alice",
-                bank_id=bank_id,
-                fact_types=fact_types,
-                limit=5,
-            )
-
-        # Must return an entry for every requested fact_type
-        assert set(results.keys()) == set(fact_types)
-
-        for ft, (sem, bm25) in results.items():
-            # Semantic and BM25 lists must be lists
-            assert isinstance(sem, list)
-            assert isinstance(bm25, list)
-            # All semantic results must declare the correct fact_type
-            for r in sem:
-                assert r.fact_type == ft, f"Semantic result has wrong fact_type: {r.fact_type}"
-            # All BM25 results must declare the correct fact_type
-            for r in bm25:
-                assert r.fact_type == ft, f"BM25 result has wrong fact_type: {r.fact_type}"
-
-    finally:
-        await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_list_documents.py

@@ -1,174 +0,0 @@
-"""
-Tests for list_documents pagination and tags filtering.
-"""
-from datetime import datetime, timezone
-
-import pytest
-
-
-async def _retain_doc(memory, bank_id, document_id, tags, request_context):
-    """Helper to retain a document with given tags. Uses gibberish content to avoid LLM
-    fact extraction (documents are persisted even with zero facts)."""
-    await memory.retain_batch_async(
-        bank_id=bank_id,
-        contents=[{"content": f"xyzabc123 !@# $$$ {document_id}"}],
-        document_id=document_id,
-        document_tags=tags or None,
-        request_context=request_context,
-    )
-
-
-@pytest.mark.asyncio
-async def test_list_documents_offset_pagination(memory, request_context):
-    """offset parameter returns the correct slice of documents."""
-    bank_id = f"test_list_docs_offset_{datetime.now(timezone.utc).timestamp()}"
-
-    try:
-        for i in range(4):
-            await _retain_doc(memory, bank_id, f"doc-{i:02d}", [], request_context)
-
-        # All documents, ordered by created_at DESC → doc-03, doc-02, doc-01, doc-00
-        all_docs = await memory.list_documents(
-            bank_id=bank_id, limit=10, offset=0, request_context=request_context
-        )
-        assert all_docs["total"] == 4
-        assert len(all_docs["items"]) == 4
-        all_ids = [d["id"] for d in all_docs["items"]]
-
-        # offset=2 should skip the first two and return the remaining two
-        page2 = await memory.list_documents(
-            bank_id=bank_id, limit=10, offset=2, request_context=request_context
-        )
-        assert page2["total"] == 4  # total is always the full count
-        assert len(page2["items"]) == 2
-        assert [d["id"] for d in page2["items"]] == all_ids[2:]
-
-        # offset beyond total returns empty items but correct total
-        beyond = await memory.list_documents(
-            bank_id=bank_id, limit=10, offset=10, request_context=request_context
-        )
-        assert beyond["total"] == 4
-        assert beyond["items"] == []
-
-    finally:
-        await memory.delete_bank(bank_id, request_context=request_context)
-
-
-@pytest.mark.asyncio
-async def test_list_documents_tags_filter_any_strict(memory, request_context):
-    """tags filter with any_strict returns only tagged documents that match."""
-    bank_id = f"test_list_docs_tags_{datetime.now(timezone.utc).timestamp()}"
-
-    try:
-        await _retain_doc(memory, bank_id, "doc-alpha", ["team-a"], request_context)
-        await _retain_doc(memory, bank_id, "doc-beta", ["team-b"], request_context)
-        await _retain_doc(memory, bank_id, "doc-both", ["team-a", "team-b"], request_context)
-        await _retain_doc(memory, bank_id, "doc-untagged", [], request_context)
-
-        # any_strict: only docs with at least one of the given tags, untagged excluded
-        result = await memory.list_documents(
-            bank_id=bank_id,
-            tags=["team-a"],
-            tags_match="any_strict",
-            request_context=request_context,
-        )
-        ids = {d["id"] for d in result["items"]}
-        assert ids == {"doc-alpha", "doc-both"}
-        assert result["total"] == 2
-
-    finally:
-        await memory.delete_bank(bank_id, request_context=request_context)
-
-
-@pytest.mark.asyncio
-async def test_list_documents_tags_filter_any_includes_untagged(memory, request_context):
-    """tags filter with 'any' mode includes untagged documents."""
-    bank_id = f"test_list_docs_tags_any_{datetime.now(timezone.utc).timestamp()}"
-
-    try:
-        await _retain_doc(memory, bank_id, "doc-tagged", ["team-a"], request_context)
-        await _retain_doc(memory, bank_id, "doc-other", ["team-b"], request_context)
-        await _retain_doc(memory, bank_id, "doc-untagged", [], request_context)
-
-        result = await memory.list_documents(
-            bank_id=bank_id,
-            tags=["team-a"],
-            tags_match="any",
-            request_context=request_context,
-        )
-        ids = {d["id"] for d in result["items"]}
-        # "any" includes untagged + matching tagged
-        assert "doc-tagged" in ids
-        assert "doc-untagged" in ids
-        assert "doc-other" not in ids
-
-    finally:
-        await memory.delete_bank(bank_id, request_context=request_context)
-
-
-@pytest.mark.asyncio
-async def test_list_documents_tags_filter_all_strict(memory, request_context):
-    """tags filter with all_strict returns only docs that have ALL the specified tags."""
-    bank_id = f"test_list_docs_tags_all_{datetime.now(timezone.utc).timestamp()}"
-
-    try:
-        await _retain_doc(memory, bank_id, "doc-a-only", ["team-a"], request_context)
-        await _retain_doc(memory, bank_id, "doc-a-and-b", ["team-a", "team-b"], request_context)
-        await _retain_doc(memory, bank_id, "doc-untagged", [], request_context)
-
-        result = await memory.list_documents(
-            bank_id=bank_id,
-            tags=["team-a", "team-b"],
-            tags_match="all_strict",
-            request_context=request_context,
-        )
-        ids = {d["id"] for d in result["items"]}
-        assert ids == {"doc-a-and-b"}
-
-    finally:
-        await memory.delete_bank(bank_id, request_context=request_context)
-
-
-@pytest.mark.asyncio
-async def test_list_documents_no_tags_filter_returns_all(memory, request_context):
-    """When no tags filter is specified, all documents are returned."""
-    bank_id = f"test_list_docs_no_tags_{datetime.now(timezone.utc).timestamp()}"
-
-    try:
-        await _retain_doc(memory, bank_id, "doc-tagged", ["team-a"], request_context)
-        await _retain_doc(memory, bank_id, "doc-untagged", [], request_context)
-
-        result = await memory.list_documents(
-            bank_id=bank_id,
-            tags=None,
-            request_context=request_context,
-        )
-        ids = {d["id"] for d in result["items"]}
-        assert ids == {"doc-tagged", "doc-untagged"}
-
-    finally:
-        await memory.delete_bank(bank_id, request_context=request_context)
-
-
-@pytest.mark.asyncio
-async def test_list_documents_tags_and_search_query_combined(memory, request_context):
-    """tags filter and q (search_query) can be combined."""
-    bank_id = f"test_list_docs_tags_q_{datetime.now(timezone.utc).timestamp()}"
-
-    try:
-        await _retain_doc(memory, bank_id, "report-2024", ["team-a"], request_context)
-        await _retain_doc(memory, bank_id, "report-2025", ["team-b"], request_context)
-        await _retain_doc(memory, bank_id, "summary-2024", ["team-a"], request_context)
-
-        result = await memory.list_documents(
-            bank_id=bank_id,
-            search_query="report",
-            tags=["team-a"],
-            tags_match="any_strict",
-            request_context=request_context,
-        )
-        ids = {d["id"] for d in result["items"]}
-        assert ids == {"report-2024"}
-
-    finally:
-        await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_llm_wrapper.py

@@ -1,37 +0,0 @@
-import pytest
-
-from hindsight_api.engine.llm_wrapper import sanitize_llm_output
-
-
-@pytest.mark.parametrize(
-    "input_text, expected",
-    [
-        # Null bytes stripped
-        ("hello\x00world", "helloworld"),
-        ("FIRST\u0000PAGE", "FIRSTPAGE"),
-        # Multiple null bytes
-        ("\x00\x00text\x00", "text"),
-        # Other control characters stripped (non-whitespace)
-        ("text\x01\x02\x03end", "textend"),
-        ("text\x08end", "textend"),  # backspace
-        ("text\x0cend", "textend"),  # form feed
-        ("text\x0bend", "textend"),  # vertical tab
-        ("text\x1fend", "textend"),  # unit separator
-        ("text\x7fend", "textend"),  # DEL
-        # Whitespace preserved
-        ("hello\tworld", "hello\tworld"),
-        ("hello\nworld", "hello\nworld"),
-        ("hello\r\nworld", "hello\r\nworld"),
-        # Unicode surrogates stripped
-        ("text\ud800end", "textend"),
-        ("text\udfffend", "textend"),
-        # Clean text unchanged
-        ("normal text", "normal text"),
-        ("unicode: café naïve", "unicode: café naïve"),
-        # Edge cases
-        ("", ""),
-        (None, None),
-    ],
-)
-def test_sanitize_llm_output(input_text, expected):
-    assert sanitize_llm_output(input_text) == expected

hindsight-api-slim/tests/test_lmstudio_tool_choice.py

@@ -1,321 +0,0 @@
-"""
-Reproduce issue #520: Reflect fails with LM Studio due to unsupported tool_choice format.
-
-The reflect agent forces tool selection via named tool_choice dicts on the first few iterations:
-  {"type": "function", "function": {"name": "search_mental_models"}}
-
-LM Studio (and Ollama) reject this format with HTTP 400:
-  "Tool choice of type 'function' is not supported. Use 'auto', 'none', or 'required'."
-
-The fix should convert named tool_choice to "required" and filter the tools list
-to only the requested tool for providers that don't support named tool_choice.
-"""
-
-import json
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-from openai import APIStatusError
-
-from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
-
-# Reflect agent tools (subset matching what agent.py uses)
-REFLECT_TOOLS = [
-    {
-        "type": "function",
-        "function": {
-            "name": "search_mental_models",
-            "description": "Search consolidated mental models",
-            "parameters": {
-                "type": "object",
-                "properties": {"query": {"type": "string"}},
-                "required": ["query"],
-            },
-        },
-    },
-    {
-        "type": "function",
-        "function": {
-            "name": "search_observations",
-            "description": "Search raw observations",
-            "parameters": {
-                "type": "object",
-                "properties": {"query": {"type": "string"}},
-                "required": ["query"],
-            },
-        },
-    },
-    {
-        "type": "function",
-        "function": {
-            "name": "recall",
-            "description": "Recall semantic memories",
-            "parameters": {
-                "type": "object",
-                "properties": {"query": {"type": "string"}},
-                "required": ["query"],
-            },
-        },
-    },
-    {
-        "type": "function",
-        "function": {
-            "name": "done",
-            "description": "Finish and return the answer",
-            "parameters": {
-                "type": "object",
-                "properties": {"answer": {"type": "string"}},
-                "required": ["answer"],
-            },
-        },
-    },
-]
-
-
-def _make_lmstudio_llm() -> OpenAICompatibleLLM:
-    return OpenAICompatibleLLM(
-        provider="lmstudio",
-        api_key="local",
-        base_url="http://localhost:1234/v1",
-        model="openai/gpt-oss-20b",
-    )
-
-
-def _lmstudio_400_error(msg: str = "Tool choice of type 'function' is not supported. Use 'auto', 'none', or 'required'.") -> APIStatusError:
-    """Simulate the HTTP 400 LM Studio returns for unsupported tool_choice format."""
-    mock_response = MagicMock()
-    mock_response.status_code = 400
-    mock_response.headers = {}
-    return APIStatusError(
-        message=msg,
-        response=mock_response,
-        body={"error": {"message": msg, "type": "invalid_request_error"}},
-    )
-
-
-def _make_tool_call_response(tool_name: str, arguments: dict) -> MagicMock:
-    """Build a mock successful tool call response from the LLM API."""
-    mock_tc = MagicMock()
-    mock_tc.id = "call_abc123"
-    mock_tc.function.name = tool_name
-    mock_tc.function.arguments = json.dumps(arguments)
-
-    mock_response = MagicMock()
-    mock_response.usage.prompt_tokens = 120
-    mock_response.usage.completion_tokens = 40
-    mock_response.usage.total_tokens = 160
-    mock_response.choices[0].finish_reason = "tool_calls"
-    mock_response.choices[0].message.content = None
-    mock_response.choices[0].message.tool_calls = [mock_tc]
-    return mock_response
-
-
-class TestLMStudioNamedToolChoiceBug:
-    """
-    Reproduces issue #520.
-
-    The reflect agent (agent.py lines 546-555) sets tool_choice to a named dict
-    on the first iterations to force sequential retrieval:
-
-        iteration=0, has_mental_models=True  → {"type": "function", "function": {"name": "search_mental_models"}}
-        iteration=0, has_mental_models=False → {"type": "function", "function": {"name": "search_observations"}}
-        iteration=1, has_mental_models=True  → {"type": "function", "function": {"name": "search_observations"}}
-        iteration=1 or (2 with models)       → {"type": "function", "function": {"name": "recall"}}
-
-    LM Studio rejects these dict formats with HTTP 400.
-    """
-
-    @pytest.mark.asyncio
-    async def test_lmstudio_named_tool_choice_no_longer_causes_400(self):
-        """
-        Regression test for issue #520: named tool_choice dict is converted to
-        "required" + filtered tools before the API call, so LM Studio never
-        sees the unsupported format and the 400 error no longer occurs.
-        """
-        llm = _make_lmstudio_llm()
-        named_tool_choice = {"type": "function", "function": {"name": "search_mental_models"}}
-        success_response = _make_tool_call_response("search_mental_models", {"query": "user name"})
-
-        with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create:
-            mock_create.return_value = success_response
-
-            # Should succeed — no 400 because the dict is converted before sending
-            result = await llm.call_with_tools(
-                messages=[{"role": "user", "content": "What is the user's name?"}],
-                tools=REFLECT_TOOLS,
-                tool_choice=named_tool_choice,
-                max_retries=0,
-            )
-
-        assert len(result.tool_calls) == 1
-        assert result.tool_calls[0].name == "search_mental_models"
-
-        sent_kwargs = mock_create.call_args.kwargs
-        assert sent_kwargs["tool_choice"] == "required"
-        assert len(sent_kwargs["tools"]) == 1
-        assert sent_kwargs["tools"][0]["function"]["name"] == "search_mental_models"
-
-    @pytest.mark.asyncio
-    @pytest.mark.parametrize(
-        "forced_tool_name",
-        ["search_mental_models", "search_observations", "recall"],
-    )
-    async def test_all_reflect_forced_tools_fail_on_lmstudio(self, forced_tool_name: str):
-        """
-        Each named tool_choice the reflect agent uses on iterations 0-2 triggers
-        the same 400 error on LM Studio.
-        """
-        llm = _make_lmstudio_llm()
-        named_tool_choice = {"type": "function", "function": {"name": forced_tool_name}}
-
-        with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create:
-            mock_create.side_effect = _lmstudio_400_error()
-
-            with pytest.raises(APIStatusError) as exc_info:
-                await llm.call_with_tools(
-                    messages=[{"role": "user", "content": "Test query"}],
-                    tools=REFLECT_TOOLS,
-                    tool_choice=named_tool_choice,
-                    max_retries=0,
-                )
-
-            assert exc_info.value.status_code == 400
-
-    @pytest.mark.asyncio
-    async def test_lmstudio_string_tool_choice_works_fine(self):
-        """
-        String tool_choice values ("auto", "none", "required") ARE supported by LM Studio.
-        Only the dict format {"type": "function", "function": {"name": "..."}} fails.
-        This test confirms the control case works.
-        """
-        llm = _make_lmstudio_llm()
-        success_response = _make_tool_call_response("search_mental_models", {"query": "user name"})
-
-        with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create:
-            mock_create.return_value = success_response
-
-            result = await llm.call_with_tools(
-                messages=[{"role": "user", "content": "What is the user's name?"}],
-                tools=REFLECT_TOOLS,
-                tool_choice="required",  # string form — LM Studio accepts this
-                max_retries=0,
-            )
-
-        assert len(result.tool_calls) == 1
-        assert result.tool_calls[0].name == "search_mental_models"
-
-        # Confirm "required" was sent, not a dict
-        sent_kwargs = mock_create.call_args.kwargs
-        assert sent_kwargs["tool_choice"] == "required"
-
-
-class TestExpectedFixBehavior:
-    """
-    Tests that document the EXPECTED behavior after the fix is applied.
-
-    For lmstudio (and ollama) providers, when tool_choice is a named dict:
-      {"type": "function", "function": {"name": "search_mental_models"}}
-
-    The fix should:
-      1. Convert tool_choice to "required"
-      2. Filter tools to only the requested tool
-
-    These tests currently FAIL (because the fix is not yet implemented).
-    After the fix is applied, they should PASS.
-    """
-
-    @pytest.mark.asyncio
-    async def test_fix_converts_named_tool_choice_to_required(self):
-        """
-        After fix: named tool_choice dict is converted to "required" for lmstudio.
-        The API receives tool_choice="required" instead of the unsupported dict.
-        """
-        llm = _make_lmstudio_llm()
-        named_tool_choice = {"type": "function", "function": {"name": "search_mental_models"}}
-        success_response = _make_tool_call_response("search_mental_models", {"query": "user name"})
-
-        with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create:
-            mock_create.return_value = success_response
-
-            result = await llm.call_with_tools(
-                messages=[{"role": "user", "content": "What is the user's name?"}],
-                tools=REFLECT_TOOLS,
-                tool_choice=named_tool_choice,
-                max_retries=0,
-            )
-
-        assert len(result.tool_calls) == 1
-        assert result.tool_calls[0].name == "search_mental_models"
-
-        sent_kwargs = mock_create.call_args.kwargs
-        # Fix: dict was converted to "required"
-        assert sent_kwargs["tool_choice"] == "required", (
-            f"Expected tool_choice='required', got {sent_kwargs['tool_choice']!r}"
-        )
-        # Fix: tools filtered to just the requested one
-        assert len(sent_kwargs["tools"]) == 1
-        assert sent_kwargs["tools"][0]["function"]["name"] == "search_mental_models"
-
-    @pytest.mark.asyncio
-    @pytest.mark.parametrize(
-        "forced_tool_name",
-        ["search_mental_models", "search_observations", "recall"],
-    )
-    async def test_fix_filters_tools_to_requested_tool(self, forced_tool_name: str):
-        """
-        After fix: tools list is filtered to only the forced tool so the model
-        can only call that one tool (equivalent to the named tool_choice behavior).
-        """
-        llm = _make_lmstudio_llm()
-        named_tool_choice = {"type": "function", "function": {"name": forced_tool_name}}
-        success_response = _make_tool_call_response(forced_tool_name, {"query": "test"})
-
-        with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create:
-            mock_create.return_value = success_response
-
-            await llm.call_with_tools(
-                messages=[{"role": "user", "content": "Test query"}],
-                tools=REFLECT_TOOLS,
-                tool_choice=named_tool_choice,
-                max_retries=0,
-            )
-
-        sent_kwargs = mock_create.call_args.kwargs
-        assert sent_kwargs["tool_choice"] == "required"
-        assert len(sent_kwargs["tools"]) == 1
-        assert sent_kwargs["tools"][0]["function"]["name"] == forced_tool_name
-
-    @pytest.mark.asyncio
-    async def test_fix_also_applies_to_openai_provider(self):
-        """
-        The fix is generalized: all providers convert named tool_choice to
-        "required" + filtered tools.  OpenAI natively supports the dict format
-        too, so the behaviour is semantically identical either way.
-        """
-        from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
-
-        openai_llm = OpenAICompatibleLLM(
-            provider="openai",
-            api_key="sk-test",
-            base_url="",
-            model="gpt-4o-mini",
-        )
-
-        named_tool_choice = {"type": "function", "function": {"name": "search_mental_models"}}
-        success_response = _make_tool_call_response("search_mental_models", {"query": "test"})
-
-        with patch.object(openai_llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create:
-            mock_create.return_value = success_response
-
-            await openai_llm.call_with_tools(
-                messages=[{"role": "user", "content": "Test"}],
-                tools=REFLECT_TOOLS,
-                tool_choice=named_tool_choice,
-                max_retries=0,
-            )
-
-        sent_kwargs = mock_create.call_args.kwargs
-        # Generalized fix applies to OpenAI too
-        assert sent_kwargs["tool_choice"] == "required"
-        assert len(sent_kwargs["tools"]) == 1
-        assert sent_kwargs["tools"][0]["function"]["name"] == "search_mental_models"

hindsight-api-slim/tests/test_migration_backsweep.py

@@ -1,154 +0,0 @@
-"""Tests for migration g7h8i9j0k1l2 (backsweep orphaned memory_units).
-
-Uses a dedicated pg0 instance (port 5562) so the test can control exactly
-which migrations have run before inserting the orphan seed data.
-"""
-
-import asyncio
-import uuid
-from pathlib import Path
-
-import pytest
-from alembic import command
-from alembic.config import Config
-from sqlalchemy import create_engine, text
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-_SCRIPT_LOCATION = str(Path(__file__).parent.parent / "hindsight_api" / "alembic")
-
-
-def _alembic_cfg(db_url: str) -> Config:
-    cfg = Config()
-    cfg.set_main_option("script_location", _SCRIPT_LOCATION)
-    cfg.set_main_option("sqlalchemy.url", db_url)
-    cfg.set_main_option("prepend_sys_path", ".")
-    cfg.set_main_option("path_separator", "os")
-    return cfg
-
-
-def _upgrade(db_url: str, revision: str) -> None:
-    command.upgrade(_alembic_cfg(db_url), revision)
-
-
-# ---------------------------------------------------------------------------
-# Fixture: fresh database at the revision just before the backsweep
-# ---------------------------------------------------------------------------
-
-@pytest.fixture(scope="module")
-def pre_backsweep_db_url():
-    """
-    Spin up a dedicated pg0 instance and run all migrations up to (but not
-    including) the backsweep revision so each test can seed orphan data and
-    then apply the backsweep itself.
-    """
-    from hindsight_api.pg0 import EmbeddedPostgres
-
-    pg0 = EmbeddedPostgres(name="hindsight-backsweep-test", port=5562)
-    loop = asyncio.new_event_loop()
-    try:
-        url = loop.run_until_complete(pg0.ensure_running())
-    finally:
-        loop.close()
-
-    # Migrate up to the revision just before the backsweep.
-    _upgrade(url, "f6g7h8i9j0k1")
-    return url
-
-
-# ---------------------------------------------------------------------------
-# The test
-# ---------------------------------------------------------------------------
-
-def test_backsweep_removes_orphans_and_preserves_legit_rows(pre_backsweep_db_url):
-    """
-    Seed four kinds of rows then apply the backsweep migration and verify:
-
-    Rows that MUST be deleted
-    ─────────────────────────
-    A. Any fact_type, bank_id missing from banks
-       → Pass 1 deletes these regardless of fact_type or source links.
-
-    B. observation, bank exists, but ALL source_memory_ids are gone
-       → Pass 2 deletes these.
-
-    Rows that MUST survive
-    ──────────────────────
-    C. observation, bank exists, at least ONE source_memory_id still live
-       → Pass 2 must not touch these.
-
-    D. Non-observation (world), bank exists, no sources (not relevant)
-       → Pass 1 must not touch these (bank exists).
-    """
-    db_url = pre_backsweep_db_url
-    engine = create_engine(db_url)
-
-    alive_bank = f"bank_{uuid.uuid4().hex[:8]}"
-    ghost_bank = f"bank_{uuid.uuid4().hex[:8]}"  # never inserted into banks
-
-    # UUIDs for memory units
-    id_pass1_world = uuid.uuid4()       # A: world unit, ghost bank
-    id_pass1_obs = uuid.uuid4()         # A: observation, ghost bank
-    id_pass2_obs = uuid.uuid4()         # B: observation, all sources gone
-    id_keep_obs = uuid.uuid4()          # C: observation with one live source
-    id_keep_world = uuid.uuid4()        # D: world unit, alive bank
-    id_live_source = uuid.uuid4()       # live source for C
-
-    with engine.connect() as conn:
-        # --- banks ---
-        conn.execute(text("INSERT INTO banks (bank_id) VALUES (:b)"), {"b": alive_bank})
-
-        # --- seed memory_units ---
-        def insert_mu(uid, bank, fact_type, sources=None):
-            src_arr = "{" + ",".join(str(s) for s in (sources or [])) + "}"
-            conn.execute(
-                text(
-                    """
-                    INSERT INTO memory_units
-                        (id, bank_id, text, fact_type, source_memory_ids)
-                    VALUES
-                        (:id, :bank, :text, :ft, CAST(:src AS uuid[]))
-                    """
-                ),
-                {"id": uid, "bank": bank, "text": "test", "ft": fact_type, "src": src_arr},
-            )
-
-        # A: ghost-bank rows (Pass 1 targets)
-        insert_mu(id_pass1_world, ghost_bank, "world")
-        insert_mu(id_pass1_obs, ghost_bank, "observation", sources=[uuid.uuid4()])
-
-        # B: observation with all-dead sources (Pass 2 target)
-        insert_mu(id_pass2_obs, alive_bank, "observation", sources=[uuid.uuid4(), uuid.uuid4()])
-
-        # C: observation with one live source (must survive)
-        insert_mu(id_live_source, alive_bank, "world")
-        insert_mu(id_keep_obs, alive_bank, "observation", sources=[id_live_source, uuid.uuid4()])
-
-        # D: world unit in alive bank (must survive)
-        insert_mu(id_keep_world, alive_bank, "world")
-
-        conn.commit()
-
-    # --- apply the backsweep ---
-    _upgrade(db_url, "g7h8i9j0k1l2")
-
-    # --- verify ---
-    with engine.connect() as conn:
-        def exists(uid):
-            return conn.execute(
-                text("SELECT 1 FROM memory_units WHERE id = :id"), {"id": uid}
-            ).fetchone() is not None
-
-        # Must be gone
-        assert not exists(id_pass1_world), "Pass 1: world unit with ghost bank should be deleted"
-        assert not exists(id_pass1_obs), "Pass 1: observation with ghost bank should be deleted"
-        assert not exists(id_pass2_obs), "Pass 2: observation with all-dead sources should be deleted"
-
-        # Must survive
-        assert exists(id_keep_obs), "observation with a live source must not be deleted"
-        assert exists(id_keep_world), "world unit in alive bank must not be deleted"
-        assert exists(id_live_source), "live source memory unit must not be deleted"
-
-    engine.dispose()

hindsight-api-slim/tests/test_migrations_thread_safety.py

@@ -1,48 +0,0 @@
-import threading
-import time
-
-from hindsight_api import migrations
-
-
-def test_run_migrations_internal_serializes_alembic_upgrade(monkeypatch):
-    max_concurrent_upgrades = 0
-    active_upgrades = 0
-    active_lock = threading.Lock()
-    start_barrier = threading.Barrier(2)
-
-    def fake_upgrade(_cfg, _revision):
-        nonlocal max_concurrent_upgrades, active_upgrades
-        with active_lock:
-            active_upgrades += 1
-            max_concurrent_upgrades = max(max_concurrent_upgrades, active_upgrades)
-        time.sleep(0.05)
-        with active_lock:
-            active_upgrades -= 1
-
-    monkeypatch.setattr(migrations.command, "upgrade", fake_upgrade)
-
-    errors = []
-
-    def run_in_thread(schema):
-        try:
-            start_barrier.wait()
-            migrations._run_migrations_internal(
-                "postgresql://user:pass@localhost/db",
-                "/tmp/alembic",
-                schema=schema,
-            )
-        except Exception as exc:  # pragma: no cover - diagnostic path
-            errors.append(exc)
-
-    threads = [
-        threading.Thread(target=run_in_thread, args=("tenant_alpha",)),
-        threading.Thread(target=run_in_thread, args=("tenant_beta",)),
-    ]
-
-    for thread in threads:
-        thread.start()
-    for thread in threads:
-        thread.join()
-
-    assert not errors
-    assert max_concurrent_upgrades == 1

hindsight-api-slim/tests/test_op_cancellation.py

@@ -1,311 +0,0 @@
-"""Tests for operation cancellation when a bank is deleted.
-
-Covers:
-- CASCADE DELETE: deleting a bank removes async_operations and webhooks rows
-- _check_op_alive: returns True when op exists, False when deleted
-- _mark_operation_completed / _mark_operation_failed: graceful no-op when row is gone
-- Consolidation checkpoint: stops early after a batch commit if op was deleted
-- Retain checkpoint: stops between sub-batches if op was deleted
-"""
-
-import uuid
-from unittest.mock import AsyncMock, patch
-
-import pytest
-import pytest_asyncio
-
-from hindsight_api.engine.memory_engine import MemoryEngine
-
-
-pytestmark = pytest.mark.xdist_group("op_cancellation_tests")
-
-_BANK_PREFIX = "test-op-cancel"
-
-
-@pytest_asyncio.fixture
-async def pool(pg0_db_url):
-    import asyncpg
-    from hindsight_api.pg0 import resolve_database_url
-
-    resolved_url = await resolve_database_url(pg0_db_url)
-    p = await asyncpg.create_pool(resolved_url, min_size=1, max_size=5, command_timeout=30)
-    yield p
-    await p.close()
-
-
-@pytest_asyncio.fixture(autouse=True)
-async def cleanup(pool):
-    """Remove test rows before and after each test."""
-    await pool.execute(f"DELETE FROM banks WHERE bank_id LIKE '{_BANK_PREFIX}%'")
-    yield
-    await pool.execute(f"DELETE FROM banks WHERE bank_id LIKE '{_BANK_PREFIX}%'")
-
-
-async def _insert_bank(pool, bank_id: str):
-    await pool.execute(
-        "INSERT INTO banks (bank_id, name) VALUES ($1, $2) ON CONFLICT DO NOTHING",
-        bank_id,
-        bank_id,
-    )
-
-
-async def _insert_op(pool, bank_id: str, op_id: uuid.UUID | None = None) -> uuid.UUID:
-    op_id = op_id or uuid.uuid4()
-    await pool.execute(
-        """
-        INSERT INTO async_operations (operation_id, bank_id, operation_type, status)
-        VALUES ($1, $2, 'consolidation', 'processing')
-        """,
-        op_id,
-        bank_id,
-    )
-    return op_id
-
-
-# ---------------------------------------------------------------------------
-# CASCADE DELETE tests
-# ---------------------------------------------------------------------------
-
-
-class TestCascadeDeleteOnBankDeletion:
-    @pytest.mark.asyncio
-    async def test_bank_deletion_cascades_to_async_operations(self, pool):
-        bank_id = f"{_BANK_PREFIX}-{uuid.uuid4().hex[:8]}"
-        await _insert_bank(pool, bank_id)
-        op_id = await _insert_op(pool, bank_id)
-
-        # Verify op exists
-        row = await pool.fetchrow("SELECT operation_id FROM async_operations WHERE operation_id = $1", op_id)
-        assert row is not None
-
-        # Delete the bank — should cascade to async_operations
-        await pool.execute("DELETE FROM banks WHERE bank_id = $1", bank_id)
-
-        row = await pool.fetchrow("SELECT operation_id FROM async_operations WHERE operation_id = $1", op_id)
-        assert row is None, "async_operations row should be deleted by CASCADE"
-
-    @pytest.mark.asyncio
-    async def test_bank_deletion_cascades_to_webhooks(self, pool):
-        bank_id = f"{_BANK_PREFIX}-{uuid.uuid4().hex[:8]}"
-        await _insert_bank(pool, bank_id)
-        webhook_id = uuid.uuid4()
-        await pool.execute(
-            """
-            INSERT INTO webhooks (id, bank_id, url, event_types)
-            VALUES ($1, $2, 'https://example.com/hook', '{}')
-            """,
-            webhook_id,
-            bank_id,
-        )
-
-        row = await pool.fetchrow("SELECT id FROM webhooks WHERE id = $1", webhook_id)
-        assert row is not None
-
-        await pool.execute("DELETE FROM banks WHERE bank_id = $1", bank_id)
-
-        row = await pool.fetchrow("SELECT id FROM webhooks WHERE id = $1", webhook_id)
-        assert row is None, "webhooks row should be deleted by CASCADE"
-
-
-# ---------------------------------------------------------------------------
-# _check_op_alive tests
-# ---------------------------------------------------------------------------
-
-
-class TestCheckOpAlive:
-    @pytest.mark.asyncio
-    async def test_returns_true_when_op_exists(self, memory: MemoryEngine, request_context):
-        bank_id = f"{_BANK_PREFIX}-{uuid.uuid4().hex[:8]}"
-        await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-        op_id = uuid.uuid4()
-        async with memory._pool.acquire() as conn:
-            await conn.execute(
-                """
-                INSERT INTO async_operations (operation_id, bank_id, operation_type, status)
-                VALUES ($1, $2, 'consolidation', 'processing')
-                """,
-                op_id,
-                bank_id,
-            )
-
-        assert await memory._check_op_alive(str(op_id)) is True
-
-    @pytest.mark.asyncio
-    async def test_returns_false_when_op_deleted(self, memory: MemoryEngine, request_context):
-        bank_id = f"{_BANK_PREFIX}-{uuid.uuid4().hex[:8]}"
-        await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-        op_id = uuid.uuid4()
-        async with memory._pool.acquire() as conn:
-            await conn.execute(
-                """
-                INSERT INTO async_operations (operation_id, bank_id, operation_type, status)
-                VALUES ($1, $2, 'consolidation', 'processing')
-                """,
-                op_id,
-                bank_id,
-            )
-            await conn.execute("DELETE FROM async_operations WHERE operation_id = $1", op_id)
-
-        assert await memory._check_op_alive(str(op_id)) is False
-
-    @pytest.mark.asyncio
-    async def test_returns_false_after_bank_cascade_delete(self, memory: MemoryEngine, request_context):
-        bank_id = f"{_BANK_PREFIX}-{uuid.uuid4().hex[:8]}"
-        await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-        op_id = uuid.uuid4()
-        async with memory._pool.acquire() as conn:
-            await conn.execute(
-                """
-                INSERT INTO async_operations (operation_id, bank_id, operation_type, status)
-                VALUES ($1, $2, 'consolidation', 'processing')
-                """,
-                op_id,
-                bank_id,
-            )
-
-        # Delete the bank — cascades to the op row
-        await memory.delete_bank(bank_id=bank_id, request_context=request_context)
-
-        assert await memory._check_op_alive(str(op_id)) is False
-
-
-# ---------------------------------------------------------------------------
-# _mark_operation_completed / _mark_operation_failed graceful no-op
-# ---------------------------------------------------------------------------
-
-
-class TestMarkOperationGracefulOnMissingRow:
-    @pytest.mark.asyncio
-    async def test_mark_completed_does_not_raise_when_row_missing(self, memory: MemoryEngine):
-        # Row never existed — should log and return cleanly
-        missing_id = str(uuid.uuid4())
-        await memory._mark_operation_completed(missing_id)  # no exception
-
-    @pytest.mark.asyncio
-    async def test_mark_failed_does_not_raise_when_row_missing(self, memory: MemoryEngine):
-        missing_id = str(uuid.uuid4())
-        await memory._mark_operation_failed(missing_id, "some error", "traceback here")  # no exception
-
-    @pytest.mark.asyncio
-    async def test_mark_completed_and_fire_webhook_does_not_raise_when_row_missing(
-        self, memory: MemoryEngine
-    ):
-        missing_id = str(uuid.uuid4())
-        await memory._mark_operation_completed_and_fire_webhook(
-            operation_id=missing_id,
-            bank_id="nonexistent-bank",
-            status="completed",
-            result=None,
-        )  # no exception
-
-
-# ---------------------------------------------------------------------------
-# Consolidation checkpoint
-# ---------------------------------------------------------------------------
-
-
-class TestConsolidationCheckpoint:
-    @pytest.mark.asyncio
-    async def test_consolidation_stops_early_when_op_cancelled(self, memory: MemoryEngine, request_context):
-        """Consolidation returns 'cancelled' status after the first batch if _check_op_alive is False."""
-        from hindsight_api.config import _get_raw_config
-        from hindsight_api.engine.consolidation.consolidator import run_consolidation_job
-
-        config = _get_raw_config()
-        original = config.enable_observations
-        config.enable_observations = True
-
-        try:
-            bank_id = f"{_BANK_PREFIX}-{uuid.uuid4().hex[:8]}"
-            await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-            # Insert a few unconsolidated memories directly so we control the batch without LLM
-            async with memory._pool.acquire() as conn:
-                for i in range(3):
-                    await conn.execute(
-                        """
-                        INSERT INTO memory_units
-                            (id, bank_id, text, fact_type, created_at, updated_at)
-                        VALUES (gen_random_uuid(), $1, $2, 'experience', NOW(), NOW())
-                        """,
-                        bank_id,
-                        f"Test memory {i} for cancellation test",
-                    )
-
-            op_id = str(uuid.uuid4())
-            call_count = 0
-
-            async def _fake_check(operation_id: str) -> bool:
-                nonlocal call_count
-                call_count += 1
-                # Return False on the very first checkpoint call
-                return False
-
-            with patch.object(memory, "_check_op_alive", side_effect=_fake_check):
-                result = await run_consolidation_job(
-                    memory_engine=memory,
-                    bank_id=bank_id,
-                    request_context=request_context,
-                    operation_id=op_id,
-                )
-
-            assert result["status"] == "cancelled"
-            assert call_count >= 1
-        finally:
-            config.enable_observations = original
-
-
-# ---------------------------------------------------------------------------
-# Retain checkpoint
-# ---------------------------------------------------------------------------
-
-
-class TestRetainCheckpoint:
-    @pytest.mark.asyncio
-    async def test_retain_stops_between_sub_batches_when_cancelled(
-        self, memory: MemoryEngine, request_context
-    ):
-        """retain_batch_async returns partial results if _check_op_alive is False between sub-batches."""
-        from hindsight_api.config import _get_raw_config
-
-        bank_id = f"{_BANK_PREFIX}-{uuid.uuid4().hex[:8]}"
-        await memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
-
-        # Force sub-batch splitting by temporarily lowering the token threshold
-        config = _get_raw_config()
-        original_tokens = config.retain_batch_tokens
-        # Set threshold very low so each item becomes its own sub-batch
-        config.retain_batch_tokens = 1
-
-        try:
-            op_id = str(uuid.uuid4())
-            check_calls = 0
-
-            async def _fake_check(operation_id: str) -> bool:
-                nonlocal check_calls
-                check_calls += 1
-                # Cancel after the first sub-batch completes
-                return check_calls <= 1
-
-            contents = [
-                {"content": f"Memory item {i} about something interesting."} for i in range(4)
-            ]
-
-            with patch.object(memory, "_check_op_alive", side_effect=_fake_check):
-                result = await memory.retain_batch_async(
-                    bank_id=bank_id,
-                    contents=contents,
-                    request_context=request_context,
-                    operation_id=op_id,
-                )
-
-            # Should have stopped early: fewer results than total items
-            assert len(result) < len(contents), (
-                f"Expected early stop but got {len(result)}/{len(contents)} results"
-            )
-            assert check_calls >= 1
-        finally:
-            config.retain_batch_tokens = original_tokens

hindsight-api-slim/tests/test_source_facts_tokens.py

@@ -1,170 +0,0 @@
-"""Tests for source_facts token limiting in recall.
-
-Covers:
-- max_source_facts_tokens: total token budget across all source facts
-- max_source_facts_tokens_per_observation: per-observation cap
-
-Both parameters are tested at the recall_async level and verified to produce
-fewer source facts when the budget is tight vs. unlimited.
-"""
-
-import pytest
-
-from hindsight_api.config import _get_raw_config
-from hindsight_api.engine.memory_engine import Budget
-
-
-@pytest.fixture(autouse=True)
-def enable_observations():
-    config = _get_raw_config()
-    original = config.enable_observations
-    config.enable_observations = True
-    yield
-    config.enable_observations = original
-
-
-async def _setup_bank_with_observations(memory, bank_id, request_context):
-    """Retain several memories and trigger consolidation to produce observations with source facts."""
-    contents = [
-        "Alice is a software engineer who loves Python programming.",
-        "Alice has been working at TechCorp for 5 years.",
-        "Alice recently completed a machine learning certification course.",
-        "Alice mentors junior developers on the team.",
-        "Alice prefers functional programming patterns in her code.",
-    ]
-    for content in contents:
-        await memory.retain_async(
-            bank_id=bank_id,
-            content=content,
-            request_context=request_context,
-        )
-    await memory.run_consolidation(bank_id=bank_id, request_context=request_context)
-
-
-class TestRecallSourceFactsPerObservationCap:
-    @pytest.mark.asyncio
-    async def test_per_observation_cap_reduces_source_facts(self, memory, request_context):
-        """A tight per-observation token cap should return fewer source facts than unlimited."""
-        bank_id = "test-sf-per-obs-cap"
-        try:
-            await _setup_bank_with_observations(memory, bank_id, request_context)
-
-            result_limited = await memory.recall_async(
-                bank_id=bank_id,
-                query="Alice engineer",
-                fact_type=["observation"],
-                max_tokens=4096,
-                include_source_facts=True,
-                max_source_facts_tokens_per_observation=1,  # Effectively cuts all source facts
-                budget=Budget.MID,
-                request_context=request_context,
-            )
-
-            result_unlimited = await memory.recall_async(
-                bank_id=bank_id,
-                query="Alice engineer",
-                fact_type=["observation"],
-                max_tokens=4096,
-                include_source_facts=True,
-                max_source_facts_tokens_per_observation=-1,
-                budget=Budget.MID,
-                request_context=request_context,
-            )
-
-            unlimited_count = len(result_unlimited.source_facts) if result_unlimited.source_facts else 0
-            limited_count = len(result_limited.source_facts) if result_limited.source_facts else 0
-
-            if unlimited_count > 0:
-                assert limited_count <= unlimited_count, (
-                    f"Per-observation cap should yield fewer source facts ({limited_count} <= {unlimited_count})"
-                )
-        finally:
-            await memory.delete_bank(bank_id, request_context=request_context)
-
-    @pytest.mark.asyncio
-    async def test_per_observation_cap_does_not_mix_between_observations(self, memory, request_context):
-        """Each observation's source facts are capped independently — not as a shared pool."""
-        bank_id = "test-sf-per-obs-independent"
-        try:
-            await _setup_bank_with_observations(memory, bank_id, request_context)
-
-            # With a generous per-observation limit each observation can have facts;
-            # with a global limit of 1 token the first observation would consume the whole budget.
-            result_per_obs = await memory.recall_async(
-                bank_id=bank_id,
-                query="Alice engineer",
-                fact_type=["observation"],
-                max_tokens=4096,
-                include_source_facts=True,
-                max_source_facts_tokens=4096,   # large global budget
-                max_source_facts_tokens_per_observation=512,  # reasonable per-obs limit
-                budget=Budget.MID,
-                request_context=request_context,
-            )
-
-            # Should not raise; source_facts may be populated for multiple observations
-            assert result_per_obs.source_facts is not None or len(result_per_obs.results) == 0
-        finally:
-            await memory.delete_bank(bank_id, request_context=request_context)
-
-
-class TestRecallSourceFactsTotalBudget:
-    @pytest.mark.asyncio
-    async def test_total_budget_limits_source_facts(self, memory, request_context):
-        """A tight total token budget should return fewer source facts than unlimited."""
-        bank_id = "test-sf-total-budget"
-        try:
-            await _setup_bank_with_observations(memory, bank_id, request_context)
-
-            result_tight = await memory.recall_async(
-                bank_id=bank_id,
-                query="Alice engineer",
-                fact_type=["observation"],
-                max_tokens=4096,
-                include_source_facts=True,
-                max_source_facts_tokens=1,  # Effectively cuts all source facts
-                budget=Budget.MID,
-                request_context=request_context,
-            )
-
-            result_unlimited = await memory.recall_async(
-                bank_id=bank_id,
-                query="Alice engineer",
-                fact_type=["observation"],
-                max_tokens=4096,
-                include_source_facts=True,
-                max_source_facts_tokens=-1,
-                budget=Budget.MID,
-                request_context=request_context,
-            )
-
-            unlimited_count = len(result_unlimited.source_facts) if result_unlimited.source_facts else 0
-            tight_count = len(result_tight.source_facts) if result_tight.source_facts else 0
-
-            if unlimited_count > 0:
-                assert tight_count <= unlimited_count, (
-                    f"Total budget should yield fewer source facts ({tight_count} <= {unlimited_count})"
-                )
-        finally:
-            await memory.delete_bank(bank_id, request_context=request_context)
-
-    @pytest.mark.asyncio
-    async def test_no_source_facts_without_flag(self, memory, request_context):
-        """source_facts should be None when include_source_facts is not set."""
-        bank_id = "test-sf-no-flag"
-        try:
-            await _setup_bank_with_observations(memory, bank_id, request_context)
-
-            result = await memory.recall_async(
-                bank_id=bank_id,
-                query="Alice engineer",
-                fact_type=["observation"],
-                max_tokens=4096,
-                include_source_facts=False,  # default
-                budget=Budget.MID,
-                request_context=request_context,
-            )
-
-            assert result.source_facts is None or len(result.source_facts) == 0
-        finally:
-            await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_webhooks.py

@@ -1,794 +0,0 @@
-"""Tests for the webhook system.
-
-Covers:
-- Unit tests for HMAC signing and retry constants (no DB required)
-- Integration tests for fire_event() using a real DB (inserts into async_operations)
-- Integration tests for _handle_webhook_delivery() on the memory engine
-- HTTP API integration tests for CRUD and delivery listing endpoints
-"""
-
-import json
-import uuid
-from datetime import datetime, timezone
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import httpx
-import pytest
-import pytest_asyncio
-
-from hindsight_api.api import create_app
-from hindsight_api.engine.memory_engine import MemoryEngine
-from hindsight_api.webhooks.manager import MAX_ATTEMPTS, RETRY_DELAYS, WebhookManager
-from hindsight_api.webhooks.models import (
-    ConsolidationEventData,
-    RetainEventData,
-    WebhookConfig,
-    WebhookEvent,
-    WebhookEventType,
-)
-from hindsight_api.worker.exceptions import RetryTaskAt
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def _make_event(bank_id: str = "bank-1") -> WebhookEvent:
-    return WebhookEvent(
-        event=WebhookEventType.CONSOLIDATION_COMPLETED,
-        bank_id=bank_id,
-        operation_id=uuid.uuid4().hex,
-        status="completed",
-        timestamp=datetime.now(timezone.utc),
-        data=ConsolidationEventData(observations_created=1),
-    )
-
-
-def _make_delivery_task(
-    bank_id: str = "bank-1",
-    url: str = "https://example.com/hook",
-    retry_count: int = 0,
-    webhook_id: str | None = None,
-) -> dict:
-    return {
-        "type": "webhook_delivery",
-        "bank_id": bank_id,
-        "url": url,
-        "secret": None,
-        "event_type": "consolidation.completed",
-        "payload": '{"event":"consolidation.completed"}',
-        "webhook_id": webhook_id,
-        "_retry_count": retry_count,
-    }
-
-
-# ---------------------------------------------------------------------------
-# Unit tests (no DB)
-# ---------------------------------------------------------------------------
-
-
-class TestHmacSigning:
-    """Unit tests for WebhookManager._sign_payload()."""
-
-    def _make_manager(self) -> WebhookManager:
-        """Create a WebhookManager with a dummy pool (not used for signing)."""
-        pool = MagicMock()
-        return WebhookManager(pool=pool, global_webhooks=[])
-
-    def test_hmac_signing_format(self):
-        """_sign_payload should return a string starting with 'sha256='."""
-        manager = self._make_manager()
-        sig = manager._sign_payload("my-secret", b"hello world")
-        assert sig.startswith("sha256="), f"Expected 'sha256=' prefix, got: {sig!r}"
-        hex_part = sig[len("sha256="):]
-        # SHA-256 hex digest is always 64 characters
-        assert len(hex_part) == 64
-        # Hex characters only
-        assert all(c in "0123456789abcdef" for c in hex_part)
-
-    def test_hmac_signing_is_deterministic(self):
-        """Same secret + payload always produces the same signature."""
-        manager = self._make_manager()
-        payload = b'{"event":"consolidation.completed"}'
-        sig1 = manager._sign_payload("secret-key", payload)
-        sig2 = manager._sign_payload("secret-key", payload)
-        assert sig1 == sig2
-
-    def test_hmac_signing_differs_with_different_secret(self):
-        """Different secrets must produce different signatures."""
-        manager = self._make_manager()
-        payload = b"payload"
-        sig1 = manager._sign_payload("secret-a", payload)
-        sig2 = manager._sign_payload("secret-b", payload)
-        assert sig1 != sig2
-
-    def test_hmac_signing_differs_with_different_payload(self):
-        """Different payloads must produce different signatures."""
-        manager = self._make_manager()
-        sig1 = manager._sign_payload("secret", b"payload-one")
-        sig2 = manager._sign_payload("secret", b"payload-two")
-        assert sig1 != sig2
-
-
-class TestRetryConstants:
-    """Unit tests to verify retry schedule constants."""
-
-    def test_retry_delays_values(self):
-        """RETRY_DELAYS must match the documented schedule."""
-        assert RETRY_DELAYS == [5, 300, 1800, 7200, 18000]
-
-    def test_max_attempts(self):
-        """MAX_ATTEMPTS should be len(RETRY_DELAYS) + 1."""
-        assert MAX_ATTEMPTS == 6
-        assert MAX_ATTEMPTS == len(RETRY_DELAYS) + 1
-
-
-# ---------------------------------------------------------------------------
-# DB integration tests
-# ---------------------------------------------------------------------------
-
-
-@pytest_asyncio.fixture
-async def webhook_manager(memory: MemoryEngine) -> WebhookManager:
-    """Return a WebhookManager backed by the test pool with no global webhooks."""
-    return WebhookManager(pool=memory._pool, global_webhooks=[])
-
-
-async def _ensure_bank(pool, bank_id: str) -> None:
-    """Upsert a minimal bank row so FK constraints on async_operations/webhooks pass."""
-    await pool.execute(
-        "INSERT INTO banks (bank_id, name) VALUES ($1, $2) ON CONFLICT DO NOTHING",
-        bank_id,
-        bank_id,
-    )
-
-
-class TestFireEvent:
-    """Integration tests for WebhookManager.fire_event()."""
-
-    @pytest.mark.asyncio
-    async def test_fire_event_creates_delivery(
-        self, memory: MemoryEngine, webhook_manager: WebhookManager
-    ):
-        """fire_event() inserts a pending webhook_delivery task in async_operations."""
-        bank_id = f"wh-test-{uuid.uuid4().hex[:8]}"
-        webhook_id = uuid.uuid4()
-
-        async with memory._pool.acquire() as conn:
-            await _ensure_bank(memory._pool, bank_id)
-            await conn.execute(
-                """
-                INSERT INTO webhooks (id, bank_id, url, secret, event_types, enabled, created_at, updated_at)
-                VALUES ($1, $2, $3, NULL, $4, true, NOW(), NOW())
-                """,
-                webhook_id,
-                bank_id,
-                "https://example.com/hook",
-                ["consolidation.completed"],
-            )
-
-        try:
-            event = _make_event(bank_id)
-            await webhook_manager.fire_event(event)
-
-            async with memory._pool.acquire() as conn:
-                rows = await conn.fetch(
-                    """
-                    SELECT status, task_payload
-                    FROM async_operations
-                    WHERE operation_type = 'webhook_delivery'
-                      AND bank_id = $1
-                      AND task_payload->>'webhook_id' = $2
-                    """,
-                    bank_id,
-                    str(webhook_id),
-                )
-
-            assert len(rows) == 1
-            assert rows[0]["status"] == "pending"
-            payload = rows[0]["task_payload"]
-            if isinstance(payload, str):
-                payload = json.loads(payload)
-            assert payload["event_type"] == "consolidation.completed"
-        finally:
-            async with memory._pool.acquire() as conn:
-                await conn.execute(
-                    "DELETE FROM async_operations WHERE operation_type = 'webhook_delivery' AND bank_id = $1",
-                    bank_id,
-                )
-                await conn.execute("DELETE FROM webhooks WHERE id = $1", webhook_id)
-
-    @pytest.mark.asyncio
-    async def test_fire_event_global_webhook(
-        self, memory: MemoryEngine
-    ):
-        """fire_event() also queues delivery tasks for global webhooks (not stored in DB)."""
-        bank_id = f"wh-global-{uuid.uuid4().hex[:8]}"
-        await _ensure_bank(memory._pool, bank_id)
-        global_webhook = WebhookConfig(
-            id="",  # No DB row
-            bank_id=None,
-            url="https://global.example.com/hook",
-            secret=None,
-            event_types=["consolidation.completed"],
-            enabled=True,
-        )
-        manager = WebhookManager(pool=memory._pool, global_webhooks=[global_webhook])
-
-        event = _make_event(bank_id)
-        await manager.fire_event(event)
-
-        async with memory._pool.acquire() as conn:
-            rows = await conn.fetch(
-                """
-                SELECT status, task_payload
-                FROM async_operations
-                WHERE operation_type = 'webhook_delivery'
-                  AND bank_id = $1
-                  AND task_payload->>'url' = 'https://global.example.com/hook'
-                ORDER BY created_at DESC
-                LIMIT 1
-                """
-                ,
-                bank_id,
-            )
-
-        assert len(rows) == 1
-        assert rows[0]["status"] == "pending"
-        payload = rows[0]["task_payload"]
-        if isinstance(payload, str):
-            payload = json.loads(payload)
-        assert payload["webhook_id"] is None  # global webhook has no DB row
-
-        # Cleanup
-        async with memory._pool.acquire() as conn:
-            await conn.execute(
-                "DELETE FROM async_operations WHERE operation_type = 'webhook_delivery' AND bank_id = $1",
-                bank_id,
-            )
-
-    @pytest.mark.asyncio
-    async def test_fire_event_no_match_if_event_type_mismatch(
-        self, memory: MemoryEngine, webhook_manager: WebhookManager
-    ):
-        """Webhooks registered for a different event type receive no delivery task."""
-        bank_id = f"wh-mismatch-{uuid.uuid4().hex[:8]}"
-        webhook_id = uuid.uuid4()
-
-        await _ensure_bank(memory._pool, bank_id)
-        async with memory._pool.acquire() as conn:
-            await conn.execute(
-                """
-                INSERT INTO webhooks (id, bank_id, url, secret, event_types, enabled, created_at, updated_at)
-                VALUES ($1, $2, $3, NULL, $4, true, NOW(), NOW())
-                """,
-                webhook_id,
-                bank_id,
-                "https://example.com/other-hook",
-                ["other.event"],
-            )
-
-        try:
-            event = _make_event(bank_id)
-            await webhook_manager.fire_event(event)
-
-            async with memory._pool.acquire() as conn:
-                count = await conn.fetchval(
-                    """
-                    SELECT COUNT(*) FROM async_operations
-                    WHERE operation_type = 'webhook_delivery' AND bank_id = $1
-                    """,
-                    bank_id,
-                )
-
-            assert count == 0
-        finally:
-            async with memory._pool.acquire() as conn:
-                await conn.execute("DELETE FROM webhooks WHERE id = $1", webhook_id)
-
-
-class TestHandleWebhookDelivery:
-    """Integration tests for MemoryEngine._handle_webhook_delivery()."""
-
-    @pytest.mark.asyncio
-    async def test_deliver_success(self, memory: MemoryEngine):
-        """A successful HTTP POST completes without raising."""
-        task_dict = _make_delivery_task(retry_count=0)
-
-        mock_response = MagicMock()
-        mock_response.raise_for_status = MagicMock()
-
-        with patch.object(memory._http_client, "post", new=AsyncMock(return_value=mock_response)):
-            # Should not raise
-            await memory._handle_webhook_delivery(task_dict)
-
-    @pytest.mark.asyncio
-    async def test_deliver_failure_raises_retry_task_at(self, memory: MemoryEngine):
-        """A failed HTTP POST raises RetryTaskAt when retries remain."""
-        task_dict = _make_delivery_task(retry_count=0)
-
-        with patch.object(
-            memory._http_client, "post", new=AsyncMock(side_effect=Exception("connection refused"))
-        ):
-            with pytest.raises(RetryTaskAt):
-                await memory._handle_webhook_delivery(task_dict)
-
-    @pytest.mark.asyncio
-    async def test_deliver_exhausted_retries_raises(self, memory: MemoryEngine):
-        """When retry_count reaches MAX_ATTEMPTS-1, a failure raises the original exception."""
-        task_dict = _make_delivery_task(retry_count=MAX_ATTEMPTS - 1)
-
-        with patch.object(
-            memory._http_client, "post", new=AsyncMock(side_effect=Exception("server error"))
-        ):
-            with pytest.raises(Exception, match="server error"):
-                await memory._handle_webhook_delivery(task_dict)
-
-    @pytest.mark.asyncio
-    async def test_deliver_retry_at_uses_delay_schedule(self, memory: MemoryEngine):
-        """RetryTaskAt.retry_at is approximately now + RETRY_DELAYS[retry_count]."""
-        from datetime import timedelta
-
-        task_dict = _make_delivery_task(retry_count=1)
-
-        with patch.object(
-            memory._http_client, "post", new=AsyncMock(side_effect=Exception("fail"))
-        ):
-            before = datetime.now(timezone.utc)
-            with pytest.raises(RetryTaskAt) as exc_info:
-                await memory._handle_webhook_delivery(task_dict)
-            after = datetime.now(timezone.utc)
-
-        retry_at = exc_info.value.retry_at
-        expected_delay = RETRY_DELAYS[1]  # retry_count=1
-        assert retry_at >= before + timedelta(seconds=expected_delay - 2)
-        assert retry_at <= after + timedelta(seconds=expected_delay + 2)
-
-    @pytest.mark.asyncio
-    async def test_execute_task_marks_operation_completed(self, memory: MemoryEngine):
-        """After a successful delivery, execute_task marks the async_operations row as completed."""
-        operation_id = str(uuid.uuid4())
-        bank_id = f"wh-exec-{uuid.uuid4().hex[:8]}"
-
-        await _ensure_bank(memory._pool, bank_id)
-        # Insert a real async_operations row so _mark_operation_completed has something to update
-        async with memory._pool.acquire() as conn:
-            await conn.execute(
-                """
-                INSERT INTO async_operations
-                  (operation_id, bank_id, operation_type, status, task_payload, result_metadata, created_at, updated_at)
-                VALUES ($1, $2, 'webhook_delivery', 'processing', '{}'::jsonb, '{}'::jsonb, NOW(), NOW())
-                """,
-                uuid.UUID(operation_id),
-                bank_id,
-            )
-
-        task_dict = {
-            **_make_delivery_task(bank_id=bank_id, retry_count=0),
-            "operation_id": operation_id,
-        }
-
-        mock_response = MagicMock()
-        mock_response.raise_for_status = MagicMock()
-
-        with patch.object(memory._http_client, "post", new=AsyncMock(return_value=mock_response)):
-            await memory.execute_task(task_dict)
-
-        async with memory._pool.acquire() as conn:
-            row = await conn.fetchrow(
-                "SELECT status FROM async_operations WHERE operation_id = $1",
-                uuid.UUID(operation_id),
-            )
-
-        assert row is not None
-        assert row["status"] == "completed", f"Expected 'completed', got '{row['status']}'"
-
-        # Cleanup
-        async with memory._pool.acquire() as conn:
-            await conn.execute(
-                "DELETE FROM async_operations WHERE operation_id = $1",
-                uuid.UUID(operation_id),
-            )
-
-
-# ---------------------------------------------------------------------------
-# HTTP API integration tests
-# ---------------------------------------------------------------------------
-
-
-@pytest_asyncio.fixture
-async def api_client(memory: MemoryEngine):
-    """Async HTTP test client wired to the FastAPI app."""
-    app = create_app(memory, initialize_memory=False)
-    transport = httpx.ASGITransport(app=app)
-    async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
-        yield client
-
-
-class TestWebhookHttpApi:
-    """HTTP API integration tests for webhook CRUD endpoints."""
-
-    @pytest.mark.asyncio
-    async def test_http_create_webhook(self, api_client: httpx.AsyncClient):
-        """POST /webhooks returns 201 and an id."""
-        bank_id = f"http-wh-{uuid.uuid4().hex[:8]}"
-        response = await api_client.post(
-            f"/v1/default/banks/{bank_id}/webhooks",
-            json={
-                "url": "https://example.com/create",
-                "event_types": ["consolidation.completed"],
-            },
-        )
-        assert response.status_code == 201, response.text
-        data = response.json()
-        assert "id" in data
-        assert data["url"] == "https://example.com/create"
-        assert data["bank_id"] == bank_id
-        assert data["secret"] is None  # secrets are never echoed back
-
-        # Cleanup
-        await api_client.delete(
-            f"/v1/default/banks/{bank_id}/webhooks/{data['id']}"
-        )
-
-    @pytest.mark.asyncio
-    async def test_http_list_webhooks(self, api_client: httpx.AsyncClient):
-        """GET /webhooks returns the webhooks registered for a bank."""
-        bank_id = f"http-wh-{uuid.uuid4().hex[:8]}"
-
-        create_resp = await api_client.post(
-            f"/v1/default/banks/{bank_id}/webhooks",
-            json={"url": "https://example.com/list", "event_types": ["consolidation.completed"]},
-        )
-        assert create_resp.status_code == 201
-        webhook_id = create_resp.json()["id"]
-
-        list_resp = await api_client.get(f"/v1/default/banks/{bank_id}/webhooks")
-        assert list_resp.status_code == 200
-        items = list_resp.json()["items"]
-        assert any(item["id"] == webhook_id for item in items)
-
-        # Cleanup
-        await api_client.delete(f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}")
-
-    @pytest.mark.asyncio
-    async def test_http_delete_webhook(self, api_client: httpx.AsyncClient):
-        """DELETE /webhooks/{id} removes the webhook; subsequent list returns empty for that bank."""
-        bank_id = f"http-wh-{uuid.uuid4().hex[:8]}"
-
-        create_resp = await api_client.post(
-            f"/v1/default/banks/{bank_id}/webhooks",
-            json={"url": "https://example.com/delete", "event_types": ["consolidation.completed"]},
-        )
-        assert create_resp.status_code == 201
-        webhook_id = create_resp.json()["id"]
-
-        delete_resp = await api_client.delete(
-            f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}"
-        )
-        assert delete_resp.status_code == 200
-        assert delete_resp.json()["success"] is True
-
-        list_resp = await api_client.get(f"/v1/default/banks/{bank_id}/webhooks")
-        assert list_resp.status_code == 200
-        ids = [item["id"] for item in list_resp.json()["items"]]
-        assert webhook_id not in ids
-
-    @pytest.mark.asyncio
-    async def test_http_delete_webhook_not_found(self, api_client: httpx.AsyncClient):
-        """DELETE with a non-existent webhook id returns 404."""
-        bank_id = f"http-wh-{uuid.uuid4().hex[:8]}"
-        missing_id = str(uuid.uuid4())
-        response = await api_client.delete(
-            f"/v1/default/banks/{bank_id}/webhooks/{missing_id}"
-        )
-        assert response.status_code == 404
-
-    @pytest.mark.asyncio
-    async def test_http_list_deliveries(
-        self, memory: MemoryEngine, api_client: httpx.AsyncClient
-    ):
-        """GET /webhooks/{id}/deliveries returns delivery records for a webhook."""
-        bank_id = f"http-wh-{uuid.uuid4().hex[:8]}"
-
-        # Create webhook via HTTP API
-        create_resp = await api_client.post(
-            f"/v1/default/banks/{bank_id}/webhooks",
-            json={
-                "url": "https://example.com/deliveries",
-                "event_types": ["consolidation.completed"],
-            },
-        )
-        assert create_resp.status_code == 201
-        webhook_id = create_resp.json()["id"]
-
-        # Insert a delivery row directly into async_operations
-        delivery_id = uuid.uuid4()
-        now = datetime.now(timezone.utc)
-        task_payload = json.dumps(
-            {
-                "type": "webhook_delivery",
-                "bank_id": bank_id,
-                "url": "https://example.com/deliveries",
-                "secret": None,
-                "event_type": "consolidation.completed",
-                "payload": '{"event":"consolidation.completed"}',
-                "webhook_id": webhook_id,
-            }
-        )
-        async with memory._pool.acquire() as conn:
-            await conn.execute(
-                """
-                INSERT INTO async_operations
-                  (operation_id, bank_id, operation_type, status, retry_count, task_payload, result_metadata, created_at, updated_at)
-                VALUES ($1, $2, 'webhook_delivery', 'completed', 0, $3::jsonb, '{}'::jsonb, $4, $4)
-                """,
-                delivery_id,
-                bank_id,
-                task_payload,
-                now,
-            )
-
-        try:
-            deliveries_resp = await api_client.get(
-                f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}/deliveries"
-            )
-            assert deliveries_resp.status_code == 200
-            items = deliveries_resp.json()["items"]
-            ids = [item["id"] for item in items]
-            assert str(delivery_id) in ids
-
-            # Verify shape of a delivery item
-            delivery = next(item for item in items if item["id"] == str(delivery_id))
-            assert delivery["status"] == "completed"
-            assert delivery["event_type"] == "consolidation.completed"
-            assert delivery["attempts"] == 1
-        finally:
-            async with memory._pool.acquire() as conn:
-                await conn.execute(
-                    "DELETE FROM async_operations WHERE operation_id = $1", delivery_id
-                )
-            await api_client.delete(
-                f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}"
-            )
-
-    @pytest.mark.asyncio
-    async def test_http_list_deliveries_webhook_not_found(self, api_client: httpx.AsyncClient):
-        """GET /webhooks/{id}/deliveries for a non-existent webhook returns 404."""
-        bank_id = f"http-wh-{uuid.uuid4().hex[:8]}"
-        missing_id = str(uuid.uuid4())
-        response = await api_client.get(
-            f"/v1/default/banks/{bank_id}/webhooks/{missing_id}/deliveries"
-        )
-        assert response.status_code == 404
-
-    @pytest.mark.asyncio
-    async def test_http_update_webhook_url(self, api_client: httpx.AsyncClient):
-        """PATCH /webhooks/{id} updates only the provided fields."""
-        bank_id = f"http-wh-{uuid.uuid4().hex[:8]}"
-
-        create_resp = await api_client.post(
-            f"/v1/default/banks/{bank_id}/webhooks",
-            json={"url": "https://example.com/original", "event_types": ["consolidation.completed"]},
-        )
-        assert create_resp.status_code == 201
-        webhook_id = create_resp.json()["id"]
-
-        patch_resp = await api_client.patch(
-            f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}",
-            json={"url": "https://example.com/updated"},
-        )
-        assert patch_resp.status_code == 200
-        data = patch_resp.json()
-        assert data["url"] == "https://example.com/updated"
-        # event_types should be unchanged
-        assert "consolidation.completed" in data["event_types"]
-
-        # Cleanup
-        await api_client.delete(f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}")
-
-    @pytest.mark.asyncio
-    async def test_http_update_webhook_event_types(self, api_client: httpx.AsyncClient):
-        """PATCH /webhooks/{id} can update event_types."""
-        bank_id = f"http-wh-{uuid.uuid4().hex[:8]}"
-
-        create_resp = await api_client.post(
-            f"/v1/default/banks/{bank_id}/webhooks",
-            json={"url": "https://example.com/hook", "event_types": ["consolidation.completed"]},
-        )
-        assert create_resp.status_code == 201
-        webhook_id = create_resp.json()["id"]
-
-        patch_resp = await api_client.patch(
-            f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}",
-            json={"event_types": ["retain.completed"]},
-        )
-        assert patch_resp.status_code == 200
-        data = patch_resp.json()
-        assert data["event_types"] == ["retain.completed"]
-
-        # Cleanup
-        await api_client.delete(f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}")
-
-    @pytest.mark.asyncio
-    async def test_http_update_webhook_enabled(self, api_client: httpx.AsyncClient):
-        """PATCH /webhooks/{id} can toggle enabled."""
-        bank_id = f"http-wh-{uuid.uuid4().hex[:8]}"
-
-        create_resp = await api_client.post(
-            f"/v1/default/banks/{bank_id}/webhooks",
-            json={"url": "https://example.com/hook", "event_types": ["consolidation.completed"]},
-        )
-        assert create_resp.status_code == 201
-        webhook_id = create_resp.json()["id"]
-        assert create_resp.json()["enabled"] is True
-
-        patch_resp = await api_client.patch(
-            f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}",
-            json={"enabled": False},
-        )
-        assert patch_resp.status_code == 200
-        assert patch_resp.json()["enabled"] is False
-
-        # Cleanup
-        await api_client.delete(f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}")
-
-    @pytest.mark.asyncio
-    async def test_http_update_webhook_http_config(self, api_client: httpx.AsyncClient):
-        """PATCH /webhooks/{id} can update http_config."""
-        bank_id = f"http-wh-{uuid.uuid4().hex[:8]}"
-
-        create_resp = await api_client.post(
-            f"/v1/default/banks/{bank_id}/webhooks",
-            json={"url": "https://example.com/hook", "event_types": ["consolidation.completed"]},
-        )
-        assert create_resp.status_code == 201
-        webhook_id = create_resp.json()["id"]
-
-        patch_resp = await api_client.patch(
-            f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}",
-            json={
-                "http_config": {
-                    "method": "POST",
-                    "timeout_seconds": 10,
-                    "headers": {"X-Custom": "value"},
-                    "params": {},
-                }
-            },
-        )
-        assert patch_resp.status_code == 200
-        data = patch_resp.json()
-        assert data["http_config"]["timeout_seconds"] == 10
-        assert data["http_config"]["headers"] == {"X-Custom": "value"}
-
-        # Cleanup
-        await api_client.delete(f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}")
-
-    @pytest.mark.asyncio
-    async def test_http_update_webhook_not_found(self, api_client: httpx.AsyncClient):
-        """PATCH /webhooks/{id} returns 404 for a non-existent webhook."""
-        bank_id = f"http-wh-{uuid.uuid4().hex[:8]}"
-        missing_id = str(uuid.uuid4())
-        response = await api_client.patch(
-            f"/v1/default/banks/{bank_id}/webhooks/{missing_id}",
-            json={"url": "https://example.com/new"},
-        )
-        assert response.status_code == 404
-
-    @pytest.mark.asyncio
-    async def test_http_update_webhook_no_fields(self, api_client: httpx.AsyncClient):
-        """PATCH /webhooks/{id} with empty body returns 422."""
-        bank_id = f"http-wh-{uuid.uuid4().hex[:8]}"
-
-        create_resp = await api_client.post(
-            f"/v1/default/banks/{bank_id}/webhooks",
-            json={"url": "https://example.com/hook", "event_types": ["consolidation.completed"]},
-        )
-        assert create_resp.status_code == 201
-        webhook_id = create_resp.json()["id"]
-
-        patch_resp = await api_client.patch(
-            f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}",
-            json={},
-        )
-        assert patch_resp.status_code == 422
-
-        # Cleanup
-        await api_client.delete(f"/v1/default/banks/{bank_id}/webhooks/{webhook_id}")
-
-
-# ---------------------------------------------------------------------------
-# retain.completed webhook tests
-# ---------------------------------------------------------------------------
-
-
-class TestRetainCompletedWebhook:
-    """Tests for the retain.completed webhook event."""
-
-    def test_retain_event_data_model(self):
-        """RetainEventData can be constructed with optional fields."""
-        data = RetainEventData(document_id="doc-123", tags=["tag1", "tag2"])
-        assert data.document_id == "doc-123"
-        assert data.tags == ["tag1", "tag2"]
-
-        empty = RetainEventData()
-        assert empty.document_id is None
-        assert empty.tags is None
-
-    def test_retain_event_type_value(self):
-        """WebhookEventType.RETAIN_COMPLETED has the correct string value."""
-        assert WebhookEventType.RETAIN_COMPLETED == "retain.completed"
-
-    @pytest.mark.asyncio
-    async def test_fire_retain_webhook_queues_per_document(
-        self, memory: MemoryEngine, webhook_manager: WebhookManager
-    ):
-        """_fire_retain_webhook queues one delivery task per content item."""
-        bank_id = f"wh-retain-{uuid.uuid4().hex[:8]}"
-        webhook_id = uuid.uuid4()
-
-        await _ensure_bank(memory._pool, bank_id)
-        async with memory._pool.acquire() as conn:
-            await conn.execute(
-                """
-                INSERT INTO webhooks (id, bank_id, url, secret, event_types, enabled, created_at, updated_at)
-                VALUES ($1, $2, $3, NULL, $4, true, NOW(), NOW())
-                """,
-                webhook_id,
-                bank_id,
-                "https://example.com/retain-hook",
-                ["retain.completed"],
-            )
-
-        try:
-            contents = [
-                {"content": "Alice works at Google", "document_id": "doc-1"},
-                {"content": "Bob loves Python", "document_id": "doc-2"},
-            ]
-            # Temporarily replace webhook manager on memory engine
-            original_manager = memory._webhook_manager
-            memory._webhook_manager = webhook_manager
-            try:
-                callback = memory._build_retain_outbox_callback(
-                    bank_id=bank_id,
-                    contents=contents,
-                    operation_id="test-op-123",
-                )
-                assert callback is not None
-                async with memory._pool.acquire() as conn:
-                    await callback(conn)
-            finally:
-                memory._webhook_manager = original_manager
-
-            async with memory._pool.acquire() as conn:
-                rows = await conn.fetch(
-                    """
-                    SELECT task_payload
-                    FROM async_operations
-                    WHERE operation_type = 'webhook_delivery'
-                      AND bank_id = $1
-                      AND task_payload->>'event_type' = 'retain.completed'
-                    ORDER BY created_at
-                    """,
-                    bank_id,
-                )
-
-            assert len(rows) == 2
-            payloads = []
-            for row in rows:
-                p = row["task_payload"]
-                if isinstance(p, str):
-                    p = json.loads(p)
-                payloads.append(p)
-
-            doc_ids_in_payloads = [json.loads(p["payload"]).get("data", {}).get("document_id") for p in payloads]
-            assert "doc-1" in doc_ids_in_payloads
-            assert "doc-2" in doc_ids_in_payloads
-        finally:
-            async with memory._pool.acquire() as conn:
-                await conn.execute(
-                    "DELETE FROM async_operations WHERE operation_type = 'webhook_delivery' AND bank_id = $1",
-                    bank_id,
-                )
-                await conn.execute("DELETE FROM webhooks WHERE id = $1", webhook_id)

hindsight-api/hindsight_api/__init__.py

@@ -46,4 +46,4 @@ __all__ = [
     "RemoteTEICrossEncoder",
     "LLMConfig",
 ]
-__version__ = "0.4.19"
+__version__ = "0.4.14"

hindsight-api/hindsight_api/admin/cli.py

@@ -14,8 +14,7 @@ from typing import Any
 import asyncpg
 import typer
 
-from ..config import DEFAULT_DATABASE_SCHEMA, HindsightConfig
-from ..extensions import TenantExtension, load_extension
+from ..config import HindsightConfig
 from ..pg0 import parse_pg0_url, resolve_database_url
 
 
@@ -215,81 +214,20 @@ def restore(
     typer.echo("Restore complete")
 
 
-async def _run_migration(
-    db_url: str,
-    schema: str | None = None,
-    base_schema: str = DEFAULT_DATABASE_SCHEMA,
-    embedding_dimension: int | None = None,
-) -> list[str]:
-    """Resolve database URL and run migrations for one schema or all discovered schemas."""
-    from ..migrations import (
-        ensure_embedding_dimension,
-        ensure_text_search_extension,
-        ensure_vector_extension,
-        run_migrations,
-    )
+async def _run_migration(db_url: str, schema: str = "public") -> None:
+    """Resolve database URL and run migrations."""
+    from ..migrations import run_migrations
 
     is_pg0, instance_name, _ = parse_pg0_url(db_url)
     if is_pg0:
         typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
     resolved_url = await resolve_database_url(db_url)
-
-    config = HindsightConfig.from_env()
-    if schema:
-        schemas = [schema]
-    else:
-        tenant_extension = load_extension("TENANT", TenantExtension)
-
-        schemas = [base_schema or DEFAULT_DATABASE_SCHEMA]
-        if tenant_extension:
-            tenants = await tenant_extension.list_tenants()
-            schemas.extend(tenant.schema for tenant in tenants if tenant.schema)
-
-        # Preserve order while removing duplicates.
-        schemas = list(dict.fromkeys(schemas))
-
-    for schema in schemas:
-        run_migrations(resolved_url, schema=schema)
-
-    if embedding_dimension is not None:
-        for schema in schemas:
-            ensure_embedding_dimension(
-                resolved_url,
-                embedding_dimension,
-                schema=schema,
-                vector_extension=config.vector_extension,
-            )
-
-    for schema in schemas:
-        ensure_vector_extension(
-            resolved_url,
-            vector_extension=config.vector_extension,
-            schema=schema,
-        )
-
-    for schema in schemas:
-        ensure_text_search_extension(
-            resolved_url,
-            text_search_extension=config.text_search_extension,
-            schema=schema,
-        )
-
-    return schemas
+    run_migrations(resolved_url, schema=schema)
 
 
 @app.command(name="run-db-migration")
 def run_db_migration(
-    schema: str | None = typer.Option(
-        None,
-        "--schema",
-        "-s",
-        help="Database schema to run migrations on. If omitted, migrate the base schema and all discovered tenant schemas.",
-    ),
-    embedding_dimension: int | None = typer.Option(
-        None,
-        "--embedding-dimension",
-        help="Expected embedding dimension to enforce after migrations. Omit to skip dimension sync.",
-    ),
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema to run migrations on"),
 ):
     """Run database migrations to the latest version."""
     config = HindsightConfig.from_env()
@@ -299,21 +237,11 @@ def run_db_migration(
         typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
         raise typer.Exit(1)
 
-    if schema:
-        typer.echo(f"Running database migrations for schema: {schema}...")
-    else:
-        typer.echo("Running database migrations for base schema and all discovered tenant schemas...")
+    typer.echo(f"Running database migrations (schema: {schema})...")
 
-    schemas = asyncio.run(
-        _run_migration(
-            config.database_url,
-            schema=schema,
-            base_schema=config.database_schema,
-            embedding_dimension=embedding_dimension,
-        )
-    )
+    asyncio.run(_run_migration(config.database_url, schema))
 
-    typer.echo(f"Database migrations completed successfully for {len(schemas)} schema(s)")
+    typer.echo("Database migrations completed successfully")
 
 
 async def _decommission_worker(db_url: str, worker_id: str, schema: str = "public") -> int:

hindsight-api/hindsight_api/alembic/versions/a1b2c3d4e5f6_add_file_storage_table.py

@@ -34,7 +34,7 @@ def upgrade() -> None:
     # Create file_storage table (minimal: just key + data)
     op.execute(
         f"""
-        CREATE TABLE IF NOT EXISTS {schema}file_storage (
+        CREATE TABLE {schema}file_storage (
             storage_key TEXT PRIMARY KEY,
             data BYTEA NOT NULL
         )

hindsight-api/hindsight_api/alembic/versions/y0t1u2v3w4x5_add_result_metadata_gin_index.py

@@ -35,7 +35,7 @@ def upgrade() -> None:
 
     # Add GIN index for JSONB containment queries (@> operator)
     op.execute(f"""
-        CREATE INDEX IF NOT EXISTS idx_async_operations_result_metadata
+        CREATE INDEX idx_async_operations_result_metadata
         ON {schema}async_operations
         USING gin(result_metadata)
     """)

hindsight-api/hindsight_api/api/mcp.py

@@ -331,7 +331,7 @@ class MCPMiddleware:
                 auth_tenant_id = auth_context.tenant_id
                 auth_api_key_id = auth_context.api_key_id
             except AuthenticationError as e:
-                await self._send_error(send, 401, str(e), extra_headers=e.headers)
+                await self._send_error(send, 401, str(e))
                 return
 
         # Set schema from tenant context so downstream DB queries use the correct schema
@@ -381,15 +381,6 @@ class MCPMiddleware:
             # Clear root_path since we're passing directly to the app
             new_scope["root_path"] = ""
 
-            # Ensure Accept header includes required MIME types for MCP SDK.
-            # Some clients (e.g., Claude Code) don't send Accept, causing
-            # the SDK to reject with 406 Not Acceptable.
-            accept_header = self._get_header(new_scope, "accept")
-            if not accept_header or "text/event-stream" not in accept_header:
-                headers = [(k, v) for k, v in new_scope.get("headers", []) if k.lower() != b"accept"]
-                headers.append((b"accept", b"application/json, text/event-stream"))
-                new_scope["headers"] = headers
-
             # Wrap send to rewrite the SSE endpoint URL to include bank_id if using path-based routing.
             # Only rewrite SSE (text/event-stream) responses to avoid corrupting tool results
             # that might contain the literal string "data: /messages".
@@ -422,17 +413,14 @@ class MCPMiddleware:
             if schema_token is not None:
                 _current_schema.reset(schema_token)
 
-    async def _send_error(self, send, status: int, message: str, extra_headers: dict[str, str] | None = None):
+    async def _send_error(self, send, status: int, message: str):
         """Send an error response."""
         body = json.dumps({"error": message}).encode()
-        headers = [(b"content-type", b"application/json")]
-        for key, value in (extra_headers or {}).items():
-            headers.append((key.encode(), value.encode()))
         await send(
             {
                 "type": "http.response.start",
                 "status": status,
-                "headers": headers,
+                "headers": [(b"content-type", b"application/json")],
             }
         )
         await send(

hindsight-api/hindsight_api/config.py

@@ -193,7 +193,6 @@ ENV_EMBEDDINGS_LITELLM_MODEL = "HINDSIGHT_API_EMBEDDINGS_LITELLM_MODEL"
 ENV_RERANKER_LITELLM_API_BASE = "HINDSIGHT_API_RERANKER_LITELLM_API_BASE"
 ENV_RERANKER_LITELLM_API_KEY = "HINDSIGHT_API_RERANKER_LITELLM_API_KEY"
 ENV_RERANKER_LITELLM_MODEL = "HINDSIGHT_API_RERANKER_LITELLM_MODEL"
-ENV_RERANKER_LITELLM_MAX_TOKENS_PER_DOC = "HINDSIGHT_API_RERANKER_LITELLM_MAX_TOKENS_PER_DOC"
 
 # LiteLLM SDK configuration (direct API access, no proxy needed)
 ENV_EMBEDDINGS_LITELLM_SDK_API_KEY = "HINDSIGHT_API_EMBEDDINGS_LITELLM_SDK_API_KEY"
@@ -212,9 +211,6 @@ ENV_RERANKER_LOCAL_MODEL = "HINDSIGHT_API_RERANKER_LOCAL_MODEL"
 ENV_RERANKER_LOCAL_FORCE_CPU = "HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU"
 ENV_RERANKER_LOCAL_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_LOCAL_MAX_CONCURRENT"
 ENV_RERANKER_LOCAL_TRUST_REMOTE_CODE = "HINDSIGHT_API_RERANKER_LOCAL_TRUST_REMOTE_CODE"
-ENV_RERANKER_LOCAL_FP16 = "HINDSIGHT_API_RERANKER_LOCAL_FP16"
-ENV_RERANKER_LOCAL_BUCKET_BATCHING = "HINDSIGHT_API_RERANKER_LOCAL_BUCKET_BATCHING"
-ENV_RERANKER_LOCAL_BATCH_SIZE = "HINDSIGHT_API_RERANKER_LOCAL_BATCH_SIZE"
 ENV_RERANKER_TEI_URL = "HINDSIGHT_API_RERANKER_TEI_URL"
 ENV_RERANKER_TEI_BATCH_SIZE = "HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE"
 ENV_RERANKER_TEI_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT"
@@ -242,7 +238,6 @@ ENV_GRAPH_RETRIEVER = "HINDSIGHT_API_GRAPH_RETRIEVER"
 ENV_MPFP_TOP_K_NEIGHBORS = "HINDSIGHT_API_MPFP_TOP_K_NEIGHBORS"
 ENV_RECALL_MAX_CONCURRENT = "HINDSIGHT_API_RECALL_MAX_CONCURRENT"
 ENV_RECALL_CONNECTION_BUDGET = "HINDSIGHT_API_RECALL_CONNECTION_BUDGET"
-ENV_RECALL_MAX_QUERY_TOKENS = "HINDSIGHT_API_RECALL_MAX_QUERY_TOKENS"
 ENV_MENTAL_MODEL_REFRESH_CONCURRENCY = "HINDSIGHT_API_MENTAL_MODEL_REFRESH_CONCURRENCY"
 
 # OpenTelemetry tracing configuration
@@ -257,9 +252,6 @@ ENV_LLM_VERTEXAI_PROJECT_ID = "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID"
 ENV_LLM_VERTEXAI_REGION = "HINDSIGHT_API_LLM_VERTEXAI_REGION"
 ENV_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY = "HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY"
 
-# Gemini safety settings
-ENV_LLM_GEMINI_SAFETY_SETTINGS = "HINDSIGHT_API_LLM_GEMINI_SAFETY_SETTINGS"
-
 # Retain settings
 ENV_RETAIN_MAX_COMPLETION_TOKENS = "HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS"
 ENV_RETAIN_CHUNK_SIZE = "HINDSIGHT_API_RETAIN_CHUNK_SIZE"
@@ -267,9 +259,7 @@ ENV_RETAIN_EXTRACT_CAUSAL_LINKS = "HINDSIGHT_API_RETAIN_EXTRACT_CAUSAL_LINKS"
 ENV_RETAIN_EXTRACTION_MODE = "HINDSIGHT_API_RETAIN_EXTRACTION_MODE"
 ENV_RETAIN_MISSION = "HINDSIGHT_API_RETAIN_MISSION"
 ENV_RETAIN_CUSTOM_INSTRUCTIONS = "HINDSIGHT_API_RETAIN_CUSTOM_INSTRUCTIONS"
-ENV_RETAIN_DEFAULT_STRATEGY = "HINDSIGHT_API_RETAIN_DEFAULT_STRATEGY"
 ENV_RETAIN_BATCH_TOKENS = "HINDSIGHT_API_RETAIN_BATCH_TOKENS"
-ENV_RETAIN_ENTITY_LOOKUP = "HINDSIGHT_API_RETAIN_ENTITY_LOOKUP"
 ENV_RETAIN_BATCH_ENABLED = "HINDSIGHT_API_RETAIN_BATCH_ENABLED"
 ENV_RETAIN_BATCH_POLL_INTERVAL_SECONDS = "HINDSIGHT_API_RETAIN_BATCH_POLL_INTERVAL_SECONDS"
 
@@ -286,7 +276,6 @@ ENV_FILE_STORAGE_AZURE_CONTAINER = "HINDSIGHT_API_FILE_STORAGE_AZURE_CONTAINER"
 ENV_FILE_STORAGE_AZURE_ACCOUNT_NAME = "HINDSIGHT_API_FILE_STORAGE_AZURE_ACCOUNT_NAME"
 ENV_FILE_STORAGE_AZURE_ACCOUNT_KEY = "HINDSIGHT_API_FILE_STORAGE_AZURE_ACCOUNT_KEY"
 ENV_FILE_PARSER = "HINDSIGHT_API_FILE_PARSER"
-ENV_FILE_PARSER_ALLOWLIST = "HINDSIGHT_API_FILE_PARSER_ALLOWLIST"
 ENV_FILE_PARSER_IRIS_TOKEN = "HINDSIGHT_API_FILE_PARSER_IRIS_TOKEN"
 ENV_FILE_PARSER_IRIS_ORG_ID = "HINDSIGHT_API_FILE_PARSER_IRIS_ORG_ID"
 ENV_FILE_CONVERSION_MAX_BATCH_SIZE_MB = "HINDSIGHT_API_FILE_CONVERSION_MAX_BATCH_SIZE_MB"
@@ -299,19 +288,7 @@ ENV_ENABLE_OBSERVATIONS = "HINDSIGHT_API_ENABLE_OBSERVATIONS"
 ENV_CONSOLIDATION_BATCH_SIZE = "HINDSIGHT_API_CONSOLIDATION_BATCH_SIZE"
 ENV_CONSOLIDATION_LLM_BATCH_SIZE = "HINDSIGHT_API_CONSOLIDATION_LLM_BATCH_SIZE"
 ENV_CONSOLIDATION_MAX_TOKENS = "HINDSIGHT_API_CONSOLIDATION_MAX_TOKENS"
-ENV_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS = "HINDSIGHT_API_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS"
-ENV_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION = (
-    "HINDSIGHT_API_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION"
-)
 ENV_OBSERVATIONS_MISSION = "HINDSIGHT_API_OBSERVATIONS_MISSION"
-ENV_ENABLE_OBSERVATION_HISTORY = "HINDSIGHT_API_ENABLE_OBSERVATION_HISTORY"
-ENV_ENABLE_MENTAL_MODEL_HISTORY = "HINDSIGHT_API_ENABLE_MENTAL_MODEL_HISTORY"
-
-# Webhook configuration (global, static - server-level only)
-ENV_WEBHOOK_URL = "HINDSIGHT_API_WEBHOOK_URL"
-ENV_WEBHOOK_SECRET = "HINDSIGHT_API_WEBHOOK_SECRET"
-ENV_WEBHOOK_EVENT_TYPES = "HINDSIGHT_API_WEBHOOK_EVENT_TYPES"
-ENV_WEBHOOK_DELIVERY_POLL_INTERVAL_SECONDS = "HINDSIGHT_API_WEBHOOK_DELIVERY_POLL_INTERVAL_SECONDS"
 
 # Optimization flags
 ENV_SKIP_LLM_VERIFICATION = "HINDSIGHT_API_SKIP_LLM_VERIFICATION"
@@ -356,7 +333,6 @@ PROVIDER_DEFAULT_MODELS = {
     "anthropic": "claude-haiku-4-5-20251001",
     "gemini": "gemini-2.5-flash",
     "groq": "openai/gpt-oss-120b",
-    "minimax": "MiniMax-M2.7",
     "ollama": "gemma3:12b",
     "lmstudio": "local-model",
     "vertexai": "google/gemini-2.5-flash-lite",
@@ -376,9 +352,6 @@ DEFAULT_LLM_VERTEXAI_PROJECT_ID = None  # Required for Vertex AI
 DEFAULT_LLM_VERTEXAI_REGION = "us-central1"
 DEFAULT_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY = None  # Optional, uses ADC if not set
 
-# Gemini safety settings defaults
-DEFAULT_LLM_GEMINI_SAFETY_SETTINGS = None  # None = use Gemini default safety settings
-
 DEFAULT_EMBEDDINGS_PROVIDER = "local"
 DEFAULT_EMBEDDINGS_LOCAL_MODEL = "BAAI/bge-small-en-v1.5"
 DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU = False  # Force CPU mode for local embeddings (avoids MPS/XPC issues on macOS)
@@ -393,9 +366,6 @@ DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT = 4  # Limit concurrent CPU-bound rerankin
 DEFAULT_RERANKER_LOCAL_TRUST_REMOTE_CODE = (
     False  # Security: disabled by default, required for some models like jina-reranker-v2
 )
-DEFAULT_RERANKER_LOCAL_FP16 = False  # FP16 inference: opt-in, faster on MPS/CUDA (not CPU)
-DEFAULT_RERANKER_LOCAL_BUCKET_BATCHING = False  # Length-sorted bucket batching: opt-in, 36-54% speedup
-DEFAULT_RERANKER_LOCAL_BATCH_SIZE = 32  # Batch size for local reranker predict() calls
 DEFAULT_RERANKER_TEI_BATCH_SIZE = 128
 DEFAULT_RERANKER_TEI_MAX_CONCURRENT = 8
 DEFAULT_RERANKER_MAX_CANDIDATES = 300
@@ -417,7 +387,6 @@ DEFAULT_TEXT_SEARCH_EXTENSION = "native"  # Options: "native", "vchord", "pg_tex
 DEFAULT_LITELLM_API_BASE = "http://localhost:4000"
 DEFAULT_EMBEDDINGS_LITELLM_MODEL = "text-embedding-3-small"
 DEFAULT_RERANKER_LITELLM_MODEL = "cohere/rerank-english-v3.0"
-DEFAULT_RERANKER_LITELLM_MAX_TOKENS_PER_DOC: int | None = None
 
 # LiteLLM SDK defaults
 DEFAULT_EMBEDDINGS_LITELLM_SDK_MODEL = "cohere/embed-english-v3.0"
@@ -436,7 +405,6 @@ DEFAULT_GRAPH_RETRIEVER = "link_expansion"  # Options: "link_expansion", "mpfp",
 DEFAULT_MPFP_TOP_K_NEIGHBORS = 20  # Fan-out limit per node in MPFP graph traversal
 DEFAULT_RECALL_MAX_CONCURRENT = 32  # Max concurrent recall operations per worker
 DEFAULT_RECALL_CONNECTION_BUDGET = 4  # Max concurrent DB connections per recall operation
-DEFAULT_RECALL_MAX_QUERY_TOKENS = 500  # Maximum tokens allowed in recall query
 DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY = 8  # Max concurrent mental model refreshes
 
 # Retain settings
@@ -444,20 +412,16 @@ DEFAULT_RETAIN_MAX_COMPLETION_TOKENS = 64000  # Max tokens for fact extraction L
 DEFAULT_RETAIN_CHUNK_SIZE = 3000  # Max chars per chunk for fact extraction
 DEFAULT_RETAIN_EXTRACT_CAUSAL_LINKS = True  # Extract causal links between facts
 DEFAULT_RETAIN_EXTRACTION_MODE = "concise"  # Extraction mode: "concise", "verbose", or "custom"
-RETAIN_EXTRACTION_MODES = ("concise", "verbose", "custom", "verbatim", "chunks")  # Allowed extraction modes
+RETAIN_EXTRACTION_MODES = ("concise", "verbose", "custom")  # Allowed extraction modes
 DEFAULT_RETAIN_MISSION = None  # Declarative spec of what to retain (injected into any extraction mode)
 DEFAULT_RETAIN_CUSTOM_INSTRUCTIONS = None  # Custom extraction guidelines (only used when mode="custom")
-DEFAULT_RETAIN_DEFAULT_STRATEGY = None  # Default strategy name (None = no strategy override)
-DEFAULT_RETAIN_STRATEGIES: dict | None = None  # Named retain strategies (dict of name → config overrides)
 DEFAULT_RETAIN_BATCH_TOKENS = 10_000  # ~40KB of text  # Max chars per sub-batch for async retain auto-splitting
-DEFAULT_RETAIN_ENTITY_LOOKUP = "trigram"  # "full" or "trigram"
 DEFAULT_RETAIN_BATCH_ENABLED = False  # Use LLM Batch API for fact extraction (only when async=True)
 DEFAULT_RETAIN_BATCH_POLL_INTERVAL_SECONDS = 60  # Batch API polling interval in seconds
 
 # File storage defaults
 DEFAULT_FILE_STORAGE_TYPE = "native"  # PostgreSQL BYTEA storage
-DEFAULT_FILE_PARSER = "markitdown"  # Default parser fallback chain (comma-separated, e.g. "iris,markitdown")
-DEFAULT_FILE_PARSER_ALLOWLIST = None  # Allowlist of parsers clients may request (None = all registered parsers)
+DEFAULT_FILE_PARSER = "markitdown"  # File parser to use (markitdown is the only supported parser)
 DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE_MB = 100  # Max total batch size in MB (all files combined)
 DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE = 10  # Max files per batch upload
 DEFAULT_ENABLE_FILE_UPLOAD_API = True  # Enable file upload endpoint
@@ -465,17 +429,9 @@ DEFAULT_FILE_DELETE_AFTER_RETAIN = True  # Delete file bytes after retain (saves
 
 # Observations defaults (consolidated knowledge from facts)
 DEFAULT_ENABLE_OBSERVATIONS = True  # Observations enabled by default
-DEFAULT_ENABLE_OBSERVATION_HISTORY = True  # Observation history tracking enabled by default
-DEFAULT_ENABLE_MENTAL_MODEL_HISTORY = True  # Mental model history tracking enabled by default
 DEFAULT_CONSOLIDATION_BATCH_SIZE = 50  # Memories to load per batch (internal memory optimization)
 DEFAULT_CONSOLIDATION_LLM_BATCH_SIZE = 8  # Facts per LLM call (1 = no batching; >1 = batch mode)
 DEFAULT_CONSOLIDATION_MAX_TOKENS = 512  # Max tokens for recall when finding related observations
-DEFAULT_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS = (
-    -1
-)  # Total token budget for source facts in consolidation recall (-1 = unlimited)
-DEFAULT_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION = (
-    256  # Max tokens of source facts per observation in consolidation prompt (-1 = unlimited)
-)
 DEFAULT_OBSERVATIONS_MISSION = None  # Declarative spec of what observations are for this bank
 
 # Database migrations
@@ -533,12 +489,6 @@ Use this tool PROACTIVELY to:
 # Default embedding dimension (used by initial migration, adjusted at runtime)
 EMBEDDING_DIMENSION = DEFAULT_EMBEDDING_DIMENSION
 
-# Webhook configuration defaults
-DEFAULT_WEBHOOK_URL = None  # None = no global webhook configured
-DEFAULT_WEBHOOK_SECRET = None  # None = no signing
-DEFAULT_WEBHOOK_EVENT_TYPES = "consolidation.completed"  # Comma-separated; default = all supported events
-DEFAULT_WEBHOOK_DELIVERY_POLL_INTERVAL_SECONDS = 30  # How often to poll for pending deliveries
-
 
 class JsonFormatter(logging.Formatter):
     """JSON formatter for structured logging.
@@ -570,11 +520,6 @@ class JsonFormatter(logging.Formatter):
         return json.dumps(log_entry)
 
 
-def _parse_str_list(value: str) -> list[str]:
-    """Parse a comma-separated string into a non-empty list of stripped tokens."""
-    return [v.strip() for v in value.split(",") if v.strip()]
-
-
 def _validate_extraction_mode(mode: str) -> str:
     """Validate and normalize extraction mode."""
     mode_lower = mode.lower()
@@ -620,9 +565,6 @@ class HindsightConfig:
     llm_vertexai_region: str
     llm_vertexai_service_account_key: str | None
 
-    # Gemini safety settings (None = use Gemini defaults; list of dicts with category/threshold)
-    llm_gemini_safety_settings: list | None
-
     # Per-operation LLM configuration (None = use default LLM config)
     retain_llm_provider: str | None
     retain_llm_api_key: str | None
@@ -677,9 +619,6 @@ class HindsightConfig:
     reranker_local_force_cpu: bool
     reranker_local_max_concurrent: int
     reranker_local_trust_remote_code: bool
-    reranker_local_fp16: bool
-    reranker_local_bucket_batching: bool
-    reranker_local_batch_size: int
     reranker_tei_url: str | None
     reranker_tei_batch_size: int
     reranker_tei_max_concurrent: int
@@ -690,7 +629,6 @@ class HindsightConfig:
     reranker_litellm_api_base: str
     reranker_litellm_api_key: str | None
     reranker_litellm_model: str
-    reranker_litellm_max_tokens_per_doc: int | None
     reranker_litellm_sdk_api_key: str | None
     reranker_litellm_sdk_model: str
     reranker_litellm_sdk_api_base: str | None
@@ -712,7 +650,6 @@ class HindsightConfig:
     mpfp_top_k_neighbors: int
     recall_max_concurrent: int
     recall_connection_budget: int
-    recall_max_query_tokens: int
     mental_model_refresh_concurrency: int
 
     # Retain settings
@@ -722,12 +659,9 @@ class HindsightConfig:
     retain_extraction_mode: str
     retain_mission: str | None
     retain_custom_instructions: str | None
-    retain_default_strategy: str | None
-    retain_strategies: dict | None
     retain_batch_tokens: int
     retain_batch_enabled: bool
     retain_batch_poll_interval_seconds: int
-    retain_entity_lookup: str  # "full" or "trigram"
 
     # File storage (static - server-level only)
     file_storage_type: str  # "native" (PostgreSQL) or "s3" (S3-compatible)
@@ -741,8 +675,7 @@ class HindsightConfig:
     file_storage_azure_container: str | None  # Azure container name (required for azure storage)
     file_storage_azure_account_name: str | None  # Azure storage account name
     file_storage_azure_account_key: str | None  # Azure storage account key
-    file_parser: list[str]  # Ordered fallback chain of parsers (e.g. ["iris", "markitdown"])
-    file_parser_allowlist: list[str] | None  # Parsers clients may request (None = all registered)
+    file_parser: str  # File parser to use (e.g., "markitdown", "iris")
     file_parser_iris_token: str | None  # Vectorize API token for iris parser (VECTORIZE_TOKEN)
     file_parser_iris_org_id: str | None  # Vectorize org ID for iris parser (VECTORIZE_ORG_ID)
     file_conversion_max_batch_size_mb: int  # Max total batch size in MB (all files combined)
@@ -752,13 +685,9 @@ class HindsightConfig:
 
     # Observations settings (consolidated knowledge from facts)
     enable_observations: bool
-    enable_observation_history: bool
-    enable_mental_model_history: bool
     consolidation_batch_size: int
     consolidation_llm_batch_size: int
     consolidation_max_tokens: int
-    consolidation_source_facts_max_tokens: int
-    consolidation_source_facts_max_tokens_per_observation: int
     observations_mission: str | None
 
     # Entity labels (controlled vocabulary of key:value classification labels extracted at retain time)
@@ -809,12 +738,6 @@ class HindsightConfig:
     otel_service_name: str
     otel_deployment_environment: str
 
-    # Webhook configuration (static - server-level only, not per-bank)
-    webhook_url: str | None  # Global webhook URL (None = disabled)
-    webhook_secret: str | None  # HMAC signing secret (None = unsigned)
-    webhook_event_types: list[str]  # Event types to deliver globally
-    webhook_delivery_poll_interval_seconds: int  # How often the delivery worker polls
-
     # Class-level sets for configuration categorization
 
     # CREDENTIAL_FIELDS: Never exposed via API, never configurable per-tenant/bank
@@ -854,16 +777,11 @@ class HindsightConfig:
         "retain_extraction_mode",
         "retain_mission",
         "retain_custom_instructions",
-        "retain_default_strategy",
-        "retain_strategies",
         # Entity labels (controlled vocabulary for entity classification)
         "entity_labels",
         "entities_allow_free_form",
         # Consolidation settings
         "enable_observations",
-        "consolidation_llm_batch_size",
-        "consolidation_source_facts_max_tokens",
-        "consolidation_source_facts_max_tokens_per_observation",
         "observations_mission",
         # Reflect settings
         "reflect_mission",
@@ -871,8 +789,6 @@ class HindsightConfig:
         "disposition_skepticism",
         "disposition_literalism",
         "disposition_empathy",
-        # Gemini safety settings (controls content filtering for Gemini/VertexAI providers)
-        "llm_gemini_safety_settings",
     }
 
     @property
@@ -993,8 +909,6 @@ class HindsightConfig:
             llm_vertexai_region=os.getenv(ENV_LLM_VERTEXAI_REGION, DEFAULT_LLM_VERTEXAI_REGION),
             llm_vertexai_service_account_key=os.getenv(ENV_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY)
             or DEFAULT_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY,
-            # Gemini safety settings (JSON-encoded list of {category, threshold} dicts)
-            llm_gemini_safety_settings=json.loads(os.getenv(ENV_LLM_GEMINI_SAFETY_SETTINGS, "null")),
             # Per-operation LLM config (None = use default)
             retain_llm_provider=os.getenv(ENV_RETAIN_LLM_PROVIDER) or None,
             retain_llm_api_key=os.getenv(ENV_RETAIN_LLM_API_KEY) or None,
@@ -1108,15 +1022,6 @@ class HindsightConfig:
                 ENV_RERANKER_LOCAL_TRUST_REMOTE_CODE, str(DEFAULT_RERANKER_LOCAL_TRUST_REMOTE_CODE)
             ).lower()
             in ("true", "1"),
-            reranker_local_fp16=os.getenv(ENV_RERANKER_LOCAL_FP16, str(DEFAULT_RERANKER_LOCAL_FP16)).lower()
-            in ("true", "1"),
-            reranker_local_bucket_batching=os.getenv(
-                ENV_RERANKER_LOCAL_BUCKET_BATCHING, str(DEFAULT_RERANKER_LOCAL_BUCKET_BATCHING)
-            ).lower()
-            in ("true", "1"),
-            reranker_local_batch_size=int(
-                os.getenv(ENV_RERANKER_LOCAL_BATCH_SIZE, str(DEFAULT_RERANKER_LOCAL_BATCH_SIZE))
-            ),
             reranker_tei_url=os.getenv(ENV_RERANKER_TEI_URL),
             reranker_tei_batch_size=int(os.getenv(ENV_RERANKER_TEI_BATCH_SIZE, str(DEFAULT_RERANKER_TEI_BATCH_SIZE))),
             reranker_tei_max_concurrent=int(
@@ -1132,9 +1037,6 @@ class HindsightConfig:
             or os.getenv(ENV_LITELLM_API_BASE, DEFAULT_LITELLM_API_BASE),
             reranker_litellm_api_key=os.getenv(ENV_RERANKER_LITELLM_API_KEY) or os.getenv(ENV_LITELLM_API_KEY),
             reranker_litellm_model=os.getenv(ENV_RERANKER_LITELLM_MODEL, DEFAULT_RERANKER_LITELLM_MODEL),
-            reranker_litellm_max_tokens_per_doc=int(v)
-            if (v := os.getenv(ENV_RERANKER_LITELLM_MAX_TOKENS_PER_DOC))
-            else DEFAULT_RERANKER_LITELLM_MAX_TOKENS_PER_DOC,
             # LiteLLM SDK reranker (direct API access)
             reranker_litellm_sdk_api_key=os.getenv(ENV_RERANKER_LITELLM_SDK_API_KEY),
             reranker_litellm_sdk_model=os.getenv(ENV_RERANKER_LITELLM_SDK_MODEL, DEFAULT_RERANKER_LITELLM_SDK_MODEL),
@@ -1161,7 +1063,6 @@ class HindsightConfig:
             recall_connection_budget=int(
                 os.getenv(ENV_RECALL_CONNECTION_BUDGET, str(DEFAULT_RECALL_CONNECTION_BUDGET))
             ),
-            recall_max_query_tokens=int(os.getenv(ENV_RECALL_MAX_QUERY_TOKENS, str(DEFAULT_RECALL_MAX_QUERY_TOKENS))),
             mental_model_refresh_concurrency=int(
                 os.getenv(ENV_MENTAL_MODEL_REFRESH_CONCURRENCY, str(DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY))
             ),
@@ -1182,10 +1083,7 @@ class HindsightConfig:
             ),
             retain_mission=os.getenv(ENV_RETAIN_MISSION) or DEFAULT_RETAIN_MISSION,
             retain_custom_instructions=os.getenv(ENV_RETAIN_CUSTOM_INSTRUCTIONS) or DEFAULT_RETAIN_CUSTOM_INSTRUCTIONS,
-            retain_default_strategy=os.getenv(ENV_RETAIN_DEFAULT_STRATEGY) or DEFAULT_RETAIN_DEFAULT_STRATEGY,
-            retain_strategies=DEFAULT_RETAIN_STRATEGIES,
             retain_batch_tokens=int(os.getenv(ENV_RETAIN_BATCH_TOKENS, str(DEFAULT_RETAIN_BATCH_TOKENS))),
-            retain_entity_lookup=os.getenv(ENV_RETAIN_ENTITY_LOOKUP, DEFAULT_RETAIN_ENTITY_LOOKUP),
             retain_batch_enabled=os.getenv(ENV_RETAIN_BATCH_ENABLED, str(DEFAULT_RETAIN_BATCH_ENABLED)).lower()
             == "true",
             retain_batch_poll_interval_seconds=int(
@@ -1203,10 +1101,7 @@ class HindsightConfig:
             file_storage_azure_container=os.getenv(ENV_FILE_STORAGE_AZURE_CONTAINER) or None,
             file_storage_azure_account_name=os.getenv(ENV_FILE_STORAGE_AZURE_ACCOUNT_NAME) or None,
             file_storage_azure_account_key=os.getenv(ENV_FILE_STORAGE_AZURE_ACCOUNT_KEY) or None,
-            file_parser=_parse_str_list(os.getenv(ENV_FILE_PARSER, DEFAULT_FILE_PARSER)),
-            file_parser_allowlist=_parse_str_list(os.getenv(ENV_FILE_PARSER_ALLOWLIST))
-            if os.getenv(ENV_FILE_PARSER_ALLOWLIST)
-            else None,
+            file_parser=os.getenv(ENV_FILE_PARSER, DEFAULT_FILE_PARSER),
             file_parser_iris_token=os.getenv(ENV_FILE_PARSER_IRIS_TOKEN) or None,
             file_parser_iris_org_id=os.getenv(ENV_FILE_PARSER_IRIS_ORG_ID) or None,
             file_conversion_max_batch_size_mb=int(
@@ -1223,14 +1118,6 @@ class HindsightConfig:
             == "true",
             # Observations settings (consolidated knowledge from facts)
             enable_observations=os.getenv(ENV_ENABLE_OBSERVATIONS, str(DEFAULT_ENABLE_OBSERVATIONS)).lower() == "true",
-            enable_observation_history=os.getenv(
-                ENV_ENABLE_OBSERVATION_HISTORY, str(DEFAULT_ENABLE_OBSERVATION_HISTORY)
-            ).lower()
-            == "true",
-            enable_mental_model_history=os.getenv(
-                ENV_ENABLE_MENTAL_MODEL_HISTORY, str(DEFAULT_ENABLE_MENTAL_MODEL_HISTORY)
-            ).lower()
-            == "true",
             consolidation_batch_size=int(
                 os.getenv(ENV_CONSOLIDATION_BATCH_SIZE, str(DEFAULT_CONSOLIDATION_BATCH_SIZE))
             ),
@@ -1240,15 +1127,6 @@ class HindsightConfig:
             consolidation_max_tokens=int(
                 os.getenv(ENV_CONSOLIDATION_MAX_TOKENS, str(DEFAULT_CONSOLIDATION_MAX_TOKENS))
             ),
-            consolidation_source_facts_max_tokens=int(
-                os.getenv(ENV_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS, str(DEFAULT_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS))
-            ),
-            consolidation_source_facts_max_tokens_per_observation=int(
-                os.getenv(
-                    ENV_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION,
-                    str(DEFAULT_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS_PER_OBSERVATION),
-                )
-            ),
             observations_mission=os.getenv(ENV_OBSERVATIONS_MISSION) or DEFAULT_OBSERVATIONS_MISSION,
             entity_labels=None,
             entities_allow_free_form=True,
@@ -1292,20 +1170,6 @@ class HindsightConfig:
             otel_exporter_otlp_headers=os.getenv(ENV_OTEL_EXPORTER_OTLP_HEADERS) or None,
             otel_service_name=os.getenv(ENV_OTEL_SERVICE_NAME, DEFAULT_OTEL_SERVICE_NAME),
             otel_deployment_environment=os.getenv(ENV_OTEL_DEPLOYMENT_ENVIRONMENT, DEFAULT_OTEL_DEPLOYMENT_ENVIRONMENT),
-            # Webhook configuration (static, server-level only)
-            webhook_url=os.getenv(ENV_WEBHOOK_URL) or DEFAULT_WEBHOOK_URL,
-            webhook_secret=os.getenv(ENV_WEBHOOK_SECRET) or DEFAULT_WEBHOOK_SECRET,
-            webhook_event_types=[
-                t.strip()
-                for t in os.getenv(ENV_WEBHOOK_EVENT_TYPES, DEFAULT_WEBHOOK_EVENT_TYPES).split(",")
-                if t.strip()
-            ],
-            webhook_delivery_poll_interval_seconds=int(
-                os.getenv(
-                    ENV_WEBHOOK_DELIVERY_POLL_INTERVAL_SECONDS,
-                    str(DEFAULT_WEBHOOK_DELIVERY_POLL_INTERVAL_SECONDS),
-                )
-            ),
         )
         config.validate()
         return config

hindsight-api/hindsight_api/config_resolver.py

@@ -10,7 +10,7 @@ multiple API servers.
 
 import json
 import logging
-from dataclasses import asdict, replace
+from dataclasses import asdict
 from typing import Any
 
 import asyncpg
@@ -239,14 +239,6 @@ class ConfigResolver:
                 logger.warning(f"Failed to check permissions for bank {bank_id}: {e}")
                 # Continue without permission check (fail open for backward compatibility)
 
-        # Validate retain_strategies: reject empty string keys
-        if "retain_strategies" in normalized_updates and normalized_updates["retain_strategies"]:
-            empty_keys = [k for k in normalized_updates["retain_strategies"] if not str(k).strip()]
-            if empty_keys:
-                raise ValueError(
-                    "Strategy names must not be empty strings. Remove entries with empty names before saving."
-                )
-
         # Merge with existing config (JSONB || operator)
         async with self.pool.acquire() as conn:
             await conn.execute(
@@ -281,35 +273,3 @@ class ConfigResolver:
             )
 
         logger.info(f"Reset bank config for {bank_id} to defaults")
-
-
-def apply_strategy(config: HindsightConfig, strategy_name: str) -> HindsightConfig:
-    """
-    Apply a named retain strategy's overrides on top of a resolved config.
-
-    A strategy is a named set of hierarchical field overrides stored in
-    config.retain_strategies. Any field in _HIERARCHICAL_FIELDS can be
-    overridden, including retain_extraction_mode, retain_chunk_size,
-    entity_labels, entities_allow_free_form, etc.
-
-    Unknown strategy names log a warning and return config unchanged.
-    Unknown or non-hierarchical fields in the strategy are silently ignored.
-    """
-    strategies = config.retain_strategies or {}
-    if strategy_name not in strategies:
-        logger.warning(f"Unknown retain strategy '{strategy_name}', using resolved config as-is")
-        return config
-
-    overrides = strategies[strategy_name]
-    if not isinstance(overrides, dict):
-        logger.warning(f"Retain strategy '{strategy_name}' is not a dict, skipping")
-        return config
-
-    configurable = HindsightConfig.get_configurable_fields()
-    filtered = {k: v for k, v in overrides.items() if k in configurable}
-
-    if not filtered:
-        return config
-
-    logger.debug(f"Applying retain strategy '{strategy_name}': {list(filtered.keys())}")
-    return replace(config, **filtered)