test: add client tests for ReflectResponse parsing

Added comprehensive tests in hindsight-clients/python/tests to verify:
- v0.4.0+ format with empty based_on object
- v0.4.0+ format with null based_on
- v0.4.0+ format with populated facts
- v0.3.0 format (list) correctly fails validation
- Missing based_on field handling

These tests document the v0.3.0 -> v0.4.0 breaking change where
based_on changed from list to object.

.dockerignore

@@ -0,0 +1,32 @@
+# Node modules (platform-specific native bindings)
+**/node_modules
+**/.next
+
+# Python
+**/__pycache__
+**/*.pyc
+**/.venv
+**/dist
+**/*.egg-info
+
+# Git
+.git
+.gitignore
+
+# IDE
+.idea
+.vscode
+*.swp
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Build artifacts
+**/target
+**/*.log
+
+# Test/Dev
+**/coverage
+**/.pytest_cache
+**/.mypy_cache

.env.example

@@ -2,10 +2,30 @@
 # Copy this file to .env and fill in your values
 
 # LLM Configuration (Required)
+# Supported providers: openai, groq, ollama, gemini, anthropic, lmstudio, vertexai
+HINDSIGHT_API_LLM_PROVIDER=openai
 HINDSIGHT_API_LLM_API_KEY=your-api-key-here
-HINDSIGHT_API_LLM_MODEL=gpt-4o-mini
+HINDSIGHT_API_LLM_MODEL=o3-mini
 HINDSIGHT_API_LLM_BASE_URL=https://api.openai.com/v1
 
+# Example: Anthropic Claude configuration
+# HINDSIGHT_API_LLM_PROVIDER=anthropic
+# HINDSIGHT_API_LLM_API_KEY=your-anthropic-api-key
+# HINDSIGHT_API_LLM_MODEL=claude-sonnet-4-20250514
+
+# Example: Google Vertex AI configuration
+# HINDSIGHT_API_LLM_PROVIDER=vertexai
+# HINDSIGHT_API_LLM_MODEL=google/gemini-2.0-flash-001
+# HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=your-gcp-project-id
+# HINDSIGHT_API_LLM_VERTEXAI_REGION=us-central1
+# HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY=/path/to/service-account-key.json  # Optional, uses ADC if not set
+
+# Example: LM Studio local configuration (Qwen 2.5 32B recommended)
+# HINDSIGHT_API_LLM_PROVIDER=lmstudio
+# HINDSIGHT_API_LLM_API_KEY=lmstudio
+# HINDSIGHT_API_LLM_BASE_URL=http://localhost:1234/v1
+# HINDSIGHT_API_LLM_MODEL=qwen2.5-32b-instruct
+
 # API Configuration (Optional)
 HINDSIGHT_API_HOST=0.0.0.0
 HINDSIGHT_API_PORT=8888
@@ -13,3 +33,35 @@ HINDSIGHT_API_LOG_LEVEL=info
 
 # Database (Optional - uses embedded pg0 by default)
 # HINDSIGHT_API_DATABASE_URL=postgresql://user:pass@host:5432/db
+# HINDSIGHT_API_DATABASE_SCHEMA=public  # PostgreSQL schema name (default: public)
+
+# Embeddings Configuration (Optional - uses local by default)
+# Provider: "local" (default) or "tei" (HuggingFace Text Embeddings Inference)
+# HINDSIGHT_API_EMBEDDINGS_PROVIDER=local
+# For local provider:
+# HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL=BAAI/bge-small-en-v1.5
+# For TEI provider:
+# HINDSIGHT_API_EMBEDDINGS_TEI_URL=http://localhost:8080
+
+# Reranker Configuration (Optional - uses local by default)
+# Provider: "local" (default) or "tei" (HuggingFace Text Embeddings Inference)
+# HINDSIGHT_API_RERANKER_PROVIDER=local
+# For local provider:
+# HINDSIGHT_API_RERANKER_LOCAL_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2
+# For TEI provider:
+# HINDSIGHT_API_RERANKER_TEI_URL=http://localhost:8081
+
+# Observability & Tracing (Optional - disabled by default)
+# Enable OpenTelemetry tracing for LLM calls (GenAI semantic conventions)
+# HINDSIGHT_API_OTEL_TRACES_ENABLED=true
+#
+# Local development with Grafana LGTM stack (recommended - see scripts/dev/grafana/README.md)
+# HINDSIGHT_API_OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+#
+# Cloud backends (Grafana Cloud, Langfuse, DataDog, etc.)
+# HINDSIGHT_API_OTEL_EXPORTER_OTLP_ENDPOINT=https://your-backend-url
+# HINDSIGHT_API_OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer your-token"
+#
+# Custom service name and environment (optional, defaults: hindsight-api, development)
+# HINDSIGHT_API_OTEL_SERVICE_NAME=hindsight-production
+# HINDSIGHT_API_OTEL_DEPLOYMENT_ENVIRONMENT=production

.githooks/pre-commit

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Pre-commit hook - runs all scripts in scripts/hooks/
+
+set -e
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+HOOKS_DIR="$REPO_ROOT/scripts/hooks"
+
+if [ ! -d "$HOOKS_DIR" ]; then
+    exit 0
+fi
+
+echo ""
+echo "=== Running pre-commit hooks ==="
+echo ""
+
+# Run all executable scripts in hooks directory
+for hook in "$HOOKS_DIR"/*.sh; do
+    if [ -x "$hook" ]; then
+        echo "[hook] $(basename "$hook")"
+        (cd "$REPO_ROOT" && "$hook")
+    fi
+done
+
+echo ""
+echo "=== Pre-commit hooks completed ==="
+echo ""

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,71 @@
+name: Bug Report
+description: Report a bug or unexpected behavior
+labels: ["bug", "triage"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to report a bug! Please fill out the sections below.
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Bug Description
+      description: A clear and concise description of the bug
+      placeholder: What happened?
+    validations:
+      required: true
+
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to Reproduce
+      description: Steps to reproduce the behavior
+      placeholder: |
+        1. Configure '...'
+        2. Call '...'
+        3. See error
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected Behavior
+      description: What did you expect to happen?
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual
+    attributes:
+      label: Actual Behavior
+      description: What actually happened?
+    validations:
+      required: true
+
+  - type: input
+    id: version
+    attributes:
+      label: Version
+      description: What version are you using?
+      placeholder: e.g., 0.1.0 or commit hash
+    validations:
+      required: false
+
+  - type: dropdown
+    id: llm-provider
+    attributes:
+      label: LLM Provider
+      description: Which LLM provider are you using?
+      options:
+        - OpenAI
+        - Anthropic
+        - Gemini
+        - Groq
+        - Ollama
+        - LM Studio
+        - Other
+    validations:
+      required: false
+

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Questions & Help
+    url: https://github.com/vectorize-io/hindsight/discussions/categories/q-a
+    about: Please ask questions and get help in Discussions instead of opening an issue.
+  - name: Ideas & Feedback
+    url: https://github.com/vectorize-io/hindsight/discussions/categories/ideas
+    about: Share ideas or give feedback in Discussions.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,82 @@
+name: Feature Request
+description: Suggest a new feature or enhancement
+labels: ["enhancement", "triage"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for suggesting a feature! Please describe what you'd like to see added.
+
+  - type: textarea
+    id: use-case
+    attributes:
+      label: Use Case
+      description: Describe your specific use case. What are you building? What's your goal?
+      placeholder: |
+        I'm building an AI agent that needs to...
+        My application handles...
+    validations:
+      required: true
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem Statement
+      description: What problem are you facing? What's missing or difficult today?
+      placeholder: Currently I have to... which causes...
+    validations:
+      required: true
+
+  - type: textarea
+    id: benefit
+    attributes:
+      label: How This Feature Would Help
+      description: Explain how this feature would improve your workflow or solve your problem
+      placeholder: With this feature, I would be able to...
+    validations:
+      required: true
+
+  - type: textarea
+    id: solution
+    attributes:
+      label: Proposed Solution
+      description: Describe your ideal solution (optional - we may have ideas too!)
+      placeholder: It would be great if Hindsight could...
+    validations:
+      required: false
+
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives Considered
+      description: Have you considered any alternative solutions or workarounds?
+    validations:
+      required: false
+
+  - type: dropdown
+    id: priority
+    attributes:
+      label: Priority
+      description: How important is this feature to you?
+      options:
+        - Nice to have
+        - Important - affects my workflow
+        - Critical - blocking my use case
+    validations:
+      required: true
+
+  - type: textarea
+    id: additional
+    attributes:
+      label: Additional Context
+      description: Any other context, mockups, or examples?
+    validations:
+      required: false
+
+  - type: checkboxes
+    id: checklist
+    attributes:
+      label: Checklist
+      options:
+        - label: I would be willing to contribute this feature
+          required: false

.github/workflows/deploy-docs.yml

@@ -20,18 +20,17 @@ concurrency:
 jobs:
   build:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: hindsight-docs
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: npm
-          cache-dependency-path: hindsight-docs/package-lock.json
-      - run: npm ci
-      - run: npm run build
+          cache-dependency-path: package-lock.json
+      - uses: astral-sh/setup-uv@v4
+      - run: npm ci --workspace=hindsight-docs
+      - run: uv run generate-llms-full
+      - run: npm run build --workspace=hindsight-docs
       - uses: actions/upload-pages-artifact@v3
         with:
           path: hindsight-docs/build

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-name: Build Release Artifacts
+name: Release
 
 on:
   push:
@@ -6,17 +6,11 @@ on:
       - 'v*'
 
 jobs:
-  build-python-packages:
+  release-python-packages:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        include:
-          - name: hindsight-all
-            path: hindsight
-          - name: hindsight-api
-            path: hindsight-api
-          - name: hindsight-client
-            path: hindsight-clients/python
+    environment: pypi
+    permissions:
+      id-token: write
 
     steps:
     - uses: actions/checkout@v4
@@ -31,19 +25,74 @@ jobs:
       with:
         python-version-file: ".python-version"
 
-    - name: Build ${{ matrix.name }} package
-      working-directory: ./${{ matrix.path }}
-      run: uv build
+    # Build all packages
+    - name: Build hindsight-client
+      working-directory: ./hindsight-clients/python
+      run: uv build --out-dir dist
 
+    - name: Build hindsight-api
+      working-directory: ./hindsight-api
+      run: uv build --out-dir dist
+
+    - name: Build hindsight-all
+      working-directory: ./hindsight
+      run: uv build --out-dir dist
+
+    - name: Build hindsight-litellm
+      working-directory: ./hindsight-integrations/litellm
+      run: uv build --out-dir dist
+
+    - name: Build hindsight-embed
+      working-directory: ./hindsight-embed
+      run: uv build --out-dir dist
+
+    # Publish in order (client and api first, then hindsight-all which depends on them)
+    - name: Publish hindsight-client to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: ./hindsight-clients/python/dist
+        skip-existing: true
+
+    - name: Publish hindsight-api to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: ./hindsight-api/dist
+        skip-existing: true
+
+    - name: Publish hindsight-all to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: ./hindsight/dist
+        skip-existing: true
+
+    - name: Publish hindsight-litellm to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: ./hindsight-integrations/litellm/dist
+        skip-existing: true
+
+    - name: Publish hindsight-embed to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: ./hindsight-embed/dist
+        skip-existing: true
+
+    # Upload artifacts for GitHub release
     - name: Upload artifacts
       uses: actions/upload-artifact@v4
       with:
-        name: python-${{ matrix.name }}-dist
-        path: ${{ matrix.path }}/dist/*
-        retention-days: 30
+        name: python-packages
+        path: |
+          hindsight-clients/python/dist/*
+          hindsight-api/dist/*
+          hindsight/dist/*
+          hindsight-integrations/litellm/dist/*
+          hindsight-embed/dist/*
+        retention-days: 1
 
-  build-typescript-client:
+  release-typescript-client:
     runs-on: ubuntu-latest
+    environment: npm
 
     steps:
     - uses: actions/checkout@v4
@@ -53,27 +102,201 @@ jobs:
       with:
         node-version: '20'
         registry-url: 'https://registry.npmjs.org'
+        cache: 'npm'
+        cache-dependency-path: package-lock.json
 
     - name: Install dependencies
-      working-directory: ./hindsight-clients/typescript
-      run: npm ci
+      run: npm ci --workspace=hindsight-clients/typescript
 
-    - name: Build TypeScript client
-      working-directory: ./hindsight-clients/typescript
-      run: npm run build
+    - name: Build
+      run: npm run build --workspace=hindsight-clients/typescript
 
-    - name: Pack npm package
+    - name: Publish to npm
+      working-directory: ./hindsight-clients/typescript
+      run: |
+        set +e
+        OUTPUT=$(npm publish --access public 2>&1)
+        EXIT_CODE=$?
+        echo "$OUTPUT"
+        if [ $EXIT_CODE -ne 0 ]; then
+          if echo "$OUTPUT" | grep -q "cannot publish over"; then
+            echo "Package version already published, skipping..."
+            exit 0
+          fi
+          exit $EXIT_CODE
+        fi
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+    - name: Pack for GitHub release
       working-directory: ./hindsight-clients/typescript
       run: npm pack
 
     - name: Upload artifacts
       uses: actions/upload-artifact@v4
       with:
-        name: typescript-client-dist
+        name: typescript-client
         path: hindsight-clients/typescript/*.tgz
-        retention-days: 30
+        retention-days: 1
 
-  build-rust-cli:
+  release-openclaw-integration:
+    runs-on: ubuntu-latest
+    environment: npm
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '22'
+        registry-url: 'https://registry.npmjs.org'
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/openclaw
+      run: npm ci
+
+    - name: Build
+      working-directory: ./hindsight-integrations/openclaw
+      run: npm run build
+
+    - name: Publish to npm
+      working-directory: ./hindsight-integrations/openclaw
+      run: |
+        set +e
+        OUTPUT=$(npm publish --access public 2>&1)
+        EXIT_CODE=$?
+        echo "$OUTPUT"
+        if [ $EXIT_CODE -ne 0 ]; then
+          if echo "$OUTPUT" | grep -q "cannot publish over"; then
+            echo "Package version already published, skipping..."
+            exit 0
+          fi
+          exit $EXIT_CODE
+        fi
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+    - name: Pack for GitHub release
+      working-directory: ./hindsight-integrations/openclaw
+      run: npm pack
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: openclaw-integration
+        path: hindsight-integrations/openclaw/*.tgz
+        retention-days: 1
+
+  release-ai-sdk-integration:
+    runs-on: ubuntu-latest
+    environment: npm
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '22'
+        registry-url: 'https://registry.npmjs.org'
+
+    - name: Install dependencies
+      working-directory: ./hindsight-integrations/ai-sdk
+      run: npm ci
+
+    - name: Build
+      working-directory: ./hindsight-integrations/ai-sdk
+      run: npm run build
+
+    - name: Publish to npm
+      working-directory: ./hindsight-integrations/ai-sdk
+      run: |
+        set +e
+        OUTPUT=$(npm publish --access public 2>&1)
+        EXIT_CODE=$?
+        echo "$OUTPUT"
+        if [ $EXIT_CODE -ne 0 ]; then
+          if echo "$OUTPUT" | grep -q "cannot publish over"; then
+            echo "Package version already published, skipping..."
+            exit 0
+          fi
+          exit $EXIT_CODE
+        fi
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+    - name: Pack for GitHub release
+      working-directory: ./hindsight-integrations/ai-sdk
+      run: npm pack
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: ai-sdk-integration
+        path: hindsight-integrations/ai-sdk/*.tgz
+        retention-days: 1
+
+  release-control-plane:
+    runs-on: ubuntu-latest
+    environment: npm
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+        registry-url: 'https://registry.npmjs.org'
+        cache: 'npm'
+        cache-dependency-path: package-lock.json
+
+    - name: Install dependencies
+      run: npm ci
+
+    - name: Build TypeScript client (dependency)
+      run: npm run build --workspace=hindsight-clients/typescript
+
+    - name: Fix platform-specific native modules
+      run: |
+        # npm ci installs from lockfile which may have wrong platform binaries
+        # Delete hoisted native modules and reinstall for current platform
+        rm -rf node_modules/lightningcss node_modules/@tailwindcss
+        npm install lightningcss @tailwindcss/postcss @tailwindcss/node
+
+    - name: Build
+      run: npm run build --workspace=hindsight-control-plane
+
+    - name: Publish to npm
+      working-directory: ./hindsight-control-plane
+      run: |
+        set +e
+        OUTPUT=$(npm publish --access public 2>&1)
+        EXIT_CODE=$?
+        echo "$OUTPUT"
+        if [ $EXIT_CODE -ne 0 ]; then
+          if echo "$OUTPUT" | grep -q "cannot publish over"; then
+            echo "Package version already published, skipping..."
+            exit 0
+          fi
+          exit $EXIT_CODE
+        fi
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+    - name: Pack for GitHub release
+      working-directory: ./hindsight-control-plane
+      run: npm pack
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: control-plane
+        path: hindsight-control-plane/*.tgz
+        retention-days: 1
+
+  release-rust-cli:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -99,24 +322,6 @@ jobs:
       with:
         targets: ${{ matrix.target }}
 
-    - name: Cache cargo registry
-      uses: actions/cache@v4
-      with:
-        path: ~/.cargo/registry
-        key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
-
-    - name: Cache cargo index
-      uses: actions/cache@v4
-      with:
-        path: ~/.cargo/git
-        key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
-
-    - name: Cache cargo build
-      uses: actions/cache@v4
-      with:
-        path: hindsight-cli/target
-        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
-
     - name: Build
       working-directory: hindsight-cli
       run: cargo build --release --target ${{ matrix.target }}
@@ -132,9 +337,10 @@ jobs:
       with:
         name: rust-cli-${{ matrix.asset_name }}
         path: artifacts/${{ matrix.asset_name }}
-        retention-days: 30
+        retention-days: 1
 
-  build-docker-images:
+  release-docker-images:
+    name: Release Docker (${{ matrix.image_name }}${{ matrix.tag_suffix }})
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -142,13 +348,30 @@ jobs:
     strategy:
       matrix:
         include:
-          # All images use the same Dockerfile with different --target
           - target: api-only
             image_name: hindsight-api
+            tag_suffix: ""
+            build_args: ""
+          - target: api-only
+            image_name: hindsight-api
+            tag_suffix: "-slim"
+            build_args: |
+              INCLUDE_LOCAL_MODELS=false
+              PRELOAD_ML_MODELS=false
           - target: cp-only
             image_name: hindsight-control-plane
+            tag_suffix: ""
+            build_args: ""
           - target: standalone
             image_name: hindsight
+            tag_suffix: ""
+            build_args: ""
+          - target: standalone
+            image_name: hindsight
+            tag_suffix: "-slim"
+            build_args: |
+              INCLUDE_LOCAL_MODELS=false
+              PRELOAD_ML_MODELS=false
 
     steps:
     - uses: actions/checkout@v4
@@ -156,7 +379,7 @@ jobs:
     - name: Free Disk Space
       uses: jlumbroso/free-disk-space@main
       with:
-        tool-cache: false
+        tool-cache: true
         android: true
         dotnet: true
         haskell: true
@@ -181,32 +404,59 @@ jobs:
       id: get_version
       run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
 
-    - name: Extract metadata
+    - name: Extract metadata for release tags
       id: meta
       uses: docker/metadata-action@v5
       with:
         images: ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}
+        flavor: |
+          latest=auto
+          suffix=${{ matrix.tag_suffix }}
         tags: |
           type=semver,pattern={{version}},value=${{ steps.get_version.outputs.VERSION }}
           type=semver,pattern={{major}}.{{minor}},value=${{ steps.get_version.outputs.VERSION }}
           type=semver,pattern={{major}},value=${{ steps.get_version.outputs.VERSION }}
           type=raw,value=latest
 
-    - name: Build and push Docker image
+    # TODO: Re-enable smoke test when disk space issue is resolved
+    # # Step 1: Build for local testing (single platform, no push)
+    # # This creates an identical image to what will be released, just for one platform
+    # - name: Build image for testing
+    #   uses: docker/build-push-action@v6
+    #   with:
+    #     context: .
+    #     file: docker/standalone/Dockerfile
+    #     target: ${{ matrix.target }}
+    #     push: false
+    #     load: true
+    #     tags: ${{ matrix.image_name }}:test
+    #     cache-from: type=gha
+    #     cache-to: type=gha,mode=max
+
+    # # Step 2: Test the image before pushing anything
+    # - name: Smoke test - verify container starts
+    #   env:
+    #     GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
+    #   run: ./docker/test-image.sh "${{ matrix.image_name }}:test" "${{ matrix.target }}"
+
+    # Build multi-platform and push to release tags
+    - name: Build and push release images
       uses: docker/build-push-action@v6
       with:
         context: .
         file: docker/standalone/Dockerfile
         target: ${{ matrix.target }}
+        build-args: ${{ matrix.build_args }}
         push: true
         platforms: linux/amd64,linux/arm64
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
 
-  package-helm-chart:
+  release-helm-chart:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
 
     steps:
     - uses: actions/checkout@v4
@@ -216,78 +466,28 @@ jobs:
       with:
         version: 'latest'
 
+    - name: Log in to GHCR
+      run: echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin
+
     - name: Lint Helm chart
-      run: |
-        helm lint helm/hindsight
+      run: helm lint helm/hindsight
 
     - name: Package Helm chart
-      run: |
-        helm package helm/hindsight --destination ./helm-packages
+      run: helm package helm/hindsight --destination ./helm-packages
 
-    - name: Upload Helm chart artifact
+    - name: Push to GHCR OCI
+      run: helm push helm-packages/*.tgz oci://ghcr.io/${{ github.repository_owner }}/charts
+
+    - name: Upload artifacts
       uses: actions/upload-artifact@v4
       with:
         name: helm-chart
         path: helm-packages/*.tgz
-        retention-days: 30
-
-  publish-python-packages:
-    runs-on: ubuntu-latest
-    needs: [build-python-packages]
-    environment: pypi
-    strategy:
-      max-parallel: 1
-      matrix:
-        include:
-          # Order matters: client and api first, then hindsight-all (which depends on them)
-          - name: hindsight-client
-          - name: hindsight-api
-          - name: hindsight-all
-
-    steps:
-    - name: Download ${{ matrix.name }}
-      uses: actions/download-artifact@v4
-      with:
-        name: python-${{ matrix.name }}-dist
-        path: ./dist
-
-    - name: Publish ${{ matrix.name }} to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        packages-dir: ./dist
-        skip-existing: true
-
-  publish-npm-package:
-    runs-on: ubuntu-latest
-    needs: [build-typescript-client]
-    environment: npm
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Node.js
-      uses: actions/setup-node@v4
-      with:
-        node-version: '20'
-        registry-url: 'https://registry.npmjs.org'
-
-    - name: Install dependencies
-      working-directory: ./hindsight-clients/typescript
-      run: npm ci
-
-    - name: Build TypeScript client
-      working-directory: ./hindsight-clients/typescript
-      run: npm run build
-
-    - name: Publish to npm
-      working-directory: ./hindsight-clients/typescript
-      run: npm publish --access public
-      env:
-        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+        retention-days: 1
 
   create-github-release:
     runs-on: ubuntu-latest
-    needs: [build-python-packages, build-typescript-client, build-rust-cli, build-docker-images, package-helm-chart]
+    needs: [release-python-packages, release-typescript-client, release-openclaw-integration, release-ai-sdk-integration, release-control-plane, release-rust-cli, release-docker-images, release-helm-chart]
     permissions:
       contents: write
 
@@ -298,47 +498,53 @@ jobs:
       id: get_version
       run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
 
-    - name: Download hindsight-all
+    - name: Download Python packages
       uses: actions/download-artifact@v4
       with:
-        name: python-hindsight-all-dist
-        path: ./artifacts/python-hindsight-all
+        name: python-packages
+        path: ./artifacts/python-packages
 
-    - name: Download hindsight-api
+    - name: Download TypeScript client
       uses: actions/download-artifact@v4
       with:
-        name: python-hindsight-api-dist
-        path: ./artifacts/python-hindsight-api
-
-    - name: Download hindsight-client
-      uses: actions/download-artifact@v4
-      with:
-        name: python-hindsight-client-dist
-        path: ./artifacts/python-hindsight-client
-
-    - name: Download TypeScript Client
-      uses: actions/download-artifact@v4
-      with:
-        name: typescript-client-dist
+        name: typescript-client
         path: ./artifacts/typescript-client
 
+    - name: Download OpenClaw Integration
+      uses: actions/download-artifact@v4
+      with:
+        name: openclaw-integration
+        path: ./artifacts/openclaw-integration
+
+    - name: Download AI SDK Integration
+      uses: actions/download-artifact@v4
+      with:
+        name: ai-sdk-integration
+        path: ./artifacts/ai-sdk-integration
+
+    - name: Download Control Plane
+      uses: actions/download-artifact@v4
+      with:
+        name: control-plane
+        path: ./artifacts/control-plane
+
     - name: Download Rust CLI (Linux)
       uses: actions/download-artifact@v4
       with:
         name: rust-cli-hindsight-linux-amd64
-        path: ./artifacts/rust-cli-hindsight-linux-amd64
+        path: ./artifacts/rust-cli-linux
 
     - name: Download Rust CLI (macOS Intel)
       uses: actions/download-artifact@v4
       with:
         name: rust-cli-hindsight-darwin-amd64
-        path: ./artifacts/rust-cli-hindsight-darwin-amd64
+        path: ./artifacts/rust-cli-darwin-amd64
 
     - name: Download Rust CLI (macOS ARM)
       uses: actions/download-artifact@v4
       with:
         name: rust-cli-hindsight-darwin-arm64
-        path: ./artifacts/rust-cli-hindsight-darwin-arm64
+        path: ./artifacts/rust-cli-darwin-arm64
 
     - name: Download Helm chart
       uses: actions/download-artifact@v4
@@ -350,119 +556,33 @@ jobs:
       run: |
         mkdir -p release-assets
         # Python packages
-        cp artifacts/python-hindsight-all/* release-assets/
-        cp artifacts/python-hindsight-api/* release-assets/
-        cp artifacts/python-hindsight-client/* release-assets/
-        # TypeScript Client
-        cp artifacts/typescript-client/*.tgz release-assets/
+        cp artifacts/python-packages/hindsight-clients/python/dist/* release-assets/ || true
+        cp artifacts/python-packages/hindsight-api/dist/* release-assets/ || true
+        cp artifacts/python-packages/hindsight/dist/* release-assets/ || true
+        cp artifacts/python-packages/hindsight-integrations/litellm/dist/* release-assets/ || true
+        cp artifacts/python-packages/hindsight-embed/dist/* release-assets/ || true
+        # TypeScript client
+        cp artifacts/typescript-client/*.tgz release-assets/ || true
+        # OpenClaw Integration
+        cp artifacts/openclaw-integration/*.tgz release-assets/ || true
+        # AI SDK Integration
+        cp artifacts/ai-sdk-integration/*.tgz release-assets/ || true
+        # Control Plane
+        cp artifacts/control-plane/*.tgz release-assets/ || true
         # Rust CLI binaries
-        cp artifacts/rust-cli-hindsight-linux-amd64/hindsight-linux-amd64 release-assets/
-        cp artifacts/rust-cli-hindsight-darwin-amd64/hindsight-darwin-amd64 release-assets/
-        cp artifacts/rust-cli-hindsight-darwin-arm64/hindsight-darwin-arm64 release-assets/
+        cp artifacts/rust-cli-linux/hindsight-linux-amd64 release-assets/ || true
+        cp artifacts/rust-cli-darwin-amd64/hindsight-darwin-amd64 release-assets/ || true
+        cp artifacts/rust-cli-darwin-arm64/hindsight-darwin-arm64 release-assets/ || true
         # Helm chart
-        cp artifacts/helm-chart/*.tgz release-assets/
-
-    - name: Generate release notes
-      id: release_notes
-      run: |
-        cat << 'EOF' > release-notes.md
-        # Hindsight v${{ steps.get_version.outputs.VERSION }}
-
-        ## Quick Start
-
-        ```bash
-        docker run -p 8888:8888 -p 9999:9999 \
-          -e HINDSIGHT_API_LLM_PROVIDER=openai \
-          -e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \
-          -e HINDSIGHT_API_LLM_MODEL=gpt-4o-mini \
-          ghcr.io/${{ github.repository_owner }}/hindsight:${{ steps.get_version.outputs.VERSION }}
-        ```
-
-        ## 📦 Release Artifacts
-
-        ### Docker Images
-        - `ghcr.io/${{ github.repository_owner }}/hindsight:${{ steps.get_version.outputs.VERSION }}` - **Standalone all-in-one** (recommended)
-        - `ghcr.io/${{ github.repository_owner }}/hindsight-api:${{ steps.get_version.outputs.VERSION }}` - API server only
-        - `ghcr.io/${{ github.repository_owner }}/hindsight-control-plane:${{ steps.get_version.outputs.VERSION }}` - Web UI only
-
-        ### Python Packages
-        - `hindsight-all` - All-in-one package (includes API + client)
-        - `hindsight-api` - API server
-        - `hindsight-client` - Client library
-
-        ### TypeScript/JavaScript
-        - `@hindsight/client` - TypeScript SDK
-
-        ### CLI Binaries
-        - `hindsight-linux-amd64` - Linux x86_64
-        - `hindsight-darwin-amd64` - macOS Intel
-        - `hindsight-darwin-arm64` - macOS Apple Silicon
-
-        ### Helm Chart
-        - `hindsight-${{ steps.get_version.outputs.VERSION }}.tgz`
-
-        ## 🚀 Installation
-
-        ### Python
-        ```bash
-        # All-in-one (recommended)
-        pip install hindsight-all==${{ steps.get_version.outputs.VERSION }}
-
-        # Or install components separately
-        pip install hindsight-api==${{ steps.get_version.outputs.VERSION }}
-        pip install hindsight-client==${{ steps.get_version.outputs.VERSION }}
-        ```
-
-        ### TypeScript/JavaScript
-        ```bash
-        npm install @hindsight/client@${{ steps.get_version.outputs.VERSION }}
-        ```
-
-        ### CLI
-        ```bash
-        # macOS (Apple Silicon)
-        curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/hindsight-darwin-arm64 -o hindsight
-        chmod +x hindsight
-        sudo mv hindsight /usr/local/bin/
-
-        # macOS (Intel)
-        curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/hindsight-darwin-amd64 -o hindsight
-        chmod +x hindsight
-        sudo mv hindsight /usr/local/bin/
-
-        # Linux
-        curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/hindsight-linux-amd64 -o hindsight
-        chmod +x hindsight
-        sudo mv hindsight /usr/local/bin/
-        ```
-
-        ### Helm (Kubernetes)
-        ```bash
-        helm install hindsight oci://ghcr.io/${{ github.repository_owner }}/charts/hindsight --version ${{ steps.get_version.outputs.VERSION }}
-        ```
-        EOF
-        cat release-notes.md
+        cp artifacts/helm-chart/*.tgz release-assets/ || true
+        ls -la release-assets/
 
     - name: Create GitHub Release
       uses: softprops/action-gh-release@v2
       with:
         files: release-assets/*
-        body_path: release-notes.md
+        generate_release_notes: true
         draft: false
         prerelease: false
-        generate_release_notes: false
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Create release summary
-      run: |
-        echo "# Release v${{ steps.get_version.outputs.VERSION }} Published Successfully" >> $GITHUB_STEP_SUMMARY
-        echo "" >> $GITHUB_STEP_SUMMARY
-        echo "## 📦 Components" >> $GITHUB_STEP_SUMMARY
-        echo "- ✅ Python packages (hindsight-all, hindsight-api, hindsight-client)" >> $GITHUB_STEP_SUMMARY
-        echo "- ✅ TypeScript Client (@hindsight/client)" >> $GITHUB_STEP_SUMMARY
-        echo "- ✅ Rust CLI (Linux amd64, macOS amd64, macOS arm64)" >> $GITHUB_STEP_SUMMARY
-        echo "- ✅ Docker images (standalone, API, Control Plane)" >> $GITHUB_STEP_SUMMARY
-        echo "- ✅ Helm chart" >> $GITHUB_STEP_SUMMARY
-        echo "" >> $GITHUB_STEP_SUMMARY
-        echo "🎉 Release is now available at: https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -5,12 +5,18 @@ build/
 dist/
 wheels/
 *.egg-info
-
+.mcp.json
+.osgrep
 # Virtual environments
 .venv
 
-# Environment variables
+# Node
+node_modules/
+
+# Environment variables and local config
 .env
+docker-compose.yml
+docker-compose.override.yml
 
 # IDE
 .idea/
@@ -21,6 +27,10 @@ wheels/
 # NLTK data (will be downloaded automatically)
 nltk_data/
 
+# Monitoring stack (Prometheus/Grafana binaries and data)
+.monitoring/
+.pgbouncer/
+
 # Large benchmark datasets (will be downloaded automatically)
 **/longmemeval_s_cleaned.json
 
@@ -29,6 +39,20 @@ logs/
 
 .DS_Store
 
+# Generated docs files
+hindsight-docs/static/llms-full.txt
+
 
 hindsight-dev/benchmarks/locomo/results/
-hindsight-dev/benchmarks/longmemeval/results/
+hindsight-dev/benchmarks/longmemeval/results/
+hindsight-dev/benchmarks/consolidation/results/
+benchmarks/results/
+hindsight-cli/target
+hindsight-clients/rust/target
+.claude
+whats-next.md
+TASK.md
+# Changelog is now tracked in hindsight-docs/src/pages/changelog.md
+# CHANGELOG.md
+
+blog-post*

AGENTS.md

@@ -0,0 +1,3 @@
+# AGENTS.md
+
+See [CLAUDE.md](./CLAUDE.md) for project documentation and coding conventions.

CLAUDE.md

@@ -0,0 +1,283 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+Hindsight is an agent memory system that provides long-term memory for AI agents using biomimetic data structures. Memories are organized as:
+- **World facts**: General knowledge ("The sky is blue")
+- **Experience facts**: Personal experiences ("I visited Paris in 2023")
+- **Mental models**: Consolidated knowledge synthesized from facts ("User prefers functional programming patterns")
+
+## Development Commands
+
+### API Server (Python/FastAPI)
+```bash
+# Start API server (loads .env automatically)
+./scripts/dev/start-api.sh
+
+# Run all tests (parallelized with pytest-xdist)
+cd hindsight-api && uv run pytest tests/
+
+# Run specific test file
+cd hindsight-api && uv run pytest tests/test_http_api_integration.py -v
+
+# Run single test function
+cd hindsight-api && uv run pytest tests/test_retain.py::test_retain_simple -v
+
+# Lint and format
+cd hindsight-api && uv run ruff check .
+cd hindsight-api && uv run ruff format .
+
+# Type checking (uses ty - extremely fast type checker from Astral)
+cd hindsight-api && uv run ty check hindsight_api/
+```
+
+### Control Plane (Next.js)
+```bash
+./scripts/dev/start-control-plane.sh
+# Or manually:
+cd hindsight-control-plane && npm run dev
+```
+
+### Documentation Site (Docusaurus)
+```bash
+./scripts/dev/start-docs.sh
+```
+
+
+### Generating Clients/OpenAPI
+```bash
+# Regenerate OpenAPI spec after API changes (REQUIRED after changing endpoints)
+./scripts/generate-openapi.sh
+
+# Regenerate all client SDKs (Python, TypeScript, Rust)
+./scripts/generate-clients.sh
+```
+
+### Benchmarks
+```bash
+./scripts/benchmarks/run-longmemeval.sh
+./scripts/benchmarks/run-locomo.sh
+./scripts/benchmarks/start-visualizer.sh  # View results at localhost:8001
+```
+
+## Architecture
+
+### Monorepo Structure
+- **hindsight-api/**: Core FastAPI server with memory engine (Python, uv)
+- **hindsight/**: Embedded Python bundle (hindsight-all package)
+- **hindsight-control-plane/**: Admin UI (Next.js, npm)
+- **hindsight-cli/**: CLI tool (Rust, cargo, uses progenitor for API client)
+- **hindsight-clients/**: Generated SDK clients (Python, TypeScript, Rust)
+- **hindsight-docs/**: Docusaurus documentation site
+- **hindsight-integrations/**: Framework integrations (LiteLLM, OpenAI)
+- **hindsight-dev/**: Development tools and benchmarks
+
+### Core Engine (hindsight-api/hindsight_api/engine/)
+- `memory_engine.py`: Main orchestrator (~170KB) for retain/recall/reflect operations
+- `llm_wrapper.py`: LLM abstraction supporting OpenAI, Anthropic, Gemini, Groq, Ollama, LM Studio
+- `embeddings.py`: Embedding generation (local sentence-transformers or TEI)
+- `cross_encoder.py`: Reranking (local or TEI)
+- `entity_resolver.py`: Entity extraction and normalization
+- `query_analyzer.py`: Query intent analysis
+
+**retain/**: Memory ingestion pipeline
+- `orchestrator.py`: Coordinates the retain flow
+- `fact_extraction.py`: LLM-based fact extraction from content
+- `link_utils.py`: Entity link creation and management
+
+**search/**: Multi-strategy retrieval
+- `retrieval.py`: Main retrieval orchestrator
+- `graph_retrieval.py`: Entity/relationship graph traversal
+- `mpfp_retrieval.py`: Multi-Path Fact Propagation retrieval
+- `fusion.py`: Reciprocal rank fusion for combining results
+- `reranking.py`: Cross-encoder reranking
+
+### API Layer (hindsight-api/hindsight_api/api/)
+- `http.py`: FastAPI HTTP routers (~80KB) for all REST endpoints
+- `mcp.py`: Model Context Protocol server implementation
+
+Main operations:
+- **Retain**: Store memories, extracts facts/entities/relationships
+- **Recall**: Retrieve memories via 4 parallel strategies (semantic, BM25, graph, temporal) + reranking
+- **Reflect**: Disposition-aware reasoning using memories and mental models.
+
+### Database
+PostgreSQL with pgvector. Schema managed via Alembic migrations in `hindsight-api/hindsight_api/alembic/`. Migrations run automatically on API startup.
+
+Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
+
+### Adding Database Migrations
+
+1. **Create a new migration file** in `hindsight-api/hindsight_api/alembic/versions/`:
+   - File name format: `<revision_id>_<description>.py` (e.g., `f1a2b3c4d5e6_add_new_index.py`)
+   - Use a unique hex revision ID (12 chars)
+   - Set `down_revision` to the previous migration's revision ID
+
+2. **Migration template**:
+   ```python
+   """Description of the migration
+
+   Revision ID: f1a2b3c4d5e6
+   Revises: <previous_revision_id>
+   Create Date: YYYY-MM-DD
+   """
+   from collections.abc import Sequence
+   from alembic import context, op
+
+   revision: str = "f1a2b3c4d5e6"
+   down_revision: str | Sequence[str] | None = "<previous_revision_id>"
+   branch_labels: str | Sequence[str] | None = None
+   depends_on: str | Sequence[str] | None = None
+
+   def _get_schema_prefix() -> str:
+       """Get schema prefix for table names (required for multi-tenant support)."""
+       schema = context.config.get_main_option("target_schema")
+       return f'"{schema}".' if schema else ""
+
+   def upgrade() -> None:
+       schema = _get_schema_prefix()
+       op.execute(f"CREATE INDEX ... ON {schema}table_name(...)")
+
+   def downgrade() -> None:
+       schema = _get_schema_prefix()
+       op.execute(f"DROP INDEX IF EXISTS {schema}index_name")
+   ```
+
+3. **Run migrations locally**:
+   ```bash
+   # Set database URL and run migrations
+   uv run hindsight-admin run-db-migration
+
+   # Run on a specific tenant schema
+   uv run hindsight-admin run-db-migration --schema tenant_xyz
+   ```
+
+## Key Conventions
+
+### Code Quality
+**Always run the lint script after making Python or TypeScript/Node changes:**
+```bash
+./scripts/hooks/lint.sh
+```
+This runs the same checks as the pre-commit hook (Ruff for Python, ESLint/Prettier for TypeScript).
+
+### Memory Banks
+- Each bank is an isolated memory store (like a "brain" for one user/agent)
+- Banks have dispositions (skepticism, literalism, empathy traits 1-5) affecting reflect
+- Banks can have background context
+- Bank isolation is strict - no cross-bank data leakage
+
+### API Design
+- All endpoints operate on a single bank per request
+- Multi-bank queries are client responsibility to orchestrate
+- Disposition traits only affect reflect, not recall
+
+### Control Plane API Routes
+
+When adding or modifying parameters in the dataplane API (hindsight-api), you must also update the control plane routes that proxy to it:
+
+1. **API Routes** (`hindsight-control-plane/src/app/api/`):
+   - `recall/route.ts` - proxies to `/v1/default/banks/{bank_id}/memories/recall`
+   - `reflect/route.ts` - proxies to `/v1/default/banks/{bank_id}/reflect`
+   - `memories/retain/route.ts` - proxies to `/v1/default/banks/{bank_id}/memories/retain`
+   - Other routes follow the same pattern
+
+2. **Client types** (`hindsight-control-plane/src/lib/api.ts`):
+   - Update the TypeScript type definitions for `recall()`, `reflect()`, `retain()` etc.
+
+3. **Checklist when adding new API parameters**:
+   - Add parameter extraction in the route handler (destructure from `body`)
+   - Pass the parameter to the SDK call
+   - Update the client type definition in `lib/api.ts`
+   - Update any UI components that need to use the new parameter
+
+### Python Style
+- Python 3.11+, type hints required
+- Async throughout (asyncpg, async FastAPI)
+- Pydantic models for request/response
+- Ruff for linting (line-length 120)
+- No Python files at project root - maintain clean directory structure
+- **Never use multi-item tuple return values** - prefer dataclass or Pydantic model for structured returns
+
+### Type Safety with Pydantic Models
+**NEVER use raw `dict` types for structured data.** Always use Pydantic models:
+- Use Pydantic `BaseModel` for all data structures passed between functions
+- Add `@field_validator` for type coercion (e.g., ensuring datetimes are timezone-aware)
+- Avoid `dict.get()` patterns - use typed model attributes instead
+- Parse external data (JSON, API responses) into Pydantic models at the boundary
+- This catches type errors at parse time, not deep in business logic
+
+```python
+# BAD - error-prone dict access
+def process(data: dict) -> str:
+    return data.get("name", "")  # No validation, silent failures
+
+# GOOD - typed and validated
+class UserData(BaseModel):
+    name: str
+    created_at: datetime
+
+    @field_validator("created_at", mode="before")
+    @classmethod
+    def ensure_tz_aware(cls, v):
+        if isinstance(v, str):
+            v = datetime.fromisoformat(v.replace("Z", "+00:00"))
+        if v.tzinfo is None:
+            return v.replace(tzinfo=timezone.utc)
+        return v
+
+def process(data: UserData) -> str:
+    return data.name  # Type-safe, validated at construction
+```
+
+### TypeScript Style
+- Next.js App Router for control plane
+- Tailwind CSS with shadcn/ui components
+
+### Adding New API Configuration Flags
+
+When adding a new environment variable configuration:
+
+1. **config.py** (`hindsight-api/hindsight_api/config.py`):
+   - Add `ENV_*` constant for the environment variable name
+   - Add `DEFAULT_*` constant for the default value
+   - Add field to `HindsightConfig` dataclass
+   - Add initialization in `from_env()` method
+
+2. **main.py** (`hindsight-api/hindsight_api/main.py`):
+   - Add field to the manual `HindsightConfig()` constructor call (search for "CLI override")
+
+3. **Use the config** in code:
+   ```python
+   from ...config import get_config
+   config = get_config()
+   value = config.your_new_field
+   ```
+
+4. **Documentation** (`hindsight-docs/docs/developer/configuration.md`):
+   - Add to appropriate section table with Variable, Description, Default
+
+## Environment Setup
+
+```bash
+cp .env.example .env
+# Edit .env with LLM API key
+
+# Python deps
+uv sync --directory hindsight-api/
+
+# Node deps (uses npm workspaces)
+npm install
+```
+
+Required env vars:
+- `HINDSIGHT_API_LLM_PROVIDER`: openai, anthropic, gemini, groq, ollama, lmstudio
+- `HINDSIGHT_API_LLM_API_KEY`: Your API key
+- `HINDSIGHT_API_LLM_MODEL`: Model name (e.g., o3-mini, claude-sonnet-4-20250514)
+
+Optional (uses local models by default):
+- `HINDSIGHT_API_EMBEDDINGS_PROVIDER`: local (default) or tei
+- `HINDSIGHT_API_RERANKER_PROVIDER`: local (default) or tei
+- `HINDSIGHT_API_DATABASE_URL`: External PostgreSQL (uses embedded pg0 by default)

CODE_OF_CONDUCT.md

@@ -0,0 +1,127 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

CONTRIBUTING.md

@@ -5,13 +5,23 @@ Thanks for your interest in contributing to Hindsight!
 ## Getting Started
 
 1. Fork and clone the repository
-2. Install dependencies:
    ```bash
-   cd hindsight-api && uv sync
+   git clone [email protected]:vectorize-io/hindsight.git
+   cd hindsight
    ```
-3. Set up your environment:
+2. Set up your environment:
    ```bash
-   export OPENAI_API_KEY=your-key
+   cp .env.example .env
+   ```
+   Edit the .env to add LLM API key and config as required
+
+3. Install dependencies:
+   ```bash
+   # Python dependencies
+   uv sync --directory hindsight-api/
+
+   # Node dependencies (uses npm workspaces)
+   npm install
    ```
 
 ## Development
@@ -41,7 +51,36 @@ cd hindsight-api
 uv run pytest tests/
 ```
 
-### Code style
+### Code Style
+
+We use [Ruff](https://docs.astral.sh/ruff/) for Python linting and formatting, and ESLint/Prettier for TypeScript.
+
+#### Setting up git hooks (recommended)
+
+Set up git hooks to automatically lint and format code before each commit:
+
+```bash
+./scripts/setup-hooks.sh
+```
+
+This configures git to use the hooks in `.githooks/`, which run all scripts in `scripts/hooks/` on commit. The lint hook runs in parallel:
+- **Python**: `ruff check --fix`, `ruff format`, `ty check`
+- **TypeScript**: `eslint --fix`, `prettier`
+
+#### Manual linting and formatting
+
+```bash
+# Run all lints (same as pre-commit)
+./scripts/hooks/lint.sh
+
+# Or run individually for Python:
+cd hindsight-api
+uv run ruff check --fix .   # Lint and auto-fix
+uv run ruff format .        # Format code
+uv run ty check hindsight_api  # Type check
+```
+
+#### Style guidelines
 
 - Use Python type hints
 - Follow existing code patterns
@@ -54,6 +93,34 @@ uv run pytest tests/
 3. Run tests to ensure nothing breaks
 4. Submit a PR with a clear description of changes
 
+## Release Process
+
+The project uses `scripts/release.sh` for creating releases. This script automates the entire release workflow:
+
+1. Bumps version in all components (API, clients, CLI, control plane, Helm)
+2. **Regenerates OpenAPI spec and client SDKs** (Python, TypeScript, Rust)
+3. Updates documentation versioning
+4. Creates a commit and git tag
+5. Pushes to GitHub (triggers CI/CD to publish packages)
+
+### Usage
+
+```bash
+./scripts/release.sh <version>
+```
+
+**Example:**
+```bash
+./scripts/release.sh 0.5.0
+```
+
+### Important for Developers
+
+- During development, version bumps in `__init__.py` do NOT require client regeneration
+- Clients are only regenerated during releases
+- Do not manually run `./scripts/generate-clients.sh` unless testing generation changes
+- Client version comments will reflect the API version from the latest release
+
 ## Reporting Issues
 
 Open an issue on GitHub with:

HINDSIGHT_PAPER.md

@@ -1,965 +0,0 @@
-# Hindsight: A Unified Memory System for AI Agents with Temporal Retrieval and Personality-Driven Reasoning
-
-## Abstract
-
-We present **Hindsight**, a comprehensive memory architecture for conversational AI agents that combines multi-strategy retrieval with personality-driven reasoning to enable both high-recall factual search and consistent, trait-based opinion formation. The system consists of two integrated components: **TEMPR (Temporal Entity Memory Priming Retrieval)** for memory recall, and **CARA (Coherent Adaptive Reasoning Agents)** for personality-aware reflection. TEMPR achieves strong retrieval performance through four parallel search strategies—semantic vector search, BM25 keyword matching, graph-based spreading activation incorporating multiple link types (entity, semantic, temporal, causal), and temporal-aware graph traversal—achieving 73.50% on LoComo and 80.60% on LongMemEval benchmarks, with particularly strong performance on multi-hop reasoning (+15.8% over baseline). CARA builds on TEMPR's four-network architecture (world facts, bank experiences, opinions, and observations) to enable personality-driven reasoning using the Big Five model, allowing agents to form and evolve opinions influenced by configurable traits while maintaining epistemic clarity between objective information and subjective beliefs. A novel observation paradigm automatically synthesizes entity-level summaries from multiple facts, creating structured mental models of people, organizations, and concepts without personality influence. The combination enables AI agents with long-term memory that can both retrieve information accurately and reason consistently with stable character traits.
-
----
-
-# Part I: Recall - TEMPR (Temporal Entity Memory Priming Retrieval)
-
-## 1. Introduction to Recall
-
-Conversational AI agents face a fundamental challenge: maintaining coherent, context-aware memories across extended interactions. Traditional search systems are optimized for human users with top-k ranking and relevance feedback, but AI agents have fundamentally different requirements: they need to retrieve variable amounts of information based on reasoning complexity while respecting LLM context windows. Existing approaches rely either on vector similarity search, which captures semantic relationships but misses entity-level connections, or on keyword matching, which provides precision but lacks conceptual understanding. Neither approach adequately handles the temporal aspects of memory or entity-based reasoning that enable multi-hop information discovery.
-
-We propose TEMPR, a memory retrieval architecture designed specifically for AI agents that combines established information retrieval techniques—semantic vector search, BM25 keyword matching, spreading activation graph traversal (Anderson 1983), and neural reranking—into a unified system optimized for agent workflows. The key architectural choices are:
-
-1. **Agent-Optimized Interface**: budget and max_tokens parameters instead of traditional top-k ranking
-2. **Comprehensive Narrative Fact Extraction with Temporal Ranges**: LLM-powered extraction that creates self-contained narrative facts preserving full conversational context, extracting temporal ranges (occurred_start/end) to distinguish point events from periods
-3. **Entity-Aware Graph Structure with Multiple Link Types**: LLM-based entity resolution and linking that connects memories through shared identities, along with temporal, semantic, and causal link types
-4. **Four-Way Parallel Retrieval**: Semantic, keyword, graph-based (spreading activation), and temporal range retrieval strategies executed in parallel and fused using RRF (Cormack et al. 2009)
-5. **Neural Cross-Encoder Reranking**: Learned query-document relevance with temporal awareness and token budget filtering
-
-This combination of techniques enables agents to discover indirectly related information through graph traversal while maintaining temporal awareness, achieving strong performance on multi-hop reasoning tasks.
-
-### 1.1 Contributions
-
-Our key contributions for the recall system are:
-
-1. **Agent-Optimized Retrieval Interface**: Unlike traditional top-k search optimized for human users, we introduce budget and max_tokens parameters that allow AI agents to dynamically trade off latency for recall based on reasoning complexity and context window constraints
-
-2. **Four-Way Parallel Retrieval**: We combine semantic vector search, BM25 keyword matching, graph-based spreading activation (Anderson 1983), and temporal-aware graph traversal into a unified parallel retrieval pipeline using Reciprocal Rank Fusion (Cormack et al. 2009) and neural cross-encoder reranking. The graph traversal incorporates multiple link types (entity, semantic, temporal, causal) with configurable weighting during activation spreading.
-
-3. **LLM-Based Knowledge Graph Construction with Temporal Ranges**: We leverage open-source LLMs for comprehensive narrative fact extraction, entity recognition, and entity disambiguation. The system extracts temporal ranges (occurred_start, occurred_end) to represent both point events and extended periods, distinguishing when facts occurred from when they were mentioned.
-
-4. **Strong Performance on Multi-Hop Reasoning**: 73.50% on LoComo and 80.60% on LongMemEval, with particularly strong performance on multi-hop queries (+15.8% over Mem0), demonstrating the effectiveness of combining these techniques for discovering indirectly related information in conversational contexts
-
-## 2. Memory Organization
-
-### 2.1 Four Memory Networks
-
-TEMPR organizes memories into four distinct networks for epistemic clarity:
-
-**World Network** (fact_type='world'): Objective information about the world
-- Example: "Alice works at Google in Mountain View on the AI team"
-- Stores facts received from external sources
-- No confidence scores (facts are information received, not beliefs)
-
-**Bank Network** (fact_type='bank'): Biographical information about the agent itself
-- Example: "I recommended Yosemite National Park to Alice for hiking"
-- Stores the agent's own actions and experiences
-- Uses first-person perspective ("I recommended..." not "The agent recommended...")
-
-**Opinion Network** (fact_type='opinion'): Subjective beliefs formed by the agent
-- Example: "Python is better for data science because of libraries like pandas (confidence: 0.85)"
-- Stores judgments and opinions with confidence scores
-- Evolved through opinion reinforcement when new evidence arrives
-- Influenced by personality traits (see Part II: Reflect)
-
-**Observation Network** (fact_type='observation'): Synthesized entity summaries
-- Example: "Alice is a software engineer at Google specializing in machine learning"
-- Objective syntheses from multiple facts about an entity
-- Generated WITHOUT personality influence (unlike opinions)
-- Automatically created and updated in background processes
-- Provides structured "mental models" of entities
-
-This separation provides:
-- **Epistemic Clarity**: Facts represent information encountered; opinions represent personality-driven judgments; observations represent objective syntheses
-- **Traceability**: Opinion reinforcement traces facts; observations trace entity-related facts
-- **Debugging**: Developers can separately inspect factual knowledge, formed beliefs, and entity models
-- **Confidence Semantics**: Facts and observations lack confidence scores; opinions have confidence scores representing conviction strength
-- **Personality Independence**: Observations remain objective while opinions reflect personality
-
-### 2.2 Memory Unit Structure
-
-Each memory is represented as a self-contained node with:
-
-- id: Unique UUID
-- bank_id: Identifier for the memory bank this memory belongs to
-- text: Self-contained comprehensive narrative fact
-- embedding: 384-dimensional vector (BAAI/bge-small-en-v1.5)
-- event_date: Timestamp when the fact became true (maintained for backward compatibility)
-- occurred_start: Timestamp when the fact/event started (temporal range support)
-- occurred_end: Timestamp when the fact/event ended (temporal range support)
-- mentioned_at: Timestamp when the fact was mentioned/learned
-- context: Optional contextual metadata
-- fact_type: One of 'world', 'bank', 'opinion'
-- confidence_score: For opinions only, strength of conviction (0.0-1.0)
-- access_count: Frequency-based importance signal
-- search_vector: Full-text search tsvector for BM25 ranking
-
-### 2.3 LLM-Powered Comprehensive Narrative Fact Extraction
-
-TEMPR employs **LLM-powered comprehensive narrative fact extraction** using open-source models. This approach provides more context-aware extraction compared to traditional rule-based NLP pipelines, though at higher computational cost.
-
-#### 2.3.1 Extraction Principles
-
-**Chunking Strategy**: TEMPR uses a coarse-grained chunking approach, extracting 2-5 comprehensive facts per conversation rather than dozens of atomic fragments. This is a deliberate tradeoff: larger chunks preserve more context and narrative flow, at the cost of reduced precision when only a small portion of the chunk is relevant.
-
-Each fact should:
-1. **Capture entire conversations or exchanges** - Include the full back-and-forth discussion
-2. **Be narrative and comprehensive** - Tell the complete story with all context
-3. **Be self-contained** - Readable without the original text
-4. **Include all participants** - WHO said/did WHAT, with their reasoning
-5. **Preserve the flow** - Keep related exchanges together in one fact
-
-**Example Comparison**:
-
-❌ **Fragmented Approach** (traditional):
-- "Bob suggested Summer Vibes"
-- "Alice wanted something unique"
-- "They considered Sunset Sessions"
-- "Alice likes Beach Beats"
-- "They chose Beach Beats"
-
-✅ **Comprehensive Approach** (TEMPR):
-- "Alice and Bob discussed naming their summer party playlist. Bob suggested 'Summer Vibes' because it's catchy and seasonal, but Alice wanted something more unique. Bob then proposed 'Sunset Sessions' and 'Beach Beats', with Alice favoring 'Beach Beats' for its playful and fun tone. They ultimately decided on 'Beach Beats' as the final name."
-
-#### 2.3.2 Open-Source LLM Extraction Pipeline
-
-The extraction process leverages open-source LLMs with structured output (Pydantic schemas). This follows the established practice of using LLMs for information extraction, which has been shown to improve context understanding compared to rule-based NLP pipelines, particularly for:
-- Coreference resolution in conversational text
-- Domain-specific entity recognition
-- Maintaining narrative coherence across multi-turn exchanges
-
-**LLM Extraction Steps**:
-1. **Pronoun Resolution**: "She loves hiking" → "Alice loves hiking"
-2. **Temporal Normalization**: "last year" → "in 2023" (absolute dates)
-3. **Temporal Range Extraction**: Identify when facts occurred vs. when mentioned
-   - Point events: "on July 14" → occurred_start = occurred_end = 2023-07-14
-   - Period events: "in February 2023" → occurred_start = 2023-02-01, occurred_end = 2023-02-28
-   - Vague periods: "lately" → estimated range based on context
-   - mentioned_at = conversation date (when fact was learned)
-4. **Participant Attribution**: Preserve WHO said/did WHAT
-5. **Reasoning Preservation**: Include WHY decisions were made
-6. **Fact Type Classification**: Determine fact categories (world, bank, opinion)
-7. **Entity Extraction**: Identify all entities (PERSON, ORG, LOCATION, PRODUCT, CONCEPT)
-
-**Temporal Augmentation**: Before embedding, facts are augmented with readable temporal information:
-- Original: "Alice started working at Google"
-- Augmented for embedding: "Alice started working at Google (happened in November 2023)"
-
-This augmentation helps semantic search understand temporal relevance without modifying the stored fact text.
-
-### 2.4 Entity Resolution and Linking
-
-Entity resolution creates strong connections between memories that share common entities, solving the problem where semantically dissimilar facts are related through shared identities.
-
-#### 2.4.1 LLM-Based Entity Recognition
-
-TEMPR uses the same open-source LLM that performs fact extraction to also identify and extract entities during the narrative fact creation process. This unified approach eliminates the brittleness of traditional NER pipelines that struggle with domain-specific entities, novel names, and context-dependent disambiguation.
-
-**Entity Types**:
-- PERSON: "Alice", "Bob Chen"
-- ORGANIZATION: "Google", "Stanford University"
-- LOCATION: "Yosemite National Park", "California"
-- PRODUCT: "Python", "pandas library"
-- CONCEPT: "machine learning", "remote work"
-- OTHER: Miscellaneous proper nouns
-
-#### 2.4.2 LLM-Based Entity Disambiguation
-
-Multiple mentions of entities (e.g., "Alice", "Alice Chen", "Alice C.") must be resolved to a single canonical entity. TEMPR uses the LLM to perform entity disambiguation, analyzing the surrounding context to determine if two entity mentions refer to the same entity. This handles complex cases like:
-- Nicknames and formal names ("Bob" vs. "Robert Chen")
-- Partial mentions ("Alice" vs. "Alice Chen")
-- Context-dependent disambiguation ("Apple the company" vs. "apple the fruit")
-
-The LLM considers multiple signals:
-- **Name Similarity**: String similarity using Levenshtein distance
-- **Co-occurrence Patterns**: Entities mentioned together frequently are likely distinct
-- **Temporal Proximity**: Recent mentions are more likely to refer to the same entity
-
-#### 2.4.3 Entity Link Structure
-
-Each entity creates a link_type='entity' edge between all memories mentioning it:
-
-**Properties**:
-- weight=1.0 (constant, no temporal decay)
-- entity_id: Reference to resolved canonical entity
-- Bidirectional connections between all mentioning memories
-
-**Impact on Retrieval**: Entity links enable graph traversal to discover indirectly related facts:
-
-**Example Query**: "What does Alice do?"
-1. **Semantic Match**: "Alice works at Google in Mountain View..." (direct match)
-2. **Entity Traversal**: Follow entity links for "Alice" →
-   - "Alice loves hiking in Yosemite..." (different semantic space)
-   - "I recommended technical books to Alice" (Bank Network, via "Alice")
-3. **Chained Traversal**: Follow "Google" entity →
-   - "Google's office in Mountain View has excellent amenities"
-
-### 2.5 Link Types and Graph Structure
-
-The memory graph contains four types of edges connecting memory units:
-
-#### 2.5.1 Temporal Links
-
-Temporal links connect memories close in time, enabling temporal reasoning:
-
-**Creation Logic**:
-
-**Properties**:
-- Decays linearly with time distance
-- Minimum weight 0.3 to maintain some connectivity
-- Enables "What happened around the same time?" queries
-
-#### 2.5.2 Semantic Links
-
-Semantic links connect memories with similar meanings:
-
-**Creation Logic**:
-
-**Properties**:
-- Uses pgvector HNSW index for efficient nearest-neighbor search
-- Higher threshold (0.7) than retrieval (0.3) to avoid over-connection
-- Weight equals cosine similarity score
-
-#### 2.5.3 Entity Links
-
-Entity links (described in Section 2.4.3) create the strongest connections:
-
-**Properties**:
-- weight=1.0 (constant, never decays)
-- Connects all memories mentioning the same resolved entity
-- Most reliable traversal path during graph search
-
-#### 2.5.4 Causal Links
-
-Causal links represent identified cause-effect relationships between facts. During fact extraction, the LLM attempts to identify causal relationships between facts extracted from the same conversation. These links are incorporated as one component of the graph retrieval system.
-
-**Causal Relationship Types**:
-- causes: This fact directly causes the target fact
-- caused_by: This fact was caused by the target fact (inverse of causes)
-- enables: This fact enables or allows the target fact to happen
-- prevents: This fact prevents or blocks the target fact
-
-**Properties**:
-- weight: Strength of causal relationship ∈ [0.0, 1.0] (default 1.0)
-- Directional edges (from cause to effect)
-- Prioritized during graph traversal with 2x activation boost
-
-**Role in Retrieval**: Causal links provide an additional signal during graph-based retrieval. When present, they allow the system to traverse explanatory relationships in addition to semantic, temporal, and entity-based connections.
-
-**Example**: For a query "Why does Alice spend time in the garden?", the system may find both direct semantic matches ("Alice spends time in the garden to find comfort") and traverse causal links to related facts ("Alice lost her friend Karlie in February 2023").
-
-**Graph Density**: Each memory unit typically has:
-- 5-10 temporal links (to nearby memories)
-- 3-5 semantic links (to similar content)
-- Variable entity links (depending on entity mention frequency)
-- 0-3 causal links (when causal relationships are identified)
-
-### 2.6 The Observation Paradigm
-
-A critical challenge in long-term memory systems is maintaining structured, high-level understanding of entities (people, organizations, places, concepts) without re-reading all individual facts each time. Traditional approaches either retrieve all entity-related facts (expensive, noisy) or maintain no entity-level state (losing structured understanding). Hindsight introduces **observations**—automatically synthesized entity summaries that provide structured "mental models" without personality influence.
-
-#### 2.6.1 Motivation and Design
-
-**The Problem**: When a system accumulates dozens of facts about an entity like "Alice," queries about Alice must either:
-1. Retrieve all 50+ individual facts (expensive, overwhelming)
-2. Rely only on top-k semantic matches (may miss key attributes)
-3. Manually maintain entity profiles (doesn't scale, requires human curation)
-
-**The Solution**: Observations provide a fourth fact type that synthesizes multiple facts into coherent, objective entity summaries, automatically maintained as new information arrives.
-
-**Key Properties**:
-- **Objective Synthesis**: Generated WITHOUT personality influence (unlike opinions)
-- **Entity-Scoped**: Each observation is about a single entity
-- **Automatic Maintenance**: Generated in background after fact ingestion
-- **Multi-Fact Fusion**: Combines information scattered across multiple facts
-- **Response Augmentation**: NOT used for retrieval/search, but returned alongside results when include_entities=True to provide entity context
-
-#### 2.6.2 Observation Generation
-
-Observations are generated through an LLM-powered synthesis process:
-
-**Trigger**: When new facts mentioning an entity are ingested via retain(), a background task is queued to regenerate observations for that entity.
-
-**Process**:
-
-**LLM Prompt Structure**:
-
-**Example Transformation**:
-
-**Input Facts**:
-- "Alice works at Google"
-- "Alice is a software engineer"
-- "Alice specializes in ML and deep learning"
-- "Alice joined Google in 2023"
-- "Alice is detail-oriented and methodical"
-
-**Generated Observations**:
-- "Alice is a software engineer at Google specializing in machine learning and deep learning"
-- "Alice joined Google in 2023"
-- "Alice is detail-oriented and methodical in her approach"
-
-#### 2.6.3 Storage and Retrieval
-
-**Storage**: Observations are stored as regular memory_units with fact_type='observation':
-
-
-**Entity Links**: Observations are linked to their entity via the entity_links table, enabling efficient lookup of all observations for an entity.
-
-**Important**: Observations are NOT used during the retrieval/search process itself. They do not participate in the 4-way parallel search (semantic, keyword, graph, temporal). Instead, they are **response augmentations**—additional context returned alongside search results.
-
-**Response Augmentation**: When calling recall() with include_entities=True:
-
-
-**Response Structure**:
-
-#### 2.6.4 Observations vs. Opinions
-
-A critical distinction separates observations from opinions:
-
-| Dimension | Observations | Opinions |
-|-----------|-------------|----------|
-| **Influence** | No personality influence | Influenced by Big Five traits |
-| **Purpose** | Objective entity summaries | Subjective beliefs and judgments |
-| **Confidence** | No confidence score | Confidence score (0.0-1.0) |
-| **Generation** | Background synthesis from facts | Formed during reflect() reasoning |
-| **Update Mechanism** | Regenerated when entity facts change | Updated via opinion reinforcement |
-| **Example** | "Alice is a software engineer at Google" | "Alice is an excellent engineer" |
-
-**Why Both?**: Observations provide factual entity understanding for retrieval contexts, while opinions represent the memory bank's personality-driven beliefs for reasoning contexts. A memory bank can have objective observations about Alice (she works at Google, specializes in ML) AND personality-influenced opinions about Alice (she's a talented engineer, she'd be great for project X).
-
-#### 2.6.5 Background Processing
-
-Observation generation is asynchronous to avoid blocking retain() operations:
-
-**Flow**:
-
-This design ensures low-latency writes while maintaining fresh entity summaries.
-
-#### 2.6.6 Benefits and Use Cases
-
-**Benefits**:
-
-1. **Contextual Entity Summaries**: After retrieving facts that mention entities, observations provide synthesized context about those entities without requiring separate queries
-2. **Structured Entity Understanding**: Provides coherent mental models of entities as response augmentation
-3. **Token Efficiency**: 3-5 observations provide more structured context than retrieving all entity-related facts
-4. **Objective Grounding**: When reflecting with personality, observations provide objective entity context
-5. **Scalability**: Automatically maintained as facts accumulate, always fresh when needed
-6. **Separation of Concerns**: Search focuses on relevant facts through semantic similarity, keyword matching, and graph traversal; observations provide entity context post-retrieval
-
-**Note on Observation Stability**: While observations are regenerated when entity facts change, the core retrieval mechanism remains grounded in the original facts. The four-way parallel search (semantic, keyword, graph, temporal) retrieves facts based on query relevance, semantic co-occurrence, and entity relationships—not based on observations. This ensures that the most relevant factual information is surfaced regardless of how observations may evolve over time.
-
-**Use Cases**:
-
-**Multi-Agent Conversations**: When retrieving facts that mention people, observations provide shared, objective entity context:
-
-**Entity-Centric Queries**: "Tell me about Alice" retrieves facts about Alice, and observations provide synthesized entity summary in the response.
-
-**Contextual Reasoning**: When forming opinions during reflect(), observations provide factual entity grounding alongside retrieved facts.
-
-**Knowledge Graph Interfaces**: Observations can be exposed as structured entity profiles in UIs or APIs via dedicated entity endpoints.
-
-## 3. Retrieval Architecture
-
-Our retrieval pipeline addresses the fundamental challenge of long-term memory: achieving both **high recall** (finding all relevant information) and **high precision** (ranking the most relevant items first).
-
-### 3.1 Four-Way Parallel Retrieval
-
-We execute four complementary retrieval strategies in parallel, each capturing different aspects of relevance:
-
-#### 3.1.1 Semantic Retrieval (Vector Similarity)
-
-**Method**: Cosine similarity between query embedding and memory embeddings
-**Index**: pgvector HNSW (Hierarchical Navigable Small World)
-**Threshold**: ≥ 0.3 similarity
-
-**Implementation**:
-
-**Advantages**:
-- Captures conceptual similarity
-- Handles synonyms and paraphrasing
-- Language-model understanding of meaning
-
-**Limitations**:
-- Misses exact proper nouns if not in training data
-- Cannot reason about temporal relationships
-- Weak at entity disambiguation
-
-#### 3.1.2 Keyword Retrieval (BM25 Full-Text Search)
-
-**Method**: PostgreSQL full-text search with BM25 ranking (ts_rank_cd)
-**Index**: GIN index on to_tsvector('english', text)
-
-**Advantages**:
-- High precision for proper nouns and technical terms
-- Exact phrase matching
-- Fast execution with GIN index
-
-**Limitations**:
-- No semantic understanding
-- Requires exact or stemmed matches
-
-**Complementarity**: Semantic + Keyword achieves >90% recall: vector search catches concepts, BM25 catches exact names.
-
-#### 3.1.3 Graph Retrieval (Spreading Activation)
-
-**Method**: Activation spreading from semantic entry points through the memory graph, following the spreading activation model of memory (Anderson 1983).
-
-**Algorithm**:
-
-**Decay Mechanism**: Activation decays by 0.8 per hop, limiting spread to ~4-5 hops.
-
-**Link Weighting with Causal Boosting**:
-- **Causal links**: Base weight × 2.0 boost (causes/caused_by) or × 1.5 boost (enables/prevents)
-- **Entity links**: weight 1.0 (no boost, already strong signal)
-- **Semantic links**: weight ∈ [0.7, 1.0] (cosine similarity, no boost)
-- **Temporal links**: weight ∈ [0.3, 1.0] (time-based decay, no boost)
-
-**Advantages**:
-- Discovers indirectly related facts through graph connectivity
-- Leverages entity links to traverse knowledge graph
-- Finds context-adjacent memories via temporal links
-- Prioritizes explanatory relationships through causal boosting
-
-#### 3.1.4 Temporal Graph Retrieval (Time-Constrained + Spreading)
-
-**Activation Condition**: Only triggered when temporal constraint detected in query
-
-**Temporal Parsing**: Uses google/flan-t5-small (80M parameters) to extract temporal constraints from natural language queries:
-- "last spring" → 2024-03-01 to 2024-05-31
-- "in June" → 2024-06-01 to 2024-06-30
-- "last year" → 2024-01-01 to 2024-12-31
-- "between March and May" → 2025-03-01 to 2025-05-31
-
-**Temporal Range Matching**: Facts are matched against time constraints using their temporal range (occurred_start, occurred_end):
-
-
-**Algorithm**:
-
-### 3.2 Reciprocal Rank Fusion (RRF)
-
-After parallel retrieval, we merge 3-4 ranked lists using Reciprocal Rank Fusion (Cormack et al. 2009):
-
-**Algorithm**:
-
-**Advantages over Score-Based Fusion**:
-- **Rank-based**: Position matters more than absolute scores
-- **Robust to missing items**: Missing from a list contributes 0, not a penalty
-- **Multi-evidence weighting**: Items appearing in multiple lists rank higher
-
-### 3.3 Neural Cross-Encoder Reranking
-
-After RRF fusion, TEMPR applies neural cross-encoder reranking to refine precision:
-
-**Model**: cross-encoder/ms-marco-MiniLM-L-6-v2 (pretrained on MS MARCO passage ranking)
-
-**Algorithm**:
-
-**Advantages**:
-- Learns query-document relevance patterns from supervised data
-- Considers full query-document interaction
-- Temporal awareness through formatted date context
-
-### 3.4 Token Budget Filtering
-
-Final stage applies token budget filtering to limit context window usage:
-
-**Algorithm**:
-
-**Purpose**: Ensures retrieved facts fit within LLM context windows while maximizing information density.
-
-### 3.5 Complete Retrieval Pipeline
-
-**End-to-End Flow**:
-
-## 4. Evaluation
-
-We evaluate TEMPR on two established long-term memory benchmarks: LoComo (Long-term Conversation Memory) and LongMemEval.
-
-### 4.1 LoComo Benchmark
-
-LoComo evaluates conversational memory systems across four dimensions: single-hop queries, multi-hop queries, open-domain queries, and temporal queries.
-
-**Results**:
-
-| Method | Single Hop J ↑ | Multi-Hop J ↑ | Open Domain J ↑ | Temporal J ↑ | Overall |
-|--------|---------------|---------------|-----------------|--------------|---------|
-| A-Mem* | 39.79 | 18.85 | 54.05 | 31.08 | 48.38 |
-| LangMem | 62.23 | 47.92 | 71.12 | 23.43 | 58.10 |
-| Zep (Mem0 paper) | 61.70 | 41.35 | 76.60 | 49.31 | 65.99 |
-| OpenAI | 63.79 | 42.92 | 62.29 | 21.71 | 52.90 |
-| Mem0 | 67.13 | 51.15 | 72.93 | 55.51 | 66.88 |
-| Mem0 w/ Graph | 65.71 | 47.19 | 75.71 | 58.13 | 68.44 |
-| **TEMPR** | **73.20** | **66.90** | **78.60** | **56.30** | **73.50** |
-
-**Analysis**: TEMPR achieves strong performance across all query types:
-- **Single-Hop (+6.1% vs Mem0)**: Superior performance due to comprehensive narrative facts and BM25 keyword matching
-- **Multi-Hop (+15.8% vs Mem0)**: Largest improvement, demonstrating effectiveness of graph-based spreading activation
-- **Open Domain (+2.9% vs Mem0)**: Strong performance through multi-strategy parallel retrieval
-- **Temporal (-1.8% vs Mem0 w/ Graph)**: Competitive temporal reasoning
-
-### 4.2 LongMemEval Benchmark
-
-LongMemEval assesses memory systems across six dimensions:
-
-**Results**:
-
-| Method | Single-Session Preference | Single-Session Assistant | Temporal Reasoning | Multi-Session | Knowledge Update | Single-Session User | Overall |
-|--------|--------------------------|-------------------------|-------------------|---------------|-----------------|-------------------|---------|
-| Zep gpt-4o-mini | 53.30% | 75.00% | 54.10% | 47.40% | 74.40% | 92.90% | 63.80% |
-| Zep gpt-4o | 56.70% | 80.40% | 62.40% | 57.90% | 83.30% | 92.90% | 71.00% |
-| **TEMPR** | **83.30%** | **80.40%** | **75.90%** | **75.20%** | **85.90%** | **92.90%** | **80.60%** |
-| Mastra gpt-4o | 46.70% | 100.00% | 75.20% | 76.70% | 84.60% | 97.10% | 80.05% |
-
-**Analysis**: TEMPR achieves competitive performance:
-- **Single-Session Preference (+26.6% vs Zep gpt-4o)**: Dramatic improvement enabled by comprehensive narrative facts
-- **Temporal Reasoning (+13.5% vs Zep gpt-4o)**: Strong performance through dedicated temporal graph retrieval
-- **Multi-Session (+17.3% vs Zep gpt-4o)**: Entity-aware graph linking maintains consistency
-
-The 80.60% overall score represents a 9.6 percentage point improvement over Zep gpt-4o (71.00%).
-
----
-
-# Part II: Reflect - CARA (Coherent Adaptive Reasoning Agents)
-
-## 5. Introduction to Reflect
-
-Conversational AI agents increasingly need to maintain consistent perspectives and form judgments that reflect stable character traits. Current systems either provide purely objective information retrieval without perspective, or generate responses that lack consistency across interactions. Human conversation partners expect agents to have stable viewpoints, preferences, and reasoning styles—characteristics that emerge from personality.
-
-We propose CARA (Coherent Adaptive Reasoning Agents), a personality framework that addresses these limitations through:
-
-1. **Big Five Personality Integration**: Configurable traits (OCEAN model) that influence how agents interpret facts and form opinions
-2. **TEMPR Memory Integration**: Leverages TEMPR's three-network architecture (world facts, bank experiences, opinions) for sophisticated memory access
-3. **Opinion Reinforcement**: Dynamic belief updating when new evidence reinforces, weakens, or contradicts existing opinions
-4. **Personality Bias Control**: Adjustable influence strength allowing agents to range from objective to strongly personality-driven
-5. **Background Merging**: LLM-powered integration of biographical information with intelligent conflict resolution
-
-This architecture enables agents to maintain consistent identities while allowing beliefs to evolve naturally with new information.
-
-### 5.1 Motivation
-
-Consider an agent discussing remote work. With high openness (0.9) and low conscientiousness (0.2), the agent might form the opinion: "Remote work enables creative flexibility and spontaneous innovation." The same facts presented to an agent with low openness (0.2) and high conscientiousness (0.9) might yield: "Remote work lacks the structure and accountability needed for consistent performance."
-
-Both agents access identical factual information, but personality traits bias how they weight different aspects (flexibility vs. structure) and what conclusions they draw. This mirrors human reasoning—our personalities influence what we attend to and how we integrate information into our worldview.
-
-### 5.2 Contributions
-
-Our key contributions for the reflect system are:
-
-1. **Personality-Aware Reasoning**: A prompt engineering framework that injects Big Five traits into LLM reasoning, demonstrating how personality consistently biases opinion formation
-
-2. **TEMPR-Based Three-Network Architecture**: Integration with TEMPR to manage three distinct networks (world facts, bank experiences, opinions), enabling architectural separation between objective information and subjective beliefs with epistemic clarity and traceability
-
-3. **Opinion Reinforcement Mechanism**: An automatic belief update system that adjusts confidence scores when new evidence arrives, creating dynamic belief systems that evolve with information
-
-4. **Background Merging with Conflict Resolution**: An LLM-powered method for maintaining coherent agent identities when new biographical information contradicts existing background
-
-5. **Bias Strength Control**: A meta-parameter that allows tuning personality influence from objective (0.0) to strongly subjective (1.0), enabling task-appropriate personality expression
-
-## 6. Personality Model
-
-### 6.1 Big Five Framework
-
-We adopt the **Big Five** personality model (OCEAN), which is empirically validated across cultures and provides continuous trait dimensions:
-
-**Trait Dimensions** (each 0.0-1.0):
-
-1. **Openness (O)**: Receptiveness to new ideas, creativity, abstract thinking
-   - High: "I embrace novel approaches", "innovation over tradition"
-   - Low: "I prefer proven methods", "tradition over experimentation"
-
-2. **Conscientiousness (C)**: Organization, goal-directed behavior, dependability
-   - High: "I plan systematically", "evidence-based decisions"
-   - Low: "I work flexibly", "intuition-based decisions"
-
-3. **Extraversion (E)**: Sociability, assertiveness, energy from interaction
-   - High: "I seek collaboration", "enthusiastic communication"
-   - Low: "I prefer solitude", "measured communication"
-
-4. **Agreeableness (A)**: Cooperation, empathy, conflict avoidance
-   - High: "I seek consensus", "consider social harmony"
-   - Low: "I express dissent", "prioritize accuracy over harmony"
-
-5. **Neuroticism (N)**: Emotional sensitivity, anxiety, stress response
-   - High: "I consider risks carefully", "emotionally engaged"
-   - Low: "I remain calm under uncertainty", "emotionally detached"
-
-**Bias Strength** (0.0-1.0): Meta-parameter controlling how much personality influences opinions
-- 0.0: Neutral, fact-based reasoning (no personality bias)
-- 0.5: Moderate personality influence, balanced with objective analysis
-- 1.0: Strong personality influence, facts filtered through trait lens
-
-### 6.2 Psychological Basis
-
-The Big Five model has several advantages for AI agents:
-
-1. **Empirical Validation**: Decades of psychological research demonstrate cross-cultural stability and predictive validity
-2. **Continuous Dimensions**: Unlike categorical types, continuous scales allow fine-grained personality tuning
-3. **Behavioral Prediction**: Traits predict information processing styles, decision-making approaches, and communication preferences
-4. **Interpretability**: Well-understood trait meanings enable users to anticipate agent behavior
-
-**Trait Influence on Reasoning**:
-- **High Openness**: Favors novel solutions, abstract thinking, considers unconventional perspectives
-- **High Conscientiousness**: Emphasizes systematic analysis, evidence quality, long-term consequences
-- **High Extraversion**: Considers social aspects, collaborative solutions, enthusiastic expression
-- **High Agreeableness**: Weights harmony, considers multiple viewpoints, seeks consensus
-- **High Neuroticism**: Attends to risks, emotional implications, uncertainty
-
-## 7. Bank Profile Structure
-
-### 7.1 Profile Schema
-
-Each memory bank has an associated profile containing identity information:
-
-
-**Name Field**: Memory bank's name used in prompts and self-reference ("Your name: Marcus")
-
-**Personality Field**: JSONB containing six continuous values (five traits + bias strength)
-
-**Background Field**: First-person narrative describing the agent's biographical context:
-- "I am a software engineer with 10 years of startup experience"
-- "I was born in Texas and value innovation over tradition"
-- "I am a creative artist interested in digital media"
-
-### 7.2 Trait Description Generation
-
-Personality traits are translated into natural language descriptions for LLM prompts:
-
-
-**Example Output** (openness=0.9, conscientiousness=0.2, extraversion=0.7, agreeableness=0.3, neuroticism=0.5):
-
-This verbalization makes traits interpretable to the LLM, enabling personality-biased reasoning.
-
-## 8. Opinion Network and Opinion Formation
-
-### 8.1 Opinion Structure
-
-Opinions are stored as memory units in the dedicated opinion network (fact_type='opinion'):
-
-**Core Attributes**:
-- text: The opinion statement with explicit reasoning
-- confidence_score: Opinion strength and resistance to change (0.0-1.0)
-- event_date: When the opinion was formed
-- bank_id: Which memory bank holds this opinion
-- entities: Mentioned entities (for reinforcement triggering)
-
-**Example Opinion**:
-
-**Fact vs. Opinion Separation**:
-
-A critical architectural distinction separates **facts** (objective information stored in world/bank networks) from **opinions** (subjective beliefs stored in the opinion network). This separation provides:
-
-1. **Epistemic Clarity**: Facts represent information encountered; opinions represent judgments formed
-2. **Traceability**: Opinion reinforcement can trace which facts influenced belief updates
-3. **Debugging**: Developers can separately inspect factual knowledge vs. formed beliefs
-4. **Confidence Semantics**: Facts lack confidence scores; opinions have confidence scores
-
-### 8.2 Opinion Formation
-
-Opinions are generated during "reflect" operations—when the agent is asked to reason about a topic and form a judgment.
-
-**Formation Process**:
-1. Retrieve relevant facts from all memory networks (world, bank, existing opinions) using TEMPR
-2. Inject bank profile (name, personality, background) into LLM prompt
-3. Generate reasoning with personality bias applied
-4. Extract new opinions from response using structured output
-5. Store opinions with confidence scores in opinion network
-
-**Prompt Structure** (bias_strength=0.8):
-
-### 8.3 System Message Adaptation
-
-The system message adjusts based on bias strength to control personality influence:
-
-**High bias (≥0.7)**:
-
-**Moderate bias (0.4-0.7)**:
-
-**Low bias (<0.4)**:
-
-### 8.4 Confidence Score Semantics
-
-Confidence scores represent opinion strength—how firmly the agent holds the belief:
-
-- **0.9-1.0**: Very strong conviction, deeply held belief
-- **0.7-0.9**: Strong conviction, firmly held opinion
-- **0.5-0.7**: Moderate conviction, open to revision
-- **0.3-0.5**: Weak conviction, easily influenced
-- **0.0-0.3**: Very weak conviction, highly malleable
-
-**LLM Generation**: Confidence scores are extracted using structured output (Pydantic schema):
-
-
-## 9. Opinion Reinforcement
-
-### 9.1 Motivation
-
-Human beliefs evolve as we encounter new information. Supporting evidence strengthens beliefs, contradictory evidence weakens them, and sufficient contradiction causes belief revision. Opinion reinforcement implements this dynamic belief updating.
-
-### 9.2 Reinforcement Mechanism
-
-When new facts are ingested (via retain), the system:
-
-1. **Identify Related Opinions**: Find existing opinions that mention entities in the new facts
-2. **Evaluate Evidence Relationship**: Use LLM to determine if new facts:
-   - **Reinforce**: Support the existing opinion (increase confidence)
-   - **Weaken**: Contradict the existing opinion (decrease confidence)
-   - **Contradict**: Strongly contradict, requiring opinion revision
-   - **Neutral**: Unrelated or no clear relationship
-3. **Update Opinions**: Adjust confidence scores or revise opinion text based on evaluation
-
-**Example Reinforcement**:
-
-**Existing Opinion** (confidence: 0.7):
-
-**New Fact**:
-
-**LLM Evaluation**: "This evidence REINFORCES the opinion with strong quantitative support."
-
-**Updated Opinion** (confidence: 0.85):
-
-### 9.3 Reinforcement Algorithm
-
-
-### 9.4 Reinforcement Guarantees
-
-**Consistency**: Opinions are only updated when new facts genuinely relate to existing beliefs
-
-**Personality Coherence**: Reinforcement evaluation incorporates bank personality, ensuring updates align with trait-driven reasoning
-
-**Transparency**: Each update records the triggering facts and reasoning, providing an audit trail
-
-**Bounded Updates**: Confidence changes are bounded (±0.1-0.15 per update) to prevent extreme swings
-
-## 10. Background Merging
-
-### 10.1 Challenge
-
-Memory bank backgrounds accumulate biographical information over time. New information may:
-- **Complement**: Add new facts without contradiction
-- **Conflict**: Contradict existing facts ("born in Texas" vs. "born in Colorado")
-- **Refine**: Provide more specific versions of existing facts
-
-Naive concatenation creates incoherent backgrounds with contradictions. We need intelligent merging.
-
-### 10.2 LLM-Powered Merging
-
-We use an LLM to merge backgrounds with conflict resolution:
-
-**Merge Rules**:
-1. **New overwrites old** when contradictory
-2. **Add non-conflicting** information
-3. **Maintain first-person** perspective ("I..." not "You...")
-4. **Keep concise** (under 500 characters)
-
-**Prompt Template**:
-
-**Example Merges**:
-
-**Conflict Resolution**:
-- Current: "I was born in Colorado"
-- New: "You were born in Texas"
-- Result: "I was born in Texas"
-
-**Addition**:
-- Current: "I was born in Texas"
-- New: "I have 10 years of startup experience"
-- Result: "I was born in Texas. I have 10 years of startup experience."
-
-### 10.3 First-Person Normalization
-
-Users may provide background in second person ("You are..."), but internal storage maintains first person for consistency in prompts.
-
-**Normalization**: LLM automatically converts:
-- "You are a creative engineer" → "I am a creative engineer"
-- "You were born in 1990" → "I was born in 1990"
-- "You value innovation" → "I value innovation"
-
-## 11. Personality-Driven Reasoning Examples
-
-### 11.1 Example: Remote Work Discussion
-
-**Scenario**: Two memory banks with opposite personalities discuss remote work given identical facts.
-
-**Facts** (both banks receive):
-- "Remote work eliminates commute time (average 1 hour/day saved)"
-- "Office work provides spontaneous collaboration and mentorship"
-- "Studies show 65% of remote workers report higher productivity"
-- "Some managers report difficulty monitoring remote employee performance"
-
-**Bank A** (High Openness=0.9, Low Conscientiousness=0.2, bias=0.8):
-
-**Bank B** (Low Openness=0.2, High Conscientiousness=0.9, bias=0.8):
-
-**Analysis**: Both banks accessed identical facts but formed opposite conclusions based on personality:
-- Bank A (high openness) weighted autonomy, flexibility, innovation
-- Bank B (high conscientiousness) weighted structure, monitoring, discipline
-
-### 11.2 Example: Opinion Evolution
-
-**Scenario**: Bank forms initial opinion, then encounters reinforcing and contradictory evidence.
-
-**Initial State** (t=0):
-
-**Reinforcement** (t=1):
-- New Fact: "Python dominates AI/ML with 75% market share; TensorFlow and PyTorch are Python-first"
-- Update: Confidence → 0.85, text adds "Python's dominance in AI/ML frameworks..."
-
-**Partial Contradiction** (t=2):
-- New Fact: "Julia offers 10x faster numerical computation; increasingly adopted in research"
-- Update: Confidence → 0.75, text revised to include nuance about specialized languages
-
-**Strong Contradiction** (t=3):
-- New Fact: "Major tech companies migrating data pipelines to Rust for performance"
-- Update: Confidence → 0.55, text revised to acknowledge Python's shifting role
-
-**Trajectory**: The opinion evolved from strong conviction (0.7 → 0.85) to weaker, more malleable belief (0.55) as evidence accumulated.
-
-## 12. Use Cases and Real-World Deployment
-
-### 12.1 Multi-Persona Sports Commentary (Production Deployment)
-
-**Application**: AI-generated sports analysis and entertainment content with multiple agent personalities
-
-**Real-World System**: A production sports content platform where AI agents with distinct personalities co-host episodic shows discussing team performance, game analysis, and sports debates.
-
-**System Architecture**:
-- **Multiple Banks**: Each bank has unique personality traits and sports background
-- **Continuous Memory**: Banks maintain persistent team/player assessments across episodes spanning months
-- **Opinion Evolution**: As games occur and statistics accumulate, banks automatically update beliefs through reinforcement
-- **Personality-Driven Commentary**: The same game results generate different perspectives based on bank traits
-
-**Key Benefits Observed**:
-1. **Viewer Engagement**: Improved audience retention with "personality diversity" as primary appeal
-2. **Content Consistency**: Banks maintain recognizable voices across episodes without manual tuning
-3. **Scalability**: New banks can be added with distinct personalities without retraining
-4. **Opinion Richness**: Opinion networks capture nuanced, evolving assessments
-
-This deployment validates that personality-driven opinion systems can operate at production scale for content generation requiring consistent yet adaptive perspectives.
-
-### 12.2 Additional Use Cases
-
-**Customer Support**: Multi-agent systems with specialized personas (empathetic, analytical, creative)
-
-**Consistent Character AI**: Conversational AI characters for entertainment or education with stable personality
-
-**Explainable AI**: Systems requiring transparent decision-making where personality traits explain reasoning style
-
----
-
-# Part III: Unified Hindsight Architecture
-
-## 13. Integration: TEMPR + CARA
-
-The Hindsight system integrates TEMPR (recall) and CARA (reflect) into a unified architecture:
-
-### 13.1 Three Core Operations
-
-**1. Retain** (retain()): Store information into memory banks
-- LLM-powered fact extraction with temporal ranges
-- Entity recognition and resolution
-- Graph link construction (temporal, semantic, entity, causal)
-- Automatic opinion reinforcement for existing beliefs
-
-**2. Recall** (recall()): Retrieve memories using multi-strategy search
-- Four-way parallel retrieval (semantic, keyword, graph, temporal)
-- Reciprocal Rank Fusion
-- Neural cross-encoder reranking
-- Token budget filtering
-
-**3. Reflect** (reflect()): Generate personality-aware responses
-- Retrieves relevant memories from all networks using TEMPR
-- Loads bank personality and background
-- Generates response influenced by Big Five traits
-- Forms new opinions with confidence scores
-- Stores opinions for future retrieval
-
-### 13.2 Unified Data Flow
-
-
-### 13.3 PostgreSQL Schema
-
-The system uses PostgreSQL with pgvector for storage:
-
-
-## 14. System Properties
-
-### 14.1 Epistemic Clarity
-
-The three-network architecture provides clear separation:
-- **World**: What the bank knows about the world
-- **Bank**: What the bank has done
-- **Opinion**: What the bank believes
-
-This enables:
-- Transparent reasoning (trace opinions back to facts)
-- Debugging (identify missing facts vs. flawed reasoning)
-- Confidence calibration (opinions have confidence, facts don't)
-
-### 14.2 Temporal Awareness
-
-Multi-dimensional temporal representation:
-- occurred_start / occurred_end: When events actually happened
-- mentioned_at: When the bank learned about it
-- event_date: Backward compatibility
-
-Enables:
-- Precise historical queries ("What happened in June?")
-- Recency-aware ranking (newer mentions prioritized)
-- Period matching (events spanning weeks or months)
-
-### 14.3 Entity-Aware Reasoning
-
-LLM-based entity resolution creates knowledge graph:
-- Connects semantically distant facts through shared entities
-- Enables multi-hop discovery ("Alice's manager's team")
-- Disambiguates mentions ("Alice" vs. "Alice Chen")
-
-### 14.4 Multiple Link Types
-
-The graph incorporates multiple relationship types:
-- Entity links connect memories mentioning the same entities
-- Semantic links connect conceptually similar memories
-- Temporal links connect temporally proximate memories
-- Causal links represent identified cause-effect relationships
-- Links are weighted differently during graph traversal
-
-### 14.5 Personality Consistency
-
-Big Five traits ensure stable reasoning style:
-- Configurable bias strength (objective to subjective)
-- Trait-appropriate opinion formation
-- Consistent voice across interactions
-
-### 14.6 Dynamic Belief Systems
-
-Opinion reinforcement enables belief evolution:
-- Confidence increases with supporting evidence
-- Confidence decreases with contradictory evidence
-- Opinion text revised when strongly contradicted
-- Audit trail of belief changes
-
-## 15. Conclusion
-
-We present Hindsight, a unified memory architecture for AI agents that combines TEMPR's multi-strategy retrieval with CARA's personality-driven reasoning. The system achieves strong performance on established benchmarks (73.50% on LoComo, 80.60% on LongMemEval) while enabling personality-consistent opinion formation through the Big Five model.
-
-The integration of four parallel search strategies (semantic, keyword, graph with multiple link types, temporal) with three-network architecture (world, bank, opinion) and opinion reinforcement creates a comprehensive memory system that:
-- Retrieves information with high recall and precision
-- Maintains epistemic clarity between facts and beliefs
-- Enables personality-driven reasoning with stable traits
-- Supports dynamic belief evolution with evidence
-
-Real-world deployment in sports content generation demonstrates the system's ability to maintain consistent yet adaptive perspectives across extended interactions. Future work will explore personality evolution, multi-agent belief systems, and richer personality models incorporating values and cultural factors.
-
-By combining temporal-aware retrieval with personality-driven reasoning, Hindsight moves toward conversational agents that exhibit not just memory and intelligence, but character—stable traits and evolving beliefs that enable more natural, trustworthy human-AI interaction.
-
-## References
-
-1. Anderson, J. R. (1983). A spreading activation theory of memory. *Journal of Verbal Learning and Verbal Behavior*, 22(3), 261-295.
-
-2. Cormack, G. V., Clarke, C. L., & Buettcher, S. (2009). Reciprocal rank fusion outperforms condorcet and individual rank learning methods. In *SIGIR'09* (pp. 758-759).
-
-3. McCrae, R. R., & Costa, P. T. (1997). Personality trait structure as a human universal. *American Psychologist*, 52(5), 509.
-
-4. Goldberg, L. R. (1993). The structure of phenotypic personality traits. *American Psychologist*, 48(1), 26.
-
-5. Malkov, Y. A., & Yashunin, D. A. (2018). Efficient and robust approximate nearest neighbor search using hierarchical navigable small world graphs. *IEEE Transactions on Pattern Analysis and Machine Intelligence*, 42(4), 824-836.
-
-6. Robertson, S., & Zaragoza, H. (2009). The probabilistic relevance framework: BM25 and beyond. *Foundations and Trends in Information Retrieval*, 3(4), 333-489.
-
-7. Brown, T. B., Mann, B., Ryder, N., Subbiah, M., Kaplan, J., Dhariwal, P., ... & Amodei, D. (2020). Language models are few-shot learners. *Advances in Neural Information Processing Systems*, 33, 1877-1901.
-
-8. Petroni, F., Rocktäschel, T., Riedel, S., Lewis, P., Bakhtin, A., Wu, Y., & Miller, A. (2019). Language models as knowledge bases?. In *Proceedings of EMNLP-IJCNLP* (pp. 2463-2473).

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Vectorize AI, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,98 +1,309 @@
-# Hindsight
+<div align="center">
 
-**Long-term memory for AI agents.**
+![Hindsight Banner](./hindsight-docs/static/img/hindsight-github-banner.png)
 
-## Why Hindsight?
+[Documentation](https://hindsight.vectorize.io) • [Paper](https://arxiv.org/abs/2512.12818) • [Cookbook](https://hindsight.vectorize.io/cookbook) • [Hindsight Cloud](https://vectorize.io/hindsight/cloud)
 
-AI assistants forget everything between sessions. Every conversation starts from zero—no context about who you are, what you've discussed, or what the memory bank has learned. This isn't just inconvenient; it fundamentally limits what AI memory banks can do.
+[![CI](https://github.com/vectorize-io/hindsight/actions/workflows/release.yml/badge.svg)](https://github.com/vectorize-io/hindsight/actions/workflows/release.yml)
+[![Slack Community](https://img.shields.io/badge/Slack-Join%20Community-4A154B?logo=slack)](https://join.slack.com/t/hindsight-space/shared_invite/zt-3nhbm4w29-LeSJ5Ixi6j8PdiYOCPlOgg)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+![PyPI - Downloads](https://img.shields.io/pypi/dm/hindsight-api?label=PyPI)
+![NPM Downloads](https://img.shields.io/npm/dm/%40vectorize-io%2Fhindsight-client?logoColor=orange&label=NPM&color=blue&link=https%3A%2F%2Fwww.npmjs.com%2Fpackage%2F%40vectorize-io%2Fhindsight-client)
 
-**The problem is harder than it looks:**
 
-- **Simple vector search isn't enough** — "What did Alice do last spring?" requires temporal reasoning, not just semantic similarity
-- **Facts get disconnected** — Knowing "Alice works at Google" and "Google is in Mountain View" should let you answer "Where does Alice work?" even if you never stored that directly
-- **Memory banks need opinions** — A coding assistant that remembers "the user prefers functional programming" should weigh that when making recommendations
-- **Context matters** — The same information means different things to different memory banks with different personalities
+</div>
 
-Hindsight solves these problems with a memory system designed specifically for AI memory banks.
+---
+
+## What is Hindsight?
+
+Hindsight™ is an agent memory system built to create smarter agents that learn over time. Most agent memory systems focus on recalling conversation history. Hindsight is focused on making agents that learn, not just remember.
+
+
+<video src="https://github.com/user-attachments/assets/923b798d-3581-4897-bb62-9cfa5a931682" controls></video>
+
+It eliminates the shortcomings of alternative techniques such as RAG and knowledge graph and delivers state-of-the-art performance on long term memory tasks.
+
+## Memory Performance & Accuracy
+
+Hindsight is the most accurate agent memory system ever tested according to benchmark performance. It has achieved state-of-the-art performance on the LongMemEval benchmark, widely used to assess memory system performance across a variety of conversational AI scenarios. The current reported performance of Hindsight and other agent memory solutions as of January 2026 is shown here:
+
+![Overview](./hindsight-docs/static/img/hindsight-bench.jpg)
+
+The benchmark performance data for Hindsight has been independently reproduced by research collaborators at the Virginia Tech [Sanghani Center for Artificial Intelligence and Data Analytics](https://sanghani.cs.vt.edu/) and The Washington Post. Other scores are self-reported by software vendors.
+
+Hindsight is being used in production at Fortune 500 enterprises and by a growing number of AI startups. 
+
+## Adding Hindsight to Your AI Agents
+
+The easiest way use Hindsight with an existing agent is with the LLM Wrapper. You can add memory to your agent with 2 lines of code. That will swap your current LLM client out with the Hindsight wrapper. After that, memories will be stored and retrieved automatically as you make LLM calls.
+
+If you need more control over how and when your agent stores and recalls memories, there's also a simple API you can integrate with using the SDKs or directly via HTTP.
+
+![Hindsight Banner](./hindsight-docs/static/img/migration-code.png)
+
+---
+
+> 🤖 **Using a coding agent?** Install the Hindsight documentation skill for instant access to docs while you code:
+> ```bash
+> npx skills add https://github.com/vectorize-io/hindsight --skill hindsight-docs
+> ```
+> Works with Claude Code, Cursor, and other AI coding assistants.
+
+---
 
 
 ## Quick Start
 
-### Option 1: Docker (recommended)
-
-Get the full experience with the API and Control Plane UI:
+### Docker (recommended)
 
 ```bash
-export OPENAI_API_KEY=your-key
-docker run -p 8888:8888 -p 9999:9999 \
-  -e HINDSIGHT_API_LLM_PROVIDER=openai \
+export OPENAI_API_KEY=sk-xxx
+
+docker run --rm -it --pull always -p 8888:8888 -p 9999:9999 \
   -e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \
-  -e HINDSIGHT_API_LLM_MODEL=gpt-4o-mini \
-  vectorize/hindsight
+  -v $HOME/.hindsight-docker:/home/hindsight/.pg0 \
+  ghcr.io/vectorize-io/hindsight:latest
 ```
 
-- **API**: http://localhost:8888
-- **Control Plane UI**: http://localhost:9999
+>API: http://localhost:8888
+>UI: http://localhost:9999
 
-Then use the Python client:
+You can modify the LLM provider by setting `HINDSIGHT_API_LLM_PROVIDER`. Valid options are `openai`, `anthropic`, `gemini`, `groq`, `ollama`, and `lmstudio`. The documentation provides more details on [supported models](https://hindsight.vectorize.io/developer/models).
+
+
+
+### Docker (external PostgreSQL)
 
 ```bash
-pip install hindsight-client
+export OPENAI_API_KEY=sk-xxx
+export HINDSIGHT_DB_PASSWORD=choose-a-password
+cd docker/docker-compose
+docker compose up 
 ```
 
+
+>API: http://localhost:8888
+>UI: http://localhost:9999
+
+### Client
+
+```bash
+pip install hindsight-client -U
+# or
+npm install @vectorize-io/hindsight-client
+```
+
+#### Python
+
 ```python
-from hindsight import HindsightClient
+from hindsight_client import Hindsight
 
-client = HindsightClient(base_url="http://localhost:8888")
+client = Hindsight(base_url="http://localhost:8888")
 
-# Store memories
-client.retain(bank_id="my-agent", content="Alice works at Google as a software engineer")
-client.retain(bank_id="my-agent", content="Alice mentioned she loves hiking in the mountains")
+# Retain: Store information
+client.retain(bank_id="my-bank", content="Alice works at Google as a software engineer")
 
-# Query with temporal reasoning
-results = client.recall(bank_id="my-agent", query="What does Alice do for work?")
+# Recall: Search memories
+client.recall(bank_id="my-bank", query="What does Alice do?")
 
-# Get a synthesized perspective
-response = client.reflect(bank_id="my-agent", query="Tell me about Alice")
-print(response.text)
+# Reflect: Generate disposition-aware response
+client.reflect(bank_id="my-bank", query="Tell me about Alice")
 ```
 
-### Option 2: Embedded (no docker/server required)
-
-For quick prototyping, run everything in-process:
+#### Node.js / TypeScript
 
 ```bash
-pip install hindsight-all
-export OPENAI_API_KEY=your-key
+npm install @vectorize-io/hindsight-client
+```
+
+```javascript
+const { HindsightClient } = require('@vectorize-io/hindsight-client');
+
+const main = async () => {
+  const client = new HindsightClient({ baseUrl: 'http://localhost:8888' });
+
+  await client.retain('my-bank', 'Alice loves hiking in Yosemite');
+
+  const results = await client.recall('my-bank', 'What does Alice like?');
+  console.log(results);
+}
+
+main();
+```
+
+
+### Python Embedded (no server required)
+
+```bash
+pip install hindsight-all -U
 ```
 
 ```python
 import os
 from hindsight import HindsightServer, HindsightClient
 
-with HindsightServer(llm_provider="openai", llm_model="gpt-4o-mini", llm_api_key=os.environ["OPENAI_API_KEY"]) as server:
+with HindsightServer(
+    llm_provider="openai",
+    llm_model="gpt-5-mini", 
+    llm_api_key=os.environ["OPENAI_API_KEY"]
+) as server:
     client = HindsightClient(base_url=server.url)
-
-    client.retain(bank_id="my-user", content="User prefers functional programming")
-    response = client.reflect(bank_id="my-user", query="What coding style should I use?")
-    print(response.text)
+    client.retain(bank_id="my-bank", content="Alice works at Google")
+    results = client.recall(bank_id="my-bank", query="Where does Alice work?")
 ```
 
 
+---
 
-## Documentation
+## Use Cases
 
-Full documentation: [vectorize-io.github.io/hindsight](https://vectorize-io.github.io/hindsight)
 
-- [Architecture](https://vectorize-io.github.io/hindsight/developer/architecture) — How ingestion, storage, and retrieval work
-- [Python Client](https://vectorize-io.github.io/hindsight/sdks/python) — Full API reference
-- [API Reference](https://vectorize-io.github.io/hindsight/api-reference) — REST API endpoints
-- [Personality](https://vectorize-io.github.io/hindsight/developer/personality) — Big Five traits and opinion formation
+Hindsight is built to support conversational AI agents as well as agents that are intended to perform tasks autonomously. The ideal use case for Hindsight are agents that require a blend of these features such as AI employees that need to handle open-ended tasks, change behavior based on user feedback, and learn to perform complex tasks to automate work at a level that approximates a human work. Hindsight can be used with simple AI workflows like those built with n8n and other similar tools, but may be overkill for such applications.
+
+### Per-User Memories and Chat History
+
+One of the simpler use cases you can use Hindsight for is to personalize AI chatbots and other conversational agents by storing and recalling memories associated with individual users.
+
+The requirements for this use case usually look something like this:
+
+![Per-User Memories](./hindsight-docs/static/img/per-user-memory-requirements.png)
+
+<video src="https://github.com/user-attachments/assets/4805e8e1-e7d1-47c6-a4f8-2344a5ec8906" controls></video>
+
+Satisfying these requirements in Hindsight is straightforward. When new user inputs and tool calls are ingested into Hindsight using the retain operation, custom metadata can be used to enrich the new memories. Metadata provides a convenient way to isolate memories that need to be restricted to a given user. Once these are fed into the retain operation, any raw memories and mental models that get created can be filtered when retrieving relevant memories. 
+
+![Per-User Memories](./hindsight-docs/static/img/per-user-memory-howto.png)
+
+---
+
+## Architecture & Operations
+
+![Overview](./hindsight-docs/static/img/hindsight-overview.webp)
+
+Most agent memory implementation rely on basic vector search or sometimes use a knowledge graph. Hindsight uses biomimetic data structures to organize agent memories in a way that is more like how human memory works:
+
+- **World:** Facts about the world ("The stove gets hot")
+- **Experiences:** Agent's own experiences ("I touched the stove and it really hurt")
+- **Mental Models:** Learned understanding of the agent's world formed by reflecting on raw memories and experiences.
+
+Memories in Hindsight are stored in banks (i.e. memory banks). When memories are added to Hindsight, they are pushed into either the world facts or experiences memory pathway. They are then represented as a combination of entities, relationships, and time series with sparse/dense vector representations to aid in later recall.
+
+Hindsight provides three simple methods to interact with the system:
+
+- **Retain:** Provide information to Hindsight that you want it to remember
+- **Recall:** Retrieve memories from Hindsight
+- **Reflect:** Reflect on memories and experiences to generate new observations and insights from existing memories.
+
+### Retain
+
+The `retain` operation is used to push new memories into Hindsight. It tells Hindsight to _retain_ the information you pass in as an input.
+
+```python
+from hindsight_client import Hindsight
+
+client = Hindsight(base_url="http://localhost:8888")
+
+# Simple
+client.retain(
+    bank_id="my-bank",
+    content="Alice works at Google as a software engineer"
+)
+
+# With context and timestamp
+client.retain(
+    bank_id="my-bank",
+    content="Alice got promoted to senior engineer",
+    context="career update",
+    timestamp="2025-06-15T10:00:00Z"
+)
+```
+
+Behind the scenes, the retain operation uses an LLM to extract key facts, temporal data, entities, and relationships. It passes these through a normalization process to transform extracted data into canonical entities, time series, and search indexes along with metadata. These representations create the pathways for accurate memory retrieval in the recall and reflect operations. 
+
+![Retain Operation](hindsight-docs/static/img/retain-operation.webp)
+
+### Recall
+
+The recall operation is used to retrieve memories. These memories can come from any of the memory types (world, experiences, etc.)
+
+```python
+from hindsight_client import Hindsight
+
+client = Hindsight(base_url="http://localhost:8888")
+
+# Simple
+client.recall(bank_id="my-bank", query="What does Alice do?")
+
+# Temporal
+client.recall(bank_id="my-bank", query="What happened in June?")
+```
+
+Recall performs 4 retrieval strategies in parallel:
+- Semantic: Vector similarity
+- Keyword: BM25 exact matching
+- Graph: Entity/temporal/causal links
+- Temporal: Time range filtering
+
+![Retain Operation](hindsight-docs/static/img/recall-operation.webp)
+
+The individual results from the retrievals are merged, then ordered by relevance using reciprocal rank fusion and a cross-encoder reranking model.
+
+The final output is trimmed as needed to fit within the token limit.
+
+### Reflect
+
+The reflect operation is used to perform a more thorough analysis of existing memories. This allows the agent to form new connections between memories and build a more thorough understanding of its world.
+
+For example, the `reflect` operation can be used to support use cases such as:
+
+- An **AI Project Manager** reflecting on what risks need to be mitigated on a project.
+- A **Sales Agent** reflecting on why certain outreach messages have gotten responses while others haven't.
+- A **Support Agent** reflecting on opportunities where customers have questions not answered by current product documentation.
+
+The `reflect` operation can also be used to handle on-demand question answering or analysis which require more deep thinking.
+
+```python
+from hindsight_client import Hindsight
+
+client = Hindsight(base_url="http://localhost:8888")
+
+client.reflect(bank_id="my-bank", query="What should I know about Alice?")
+```
+
+![Retain Operation](hindsight-docs/static/img/reflect-operation.webp)
+
+---
+
+## Resources
+
+**Documentation:** 
+- [https://hindsight.vectorize.io](https://hindsight.vectorize.io)
+
+**Clients:**
+- [Python](http://hindsight.vectorize.io/sdks/python)
+- [Node.js](http://hindsight.vectorize.io/sdks/nodejs)
+- [REST API](https://hindsight.vectorize.io/api-reference)
+- [CLI](https://hindsight.vectorize.io/sdks/cli)
+
+**Community:**
+- [Slack](https://join.slack.com/t/hindsight-space/shared_invite/zt-3nhbm4w29-LeSJ5Ixi6j8PdiYOCPlOgg)
+- [GitHub Issues](https://github.com/vectorize-io/hindsight/issues)
+
+---
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=vectorize-io/hindsight&type=date&legend=top-left)](https://www.star-history.com/#vectorize-io/hindsight&type=date&legend=top-left)
+---
 
 ## Contributing
 
-We welcome contributions! See [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines.
+See [CONTRIBUTING.md](./CONTRIBUTING.md).
 
 ## License
 
-MIT
+MIT — see [LICENSE](./LICENSE)
+
+---
+
+Built by [Vectorize.io](https://vectorize.io)

SECURITY.md

@@ -0,0 +1,39 @@
+# Security Policy
+
+## Supported Versions
+
+We release patches for security vulnerabilities. Which versions are eligible for
+receiving such patches depends on the CVSS v3.0 Rating:
+
+| Version | Supported          |
+| ------- | ------------------ |
+| latest  | :white_check_mark: |
+
+## Reporting a Vulnerability
+
+Please report (suspected) security vulnerabilities to the maintainers privately.
+You can do this by opening a [GitHub Security Advisory](https://github.com/vectorize-io/hindsight/security/advisories/new).
+
+You will receive a response from us within 48 hours. If the issue is confirmed,
+we will release a patch as soon as possible depending on complexity but
+typically within a few days.
+
+Please include the following information in your report:
+
+- Type of issue (e.g., buffer overflow, SQL injection, cross-site scripting, etc.)
+- Full paths of source file(s) related to the manifestation of the issue
+- The location of the affected source code (tag/branch/commit or direct URL)
+- Any special configuration required to reproduce the issue
+- Step-by-step instructions to reproduce the issue
+- Proof-of-concept or exploit code (if possible)
+- Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+We follow the principle of [Coordinated Vulnerability Disclosure](https://www.cisa.gov/resources-tools/programs/coordinated-vulnerability-disclosure-program).

docker/README.md

@@ -1,155 +0,0 @@
-# Hindsight Docker
-
-Run Hindsight with Docker in standalone or distributed mode.
-
-## Quick Start (Standalone)
-
-```bash
-cd docker
-./start.sh
-```
-
-**Force rebuild after code changes:**
-```bash
-./start.sh --build        # Quick: rebuild and start
-# or
-./rebuild.sh              # Complete: rebuild from scratch (no cache)
-```
-
-Access:
-- **Control Plane**: http://localhost:3000
-- **API**: http://localhost:8888
-
-Press `Ctrl+C` to stop.
-
-## What You Get
-
-**Standalone** (default, simple):
-- One container with API + Control Plane + embedded database
-- Perfect for local development and simple deployments
-
-**Distributed** (advanced):
-- Separate containers for API and Control Plane
-- Better for production, scaling, or custom configurations
-
-## Deployment Modes
-
-### 1. Standalone (Recommended)
-
-All-in-one container with embedded pg0 database.
-
-```bash
-./start.sh
-# or
-cd standalone
-docker-compose up
-```
-
-**Data storage:** `/app/data` volume
-
-### 2. Distributed (Advanced)
-
-Separate API and Control Plane containers.
-
-```bash
-cd services
-docker-compose up
-```
-
-**Data storage:** `api_data` volume
-
-See `services/README.md` for details.
-
-## Data Management
-
-**Reset data:**
-```bash
-# Standalone
-cd standalone && docker-compose down -v
-
-# Distributed
-cd services && docker-compose down -v
-```
-
-## Building Images
-
-```bash
-# Standalone
-cd standalone
-docker build -f Dockerfile -t hindsight:latest ../..
-
-# Services
-cd services
-./build-all.sh
-```
-
-## Using External Database
-
-Both modes use embedded pg0 by default. To use external PostgreSQL:
-
-```bash
-export HINDSIGHT_API_DATABASE_URL=postgresql://user:pass@host:5432/db
-```
-
-## Directory Structure
-
-```
-docker/
-├── start.sh              # Quick start (standalone)
-├── README.md             # This file
-├── standalone/           # All-in-one deployment
-│   ├── Dockerfile
-│   ├── docker-compose.yml
-│   └── start-all.sh
-└── services/             # Distributed deployment
-    ├── docker-compose.yml
-    ├── api.Dockerfile
-    ├── control-plane.Dockerfile
-    ├── build-all.sh
-    └── README.md
-```
-
-## Advanced Usage
-
-**Background mode:**
-```bash
-cd standalone
-docker-compose up -d
-docker-compose logs -f
-docker-compose down
-```
-
-**Custom configuration:**
-Edit `standalone/docker-compose.yml` or `services/docker-compose.yml`
-
-## Environment Variables
-
-Hindsight requires configuration through environment variables (all prefixed with `HINDSIGHT_`).
-
-### Required:
-- `HINDSIGHT_API_LLM_API_KEY` - Your LLM API key (OpenAI, Anthropic, etc.)
-
-### Optional:
-- `HINDSIGHT_API_LLM_MODEL` - Model name (default: gpt-4o-mini)
-- `HINDSIGHT_API_LLM_BASE_URL` - API base URL (default: https://api.openai.com/v1)
-- `HINDSIGHT_API_LOG_LEVEL` - Logging level: debug, info, warning, error
-- `HINDSIGHT_API_DATABASE_URL` - External PostgreSQL connection (uses embedded pg0 by default)
-
-### Setup Options:
-
-**Option 1: .env file (recommended)**
-```bash
-# Copy example file
-cp .env.example .env
-
-# Edit .env and add your API key
-HINDSIGHT_API_LLM_API_KEY=sk-...
-```
-
-**Option 2: Export in shell**
-```bash
-export HINDSIGHT_API_LLM_API_KEY=sk-...
-export HINDSIGHT_API_LLM_MODEL=gpt-4o-mini
-```
-
-The `start.sh` script automatically loads `.env` if it exists and validates the API key is set.

docker/docker-compose/docker-compose.yaml

@@ -0,0 +1,54 @@
+# Docker Compose file for Hindsight with PostgreSQL and pgvector
+#
+# Make sure to set the required environment variables before running:
+# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
+# - Configure LLM provider variables as needed (see below in the hindsight service)
+#
+# Usage:
+# docker compose up -d
+#
+# Optional environment variables with defaults:
+# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
+# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
+# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
+# - HINDSIGHT_DB_VERSION: PostgreSQL version (default: 18)
+
+services:
+  db:
+    # Use a PostgreSQL-Image with pgvector extension pre-installed
+    # see https://hub.docker.com/r/pgvector/pgvector 
+    image: pgvector/pgvector:pg${HINDSIGHT_DB_VERSION:-18}
+    container_name: hindsight-db
+    restart: always
+    # Expose PostgreSQL port
+    # ports:
+      # - "5432:5432"
+    environment:
+      POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
+      POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}
+      POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
+    volumes:
+      - pg_data:/var/lib/postgresql/${HINDSIGHT_DB_VERSION:-18}/docker
+    networks:
+      - hindsight-net
+
+  hindsight:
+    image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
+    container_name: hindsight-app
+    ports:
+      - "8888:8888"
+      - "9999:9999"
+    environment:
+      - HINDSIGHT_API_LLM_API_KEY=${OPENAI_API_KEY?Please set the OPENAI_API_KEY env variable}
+      - HINDSIGHT_API_DATABASE_URL=postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
+    depends_on:
+      - db
+    networks:
+      - hindsight-net
+
+networks:
+  hindsight-net:
+    driver: bridge
+
+volumes:
+  pg_data:

docker/standalone/Dockerfile

@@ -2,16 +2,25 @@
 # Supports building API-only, Control Plane-only, or both
 #
 # Build args:
-#   INCLUDE_API=true/false  - Include API (default: true)
-#   INCLUDE_CP=true/false   - Include Control Plane (default: true)
+#   INCLUDE_API=true/false         - Include API (default: true)
+#   INCLUDE_CP=true/false          - Include Control Plane (default: true)
+#   INCLUDE_LOCAL_MODELS=true/false - Include local ML models for embeddings/reranking (default: true)
+#                                     Set to false when using external providers (TEI, OpenAI, Cohere)
+#   PRELOAD_ML_MODELS=true/false   - Pre-download ML models during build (default: true)
+#                                     Only effective when INCLUDE_LOCAL_MODELS=true
+#                                     NOTE: tiktoken encodings are ALWAYS preloaded (required for air-gapped deployments)
 #
 # Examples:
-#   docker build -t hindsight .                                    # Both (standalone)
-#   docker build -t hindsight-api --build-arg INCLUDE_CP=false .   # API only
-#   docker build -t hindsight-cp --build-arg INCLUDE_API=false .   # Control Plane only
+#   docker build -t hindsight .                                          # Both (standalone)
+#   docker build -t hindsight-api --build-arg INCLUDE_CP=false .         # API only
+#   docker build -t hindsight-cp --build-arg INCLUDE_API=false .         # Control Plane only
+#   docker build -t hindsight --build-arg PRELOAD_ML_MODELS=false .      # Skip ML model preload
+#   docker build -t hindsight --build-arg INCLUDE_LOCAL_MODELS=false .   # Skip local ML deps (for external providers)
 
 ARG INCLUDE_API=true
 ARG INCLUDE_CP=true
+ARG PRELOAD_ML_MODELS=true
+ARG INCLUDE_LOCAL_MODELS=true
 
 # =============================================================================
 # Stage: API Builder
@@ -19,6 +28,7 @@ ARG INCLUDE_CP=true
 FROM python:3.11-slim AS api-builder
 
 ARG INCLUDE_API
+ARG INCLUDE_LOCAL_MODELS
 RUN if [ "$INCLUDE_API" != "true" ]; then echo "Skipping API build" && exit 0; fi
 
 WORKDIR /app
@@ -37,57 +47,92 @@ COPY hindsight-api/README.md ./api/
 
 WORKDIR /app/api
 
+# Remove local ML model dependencies if INCLUDE_LOCAL_MODELS=false
+# This creates a smaller image when using external providers (TEI, OpenAI, Cohere)
+RUN if [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then \
+        echo "Removing local-models dependencies (sentence-transformers, torch, transformers)..." && \
+        sed -i '/"sentence-transformers/d' pyproject.toml && \
+        sed -i '/"transformers/d' pyproject.toml && \
+        sed -i '/"torch/d' pyproject.toml; \
+    fi
+
 # Sync dependencies (will create lock file if needed)
 RUN uv sync
 
-# Copy source code and alembic migrations
+# Copy source code (alembic migrations are inside hindsight_api/)
 COPY hindsight-api/hindsight_api ./hindsight_api
-COPY hindsight-api/alembic ./alembic
+
+# Install the local package (uv sync only installed dependencies, not the package itself)
+RUN uv pip install -e .
 
 # =============================================================================
 # Stage: SDK Builder (needed for Control Plane)
 # =============================================================================
-FROM node:20-alpine AS sdk-builder
+FROM node:20-slim AS sdk-builder
 
 ARG INCLUDE_CP
 RUN if [ "$INCLUDE_CP" != "true" ]; then echo "Skipping SDK build" && exit 0; fi
 
-WORKDIR /app/sdk
+WORKDIR /app
 
-COPY hindsight-clients/typescript/package*.json ./
-RUN npm ci
+# Copy root package files for npm workspaces
+COPY package.json package-lock.json ./
+COPY hindsight-clients/typescript/ ./hindsight-clients/typescript/
 
-COPY hindsight-clients/typescript/ ./
-RUN npm run build
+# Install and build SDK using workspace (--ignore-scripts skips git hooks setup)
+RUN npm ci --ignore-scripts -w @vectorize-io/hindsight-client
+RUN npm run build -w @vectorize-io/hindsight-client
 
 # =============================================================================
 # Stage: Control Plane Builder
 # =============================================================================
-FROM node:20-alpine AS cp-builder
+FROM node:20-slim AS cp-builder
 
 ARG INCLUDE_CP
 RUN if [ "$INCLUDE_CP" != "true" ]; then echo "Skipping CP build" && exit 0; fi
 
-WORKDIR /app
-
-# Copy built SDK
-COPY --from=sdk-builder /app/sdk /app/sdk
+# Create directory structure matching the monorepo layout
+# This is required because build:standalone script expects .next/standalone/memory-poc/hindsight-control-plane
+WORKDIR /app/memory-poc/hindsight-control-plane
 
 # Install Control Plane dependencies
-COPY hindsight-control-plane/package*.json ./
-RUN npm ci
+# Only copy package.json (not package-lock.json) to ensure npm installs
+# correct platform-specific native bindings for lightningcss/tailwindcss
+COPY hindsight-control-plane/package.json ./
+# Remove the file: dependency on SDK (we'll copy it directly later)
+RUN sed -i '/"@vectorize-io\/hindsight-client":/d' package.json
+RUN npm install
 
-# Copy Control Plane source
+# Copy Control Plane source (excluding node_modules via .dockerignore)
 COPY hindsight-control-plane/ ./
+# Remove package-lock.json to avoid conflicts with installed native bindings
+# Also remove the file: dependency from package.json (restored by COPY above)
+RUN rm -f package-lock.json && sed -i '/"@vectorize-io\/hindsight-client":/d' package.json
 
-# Link SDK (temporary for build)
-RUN cd /app/sdk && npm link && cd /app && npm link @hindsight/client
+# Copy built SDK directly into node_modules (more reliable than npm link in Docker)
+COPY --from=sdk-builder /app/hindsight-clients/typescript ./node_modules/@vectorize-io/hindsight-client
 
-# Build Control Plane
-RUN npm run build
+# Build Control Plane - run next build first, then custom standalone copy
+# (The build:standalone script expects a specific path structure that differs in Docker)
+RUN npm exec -- next build
 
-# Create public directory if it doesn't exist
-RUN mkdir -p public
+# Create standalone directory structure manually
+# Note: Must exclude node_modules from find to avoid wrong server.js from next/dist/experimental/testmode/
+# Note: Must explicitly copy .next since glob * doesn't match hidden directories
+RUN STANDALONE_ROOT=$(find .next/standalone -path '*/node_modules' -prune -o -name 'server.js' -print | head -1 | xargs dirname) && \
+    mkdir -p standalone && \
+    cp -r "$STANDALONE_ROOT"/* standalone/ && \
+    cp -r "$STANDALONE_ROOT"/.next standalone/.next && \
+    # Copy node_modules if separate from app dir (monorepo structure)
+    if [ -d ".next/standalone/node_modules" ] && [ "$STANDALONE_ROOT" != ".next/standalone" ]; then \
+      cp -r .next/standalone/node_modules standalone/node_modules; \
+    fi && \
+    cp -r .next/static standalone/.next/static && \
+    mkdir -p standalone/public && \
+    cp -r public/* standalone/public/ 2>/dev/null || true && \
+    # Verify required files exist
+    test -f standalone/server.js || (echo "ERROR: server.js missing!" && exit 1) && \
+    test -f standalone/.next/BUILD_ID || (echo "ERROR: BUILD_ID missing!" && exit 1)
 
 # =============================================================================
 # Stage: Final Image - API Only
@@ -96,18 +141,18 @@ FROM python:3.11-slim AS api-only
 
 WORKDIR /app
 
-# Install pg0 dependencies
+# Note: libicu version varies by Debian version - try common versions in order
 RUN apt-get update && apt-get install -y \
     curl \
+    procps \
     libxml2 \
     libssl3 \
     libgssapi-krb5-2 \
     libossp-uuid16 \
-    && apt-get install -y libicu72 || apt-get install -y libicu74 || apt-get install -y libicu* \
+    && (apt-get install -y libicu72 2>/dev/null || apt-get install -y libicu74 2>/dev/null || apt-get install -y libicu76 2>/dev/null || true) \
     && rm -rf /var/lib/apt/lists/* \
     && pip install --no-cache-dir uv
 
-# Create non-root user (PostgreSQL cannot run as root)
 RUN useradd -m -s /bin/bash hindsight
 
 # Copy API with virtual environment from builder
@@ -117,43 +162,65 @@ COPY --from=api-builder /app/api /app/api
 COPY docker/standalone/start-all.sh /app/start-all.sh
 RUN chmod +x /app/start-all.sh
 
-# Create data directory for pg0 and set ownership
-RUN mkdir -p /app/data && chown -R hindsight:hindsight /app
+RUN chown -R hindsight:hindsight /app
 
-# Switch to non-root user
 USER hindsight
 
-# Set PATH for hindsight user
-ENV PATH="/home/hindsight/.hindsight/bin:/app/api/.venv/bin:${PATH}"
+ENV PATH="/app/api/.venv/bin:${PATH}"
 
-# Install pg0 binary
-RUN mkdir -p /home/hindsight/.hindsight/bin && \
-    ARCH=$(uname -m) && \
-    if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \
-        PG0_BINARY="pg0-linux-aarch64-gnu"; \
-    elif [ "$ARCH" = "x86_64" ]; then \
-        PG0_BINARY="pg0-linux-x86_64-gnu"; \
-    else \
-        echo "Unsupported architecture: $ARCH" && exit 1; \
-    fi && \
-    echo "Installing pg0 binary: $PG0_BINARY" && \
-    for i in 1 2 3 4 5; do \
-        curl -fsSL -o /home/hindsight/.hindsight/bin/pg0 \
-            "https://github.com/vectorize-io/pg0/releases/latest/download/$PG0_BINARY" && \
-        chmod +x /home/hindsight/.hindsight/bin/pg0 && \
-        break || (echo "Retry $i failed, waiting..." && sleep 10); \
-    done && \
-    /home/hindsight/.hindsight/bin/pg0 --version
+# Pre-download tiktoken encoding (ALWAYS - required for token counting even in air-gapped envs)
+# Tiktoken is a core runtime dependency, not an optional ML model
+RUN MAX_RETRIES=3; \
+    RETRY_DELAY=5; \
+    for i in $(seq 1 $MAX_RETRIES); do \
+      echo "Attempt $i/$MAX_RETRIES: Downloading tiktoken encoding..."; \
+      /app/api/.venv/bin/python -c "\
+import tiktoken; \
+print('Downloading cl100k_base encoding...'); \
+tiktoken.get_encoding('cl100k_base'); \
+print('Tiktoken encoding cached successfully')" && break; \
+      if [ $i -lt $MAX_RETRIES ]; then \
+        echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
+        sleep $RETRY_DELAY; \
+        RETRY_DELAY=$((RETRY_DELAY * 2)); \
+      fi; \
+    done; \
+    if [ $i -eq $MAX_RETRIES ]; then \
+      echo "ERROR: Failed to download tiktoken encoding after $MAX_RETRIES attempts"; \
+      exit 1; \
+    fi
 
-# Pre-download PostgreSQL binaries
-ENV PG0_HOME=/home/hindsight/.pg0-cache
-RUN pg0 start --help && \
-    (pg0 start --name hindsight --port 5555 --username hindsight --password hindsight --database hindsight && \
-    sleep 2 && \
-    pg0 stop --name hindsight && \
-    echo "PostgreSQL pre-cached to $PG0_HOME") || echo "Pre-download skipped"
-
-ENV PG0_HOME=/home/hindsight/.pg0
+# Pre-download ML models to avoid runtime download (conditional)
+# Only runs if both PRELOAD_ML_MODELS=true AND INCLUDE_LOCAL_MODELS=true
+# Includes retry logic with exponential backoff for transient network failures
+ARG PRELOAD_ML_MODELS
+ARG INCLUDE_LOCAL_MODELS
+ENV HF_HUB_DOWNLOAD_TIMEOUT=600
+RUN if [ "$PRELOAD_ML_MODELS" = "true" ] && [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
+    MAX_RETRIES=3; \
+    RETRY_DELAY=10; \
+    for i in $(seq 1 $MAX_RETRIES); do \
+      echo "Attempt $i/$MAX_RETRIES: Downloading ML models..."; \
+      /app/api/.venv/bin/python -c "\
+import os; os.environ['HF_HUB_DOWNLOAD_TIMEOUT'] = '600'; \
+from sentence_transformers import SentenceTransformer, CrossEncoder; \
+print('Downloading embedding model...'); \
+SentenceTransformer('BAAI/bge-small-en-v1.5'); \
+print('Downloading cross-encoder model...'); \
+CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'); \
+print('Models cached successfully')" && break; \
+      if [ $i -lt $MAX_RETRIES ]; then \
+        echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
+        sleep $RETRY_DELAY; \
+        RETRY_DELAY=$((RETRY_DELAY * 2)); \
+      fi; \
+    done; \
+    if [ $i -eq $MAX_RETRIES ] && ! /app/api/.venv/bin/python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('BAAI/bge-small-en-v1.5')" 2>/dev/null; then \
+      echo "ERROR: Failed to download models after $MAX_RETRIES attempts"; \
+      exit 1; \
+    fi; \
+    elif [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then echo "Skipping ML model preload (local-models not included)"; \
+    else echo "Skipping ML model preload"; fi
 
 EXPOSE 8888
 
@@ -162,6 +229,11 @@ ENV HINDSIGHT_API_PORT=8888
 ENV HINDSIGHT_API_LOG_LEVEL=info
 ENV HINDSIGHT_ENABLE_API=true
 ENV HINDSIGHT_ENABLE_CP=false
+ENV PYTHONUNBUFFERED=1
+# Suppress verbose transformers/HuggingFace model loading warnings
+ENV TRANSFORMERS_VERBOSITY=error
+ENV HF_HUB_VERBOSITY=error
+ENV TOKENIZERS_PARALLELISM=false
 
 CMD ["/app/start-all.sh"]
 
@@ -173,13 +245,13 @@ FROM node:20-alpine AS cp-only
 WORKDIR /app
 
 # Copy built SDK
-COPY --from=sdk-builder /app/sdk /app/sdk
+COPY --from=sdk-builder /app/hindsight-clients/typescript /app/sdk
 
 # Copy Control Plane standalone build
 WORKDIR /app/control-plane
-COPY --from=cp-builder /app/.next/standalone ./
-COPY --from=cp-builder /app/.next/static ./.next/static
-COPY --from=cp-builder /app/public ./public
+COPY --from=cp-builder /app/memory-poc/hindsight-control-plane/standalone ./
+COPY --from=cp-builder /app/memory-poc/hindsight-control-plane/.next/static ./.next/static
+COPY --from=cp-builder /app/memory-poc/hindsight-control-plane/public ./public
 
 WORKDIR /app
 
@@ -206,33 +278,34 @@ FROM python:3.11-slim AS standalone
 
 WORKDIR /app
 
-# Install Node.js, curl, uv, and pg0 dependencies
+# Install Node.js, curl, uv, and system dependencies
+# Note: libicu version varies by Debian version - try common versions in order
 RUN apt-get update && apt-get install -y \
     curl \
+    procps \
     libxml2 \
     libssl3 \
     libgssapi-krb5-2 \
     libossp-uuid16 \
-    && apt-get install -y libicu72 || apt-get install -y libicu74 || apt-get install -y libicu* \
+    && (apt-get install -y libicu72 2>/dev/null || apt-get install -y libicu74 2>/dev/null || apt-get install -y libicu76 2>/dev/null || true) \
     && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
     && apt-get install -y nodejs \
     && rm -rf /var/lib/apt/lists/* \
     && pip install --no-cache-dir uv
 
-# Create non-root user (PostgreSQL cannot run as root)
 RUN useradd -m -s /bin/bash hindsight
 
 # Copy API with virtual environment from builder
 COPY --from=api-builder /app/api /app/api
 
 # Copy built SDK
-COPY --from=sdk-builder /app/sdk /app/sdk
+COPY --from=sdk-builder /app/hindsight-clients/typescript /app/sdk
 
 # Copy Control Plane standalone build
 WORKDIR /app/control-plane
-COPY --from=cp-builder /app/.next/standalone ./
-COPY --from=cp-builder /app/.next/static ./.next/static
-COPY --from=cp-builder /app/public ./public
+COPY --from=cp-builder /app/memory-poc/hindsight-control-plane/standalone ./
+COPY --from=cp-builder /app/memory-poc/hindsight-control-plane/.next/static ./.next/static
+COPY --from=cp-builder /app/memory-poc/hindsight-control-plane/public ./public
 
 WORKDIR /app
 
@@ -240,43 +313,65 @@ WORKDIR /app
 COPY docker/standalone/start-all.sh /app/start-all.sh
 RUN chmod +x /app/start-all.sh
 
-# Create data directory for pg0 and set ownership
-RUN mkdir -p /app/data && chown -R hindsight:hindsight /app
+RUN chown -R hindsight:hindsight /app
 
-# Switch to non-root user
 USER hindsight
 
-# Set PATH for hindsight user
-ENV PATH="/home/hindsight/.hindsight/bin:/app/api/.venv/bin:${PATH}"
+ENV PATH="/app/api/.venv/bin:${PATH}"
 
-# Install pg0 binary
-RUN mkdir -p /home/hindsight/.hindsight/bin && \
-    ARCH=$(uname -m) && \
-    if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \
-        PG0_BINARY="pg0-linux-aarch64-gnu"; \
-    elif [ "$ARCH" = "x86_64" ]; then \
-        PG0_BINARY="pg0-linux-x86_64-gnu"; \
-    else \
-        echo "Unsupported architecture: $ARCH" && exit 1; \
-    fi && \
-    echo "Installing pg0 binary: $PG0_BINARY" && \
-    for i in 1 2 3 4 5; do \
-        curl -fsSL -o /home/hindsight/.hindsight/bin/pg0 \
-            "https://github.com/vectorize-io/pg0/releases/latest/download/$PG0_BINARY" && \
-        chmod +x /home/hindsight/.hindsight/bin/pg0 && \
-        break || (echo "Retry $i failed, waiting..." && sleep 10); \
-    done && \
-    /home/hindsight/.hindsight/bin/pg0 --version
+# Pre-download tiktoken encoding (ALWAYS - required for token counting even in air-gapped envs)
+# Tiktoken is a core runtime dependency, not an optional ML model
+RUN MAX_RETRIES=3; \
+    RETRY_DELAY=5; \
+    for i in $(seq 1 $MAX_RETRIES); do \
+      echo "Attempt $i/$MAX_RETRIES: Downloading tiktoken encoding..."; \
+      /app/api/.venv/bin/python -c "\
+import tiktoken; \
+print('Downloading cl100k_base encoding...'); \
+tiktoken.get_encoding('cl100k_base'); \
+print('Tiktoken encoding cached successfully')" && break; \
+      if [ $i -lt $MAX_RETRIES ]; then \
+        echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
+        sleep $RETRY_DELAY; \
+        RETRY_DELAY=$((RETRY_DELAY * 2)); \
+      fi; \
+    done; \
+    if [ $i -eq $MAX_RETRIES ]; then \
+      echo "ERROR: Failed to download tiktoken encoding after $MAX_RETRIES attempts"; \
+      exit 1; \
+    fi
 
-# Pre-download PostgreSQL binaries
-ENV PG0_HOME=/home/hindsight/.pg0-cache
-RUN pg0 start --help && \
-    (pg0 start --name hindsight --port 5555 --username hindsight --password hindsight --database hindsight && \
-    sleep 2 && \
-    pg0 stop --name hindsight && \
-    echo "PostgreSQL pre-cached to $PG0_HOME") || echo "Pre-download skipped"
-
-ENV PG0_HOME=/home/hindsight/.pg0
+# Pre-download ML models to avoid runtime download (conditional)
+# Only runs if both PRELOAD_ML_MODELS=true AND INCLUDE_LOCAL_MODELS=true
+# Includes retry logic with exponential backoff for transient network failures
+ARG PRELOAD_ML_MODELS
+ARG INCLUDE_LOCAL_MODELS
+ENV HF_HUB_DOWNLOAD_TIMEOUT=600
+RUN if [ "$PRELOAD_ML_MODELS" = "true" ] && [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
+    MAX_RETRIES=3; \
+    RETRY_DELAY=10; \
+    for i in $(seq 1 $MAX_RETRIES); do \
+      echo "Attempt $i/$MAX_RETRIES: Downloading ML models..."; \
+      /app/api/.venv/bin/python -c "\
+import os; os.environ['HF_HUB_DOWNLOAD_TIMEOUT'] = '600'; \
+from sentence_transformers import SentenceTransformer, CrossEncoder; \
+print('Downloading embedding model...'); \
+SentenceTransformer('BAAI/bge-small-en-v1.5'); \
+print('Downloading cross-encoder model...'); \
+CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'); \
+print('Models cached successfully')" && break; \
+      if [ $i -lt $MAX_RETRIES ]; then \
+        echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
+        sleep $RETRY_DELAY; \
+        RETRY_DELAY=$((RETRY_DELAY * 2)); \
+      fi; \
+    done; \
+    if [ $i -eq $MAX_RETRIES ] && ! /app/api/.venv/bin/python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('BAAI/bge-small-en-v1.5')" 2>/dev/null; then \
+      echo "ERROR: Failed to download models after $MAX_RETRIES attempts"; \
+      exit 1; \
+    fi; \
+    elif [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then echo "Skipping ML model preload (local-models not included)"; \
+    else echo "Skipping ML model preload"; fi
 
 EXPOSE 8888 9999
 
@@ -287,6 +382,11 @@ ENV NODE_ENV=production
 ENV HINDSIGHT_CP_DATAPLANE_API_URL=http://localhost:8888
 ENV HINDSIGHT_ENABLE_API=true
 ENV HINDSIGHT_ENABLE_CP=true
+ENV PYTHONUNBUFFERED=1
+# Suppress verbose transformers/HuggingFace model loading warnings
+ENV TRANSFORMERS_VERBOSITY=error
+ENV HF_HUB_VERBOSITY=error
+ENV TOKENIZERS_PARALLELISM=false
 
 CMD ["/app/start-all.sh"]
 

docker/standalone/docker-compose.yml

@@ -1,25 +0,0 @@
-services:
-  hindsight:
-    image: hindsight
-    build:
-      context: ../..
-      dockerfile: docker/standalone/Dockerfile
-    env_file:
-      - ../../.env
-    ports:
-      - "9999:9999"
-      - "8888:8888"
-    environment:
-      # These override env_file values only when set in host shell
-      # Default values are applied only when not set in env_file or host
-      HINDSIGHT_API_HOST: ${HINDSIGHT_API_HOST:-0.0.0.0}
-      HINDSIGHT_API_PORT: ${HINDSIGHT_API_PORT:-8888}
-      HINDSIGHT_API_LOG_LEVEL: ${HINDSIGHT_API_LOG_LEVEL:-info}
-      # HINDSIGHT_API_DATABASE_URL can be set if you want to use an external database
-      # If not set, embedded pg0 will be used automatically
-    volumes:
-      - hindsight_data:/home/hindsight/.pg0
-    restart: unless-stopped
-
-volumes:
-  hindsight_data:

docker/standalone/start-all.sh

@@ -1,23 +1,74 @@
 #!/bin/bash
 set -e
 
-echo "🚀 Starting Hindsight..."
-echo ""
-
 # Service flags (default to true if not set)
 ENABLE_API="${HINDSIGHT_ENABLE_API:-true}"
 ENABLE_CP="${HINDSIGHT_ENABLE_CP:-true}"
 
-# Copy pre-cached PostgreSQL data if runtime directory is empty (first run with volume)
-if [ "$ENABLE_API" = "true" ]; then
-    PG0_CACHE="/home/hindsight/.pg0-cache"
-    PG0_HOME="/home/hindsight/.pg0"
-    if [ -d "$PG0_CACHE" ] && [ "$(ls -A $PG0_CACHE 2>/dev/null)" ]; then
-        if [ ! "$(ls -A $PG0_HOME 2>/dev/null)" ]; then
-            echo "📦 Copying pre-cached PostgreSQL data..."
-            cp -r "$PG0_CACHE"/* "$PG0_HOME"/ 2>/dev/null || true
-        fi
+# =============================================================================
+# Dependency waiting (opt-in via HINDSIGHT_WAIT_FOR_DEPS=true)
+#
+# Problem: When running with LM Studio, the LLM may take time to load models.
+# If Hindsight starts before LM Studio is ready, it fails on LLM verification.
+# This wait loop ensures dependencies are ready before starting.
+# =============================================================================
+if [ "${HINDSIGHT_WAIT_FOR_DEPS:-false}" = "true" ]; then
+    LLM_BASE_URL="${HINDSIGHT_API_LLM_BASE_URL:-http://host.docker.internal:1234/v1}"
+    MAX_RETRIES="${HINDSIGHT_RETRY_MAX:-0}"  # 0 = infinite
+    RETRY_INTERVAL="${HINDSIGHT_RETRY_INTERVAL:-10}"
+
+    # Check if external database is configured (skip check for embedded pg0)
+    SKIP_DB_CHECK=false
+    if [ -z "${HINDSIGHT_API_DATABASE_URL}" ]; then
+        SKIP_DB_CHECK=true
+    else
+        DB_CHECK_HOST=$(echo "$HINDSIGHT_API_DATABASE_URL" | sed -E 's|.*@([^:/]+):([0-9]+)/.*|\1 \2|')
     fi
+
+    check_db() {
+        if $SKIP_DB_CHECK; then
+            return 0
+        fi
+        if command -v pg_isready &> /dev/null; then
+            pg_isready -h $(echo $DB_CHECK_HOST | cut -d' ' -f1) -p $(echo $DB_CHECK_HOST | cut -d' ' -f2) &>/dev/null
+        else
+            python3 -c "import socket; s=socket.socket(); s.settimeout(5); exit(0 if s.connect_ex(('$(echo $DB_CHECK_HOST | cut -d' ' -f1)', $(echo $DB_CHECK_HOST | cut -d' ' -f2))) == 0 else 1)" 2>/dev/null
+        fi
+    }
+
+    check_llm() {
+        curl -sf "${LLM_BASE_URL}/models" --connect-timeout 5 &>/dev/null
+    }
+
+    echo "⏳ Waiting for dependencies to be ready..."
+    attempt=1
+
+    while true; do
+        db_ok=false
+        llm_ok=false
+
+        if check_db; then
+            db_ok=true
+        fi
+
+        if check_llm; then
+            llm_ok=true
+        fi
+
+        if $db_ok && $llm_ok; then
+            echo "✅ Dependencies ready!"
+            break
+        fi
+
+        if [ "$MAX_RETRIES" -ne 0 ] && [ "$attempt" -ge "$MAX_RETRIES" ]; then
+            echo "❌ Max retries ($MAX_RETRIES) reached. Dependencies not available."
+            exit 1
+        fi
+
+        echo "   Attempt $attempt: DB=$( $db_ok && echo 'ok' || echo 'waiting' ), LLM=$( $llm_ok && echo 'ok' || echo 'waiting' )"
+        sleep "$RETRY_INTERVAL"
+        ((attempt++))
+    done
 fi
 
 # Track PIDs for wait
@@ -26,32 +77,31 @@ PIDS=()
 # Start API if enabled
 if [ "$ENABLE_API" = "true" ]; then
     cd /app/api
-    python -m hindsight_api.web.server 2>&1 | sed -u 's/^/[api] /' &
+    # Run API directly - Python's PYTHONUNBUFFERED=1 handles output buffering
+    hindsight-api &
     API_PID=$!
     PIDS+=($API_PID)
 
     # Wait for API to be ready
-    echo "⏳ Waiting for API..."
     for i in {1..60}; do
         if curl -sf http://localhost:8888/health &>/dev/null; then
-            echo "✅ API is ready"
             break
         fi
         sleep 1
     done
 else
-    echo "⏭️  API disabled (HINDSIGHT_ENABLE_API=false)"
+    echo "API disabled (HINDSIGHT_ENABLE_API=false)"
 fi
 
 # Start Control Plane if enabled
 if [ "$ENABLE_CP" = "true" ]; then
     echo "🎛️  Starting Control Plane..."
     cd /app/control-plane
-    PORT=9999 node server.js 2>&1 | grep -v -E "^[[:space:]]*(▲|✓|-|$)" | sed -u 's/^/[control-plane] /' &
+    PORT=9999 node server.js &
     CP_PID=$!
     PIDS+=($CP_PID)
 else
-    echo "⏭️  Control Plane disabled (HINDSIGHT_ENABLE_CP=false)"
+    echo "Control Plane disabled (HINDSIGHT_ENABLE_CP=false)"
 fi
 
 # Print status

docker/start.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-# Start Hindsight (standalone all-in-one)
-
-cd "$(dirname "$0")"
-
-# Check for --build flag
-BUILD_FLAG=""
-if [[ "$1" == "--build" ]] || [[ "$1" == "-b" ]]; then
-  BUILD_FLAG="--build"
-  echo "🔨 Forcing rebuild of images..."
-  echo ""
-fi
-
-echo "🚀 Starting Hindsight..."
-echo ""
-
-# Load .env file from project root if it exists
-if [ -f ../.env ]; then
-  echo "📝 Loading environment variables from .env file..."
-  export $(grep -v '^#' ../.env | grep -v '^$' | xargs)
-fi
-
-# Check for required HINDSIGHT_API_LLM_API_KEY
-if [ -z "$HINDSIGHT_API_LLM_API_KEY" ]; then
-  echo "⚠️  Warning: HINDSIGHT_API_LLM_API_KEY is not set"
-  echo ""
-  echo "Set it by either:"
-  echo "  1. Creating a .env file in the project root with: HINDSIGHT_API_LLM_API_KEY=your-key"
-  echo "  2. Exporting: export HINDSIGHT_API_LLM_API_KEY=your-key"
-  echo ""
-  read -p "Continue anyway? (y/N) " -n 1 -r
-  echo
-  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
-    exit 1
-  fi
-fi
-
-cd standalone
-
-# Run docker-compose with optional --build flag
-docker-compose up $BUILD_FLAG

docker/test-image.sh

@@ -0,0 +1,201 @@
+#!/bin/bash
+#
+# Docker Smoke Test Script
+#
+# Tests that a Hindsight Docker image starts correctly and becomes healthy.
+# Can be run locally or in CI pipelines.
+#
+# Usage:
+#   ./docker/test-image.sh <image> [target]
+#
+# Arguments:
+#   image   - Docker image to test (e.g., hindsight-api:test, ghcr.io/vectorize-io/hindsight:latest)
+#   target  - Optional: 'cp-only' for control plane, otherwise assumes API image (default: api)
+#
+# Environment variables:
+#   GROQ_API_KEY                                - Required for API/standalone images (LLM verification)
+#   HINDSIGHT_API_LLM_PROVIDER                  - LLM provider (default: groq)
+#   HINDSIGHT_API_LLM_MODEL                     - LLM model (default: llama-3.3-70b-versatile)
+#   HINDSIGHT_API_EMBEDDINGS_PROVIDER           - Embeddings provider (optional, for slim images: openai, cohere, tei)
+#   HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY     - OpenAI API key for embeddings (optional)
+#   HINDSIGHT_API_RERANKER_PROVIDER             - Reranker provider (optional, for slim images: cohere, tei)
+#   HINDSIGHT_API_COHERE_API_KEY                - Cohere API key for reranking (optional)
+#   SMOKE_TEST_TIMEOUT                          - Timeout in seconds (default: 120)
+#   SMOKE_TEST_CONTAINER_NAME                   - Container name (default: hindsight-smoke-test)
+#
+# Examples:
+#   # Test a locally built full image
+#   ./docker/test-image.sh hindsight-api:test
+#
+#   # Test a released image
+#   ./docker/test-image.sh ghcr.io/vectorize-io/hindsight:latest
+#
+#   # Test control plane image
+#   ./docker/test-image.sh hindsight-control-plane:test cp-only
+#
+#   # Test slim image with external providers
+#   export GROQ_API_KEY=gsk_xxx
+#   export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai
+#   export HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=sk-xxx
+#   export HINDSIGHT_API_RERANKER_PROVIDER=cohere
+#   export HINDSIGHT_API_COHERE_API_KEY=xxx
+#   ./docker/test-image.sh hindsight-slim:test
+#
+# Exit codes:
+#   0 - Success (container healthy)
+#   1 - Failure (container not healthy within timeout)
+#   2 - Invalid arguments
+#
+
+set -euo pipefail
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
+NC='\033[0m' # No Color
+
+# Configuration
+IMAGE="${1:-}"
+TARGET="${2:-api}"
+TIMEOUT="${SMOKE_TEST_TIMEOUT:-120}"
+CONTAINER_NAME="${SMOKE_TEST_CONTAINER_NAME:-hindsight-smoke-test}"
+LLM_PROVIDER="${HINDSIGHT_API_LLM_PROVIDER:-groq}"
+LLM_MODEL="${HINDSIGHT_API_LLM_MODEL:-llama-3.3-70b-versatile}"
+
+# Validate arguments
+if [ -z "$IMAGE" ]; then
+    echo -e "${RED}Error: Image argument is required${NC}"
+    echo ""
+    echo "Usage: $0 <image> [target]"
+    echo ""
+    echo "Examples:"
+    echo "  $0 hindsight-api:test"
+    echo "  $0 ghcr.io/vectorize-io/hindsight:latest"
+    echo "  $0 hindsight-control-plane:test cp-only"
+    exit 2
+fi
+
+# Determine health endpoint based on target
+if [ "$TARGET" = "cp-only" ]; then
+    HEALTH_PORT=9999
+    HEALTH_PATH="/api/health"
+    NEEDS_LLM=false
+else
+    HEALTH_PORT=8888
+    HEALTH_PATH="/health"
+    NEEDS_LLM=true
+fi
+
+# Check for required environment variables
+if [ "$NEEDS_LLM" = true ] && [ -z "${GROQ_API_KEY:-}" ]; then
+    echo -e "${RED}Error: GROQ_API_KEY environment variable is required for API/standalone images${NC}"
+    echo "Set it with: export GROQ_API_KEY=your-api-key"
+    exit 2
+fi
+
+# Cleanup function
+cleanup() {
+    echo "Cleaning up..."
+    docker stop "$CONTAINER_NAME" 2>/dev/null || true
+    docker rm "$CONTAINER_NAME" 2>/dev/null || true
+}
+
+# Set trap to cleanup on exit
+trap cleanup EXIT
+
+echo -e "${YELLOW}Starting smoke test for: ${IMAGE}${NC}"
+echo "  Target: $TARGET"
+echo "  Health endpoint: http://localhost:${HEALTH_PORT}${HEALTH_PATH}"
+echo "  Timeout: ${TIMEOUT}s"
+echo ""
+
+# Remove any existing container with the same name
+docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
+
+# Start container based on target type
+echo "Starting container..."
+if [ "$TARGET" = "cp-only" ]; then
+    docker run -d --name "$CONTAINER_NAME" \
+        -p "${HEALTH_PORT}:${HEALTH_PORT}" \
+        "$IMAGE"
+else
+    # Build docker run command with required and optional env vars
+    DOCKER_CMD="docker run -d --name $CONTAINER_NAME"
+    DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_PROVIDER=$LLM_PROVIDER"
+    DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_API_KEY=${GROQ_API_KEY}"
+    DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_MODEL=$LLM_MODEL"
+
+    # Add optional embeddings provider config
+    if [ -n "${HINDSIGHT_API_EMBEDDINGS_PROVIDER:-}" ]; then
+        DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_EMBEDDINGS_PROVIDER=${HINDSIGHT_API_EMBEDDINGS_PROVIDER}"
+    fi
+    if [ -n "${HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY:-}" ]; then
+        DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=${HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY}"
+    fi
+
+    # Add optional reranker provider config
+    if [ -n "${HINDSIGHT_API_RERANKER_PROVIDER:-}" ]; then
+        DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_RERANKER_PROVIDER=${HINDSIGHT_API_RERANKER_PROVIDER}"
+    fi
+    if [ -n "${HINDSIGHT_API_COHERE_API_KEY:-}" ]; then
+        DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_COHERE_API_KEY=${HINDSIGHT_API_COHERE_API_KEY}"
+    fi
+
+    DOCKER_CMD="$DOCKER_CMD -p ${HEALTH_PORT}:${HEALTH_PORT}"
+    DOCKER_CMD="$DOCKER_CMD $IMAGE"
+
+    eval $DOCKER_CMD
+fi
+
+# Wait for health endpoint
+echo "Waiting for health endpoint at http://localhost:${HEALTH_PORT}${HEALTH_PATH}..."
+start_time=$(date +%s)
+
+for i in $(seq 1 "$TIMEOUT"); do
+    if curl -sf "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" > /dev/null 2>&1; then
+        end_time=$(date +%s)
+        duration=$((end_time - start_time))
+        echo ""
+        echo -e "${GREEN}Container is healthy after ${duration}s${NC}"
+        echo ""
+        echo "=== Health Response ==="
+        curl -s "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" | python3 -m json.tool 2>/dev/null || curl -s "http://localhost:${HEALTH_PORT}${HEALTH_PATH}"
+        echo ""
+        echo ""
+        echo "=== Container Logs (last 50 lines) ==="
+        docker logs "$CONTAINER_NAME" 2>&1 | tail -50
+        echo ""
+        echo -e "${GREEN}Smoke test PASSED${NC}"
+        exit 0
+    fi
+
+    # Show progress every 10 seconds
+    if [ $((i % 10)) -eq 0 ]; then
+        echo "  Still waiting... (${i}s)"
+    fi
+
+    # Check if container is still running
+    if ! docker ps -q -f "name=$CONTAINER_NAME" | grep -q .; then
+        echo ""
+        echo -e "${RED}Container exited unexpectedly!${NC}"
+        echo ""
+        echo "=== Container Logs ==="
+        docker logs "$CONTAINER_NAME" 2>&1
+        echo ""
+        echo -e "${RED}Smoke test FAILED${NC}"
+        exit 1
+    fi
+
+    sleep 1
+done
+
+# Timeout reached
+echo ""
+echo -e "${RED}Container failed to become healthy after ${TIMEOUT}s${NC}"
+echo ""
+echo "=== Container Logs ==="
+docker logs "$CONTAINER_NAME" 2>&1
+echo ""
+echo -e "${RED}Smoke test FAILED${NC}"
+exit 1

docker/test-slim-local.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Local Test Script for Slim Docker Images
+#
+# This script makes it easy to test slim images locally with external providers.
+# It expects API keys to be set in environment variables.
+#
+# Usage:
+#   export GROQ_API_KEY=gsk_xxx
+#   export OPENAI_API_KEY=sk-xxx
+#   export COHERE_API_KEY=xxx
+#   ./docker/test-slim-local.sh
+#
+# Or inline:
+#   GROQ_API_KEY=gsk_xxx OPENAI_API_KEY=sk_xxx COHERE_API_KEY=xxx ./docker/test-slim-local.sh
+#
+
+set -euo pipefail
+
+# Check for required API keys
+if [ -z "${GROQ_API_KEY:-}" ]; then
+    echo "❌ Error: GROQ_API_KEY environment variable is required"
+    echo "Set it with: export GROQ_API_KEY=gsk_xxx"
+    exit 1
+fi
+
+if [ -z "${OPENAI_API_KEY:-}" ]; then
+    echo "❌ Error: OPENAI_API_KEY environment variable is required"
+    echo "Set it with: export OPENAI_API_KEY=sk-xxx"
+    exit 1
+fi
+
+if [ -z "${COHERE_API_KEY:-}" ]; then
+    echo "❌ Error: COHERE_API_KEY environment variable is required"
+    echo "Set it with: export COHERE_API_KEY=xxx"
+    exit 1
+fi
+
+# Configuration
+IMAGE="${1:-hindsight-slim:test}"
+echo "Testing image: $IMAGE"
+echo ""
+
+# Set up external providers
+export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai
+export HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=$OPENAI_API_KEY
+export HINDSIGHT_API_RERANKER_PROVIDER=cohere
+export HINDSIGHT_API_COHERE_API_KEY=$COHERE_API_KEY
+
+# Run the test
+exec "$(dirname "$0")/test-image.sh" "$IMAGE" standalone

helm/INSTALL.txt

@@ -1,135 +0,0 @@
-HINDSIGHT HELM CHART INSTALLATION GUIDE
-=====================================
-
-PREREQUISITES
--------------
-- Kubernetes cluster (1.19+)
-- kubectl configured
-- Helm 3.x installed
-- PostgreSQL database with pgvector extension (if not using bundled PostgreSQL)
-
-BASIC INSTALLATION
-------------------
-
-1. Install with default values (requires external PostgreSQL):
-
-   helm install hindsight ./hindsight \
-     --set postgresql.external.host=your-postgres-host \
-     --set postgresql.external.password=your-password \
-     --set api.secrets.MEMORY_LLM_API_KEY=your-api-key
-
-2. Install with custom values file:
-
-   helm install hindsight ./hindsight -f hindsight/values-production.yaml
-
-3. Install in a specific namespace:
-
-   kubectl create namespace hindsight
-   helm install hindsight ./hindsight -n hindsight
-
-CONFIGURATION OPTIONS
----------------------
-
-Development setup (using values-development.yaml):
-   helm install hindsight ./hindsight -f hindsight/values-development.yaml
-
-Production setup (using values-production.yaml):
-   helm install hindsight ./hindsight -f hindsight/values-production.yaml
-
-Custom LLM provider:
-   helm install hindsight ./hindsight \
-     --set api.env.MEMORY_LLM_PROVIDER=openai \
-     --set api.env.MEMORY_LLM_MODEL=gpt-4 \
-     --set api.secrets.MEMORY_LLM_API_KEY=sk-your-key
-
-Enable ingress:
-   helm install hindsight ./hindsight \
-     --set ingress.enabled=true \
-     --set ingress.hosts[0].host=hindsight.example.com
-
-Enable autoscaling:
-   helm install hindsight ./hindsight \
-     --set autoscaling.enabled=true \
-     --set autoscaling.minReplicas=2 \
-     --set autoscaling.maxReplicas=10
-
-UPGRADE
--------
-
-Upgrade existing installation:
-   helm upgrade hindsight ./hindsight
-
-Upgrade with new values:
-   helm upgrade hindsight ./hindsight -f hindsight/values-production.yaml
-
-UNINSTALL
----------
-
-Remove the Helm release:
-   helm uninstall hindsight
-
-Remove with namespace:
-   helm uninstall hindsight -n hindsight
-
-TESTING
--------
-
-Test the installation with dry-run:
-   helm install hindsight ./hindsight --dry-run --debug
-
-Validate templates:
-   helm template hindsight ./hindsight
-
-Lint the chart:
-   helm lint ./hindsight
-
-ACCESSING THE SERVICES
-----------------------
-
-Port-forward control plane:
-   kubectl port-forward svc/hindsight-control-plane 3000:3000
-
-Port-forward API:
-   kubectl port-forward svc/hindsight-api 8888:8888
-
-Get service URLs:
-   helm status hindsight
-
-DATABASE INITIALIZATION
------------------------
-
-NOTE: Database migrations now run automatically when the API service starts.
-      You typically don't need to run migrations manually.
-
-If you want to pre-initialize the database before deploying (optional):
-   kubectl run hindsight-init --rm -it --restart=Never \
-     --image=hindsight/api:latest \
-     --env="DATABASE_URL=postgresql://user:pass@host:5432/hindsight" \
-     -- python -c "from hindsight.migrations import run_migrations; run_migrations()"
-
-TROUBLESHOOTING
----------------
-
-Check pod status:
-   kubectl get pods -l app.kubernetes.io/name=hindsight
-
-View logs for API:
-   kubectl logs -l app.kubernetes.io/component=api
-
-View logs for control plane:
-   kubectl logs -l app.kubernetes.io/component=control-plane
-
-Describe a pod:
-   kubectl describe pod <pod-name>
-
-Check configuration:
-   kubectl get configmap hindsight-config -o yaml
-   kubectl get secret hindsight-secret -o yaml
-
-NOTES
------
-- Make sure PostgreSQL has pgvector extension enabled
-- Run database migrations before first use
-- Configure proper resource limits for production
-- Use external secrets management for production
-- Enable TLS/SSL for production deployments

helm/hindsight/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: postgresql
+  repository: https://charts.bitnami.com/bitnami
+  version: 15.5.38
+digest: sha256:f67c7612736803ece8a669f8ca6b0555f3b78557bc0ecb732aa2e43f0df7750d
+generated: "2025-12-10T17:20:57.058794+01:00"

helm/hindsight/Chart.yaml

@@ -1,9 +1,9 @@
 apiVersion: v2
 name: hindsight
-description: A Helm chart for Hindsight - temporal-semantic-entity memory system for AI agents
+description: Hindsight helm chart
 type: application
-version: 0.0.9
-appVersion: "0.0.9"
+version: 0.4.10
+appVersion: "0.4.10"
 keywords:
   - ai
   - memory

helm/hindsight/README.md

@@ -0,0 +1,182 @@
+# Hindsight Helm Chart
+
+Helm chart for deploying Hindsight - a temporal-semantic-entity memory system for AI agents.
+
+## Prerequisites
+
+- Kubernetes 1.19+
+- Helm 3.0+
+- PostgreSQL database (external or bundled)
+
+## Quick Start
+
+```bash
+# Update dependencies first
+helm dependency update ./helm/hindsight
+
+# Install (PostgreSQL included by default)
+export OPENAI_API_KEY="sk-your-openai-key"
+helm upgrade hindsight --install ./helm/hindsight -n hindsight --create-namespace \
+  --set api.secrets.HINDSIGHT_API_LLM_API_KEY="$OPENAI_API_KEY"
+```
+
+To use an external database instead:
+
+```bash
+helm install hindsight ./helm/hindsight -n hindsight --create-namespace \
+  --set api.secrets.HINDSIGHT_API_LLM_API_KEY="sk-your-openai-key" \
+  --set postgresql.enabled=false \
+  --set postgresql.external.host=my-postgres.example.com \
+  --set postgresql.external.password=mypassword
+```
+
+## Installation
+
+### Add the repository (if published)
+
+```bash
+helm repo add hindsight https://your-helm-repo.com
+helm repo update
+```
+
+### Install with custom values file
+
+Create a `values-override.yaml`:
+
+```yaml
+api:
+  secrets:
+    HINDSIGHT_API_LLM_API_KEY: "sk-your-openai-key"
+
+postgresql:
+  external:
+    host: "my-postgres.example.com"
+    password: "mypassword"
+```
+
+Then install:
+
+```bash
+helm install hindsight ./helm/hindsight -n hindsight --create-namespace -f values-override.yaml
+```
+
+## Configuration
+
+### Key Values
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `version` | Default image tag for all components | `0.1.0` |
+| `api.enabled` | Enable the API component | `true` |
+| `api.image.repository` | API image repository | `hindsight/api` |
+| `api.image.tag` | API image tag (defaults to `version`) | - |
+| `api.service.port` | API service port | `8888` |
+| `controlPlane.enabled` | Enable the control plane | `true` |
+| `controlPlane.image.repository` | Control plane image repository | `hindsight/control-plane` |
+| `controlPlane.image.tag` | Control plane image tag (defaults to `version`) | - |
+| `controlPlane.service.port` | Control plane service port | `3000` |
+| `postgresql.enabled` | Deploy PostgreSQL as subchart | `true` |
+| `postgresql.external.host` | External PostgreSQL host | `postgresql` |
+| `postgresql.external.port` | External PostgreSQL port | `5432` |
+| `postgresql.external.database` | Database name | `hindsight` |
+| `postgresql.external.username` | Database username | `hindsight` |
+| `ingress.enabled` | Enable ingress | `false` |
+| `autoscaling.enabled` | Enable HPA | `false` |
+
+### Environment Variables
+
+All environment variables in `api.env` and `controlPlane.env` are automatically added to the respective pods. Sensitive values should go in `api.secrets` or `controlPlane.secrets`.
+
+```yaml
+api:
+  env:
+    HINDSIGHT_API_LLM_PROVIDER: "openai"
+    HINDSIGHT_API_LLM_MODEL: "gpt-4"
+  secrets:
+    HINDSIGHT_API_LLM_API_KEY: "your-api-key"
+    HINDSIGHT_API_LLM_BASE_URL: "https://api.openai.com/v1"
+
+controlPlane:
+  env:
+    NODE_ENV: "production"
+  secrets: {}
+```
+
+### External Database
+
+To connect to an external PostgreSQL database:
+
+```yaml
+postgresql:
+  enabled: false
+  external:
+    host: "my-postgres.example.com"
+    port: 5432
+    database: "hindsight"
+    username: "hindsight"
+    password: "your-password"
+```
+
+### Ingress
+
+To expose the services via ingress:
+
+```yaml
+ingress:
+  enabled: true
+  className: "nginx"
+  annotations:
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+  hosts:
+    - host: hindsight.example.com
+      paths:
+        - path: /
+          pathType: Prefix
+          service: controlPlane
+        - path: /api
+          pathType: Prefix
+          service: api
+  tls:
+    - secretName: hindsight-tls
+      hosts:
+        - hindsight.example.com
+```
+
+## Upgrading
+
+```bash
+helm upgrade hindsight ./helm/hindsight -n hindsight
+```
+
+## Uninstalling
+
+```bash
+helm uninstall hindsight -n hindsight
+```
+
+## Components
+
+The chart deploys:
+
+- **API**: The main Hindsight API server for memory operations
+- **Control Plane**: Web UI for managing agents and viewing memories
+
+## Development
+
+### Lint the chart
+
+```bash
+helm lint ./helm/hindsight
+```
+
+### Template locally
+
+```bash
+helm template hindsight ./helm/hindsight --debug
+```
+
+### Dry run installation
+
+```bash
+helm install hindsight ./helm/hindsight --dry-run --debug
+```

helm/hindsight/templates/NOTES.txt

@@ -1,71 +1,2 @@
-Thank you for installing {{ .Chart.Name }}!
-
-Your release is named {{ .Release.Name }}.
-
-To learn more about the release, try:
-
-  $ helm status {{ .Release.Name }}
-  $ helm get all {{ .Release.Name }}
-
-{{- if .Values.ingress.enabled }}
-
-The application is accessible via the following URL(s):
-{{- range .Values.ingress.hosts }}
-  - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .host }}
-{{- end }}
-
-{{- else }}
-
-1. Get the Control Plane URL by running these commands:
-{{- if contains "NodePort" .Values.controlPlane.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "hindsight.fullname" . }}-control-plane)
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo "Control Plane URL: http://$NODE_IP:$NODE_PORT"
-{{- else if contains "LoadBalancer" .Values.controlPlane.service.type }}
-  NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-  You can watch the status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "hindsight.fullname" . }}-control-plane'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "hindsight.fullname" . }}-control-plane --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo "Control Plane URL: http://$SERVICE_IP:{{ .Values.controlPlane.service.port }}"
-{{- else if contains "ClusterIP" .Values.controlPlane.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/component=control-plane,app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
-  echo "Control Plane URL: http://127.0.0.1:3000"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 3000:$CONTAINER_PORT
-{{- end }}
-
-2. Get the API URL by running these commands:
-{{- if contains "NodePort" .Values.api.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "hindsight.fullname" . }}-api)
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo "API URL: http://$NODE_IP:$NODE_PORT"
-{{- else if contains "LoadBalancer" .Values.api.service.type }}
-  NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-  You can watch the status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "hindsight.fullname" . }}-api'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "hindsight.fullname" . }}-api --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo "API URL: http://$SERVICE_IP:{{ .Values.api.service.port }}"
-{{- else if contains "ClusterIP" .Values.api.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/component=api,app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
-  echo "API URL: http://127.0.0.1:8888"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8888:$CONTAINER_PORT
-{{- end }}
-
-{{- end }}
-
-{{- if not .Values.postgresql.enabled }}
-
-NOTE: You are using an external PostgreSQL database.
-Please ensure that:
-  1. The database is accessible from the cluster
-  2. The pgvector extension is enabled
-
-Database migrations run automatically when the API service starts.
-
-If you want to pre-initialize the database before deploying (optional):
-  kubectl run --namespace {{ .Release.Namespace }} hindsight-init --rm -it --restart=Never \
-    --image={{ .Values.api.image.repository }}:{{ .Values.api.image.tag }} \
-    --env="DATABASE_URL={{ include "hindsight.databaseUrl" . }}" \
-    -- python -c "from hindsight.migrations import run_migrations; run_migrations()"
-{{- end }}
-
-For more information, visit: https://github.com/yourusername/hindsight
+Hindsight installed. Access the control plane:
+  kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "hindsight.fullname" . }}-control-plane 3000:3000

helm/hindsight/templates/_helpers.tpl

@@ -80,6 +80,22 @@ Control plane selector labels
 app.kubernetes.io/component: control-plane
 {{- end }}
 
+{{/*
+Worker labels
+*/}}
+{{- define "hindsight.worker.labels" -}}
+{{ include "hindsight.labels" . }}
+app.kubernetes.io/component: worker
+{{- end }}
+
+{{/*
+Worker selector labels
+*/}}
+{{- define "hindsight.worker.selectorLabels" -}}
+{{ include "hindsight.selectorLabels" . }}
+app.kubernetes.io/component: worker
+{{- end }}
+
 {{/*
 Create the name of the service account to use
 */}}
@@ -98,7 +114,7 @@ Generate database URL
 {{- if .Values.databaseUrl }}
 {{- .Values.databaseUrl }}
 {{- else if .Values.postgresql.enabled }}
-{{- printf "postgresql://%s:%s@%s-postgresql:%d/%s" .Values.postgresql.auth.username .Values.postgresql.auth.password (include "hindsight.fullname" .) (.Values.postgresql.primary.service.port | int) .Values.postgresql.auth.database }}
+{{- printf "postgresql://%s:%s@%s-postgresql:%d/%s" .Values.postgresql.auth.username .Values.postgresql.auth.password (include "hindsight.fullname" .) (.Values.postgresql.service.port | int) .Values.postgresql.auth.database }}
 {{- else }}
 {{- printf "postgresql://%s:$(POSTGRES_PASSWORD)@%s:%d/%s" .Values.postgresql.external.username .Values.postgresql.external.host (.Values.postgresql.external.port | int) .Values.postgresql.external.database }}
 {{- end }}
@@ -110,3 +126,46 @@ API URL for control plane
 {{- define "hindsight.apiUrl" -}}
 {{- printf "http://%s-api:%d" (include "hindsight.fullname" .) (.Values.api.service.port | int) }}
 {{- end }}
+
+{{/*
+TEI reranker labels
+*/}}
+{{- define "hindsight.tei.reranker.labels" -}}
+{{ include "hindsight.labels" . }}
+app.kubernetes.io/component: tei-reranker
+{{- end }}
+
+{{/*
+TEI reranker selector labels
+*/}}
+{{- define "hindsight.tei.reranker.selectorLabels" -}}
+{{ include "hindsight.selectorLabels" . }}
+app.kubernetes.io/component: tei-reranker
+{{- end }}
+
+{{/*
+TEI embedding labels
+*/}}
+{{- define "hindsight.tei.embedding.labels" -}}
+{{ include "hindsight.labels" . }}
+app.kubernetes.io/component: tei-embedding
+{{- end }}
+
+{{/*
+TEI embedding selector labels
+*/}}
+{{- define "hindsight.tei.embedding.selectorLabels" -}}
+{{ include "hindsight.selectorLabels" . }}
+app.kubernetes.io/component: tei-embedding
+{{- end }}
+
+{{/*
+Get the name of the secret to use
+*/}}
+{{- define "hindsight.secretName" -}}
+{{- if .Values.existingSecret }}
+{{- .Values.existingSecret }}
+{{- else }}
+{{- printf "%s-secret" (include "hindsight.fullname" .) }}
+{{- end }}
+{{- end }}

helm/hindsight/templates/api-deployment.yaml

@@ -15,8 +15,9 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- if not .Values.existingSecret }}
         checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+        {{- end }}
         {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -32,45 +33,61 @@ spec:
       - name: api
         securityContext:
           {{- toYaml .Values.securityContext | nindent 10 }}
-        image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag }}"
+        image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag | default .Values.version | default .Chart.AppVersion }}"
         imagePullPolicy: {{ .Values.api.image.pullPolicy }}
         ports:
         - name: http
           containerPort: {{ .Values.api.service.targetPort }}
           protocol: TCP
+        {{- if .Values.existingSecret }}
+        envFrom:
+        - secretRef:
+            name: {{ .Values.existingSecret }}
+        {{- end }}
         env:
-        - name: HINDSIGHT_API_DATABASE_URL
-          value: {{ include "hindsight.databaseUrl" . | quote }}
+        {{- /* POSTGRES_PASSWORD must be defined before DATABASE_URL for $(VAR) interpolation */}}
         {{- if not .Values.postgresql.enabled }}
         - name: POSTGRES_PASSWORD
           valueFrom:
             secretKeyRef:
-              name: {{ include "hindsight.fullname" . }}-secret
+              name: {{ include "hindsight.secretName" . }}
               key: postgres-password
         {{- end }}
-        - name: HINDSIGHT_API_LLM_PROVIDER
-          valueFrom:
-            configMapKeyRef:
-              name: {{ include "hindsight.fullname" . }}-config
-              key: llm-provider
-        - name: HINDSIGHT_API_LLM_MODEL
-          valueFrom:
-            configMapKeyRef:
-              name: {{ include "hindsight.fullname" . }}-config
-              key: llm-model
-        {{- if and .Values.api.secrets (hasKey .Values.api.secrets "HINDSIGHT_API_LLM_API_KEY") }}
-        - name: HINDSIGHT_API_LLM_API_KEY
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "hindsight.fullname" . }}-secret
-              key: llm-api-key
+        - name: HINDSIGHT_API_DATABASE_URL
+          value: {{ include "hindsight.databaseUrl" . | quote }}
+        {{- /* Disable internal worker when dedicated workers are enabled */}}
+        {{- if .Values.worker.enabled }}
+        - name: HINDSIGHT_API_WORKER_ENABLED
+          value: "false"
         {{- end }}
-        {{- if and .Values.api.secrets (hasKey .Values.api.secrets "HINDSIGHT_API_LLM_BASE_URL") }}
-        - name: HINDSIGHT_API_LLM_BASE_URL
+        {{- /* Explicitly set port to override K8s service discovery env var (HINDSIGHT_API_PORT) */}}
+        - name: HINDSIGHT_API_PORT
+          value: {{ .Values.api.service.targetPort | quote }}
+        {{- range $key, $value := .Values.api.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- if .Values.tei.reranker.enabled }}
+        - name: HINDSIGHT_API_RERANKER_PROVIDER
+          value: "tei"
+        - name: HINDSIGHT_API_RERANKER_TEI_URL
+          value: "http://{{ include "hindsight.fullname" . }}-tei-reranker:{{ .Values.tei.reranker.port }}"
+        {{- end }}
+        {{- if .Values.tei.embedding.enabled }}
+        - name: HINDSIGHT_API_EMBEDDINGS_PROVIDER
+          value: "tei"
+        - name: HINDSIGHT_API_EMBEDDINGS_TEI_URL
+          value: "http://{{ include "hindsight.fullname" . }}-tei-embedding:{{ .Values.tei.embedding.port }}"
+        {{- end }}
+        {{- /* Only use api.secrets when not using existingSecret (for chart-managed secrets) */}}
+        {{- if not .Values.existingSecret }}
+        {{- range $key, $value := .Values.api.secrets }}
+        - name: {{ $key }}
           valueFrom:
             secretKeyRef:
-              name: {{ include "hindsight.fullname" . }}-secret
-              key: llm-base-url
+              name: {{ include "hindsight.secretName" $ }}
+              key: {{ $key }}
+        {{- end }}
         {{- end }}
         livenessProbe:
           {{- toYaml .Values.api.livenessProbe | nindent 10 }}
@@ -82,7 +99,7 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.affinity }}
+      {{- with (.Values.api.affinity | default .Values.affinity) }}
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}

helm/hindsight/templates/configmap.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "hindsight.fullname" . }}-config
-  labels:
-    {{- include "hindsight.labels" . | nindent 4 }}
-data:
-  # API configuration
-  llm-provider: {{ .Values.api.env.HINDSIGHT_API_LLM_PROVIDER | quote }}
-  llm-model: {{ .Values.api.env.HINDSIGHT_API_LLM_MODEL | quote }}
-
-  # Control plane configuration
-  node-env: {{ .Values.controlPlane.env.NODE_ENV | quote }}
-  hostname: {{ .Values.controlPlane.env.HINDSIGHT_CP_HOSTNAME | quote }}
-  control-plane-port: {{ .Values.controlPlane.env.HINDSIGHT_CP_PORT | quote }}

helm/hindsight/templates/controlplane-deployment.yaml

@@ -15,7 +15,9 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- if not .Values.existingSecret }}
+        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+        {{- end }}
         {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -31,30 +33,34 @@ spec:
       - name: control-plane
         securityContext:
           {{- toYaml .Values.securityContext | nindent 10 }}
-        image: "{{ .Values.controlPlane.image.repository }}:{{ .Values.controlPlane.image.tag }}"
+        image: "{{ .Values.controlPlane.image.repository }}:{{ .Values.controlPlane.image.tag | default .Values.version | default .Chart.AppVersion }}"
         imagePullPolicy: {{ .Values.controlPlane.image.pullPolicy }}
         ports:
         - name: http
           containerPort: {{ .Values.controlPlane.service.targetPort }}
           protocol: TCP
+        {{- if .Values.existingSecret }}
+        envFrom:
+        - secretRef:
+            name: {{ .Values.existingSecret }}
+        {{- end }}
         env:
-        - name: NODE_ENV
-          valueFrom:
-            configMapKeyRef:
-              name: {{ include "hindsight.fullname" . }}-config
-              key: node-env
-        - name: HINDSIGHT_CP_HOSTNAME
-          valueFrom:
-            configMapKeyRef:
-              name: {{ include "hindsight.fullname" . }}-config
-              key: hostname
-        - name: HINDSIGHT_CP_PORT
-          valueFrom:
-            configMapKeyRef:
-              name: {{ include "hindsight.fullname" . }}-config
-              key: control-plane-port
         - name: HINDSIGHT_CP_DATAPLANE_API_URL
           value: {{ include "hindsight.apiUrl" . | quote }}
+        {{- range $key, $value := .Values.controlPlane.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- /* Only use controlPlane.secrets when not using existingSecret (for chart-managed secrets) */}}
+        {{- if not .Values.existingSecret }}
+        {{- range $key, $value := .Values.controlPlane.secrets }}
+        - name: {{ $key }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "hindsight.secretName" $ }}
+              key: {{ $key }}
+        {{- end }}
+        {{- end }}
         livenessProbe:
           {{- toYaml .Values.controlPlane.livenessProbe | nindent 10 }}
         readinessProbe:
@@ -65,7 +71,7 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.affinity }}
+      {{- with (.Values.controlPlane.affinity | default .Values.affinity) }}
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}

helm/hindsight/templates/pdb.yaml

@@ -0,0 +1,56 @@
+{{- if and .Values.api.enabled .Values.api.podDisruptionBudget.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "hindsight.fullname" . }}-api
+  labels:
+    {{- include "hindsight.api.labels" . | nindent 4 }}
+spec:
+  {{- if .Values.api.podDisruptionBudget.minAvailable }}
+  minAvailable: {{ .Values.api.podDisruptionBudget.minAvailable }}
+  {{- end }}
+  {{- if .Values.api.podDisruptionBudget.maxUnavailable }}
+  maxUnavailable: {{ .Values.api.podDisruptionBudget.maxUnavailable }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "hindsight.api.selectorLabels" . | nindent 6 }}
+{{- end }}
+---
+{{- if and .Values.controlPlane.enabled .Values.controlPlane.podDisruptionBudget.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "hindsight.fullname" . }}-control-plane
+  labels:
+    {{- include "hindsight.controlPlane.labels" . | nindent 4 }}
+spec:
+  {{- if .Values.controlPlane.podDisruptionBudget.minAvailable }}
+  minAvailable: {{ .Values.controlPlane.podDisruptionBudget.minAvailable }}
+  {{- end }}
+  {{- if .Values.controlPlane.podDisruptionBudget.maxUnavailable }}
+  maxUnavailable: {{ .Values.controlPlane.podDisruptionBudget.maxUnavailable }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "hindsight.controlPlane.selectorLabels" . | nindent 6 }}
+{{- end }}
+---
+{{- if and .Values.worker.enabled .Values.worker.podDisruptionBudget.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "hindsight.fullname" . }}-worker
+  labels:
+    {{- include "hindsight.worker.labels" . | nindent 4 }}
+spec:
+  {{- if .Values.worker.podDisruptionBudget.minAvailable }}
+  minAvailable: {{ .Values.worker.podDisruptionBudget.minAvailable }}
+  {{- end }}
+  {{- if .Values.worker.podDisruptionBudget.maxUnavailable }}
+  maxUnavailable: {{ .Values.worker.podDisruptionBudget.maxUnavailable }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "hindsight.worker.selectorLabels" . | nindent 6 }}
+{{- end }}

helm/hindsight/templates/postgresql-service.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.postgresql.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hindsight.fullname" . }}-postgresql
+  labels:
+    {{- include "hindsight.labels" . | nindent 4 }}
+    app.kubernetes.io/component: postgresql
+spec:
+  type: ClusterIP
+  ports:
+  - port: {{ .Values.postgresql.service.port }}
+    targetPort: postgresql
+    protocol: TCP
+    name: postgresql
+  selector:
+    {{- include "hindsight.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: postgresql
+{{- end }}

helm/hindsight/templates/postgresql-statefulset.yaml

@@ -0,0 +1,85 @@
+{{- if .Values.postgresql.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "hindsight.fullname" . }}-postgresql
+  labels:
+    {{- include "hindsight.labels" . | nindent 4 }}
+    app.kubernetes.io/component: postgresql
+spec:
+  serviceName: {{ include "hindsight.fullname" . }}-postgresql
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "hindsight.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: postgresql
+  template:
+    metadata:
+      labels:
+        {{- include "hindsight.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: postgresql
+    spec:
+      containers:
+      - name: postgresql
+        image: "{{ .Values.postgresql.image.repository }}:{{ .Values.postgresql.image.tag }}"
+        imagePullPolicy: {{ .Values.postgresql.image.pullPolicy }}
+        ports:
+        - name: postgresql
+          containerPort: 5432
+          protocol: TCP
+        env:
+        - name: POSTGRES_USER
+          value: {{ .Values.postgresql.auth.username | quote }}
+        - name: POSTGRES_PASSWORD
+          value: {{ .Values.postgresql.auth.password | quote }}
+        - name: POSTGRES_DB
+          value: {{ .Values.postgresql.auth.database | quote }}
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        livenessProbe:
+          exec:
+            command:
+            - pg_isready
+            - -U
+            - {{ .Values.postgresql.auth.username }}
+            - -d
+            - {{ .Values.postgresql.auth.database }}
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 3
+        readinessProbe:
+          exec:
+            command:
+            - pg_isready
+            - -U
+            - {{ .Values.postgresql.auth.username }}
+            - -d
+            - {{ .Values.postgresql.auth.database }}
+          initialDelaySeconds: 5
+          periodSeconds: 5
+          timeoutSeconds: 3
+          failureThreshold: 3
+        resources:
+          {{- toYaml .Values.postgresql.resources | nindent 10 }}
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/postgresql/data
+  {{- if .Values.postgresql.persistence.enabled }}
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      {{- if .Values.postgresql.persistence.storageClass }}
+      storageClassName: {{ .Values.postgresql.persistence.storageClass | quote }}
+      {{- end }}
+      resources:
+        requests:
+          storage: {{ .Values.postgresql.persistence.size }}
+  {{- else }}
+      volumes:
+      - name: data
+        emptyDir: {}
+  {{- end }}
+{{- end }}

helm/hindsight/templates/secret.yaml

@@ -1,19 +1,19 @@
+{{- if not .Values.existingSecret }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ include "hindsight.fullname" . }}-secret
+  name: {{ include "hindsight.secretName" . }}
   labels:
     {{- include "hindsight.labels" . | nindent 4 }}
 type: Opaque
 data:
-  {{- if and .Values.api.secrets (hasKey .Values.api.secrets "MEMORY_LLM_API_KEY") }}
-  llm-api-key: {{ .Values.api.secrets.MEMORY_LLM_API_KEY | b64enc | quote }}
+  {{- range $key, $value := .Values.api.secrets }}
+  {{ $key }}: {{ $value | b64enc | quote }}
   {{- end }}
-  {{- if and .Values.api.secrets (hasKey .Values.api.secrets "MEMORY_LLM_BASE_URL") }}
-  llm-base-url: {{ .Values.api.secrets.MEMORY_LLM_BASE_URL | b64enc | quote }}
+  {{- range $key, $value := .Values.controlPlane.secrets }}
+  {{ $key }}: {{ $value | b64enc | quote }}
   {{- end }}
-  {{- if not .Values.postgresql.enabled }}
-  {{- if .Values.postgresql.external.password }}
+  {{- if and (not .Values.postgresql.enabled) .Values.postgresql.external.password }}
   postgres-password: {{ .Values.postgresql.external.password | b64enc | quote }}
   {{- end }}
-  {{- end }}
+{{- end }}

helm/hindsight/templates/tei-embedding-deployment.yaml

@@ -0,0 +1,76 @@
+{{- if .Values.tei.embedding.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "hindsight.fullname" . }}-tei-embedding
+  labels:
+    {{- include "hindsight.tei.embedding.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.tei.embedding.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "hindsight.tei.embedding.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "hindsight.tei.embedding.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- if .Values.serviceAccount.create }}
+      serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+      - name: tei-embedding
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 10 }}
+        image: "{{ .Values.tei.embedding.image.repository }}:{{ .Values.tei.embedding.image.tag }}"
+        imagePullPolicy: {{ .Values.tei.embedding.image.pullPolicy }}
+        args:
+          - "--model-id"
+          - {{ .Values.tei.embedding.model | quote }}
+          - "--hostname"
+          - "0.0.0.0"
+          {{- range .Values.tei.embedding.args }}
+          - {{ . | quote }}
+          {{- end }}
+        ports:
+        - name: http
+          containerPort: {{ .Values.tei.embedding.port }}
+          protocol: TCP
+        env:
+        - name: PORT
+          value: {{ .Values.tei.embedding.port | quote }}
+        {{- range $key, $value := .Values.tei.embedding.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        livenessProbe:
+          {{- toYaml .Values.tei.embedding.livenessProbe | nindent 10 }}
+        readinessProbe:
+          {{- toYaml .Values.tei.embedding.readinessProbe | nindent 10 }}
+        resources:
+          {{- toYaml .Values.tei.embedding.resources | nindent 10 }}
+        volumeMounts:
+        - name: model-cache
+          mountPath: /data
+      volumes:
+      - name: model-cache
+        emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

helm/hindsight/templates/tei-embedding-service.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.tei.embedding.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hindsight.fullname" . }}-tei-embedding
+  labels:
+    {{- include "hindsight.tei.embedding.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+  - port: {{ .Values.tei.embedding.port }}
+    targetPort: http
+    protocol: TCP
+    name: http
+  selector:
+    {{- include "hindsight.tei.embedding.selectorLabels" . | nindent 4 }}
+{{- end }}

helm/hindsight/templates/tei-reranker-deployment.yaml

@@ -0,0 +1,76 @@
+{{- if .Values.tei.reranker.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "hindsight.fullname" . }}-tei-reranker
+  labels:
+    {{- include "hindsight.tei.reranker.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.tei.reranker.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "hindsight.tei.reranker.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "hindsight.tei.reranker.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- if .Values.serviceAccount.create }}
+      serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+      - name: tei-reranker
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 10 }}
+        image: "{{ .Values.tei.reranker.image.repository }}:{{ .Values.tei.reranker.image.tag }}"
+        imagePullPolicy: {{ .Values.tei.reranker.image.pullPolicy }}
+        args:
+          - "--model-id"
+          - {{ .Values.tei.reranker.model | quote }}
+          - "--hostname"
+          - "0.0.0.0"
+          {{- range .Values.tei.reranker.args }}
+          - {{ . | quote }}
+          {{- end }}
+        ports:
+        - name: http
+          containerPort: {{ .Values.tei.reranker.port }}
+          protocol: TCP
+        env:
+        - name: PORT
+          value: {{ .Values.tei.reranker.port | quote }}
+        {{- range $key, $value := .Values.tei.reranker.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        livenessProbe:
+          {{- toYaml .Values.tei.reranker.livenessProbe | nindent 10 }}
+        readinessProbe:
+          {{- toYaml .Values.tei.reranker.readinessProbe | nindent 10 }}
+        resources:
+          {{- toYaml .Values.tei.reranker.resources | nindent 10 }}
+        volumeMounts:
+        - name: model-cache
+          mountPath: /data
+      volumes:
+      - name: model-cache
+        emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

helm/hindsight/templates/tei-reranker-service.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.tei.reranker.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hindsight.fullname" . }}-tei-reranker
+  labels:
+    {{- include "hindsight.tei.reranker.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+  - port: {{ .Values.tei.reranker.port }}
+    targetPort: http
+    protocol: TCP
+    name: http
+  selector:
+    {{- include "hindsight.tei.reranker.selectorLabels" . | nindent 4 }}
+{{- end }}

helm/hindsight/templates/worker-service.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.worker.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hindsight.fullname" . }}-worker
+  labels:
+    {{- include "hindsight.worker.labels" . | nindent 4 }}
+  {{- if .Values.podAnnotations }}
+  annotations:
+    {{- /* Common Prometheus annotations for metrics scraping */}}
+    prometheus.io/scrape: "true"
+    prometheus.io/port: {{ .Values.worker.service.port | quote }}
+    prometheus.io/path: "/metrics"
+  {{- end }}
+spec:
+  # Headless service for StatefulSet (enables stable DNS names like worker-0.worker.namespace)
+  clusterIP: None
+  ports:
+  - port: {{ .Values.worker.service.port }}
+    targetPort: {{ .Values.worker.service.targetPort }}
+    protocol: TCP
+    name: http
+  selector:
+    {{- include "hindsight.worker.selectorLabels" . | nindent 4 }}
+{{- end }}

helm/hindsight/templates/worker-statefulset.yaml

@@ -0,0 +1,110 @@
+{{- if .Values.worker.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "hindsight.fullname" . }}-worker
+  labels:
+    {{- include "hindsight.worker.labels" . | nindent 4 }}
+spec:
+  serviceName: {{ include "hindsight.fullname" . }}-worker
+  replicas: {{ .Values.worker.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "hindsight.worker.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        {{- if not .Values.existingSecret }}
+        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+        {{- end }}
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "hindsight.worker.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- if .Values.serviceAccount.create }}
+      serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+      - name: worker
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 10 }}
+        image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Values.version | default .Chart.AppVersion }}"
+        imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+        command: ["hindsight-worker"]
+        ports:
+        - name: http
+          containerPort: {{ .Values.worker.service.targetPort }}
+          protocol: TCP
+        {{- if .Values.existingSecret }}
+        envFrom:
+        - secretRef:
+            name: {{ .Values.existingSecret }}
+        {{- end }}
+        env:
+        {{- /* POSTGRES_PASSWORD must be defined before DATABASE_URL for $(VAR) interpolation */}}
+        {{- if not .Values.postgresql.enabled }}
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "hindsight.secretName" . }}
+              key: postgres-password
+        {{- end }}
+        - name: HINDSIGHT_API_DATABASE_URL
+          value: {{ include "hindsight.databaseUrl" . | quote }}
+        {{- /* Worker ID uses pod name (StatefulSet provides stable names like worker-0, worker-1) */}}
+        - name: HINDSIGHT_API_WORKER_ID
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        {{- /* Inherit LLM config from api.env */}}
+        {{- range $key, $value := .Values.api.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- /* Worker-specific env vars */}}
+        {{- range $key, $value := .Values.worker.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- /* Only use secrets when not using existingSecret */}}
+        {{- if not .Values.existingSecret }}
+        {{- /* Inherit secrets from api.secrets */}}
+        {{- range $key, $value := .Values.api.secrets }}
+        - name: {{ $key }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "hindsight.secretName" $ }}
+              key: {{ $key }}
+        {{- end }}
+        {{- /* Worker-specific secrets (can override api.secrets) */}}
+        {{- range $key, $value := .Values.worker.secrets }}
+        - name: {{ $key }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "hindsight.secretName" $ }}
+              key: {{ $key }}
+        {{- end }}
+        {{- end }}
+        livenessProbe:
+          {{- toYaml .Values.worker.livenessProbe | nindent 10 }}
+        readinessProbe:
+          {{- toYaml .Values.worker.readinessProbe | nindent 10 }}
+        resources:
+          {{- toYaml .Values.worker.resources | nindent 10 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with (.Values.worker.affinity | default .Values.affinity) }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

helm/hindsight/values.yaml

@@ -1,5 +1,18 @@
 # Default values for hindsight
 
+# Global version override - use this to set a consistent image tag across all components
+# If not set, defaults to Chart.appVersion from Chart.yaml
+# version: ""
+
+# Use an existing secret instead of creating one from values
+# When set, all keys from this secret are injected as environment variables via envFrom
+# Required keys:
+#   - postgres-password: PostgreSQL password (when postgresql.enabled=false)
+# Optional keys (any key becomes an env var):
+#   - HINDSIGHT_API_LLM_API_KEY: API key for LLM provider
+#   - Any other env vars you want to inject
+# existingSecret: "my-hindsight-secret"
+
 # Global settings
 replicaCount: 1
 
@@ -8,9 +21,9 @@ api:
   enabled: true
   replicaCount: 1
   image:
-    repository: hindsight/api
+    repository: ghcr.io/vectorize-io/hindsight-api
     pullPolicy: IfNotPresent
-    tag: "latest"
+    # tag defaults to .Values.version if not specified
 
   service:
     type: ClusterIP
@@ -29,7 +42,7 @@ api:
   # Liveness and readiness probes
   livenessProbe:
     httpGet:
-      path: /
+      path: /health
       port: 8888
     initialDelaySeconds: 30
     periodSeconds: 10
@@ -38,16 +51,25 @@ api:
 
   readinessProbe:
     httpGet:
-      path: /
+      path: /health
       port: 8888
     initialDelaySeconds: 10
     periodSeconds: 5
     timeoutSeconds: 3
     failureThreshold: 3
 
+  # Pod disruption budget
+  podDisruptionBudget:
+    enabled: false
+    minAvailable: 1
+    # maxUnavailable: 1
+
+  # Pod affinity/anti-affinity (overrides global affinity for this component)
+  # affinity: {}
+
   # Environment variables
   env:
-    HINDSIGHT_API_LLM_PROVIDER: "groq"
+    #HINDSIGHT_API_LLM_PROVIDER: "groq"
     HINDSIGHT_API_LLM_MODEL: "openai/gpt-oss-120b"
 
   # Secret environment variables
@@ -55,14 +77,80 @@ api:
     # HINDSIGHT_API_LLM_API_KEY: "your-api-key"
     # HINDSIGHT_API_LLM_BASE_URL: "https://api.groq.com/openai/v1"
 
+# Worker settings (distributed task processing)
+# When enabled, dedicated worker pods process tasks and the API's internal worker is disabled
+worker:
+  enabled: false
+  replicaCount: 2
+  image:
+    repository: ghcr.io/vectorize-io/hindsight-api
+    pullPolicy: IfNotPresent
+    # tag: ""  # defaults to .Values.version, then Chart.appVersion if not specified
+
+  service:
+    # Service for metrics scraping (headless for StatefulSet)
+    port: 8889
+    targetPort: 8889
+
+  # Resource limits and requests
+  resources:
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+    requests:
+      cpu: 500m
+      memory: 1Gi
+
+  # Liveness and readiness probes
+  livenessProbe:
+    httpGet:
+      path: /health
+      port: 8889
+    initialDelaySeconds: 30
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 3
+
+  readinessProbe:
+    httpGet:
+      path: /health
+      port: 8889
+    initialDelaySeconds: 10
+    periodSeconds: 5
+    timeoutSeconds: 3
+    failureThreshold: 3
+
+  # Worker-specific environment variables
+  env:
+    # Poll interval in milliseconds (how often to check for new tasks)
+    HINDSIGHT_API_WORKER_POLL_INTERVAL_MS: "500"
+    # Number of tasks to claim per poll cycle
+    HINDSIGHT_API_WORKER_BATCH_SIZE: "10"
+    # Max retries before marking a task as failed
+    HINDSIGHT_API_WORKER_MAX_RETRIES: "3"
+    # HTTP port for metrics/health (matches service.targetPort)
+    HINDSIGHT_API_WORKER_HTTP_PORT: "8889"
+
+  # Pod disruption budget
+  podDisruptionBudget:
+    enabled: false
+    minAvailable: 1
+    # maxUnavailable: 1
+
+  # Pod affinity/anti-affinity (overrides global affinity for this component)
+  # affinity: {}
+
+  # Secret environment variables (inherited from api.secrets if not specified)
+  secrets: {}
+
 # Image settings for control plane
 controlPlane:
   enabled: true
   replicaCount: 1
   image:
-    repository: hindsight/hindsight-control-plane
+    repository: ghcr.io/vectorize-io/hindsight-control-plane
     pullPolicy: IfNotPresent
-    tag: "latest"
+    # tag defaults to .Values.version if not specified
 
   service:
     type: ClusterIP
@@ -78,10 +166,9 @@ controlPlane:
       cpu: 250m
       memory: 512Mi
 
-  # Liveness and readiness probes
+  # Liveness and readiness probes (TCP check)
   livenessProbe:
-    httpGet:
-      path: /
+    tcpSocket:
       port: 3000
     initialDelaySeconds: 30
     periodSeconds: 10
@@ -89,14 +176,22 @@ controlPlane:
     failureThreshold: 3
 
   readinessProbe:
-    httpGet:
-      path: /
+    tcpSocket:
       port: 3000
     initialDelaySeconds: 10
     periodSeconds: 5
     timeoutSeconds: 3
     failureThreshold: 3
 
+  # Pod disruption budget
+  podDisruptionBudget:
+    enabled: false
+    minAvailable: 1
+    # maxUnavailable: 1
+
+  # Pod affinity/anti-affinity (overrides global affinity for this component)
+  # affinity: {}
+
   # Environment variables
   env:
     NODE_ENV: "production"
@@ -106,21 +201,43 @@ controlPlane:
 # PostgreSQL configuration
 postgresql:
   # Set to true to deploy PostgreSQL as part of this chart
-  enabled: false
+  enabled: true
+
+  image:
+    repository: ankane/pgvector
+    tag: latest
+    pullPolicy: IfNotPresent
+
+  auth:
+    username: "hindsight"
+    password: "hindsight"
+    database: "hindsight"
+
+  service:
+    port: 5432
+
+  persistence:
+    enabled: true
+    size: 8Gi
+    # storageClass: ""
+
+  resources:
+    limits:
+      cpu: 1000m
+      memory: 1Gi
+    requests:
+      cpu: 250m
+      memory: 256Mi
 
   # External PostgreSQL connection details
-  # If postgresql.enabled is false, provide external database details
+  # Only used if postgresql.enabled is false
   external:
     host: "postgresql"
     port: 5432
     database: "hindsight"
     username: "hindsight"
-    # Password should be provided via secret
     # password: ""
 
-# Database URL (auto-generated from postgresql config if not provided)
-# databaseUrl: "postgresql://user:pass@host:5432/database"
-
 # Ingress configuration
 ingress:
   enabled: false
@@ -173,9 +290,87 @@ nodeSelector: {}
 # Tolerations
 tolerations: []
 
-# Affinity
+# Affinity (applied to all components unless overridden per-component)
 affinity: {}
 
+# TEI (Text Embeddings Inference) - optional standalone deployments
+# for reranking and/or embedding models
+tei:
+  reranker:
+    enabled: false
+    replicaCount: 1
+    image:
+      repository: ghcr.io/huggingface/text-embeddings-inference
+      tag: cpu-1.8.3
+      pullPolicy: IfNotPresent
+    model: "cross-encoder/ms-marco-MiniLM-L-6-v2"
+    port: 8090
+    args:
+      - "--auto-truncate"
+    env:
+      PAYLOAD_LIMIT: "10000000"
+      MAX_CLIENT_BATCH_SIZE: "256"
+    resources:
+      limits:
+        cpu: 2000m
+        memory: 2Gi
+      requests:
+        cpu: 500m
+        memory: 1Gi
+    livenessProbe:
+      httpGet:
+        path: /health
+        port: 8090
+      initialDelaySeconds: 30
+      periodSeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 6
+    readinessProbe:
+      httpGet:
+        path: /health
+        port: 8090
+      initialDelaySeconds: 15
+      periodSeconds: 5
+      timeoutSeconds: 3
+      failureThreshold: 3
+
+  embedding:
+    enabled: false
+    replicaCount: 1
+    image:
+      repository: ghcr.io/huggingface/text-embeddings-inference
+      tag: cpu-1.8.3
+      pullPolicy: IfNotPresent
+    model: "sentence-transformers/all-MiniLM-L6-v2"
+    port: 8091
+    args: []
+    env:
+      PAYLOAD_LIMIT: "10000000"
+      MAX_CLIENT_BATCH_SIZE: "256"
+    resources:
+      limits:
+        cpu: 2000m
+        memory: 2Gi
+      requests:
+        cpu: 500m
+        memory: 1Gi
+    livenessProbe:
+      httpGet:
+        path: /health
+        port: 8091
+      initialDelaySeconds: 30
+      periodSeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 6
+    readinessProbe:
+      httpGet:
+        path: /health
+        port: 8091
+      initialDelaySeconds: 15
+      periodSeconds: 5
+      timeoutSeconds: 3
+      failureThreshold: 3
+
 # Autoscaling
 autoscaling:
   enabled: false

hindsight-api/README.md

@@ -1 +1,137 @@
-# Memory 
+# Hindsight API
+
+**Memory System for AI Agents** — Temporal + Semantic + Entity Memory Architecture using PostgreSQL with pgvector.
+
+Hindsight gives AI agents persistent memory that works like human memory: it stores facts, tracks entities and relationships, handles temporal reasoning ("what happened last spring?"), and forms opinions based on configurable disposition traits.
+
+## Installation
+
+```bash
+pip install hindsight-api
+```
+
+## Quick Start
+
+### Run the Server
+
+```bash
+# Set your LLM provider
+export HINDSIGHT_API_LLM_PROVIDER=openai
+export HINDSIGHT_API_LLM_API_KEY=sk-xxxxxxxxxxxx
+
+# Start the server (uses embedded PostgreSQL by default)
+hindsight-api
+```
+
+The server starts at http://localhost:8888 with:
+- REST API for memory operations
+- MCP server at `/mcp` for tool-use integration
+
+### Use the Python API
+
+```python
+from hindsight_api import MemoryEngine
+
+# Create and initialize the memory engine
+memory = MemoryEngine()
+await memory.initialize()
+
+# Create a memory bank for your agent
+bank = await memory.create_memory_bank(
+    name="my-assistant",
+    background="A helpful coding assistant"
+)
+
+# Store a memory
+await memory.retain(
+    memory_bank_id=bank.id,
+    content="The user prefers Python for data science projects"
+)
+
+# Recall memories
+results = await memory.recall(
+    memory_bank_id=bank.id,
+    query="What programming language does the user prefer?"
+)
+
+# Reflect with reasoning
+response = await memory.reflect(
+    memory_bank_id=bank.id,
+    query="Should I recommend Python or R for this ML project?"
+)
+```
+
+## CLI Options
+
+```bash
+hindsight-api --help
+
+# Common options
+hindsight-api --port 9000          # Custom port (default: 8888)
+hindsight-api --host 127.0.0.1     # Bind to localhost only
+hindsight-api --workers 4          # Multiple worker processes
+hindsight-api --log-level debug    # Verbose logging
+```
+
+## Configuration
+
+Configure via environment variables:
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `HINDSIGHT_API_DATABASE_URL` | PostgreSQL connection string | `pg0` (embedded) |
+| `HINDSIGHT_API_LLM_PROVIDER` | `openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio` | `openai` |
+| `HINDSIGHT_API_LLM_API_KEY` | API key for LLM provider | - |
+| `HINDSIGHT_API_LLM_MODEL` | Model name | `gpt-4o-mini` |
+| `HINDSIGHT_API_HOST` | Server bind address | `0.0.0.0` |
+| `HINDSIGHT_API_PORT` | Server port | `8888` |
+
+### Example with External PostgreSQL
+
+```bash
+export HINDSIGHT_API_DATABASE_URL=postgresql://user:pass@localhost:5432/hindsight
+export HINDSIGHT_API_LLM_PROVIDER=groq
+export HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx
+
+hindsight-api
+```
+
+## Docker
+
+```bash
+docker run --rm -it -p 8888:8888 \
+  -e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \
+  -v $HOME/.hindsight-docker:/home/hindsight/.pg0 \
+  ghcr.io/vectorize-io/hindsight:latest
+```
+
+## MCP Server
+
+For local MCP integration without running the full API server:
+
+```bash
+hindsight-local-mcp
+```
+
+This runs a stdio-based MCP server that can be used directly with MCP-compatible clients.
+
+## Key Features
+
+- **Multi-Strategy Retrieval (TEMPR)** — Semantic, keyword, graph, and temporal search combined with RRF fusion
+- **Entity Graph** — Automatic entity extraction and relationship tracking
+- **Temporal Reasoning** — Native support for time-based queries
+- **Disposition Traits** — Configurable skepticism, literalism, and empathy influence opinion formation
+- **Three Memory Types** — World facts, bank actions, and formed opinions with confidence scores
+
+## Documentation
+
+Full documentation: [https://hindsight.vectorize.io](https://hindsight.vectorize.io)
+
+- [Installation Guide](https://hindsight.vectorize.io/developer/installation)
+- [Configuration Reference](https://hindsight.vectorize.io/developer/configuration)
+- [API Reference](https://hindsight.vectorize.io/api-reference)
+- [Python SDK](https://hindsight.vectorize.io/sdks/python)
+
+## License
+
+Apache 2.0

hindsight-api/alembic/env.py

@@ -1,129 +0,0 @@
-"""
-Alembic environment configuration for SQLAlchemy with pgvector.
-Uses synchronous psycopg2 driver for migrations to avoid pgbouncer issues.
-"""
-import logging
-import os
-import sys
-from pathlib import Path
-
-from sqlalchemy import pool, engine_from_config
-from sqlalchemy.engine import Connection
-
-from alembic import context
-from dotenv import load_dotenv
-
-# Import your models here
-from hindsight_api.models import Base
-
-# Load environment variables based on HINDSIGHT_API_DATABASE_URL env var or default to local
-def load_env():
-    """Load environment variables from .env"""
-    # Check if HINDSIGHT_API_DATABASE_URL is already set (e.g., by CI/CD)
-    if os.getenv("HINDSIGHT_API_DATABASE_URL"):
-        return
-
-    # Look for .env file in the parent directory (root of the workspace)
-    root_dir = Path(__file__).parent.parent.parent
-    env_file = root_dir / ".env"
-
-    if env_file.exists():
-        load_dotenv(env_file)
-
-load_env()
-
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
-config = context.config
-
-# Note: We don't call fileConfig() here to avoid overriding the application's logging configuration.
-# Alembic will use the existing logging configuration from the application.
-
-# add your model's MetaData object here
-# for 'autogenerate' support
-target_metadata = Base.metadata
-
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# my_important_option = config.get_main_option("my_important_option")
-# ... etc.
-
-
-def get_database_url() -> str:
-    """
-    Get and process the database URL from config or environment.
-
-    Returns the URL with the correct driver (psycopg2) for migrations.
-    """
-    # Get database URL from config (set programmatically) or environment
-    database_url = config.get_main_option("sqlalchemy.url")
-    if not database_url:
-        database_url = os.getenv("HINDSIGHT_API_DATABASE_URL")
-        if not database_url:
-            raise ValueError(
-                "Database URL not found. "
-                "Set HINDSIGHT_API_DATABASE_URL environment variable or pass database_url to run_migrations()."
-            )
-
-    # For migrations, use psycopg2 (sync driver) to avoid pgbouncer prepared statement issues
-    if database_url.startswith("postgresql+asyncpg://"):
-        database_url = database_url.replace("postgresql+asyncpg://", "postgresql://", 1)
-    elif database_url.startswith("postgres+asyncpg://"):
-        database_url = database_url.replace("postgres+asyncpg://", "postgresql://", 1)
-
-    # Update config with processed URL for engine_from_config to use
-    config.set_main_option("sqlalchemy.url", database_url)
-
-    return database_url
-
-
-def run_migrations_offline() -> None:
-    """Run migrations in 'offline' mode.
-
-    This configures the context with just a URL
-    and not an Engine, though an Engine is acceptable
-    here as well.  By skipping the Engine creation
-    we don't even need a DBAPI to be available.
-
-    Calls to context.execute() here emit the given string to the
-    script output.
-
-    """
-    logging.info("running offline")
-    database_url = get_database_url()
-
-    context.configure(
-        url=database_url,
-        target_metadata=target_metadata,
-        literal_binds=True,
-        dialect_opts={"paramstyle": "named"},
-    )
-
-    with context.begin_transaction():
-        context.run_migrations()
-
-
-def run_migrations_online() -> None:
-    """Run migrations in 'online' mode with synchronous engine."""
-    get_database_url()  # Process and set the database URL in config
-
-    connectable = engine_from_config(
-        config.get_section(config.config_ini_section, {}),
-        prefix="sqlalchemy.",
-        poolclass=pool.NullPool,
-    )
-
-    with connectable.connect() as connection:
-        context.configure(
-            connection=connection,
-            target_metadata=target_metadata
-        )
-
-        with context.begin_transaction():
-            context.run_migrations()
-
-
-if context.is_offline_mode():
-    run_migrations_offline()
-else:
-    run_migrations_online()

hindsight-api/alembic/versions/5a366d414dce_initial_schema.py

@@ -1,275 +0,0 @@
-"""initial_schema
-
-Revision ID: 5a366d414dce
-Revises:
-Create Date: 2025-11-27 11:54:19.228030
-
-"""
-from typing import Sequence, Union
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-from pgvector.sqlalchemy import Vector
-
-
-# revision identifiers, used by Alembic.
-revision: str = '5a366d414dce'
-down_revision: Union[str, Sequence[str], None] = None
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    """Upgrade schema - create all tables from scratch."""
-
-    # Enable required extensions
-    op.execute('CREATE EXTENSION IF NOT EXISTS "uuid-ossp"')
-    op.execute('CREATE EXTENSION IF NOT EXISTS vector')
-
-    # Create banks table
-    op.create_table(
-        'banks',
-        sa.Column('bank_id', sa.Text(), nullable=False),
-        sa.Column('name', sa.Text(), nullable=True),
-        sa.Column('personality', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
-        sa.Column('background', sa.Text(), nullable=True),
-        sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
-        sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
-        sa.PrimaryKeyConstraint('bank_id', name=op.f('pk_banks'))
-    )
-
-    # Create documents table
-    op.create_table(
-        'documents',
-        sa.Column('id', sa.Text(), nullable=False),
-        sa.Column('bank_id', sa.Text(), nullable=False),
-        sa.Column('original_text', sa.Text(), nullable=True),
-        sa.Column('content_hash', sa.Text(), nullable=True),
-        sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
-        sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
-        sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
-        sa.PrimaryKeyConstraint('id', 'bank_id', name=op.f('pk_documents'))
-    )
-    op.create_index('idx_documents_bank_id', 'documents', ['bank_id'])
-    op.create_index('idx_documents_content_hash', 'documents', ['content_hash'])
-
-    # Create async_operations table
-    op.create_table(
-        'async_operations',
-        sa.Column('operation_id', postgresql.UUID(as_uuid=True), server_default=sa.text('uuid_generate_v4()'), nullable=False),
-        sa.Column('bank_id', sa.Text(), nullable=False),
-        sa.Column('operation_type', sa.Text(), nullable=False),
-        sa.Column('status', sa.Text(), server_default='pending', nullable=False),
-        sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
-        sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
-        sa.Column('completed_at', postgresql.TIMESTAMP(timezone=True), nullable=True),
-        sa.Column('error_message', sa.Text(), nullable=True),
-        sa.Column('result_metadata', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
-        sa.PrimaryKeyConstraint('operation_id', name=op.f('pk_async_operations')),
-        sa.CheckConstraint("status IN ('pending', 'processing', 'completed', 'failed')", name='async_operations_status_check')
-    )
-    op.create_index('idx_async_operations_bank_id', 'async_operations', ['bank_id'])
-    op.create_index('idx_async_operations_status', 'async_operations', ['status'])
-    op.create_index('idx_async_operations_bank_status', 'async_operations', ['bank_id', 'status'])
-
-    # Create entities table
-    op.create_table(
-        'entities',
-        sa.Column('id', postgresql.UUID(as_uuid=True), server_default=sa.text('uuid_generate_v4()'), nullable=False),
-        sa.Column('canonical_name', sa.Text(), nullable=False),
-        sa.Column('bank_id', sa.Text(), nullable=False),
-        sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
-        sa.Column('first_seen', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
-        sa.Column('last_seen', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
-        sa.Column('mention_count', sa.Integer(), server_default='1', nullable=False),
-        sa.PrimaryKeyConstraint('id', name=op.f('pk_entities'))
-    )
-    op.create_index('idx_entities_bank_id', 'entities', ['bank_id'])
-    op.create_index('idx_entities_canonical_name', 'entities', ['canonical_name'])
-    op.create_index('idx_entities_bank_name', 'entities', ['bank_id', 'canonical_name'])
-    # Create unique index on (bank_id, LOWER(canonical_name)) for entity resolution
-    op.execute('CREATE UNIQUE INDEX idx_entities_bank_lower_name ON entities (bank_id, LOWER(canonical_name))')
-
-    # Create memory_units table
-    op.create_table(
-        'memory_units',
-        sa.Column('id', postgresql.UUID(as_uuid=True), server_default=sa.text('uuid_generate_v4()'), nullable=False),
-        sa.Column('bank_id', sa.Text(), nullable=False),
-        sa.Column('document_id', sa.Text(), nullable=True),
-        sa.Column('text', sa.Text(), nullable=False),
-        sa.Column('embedding', Vector(384), nullable=True),
-        sa.Column('context', sa.Text(), nullable=True),
-        sa.Column('event_date', postgresql.TIMESTAMP(timezone=True), nullable=False),
-        sa.Column('occurred_start', postgresql.TIMESTAMP(timezone=True), nullable=True),
-        sa.Column('occurred_end', postgresql.TIMESTAMP(timezone=True), nullable=True),
-        sa.Column('mentioned_at', postgresql.TIMESTAMP(timezone=True), nullable=True),
-        sa.Column('fact_type', sa.Text(), server_default='world', nullable=False),
-        sa.Column('confidence_score', sa.Float(), nullable=True),
-        sa.Column('access_count', sa.Integer(), server_default='0', nullable=False),
-        sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
-        sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
-        sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
-        sa.ForeignKeyConstraint(['document_id', 'bank_id'], ['documents.id', 'documents.bank_id'], name='memory_units_document_fkey', ondelete='CASCADE'),
-        sa.PrimaryKeyConstraint('id', name=op.f('pk_memory_units')),
-        sa.CheckConstraint("fact_type IN ('world', 'bank', 'opinion', 'observation')", name='memory_units_fact_type_check'),
-        sa.CheckConstraint("confidence_score IS NULL OR (confidence_score >= 0.0 AND confidence_score <= 1.0)", name='memory_units_confidence_range_check'),
-        sa.CheckConstraint(
-            "(fact_type = 'opinion' AND confidence_score IS NOT NULL) OR "
-            "(fact_type = 'observation') OR "
-            "(fact_type NOT IN ('opinion', 'observation') AND confidence_score IS NULL)",
-            name='confidence_score_fact_type_check'
-        )
-    )
-
-    # Add search_vector column for full-text search
-    op.execute("""
-        ALTER TABLE memory_units
-        ADD COLUMN search_vector tsvector
-        GENERATED ALWAYS AS (to_tsvector('english', COALESCE(text, '') || ' ' || COALESCE(context, ''))) STORED
-    """)
-
-    op.create_index('idx_memory_units_bank_id', 'memory_units', ['bank_id'])
-    op.create_index('idx_memory_units_document_id', 'memory_units', ['document_id'])
-    op.create_index('idx_memory_units_event_date', 'memory_units', [sa.text('event_date DESC')])
-    op.create_index('idx_memory_units_bank_date', 'memory_units', ['bank_id', sa.text('event_date DESC')])
-    op.create_index('idx_memory_units_access_count', 'memory_units', [sa.text('access_count DESC')])
-    op.create_index('idx_memory_units_fact_type', 'memory_units', ['fact_type'])
-    op.create_index('idx_memory_units_bank_fact_type', 'memory_units', ['bank_id', 'fact_type'])
-    op.create_index('idx_memory_units_bank_type_date', 'memory_units', ['bank_id', 'fact_type', sa.text('event_date DESC')])
-    op.create_index('idx_memory_units_opinion_confidence', 'memory_units', ['bank_id', sa.text('confidence_score DESC')], postgresql_where=sa.text("fact_type = 'opinion'"))
-    op.create_index('idx_memory_units_opinion_date', 'memory_units', ['bank_id', sa.text('event_date DESC')], postgresql_where=sa.text("fact_type = 'opinion'"))
-    op.create_index('idx_memory_units_observation_date', 'memory_units', ['bank_id', sa.text('event_date DESC')], postgresql_where=sa.text("fact_type = 'observation'"))
-    op.create_index('idx_memory_units_embedding', 'memory_units', ['embedding'], postgresql_using='hnsw', postgresql_ops={'embedding': 'vector_cosine_ops'})
-
-    # Create BM25 full-text search index on search_vector
-    op.execute("""
-        CREATE INDEX idx_memory_units_text_search ON memory_units
-        USING gin(search_vector)
-    """)
-
-    op.execute("""
-        CREATE MATERIALIZED VIEW memory_units_bm25 AS
-        SELECT
-            id,
-            bank_id,
-            text,
-            to_tsvector('english', text) AS text_vector,
-            log(1.0 + length(text)::float / (SELECT avg(length(text)) FROM memory_units)) AS doc_length_factor
-        FROM memory_units
-    """)
-
-    op.create_index('idx_memory_units_bm25_bank', 'memory_units_bm25', ['bank_id'])
-    op.create_index('idx_memory_units_bm25_text_vector', 'memory_units_bm25', ['text_vector'], postgresql_using='gin')
-
-    # Create entity_cooccurrences table
-    op.create_table(
-        'entity_cooccurrences',
-        sa.Column('entity_id_1', postgresql.UUID(as_uuid=True), nullable=False),
-        sa.Column('entity_id_2', postgresql.UUID(as_uuid=True), nullable=False),
-        sa.Column('cooccurrence_count', sa.Integer(), server_default='1', nullable=False),
-        sa.Column('last_cooccurred', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
-        sa.ForeignKeyConstraint(['entity_id_1'], ['entities.id'], name=op.f('fk_entity_cooccurrences_entity_id_1_entities'), ondelete='CASCADE'),
-        sa.ForeignKeyConstraint(['entity_id_2'], ['entities.id'], name=op.f('fk_entity_cooccurrences_entity_id_2_entities'), ondelete='CASCADE'),
-        sa.PrimaryKeyConstraint('entity_id_1', 'entity_id_2', name=op.f('pk_entity_cooccurrences')),
-        sa.CheckConstraint('entity_id_1 < entity_id_2', name='entity_cooccurrence_order_check')
-    )
-    op.create_index('idx_entity_cooccurrences_entity1', 'entity_cooccurrences', ['entity_id_1'])
-    op.create_index('idx_entity_cooccurrences_entity2', 'entity_cooccurrences', ['entity_id_2'])
-    op.create_index('idx_entity_cooccurrences_count', 'entity_cooccurrences', [sa.text('cooccurrence_count DESC')])
-
-    # Create memory_links table
-    op.create_table(
-        'memory_links',
-        sa.Column('from_unit_id', postgresql.UUID(as_uuid=True), nullable=False),
-        sa.Column('to_unit_id', postgresql.UUID(as_uuid=True), nullable=False),
-        sa.Column('link_type', sa.Text(), nullable=False),
-        sa.Column('entity_id', postgresql.UUID(as_uuid=True), nullable=True),
-        sa.Column('weight', sa.Float(), server_default='1.0', nullable=False),
-        sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
-        sa.ForeignKeyConstraint(['entity_id'], ['entities.id'], name=op.f('fk_memory_links_entity_id_entities'), ondelete='CASCADE'),
-        sa.ForeignKeyConstraint(['from_unit_id'], ['memory_units.id'], name=op.f('fk_memory_links_from_unit_id_memory_units'), ondelete='CASCADE'),
-        sa.ForeignKeyConstraint(['to_unit_id'], ['memory_units.id'], name=op.f('fk_memory_links_to_unit_id_memory_units'), ondelete='CASCADE'),
-        sa.CheckConstraint("link_type IN ('temporal', 'semantic', 'entity', 'causes', 'caused_by', 'enables', 'prevents')", name='memory_links_link_type_check'),
-        sa.CheckConstraint('weight >= 0.0 AND weight <= 1.0', name='memory_links_weight_check')
-    )
-    # Create unique constraint using COALESCE for nullable entity_id
-    op.execute("CREATE UNIQUE INDEX idx_memory_links_unique ON memory_links (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid))")
-    op.create_index('idx_memory_links_from_unit', 'memory_links', ['from_unit_id'])
-    op.create_index('idx_memory_links_to_unit', 'memory_links', ['to_unit_id'])
-    op.create_index('idx_memory_links_entity', 'memory_links', ['entity_id'])
-    op.create_index('idx_memory_links_link_type', 'memory_links', ['link_type'])
-
-    # Create unit_entities table
-    op.create_table(
-        'unit_entities',
-        sa.Column('unit_id', postgresql.UUID(as_uuid=True), nullable=False),
-        sa.Column('entity_id', postgresql.UUID(as_uuid=True), nullable=False),
-        sa.ForeignKeyConstraint(['entity_id'], ['entities.id'], name=op.f('fk_unit_entities_entity_id_entities'), ondelete='CASCADE'),
-        sa.ForeignKeyConstraint(['unit_id'], ['memory_units.id'], name=op.f('fk_unit_entities_unit_id_memory_units'), ondelete='CASCADE'),
-        sa.PrimaryKeyConstraint('unit_id', 'entity_id', name=op.f('pk_unit_entities'))
-    )
-    op.create_index('idx_unit_entities_unit', 'unit_entities', ['unit_id'])
-    op.create_index('idx_unit_entities_entity', 'unit_entities', ['entity_id'])
-
-
-def downgrade() -> None:
-    """Downgrade schema - drop all tables."""
-
-    # Drop tables in reverse dependency order
-    op.drop_index('idx_unit_entities_entity', table_name='unit_entities')
-    op.drop_index('idx_unit_entities_unit', table_name='unit_entities')
-    op.drop_table('unit_entities')
-
-    op.drop_index('idx_memory_links_link_type', table_name='memory_links')
-    op.drop_index('idx_memory_links_entity', table_name='memory_links')
-    op.drop_index('idx_memory_links_to_unit', table_name='memory_links')
-    op.drop_index('idx_memory_links_from_unit', table_name='memory_links')
-    op.execute('DROP INDEX IF EXISTS idx_memory_links_unique')
-    op.drop_table('memory_links')
-
-    op.drop_index('idx_entity_cooccurrences_count', table_name='entity_cooccurrences')
-    op.drop_index('idx_entity_cooccurrences_entity2', table_name='entity_cooccurrences')
-    op.drop_index('idx_entity_cooccurrences_entity1', table_name='entity_cooccurrences')
-    op.drop_table('entity_cooccurrences')
-
-    # Drop BM25 materialized view and index
-    op.drop_index('idx_memory_units_bm25_text_vector', table_name='memory_units_bm25')
-    op.drop_index('idx_memory_units_bm25_bank', table_name='memory_units_bm25')
-    op.execute('DROP MATERIALIZED VIEW IF EXISTS memory_units_bm25')
-
-    op.drop_index('idx_memory_units_embedding', table_name='memory_units')
-    op.drop_index('idx_memory_units_observation_date', table_name='memory_units')
-    op.drop_index('idx_memory_units_opinion_date', table_name='memory_units')
-    op.drop_index('idx_memory_units_opinion_confidence', table_name='memory_units')
-    op.drop_index('idx_memory_units_bank_type_date', table_name='memory_units')
-    op.drop_index('idx_memory_units_bank_fact_type', table_name='memory_units')
-    op.drop_index('idx_memory_units_fact_type', table_name='memory_units')
-    op.drop_index('idx_memory_units_access_count', table_name='memory_units')
-    op.drop_index('idx_memory_units_bank_date', table_name='memory_units')
-    op.drop_index('idx_memory_units_event_date', table_name='memory_units')
-    op.drop_index('idx_memory_units_document_id', table_name='memory_units')
-    op.drop_index('idx_memory_units_bank_id', table_name='memory_units')
-    op.execute('DROP INDEX IF EXISTS idx_memory_units_text_search')
-    op.drop_table('memory_units')
-
-    op.execute('DROP INDEX IF EXISTS idx_entities_bank_lower_name')
-    op.drop_index('idx_entities_bank_name', table_name='entities')
-    op.drop_index('idx_entities_canonical_name', table_name='entities')
-    op.drop_index('idx_entities_bank_id', table_name='entities')
-    op.drop_table('entities')
-
-    op.drop_index('idx_async_operations_bank_status', table_name='async_operations')
-    op.drop_index('idx_async_operations_status', table_name='async_operations')
-    op.drop_index('idx_async_operations_bank_id', table_name='async_operations')
-    op.drop_table('async_operations')
-
-    op.drop_index('idx_documents_content_hash', table_name='documents')
-    op.drop_index('idx_documents_bank_id', table_name='documents')
-    op.drop_table('documents')
-
-    op.drop_table('banks')
-
-    # Drop extensions (optional - comment out if you want to keep them)
-    # op.execute('DROP EXTENSION IF EXISTS vector')
-    # op.execute('DROP EXTENSION IF EXISTS "uuid-ossp"')

hindsight-api/alembic/versions/b7c4d8e9f1a2_add_chunks_table.py

@@ -1,70 +0,0 @@
-"""add_chunks_table
-
-Revision ID: b7c4d8e9f1a2
-Revises: 5a366d414dce
-Create Date: 2025-11-28 00:00:00.000000
-
-"""
-from typing import Sequence, Union
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-
-# revision identifiers, used by Alembic.
-revision: str = 'b7c4d8e9f1a2'
-down_revision: Union[str, Sequence[str], None] = '5a366d414dce'
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    """Add chunks table and link memory_units to chunks."""
-
-    # Create chunks table with single text PK (bank_id_document_id_chunk_index)
-    op.create_table(
-        'chunks',
-        sa.Column('chunk_id', sa.Text(), nullable=False),
-        sa.Column('document_id', sa.Text(), nullable=False),
-        sa.Column('bank_id', sa.Text(), nullable=False),
-        sa.Column('chunk_index', sa.Integer(), nullable=False),
-        sa.Column('chunk_text', sa.Text(), nullable=False),
-        sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
-        sa.ForeignKeyConstraint(['document_id', 'bank_id'], ['documents.id', 'documents.bank_id'], name='chunks_document_fkey', ondelete='CASCADE'),
-        sa.PrimaryKeyConstraint('chunk_id', name=op.f('pk_chunks'))
-    )
-
-    # Add indexes for efficient queries
-    op.create_index('idx_chunks_document_id', 'chunks', ['document_id'])
-    op.create_index('idx_chunks_bank_id', 'chunks', ['bank_id'])
-
-    # Add chunk_id column to memory_units (nullable, as existing records won't have chunks)
-    op.add_column('memory_units', sa.Column('chunk_id', sa.Text(), nullable=True))
-
-    # Add foreign key constraint to chunks table
-    op.create_foreign_key(
-        'memory_units_chunk_fkey',
-        'memory_units',
-        'chunks',
-        ['chunk_id'],
-        ['chunk_id'],
-        ondelete='SET NULL'
-    )
-
-    # Add index on chunk_id for efficient lookups
-    op.create_index('idx_memory_units_chunk_id', 'memory_units', ['chunk_id'])
-
-
-def downgrade() -> None:
-    """Remove chunks table and chunk_id from memory_units."""
-
-    # Drop index and foreign key from memory_units
-    op.drop_index('idx_memory_units_chunk_id', table_name='memory_units')
-    op.drop_constraint('memory_units_chunk_fkey', 'memory_units', type_='foreignkey')
-    op.drop_column('memory_units', 'chunk_id')
-
-    # Drop chunks table indexes and table
-    op.drop_index('idx_chunks_bank_id', table_name='chunks')
-    op.drop_index('idx_chunks_document_id', table_name='chunks')
-    op.drop_table('chunks')

hindsight-api/alembic/versions/c8e5f2a3b4d1_add_retain_params_to_documents.py

@@ -1,39 +0,0 @@
-"""add_retain_params_to_documents
-
-Revision ID: c8e5f2a3b4d1
-Revises: b7c4d8e9f1a2
-Create Date: 2025-12-02 00:00:00.000000
-
-"""
-from typing import Sequence, Union
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-
-# revision identifiers, used by Alembic.
-revision: str = 'c8e5f2a3b4d1'
-down_revision: Union[str, Sequence[str], None] = 'b7c4d8e9f1a2'
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    """Add retain_params JSONB column to documents table."""
-
-    # Add retain_params column to store parameters passed during retain
-    op.add_column('documents', sa.Column('retain_params', postgresql.JSONB(), nullable=True))
-
-    # Add index for efficient queries on retain_params
-    op.create_index('idx_documents_retain_params', 'documents', ['retain_params'], postgresql_using='gin')
-
-
-def downgrade() -> None:
-    """Remove retain_params column from documents table."""
-
-    # Drop index
-    op.drop_index('idx_documents_retain_params', table_name='documents')
-
-    # Drop column
-    op.drop_column('documents', 'retain_params')

hindsight-api/hindsight_api/__init__.py

@@ -3,24 +3,31 @@ Memory System for AI Agents.
 
 Temporal + Semantic Memory Architecture using PostgreSQL with pgvector.
 """
+
+from .config import HindsightConfig, get_config
+from .engine.cross_encoder import CrossEncoderModel, LocalSTCrossEncoder, RemoteTEICrossEncoder
+from .engine.embeddings import Embeddings, LocalSTEmbeddings, RemoteTEIEmbeddings
+from .engine.llm_wrapper import LLMConfig
 from .engine.memory_engine import MemoryEngine
 from .engine.search.trace import (
-    SearchTrace,
-    QueryInfo,
     EntryPoint,
-    NodeVisit,
-    WeightComponents,
     LinkInfo,
+    NodeVisit,
     PruningDecision,
-    SearchSummary,
+    QueryInfo,
     SearchPhaseMetrics,
+    SearchSummary,
+    SearchTrace,
+    WeightComponents,
 )
 from .engine.search.tracer import SearchTracer
-from .engine.embeddings import Embeddings, SentenceTransformersEmbeddings
-from .engine.llm_wrapper import LLMConfig
+from .models import RequestContext
 
 __all__ = [
     "MemoryEngine",
+    "RequestContext",
+    "HindsightConfig",
+    "get_config",
     "SearchTrace",
     "SearchTracer",
     "QueryInfo",
@@ -32,7 +39,11 @@ __all__ = [
     "SearchSummary",
     "SearchPhaseMetrics",
     "Embeddings",
-    "SentenceTransformersEmbeddings",
+    "LocalSTEmbeddings",
+    "RemoteTEIEmbeddings",
+    "CrossEncoderModel",
+    "LocalSTCrossEncoder",
+    "RemoteTEICrossEncoder",
     "LLMConfig",
 ]
-__version__ = "0.1.0"
+__version__ = "0.4.10"

hindsight-api/hindsight_api/admin/__init__.py

@@ -0,0 +1 @@
+# Admin CLI for Hindsight

hindsight-api/hindsight_api/admin/cli.py

@@ -0,0 +1,311 @@
+"""
+Hindsight Admin CLI - backup and restore operations.
+"""
+
+import asyncio
+import io
+import json
+import logging
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import asyncpg
+import typer
+
+from ..config import HindsightConfig
+from ..pg0 import parse_pg0_url, resolve_database_url
+
+
+def _fq_table(table: str, schema: str) -> str:
+    """Get fully-qualified table name with schema prefix."""
+    return f"{schema}.{table}"
+
+
+# Setup logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(message)s",
+)
+logger = logging.getLogger(__name__)
+
+app = typer.Typer(name="hindsight-admin", help="Hindsight administrative commands")
+
+# Tables to backup/restore in dependency order
+# Import must happen in this order due to foreign key constraints
+BACKUP_TABLES = [
+    "banks",
+    "documents",
+    "entities",
+    "chunks",
+    "memory_units",
+    "unit_entities",
+    "entity_cooccurrences",
+    "memory_links",
+]
+
+MANIFEST_VERSION = "1"
+
+
+async def _backup(database_url: str, output_path: Path, schema: str = "public") -> dict[str, Any]:
+    """Backup all tables to a zip file using binary COPY protocol."""
+    conn = await asyncpg.connect(database_url)
+    try:
+        tables: dict[str, Any] = {}
+        manifest: dict[str, Any] = {
+            "version": MANIFEST_VERSION,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+            "schema": schema,
+            "tables": tables,
+        }
+
+        # Use a transaction with REPEATABLE READ isolation to get a consistent
+        # snapshot across all tables. This prevents race conditions where
+        # entity_cooccurrences could reference entities created after the
+        # entities table was backed up.
+        async with conn.transaction(isolation="repeatable_read"):
+            with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zf:
+                for i, table in enumerate(BACKUP_TABLES, 1):
+                    typer.echo(f"  [{i}/{len(BACKUP_TABLES)}] Backing up {table}...", nl=False)
+
+                    buffer = io.BytesIO()
+
+                    # Use binary COPY for exact type preservation
+                    # asyncpg requires schema_name as separate parameter
+                    await conn.copy_from_table(table, schema_name=schema, output=buffer, format="binary")
+
+                    data = buffer.getvalue()
+                    zf.writestr(f"{table}.bin", data)
+
+                    # Get row count for manifest
+                    qualified_table = _fq_table(table, schema)
+                    row_count = await conn.fetchval(f"SELECT COUNT(*) FROM {qualified_table}")
+                    tables[table] = {
+                        "rows": row_count,
+                        "size_bytes": len(data),
+                    }
+
+                    typer.echo(f" {row_count} rows")
+
+                zf.writestr("manifest.json", json.dumps(manifest, indent=2))
+
+        return manifest
+    finally:
+        await conn.close()
+
+
+async def _restore(database_url: str, input_path: Path, schema: str = "public") -> dict[str, Any]:
+    """Restore all tables from a zip file using binary COPY protocol."""
+    conn = await asyncpg.connect(database_url)
+    try:
+        with zipfile.ZipFile(input_path, "r") as zf:
+            # Read and validate manifest
+            manifest: dict[str, Any] = json.loads(zf.read("manifest.json"))
+            if manifest.get("version") != MANIFEST_VERSION:
+                raise ValueError(f"Unsupported backup version: {manifest.get('version')}")
+
+            # Use a transaction for atomic restore - either all tables are
+            # restored or none are, preventing partial/inconsistent state.
+            async with conn.transaction():
+                typer.echo("  Clearing existing data...")
+                # Truncate tables in reverse order (respects FK constraints)
+                for table in reversed(BACKUP_TABLES):
+                    qualified_table = _fq_table(table, schema)
+                    await conn.execute(f"TRUNCATE TABLE {qualified_table} CASCADE")
+
+                # Restore tables in forward order
+                for i, table in enumerate(BACKUP_TABLES, 1):
+                    filename = f"{table}.bin"
+                    if filename not in zf.namelist():
+                        typer.echo(f"  [{i}/{len(BACKUP_TABLES)}] {table}: skipped (not in backup)")
+                        continue
+
+                    expected_rows = manifest["tables"].get(table, {}).get("rows", "?")
+                    typer.echo(f"  [{i}/{len(BACKUP_TABLES)}] Restoring {table}... {expected_rows} rows")
+
+                    data = zf.read(filename)
+                    buffer = io.BytesIO(data)
+                    # asyncpg requires schema_name as separate parameter
+                    await conn.copy_to_table(table, schema_name=schema, source=buffer, format="binary")
+
+                # Refresh materialized view
+                typer.echo("  Refreshing materialized views...")
+                await conn.execute(f"REFRESH MATERIALIZED VIEW {_fq_table('memory_units_bm25', schema)}")
+
+        return manifest
+    finally:
+        await conn.close()
+
+
+async def _run_backup(db_url: str, output: Path, schema: str = "public") -> dict[str, Any]:
+    """Resolve database URL and run backup."""
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+    return await _backup(resolved_url, output, schema)
+
+
+async def _run_restore(db_url: str, input_file: Path, schema: str = "public") -> dict[str, Any]:
+    """Resolve database URL and run restore."""
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+    return await _restore(resolved_url, input_file, schema)
+
+
+@app.command()
+def backup(
+    output: Path = typer.Argument(..., help="Output file path (.zip)"),
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema to backup"),
+):
+    """Backup the Hindsight database to a zip file."""
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    if output.suffix != ".zip":
+        output = output.with_suffix(".zip")
+
+    typer.echo(f"Backing up database (schema: {schema}) to {output}...")
+
+    manifest = asyncio.run(_run_backup(config.database_url, output, schema))
+
+    total_rows = sum(t["rows"] for t in manifest["tables"].values())
+    typer.echo(f"Backed up {total_rows} rows across {len(BACKUP_TABLES)} tables")
+    typer.echo(f"Backup saved to {output}")
+
+
+@app.command()
+def restore(
+    input_file: Path = typer.Argument(..., help="Input backup file (.zip)"),
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema to restore to"),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"),
+):
+    """Restore the database from a backup file. WARNING: This deletes all existing data."""
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    if not input_file.exists():
+        typer.echo(f"Error: File not found: {input_file}", err=True)
+        raise typer.Exit(1)
+
+    if not yes:
+        typer.confirm(
+            "This will DELETE all existing data and replace it with the backup. Continue?",
+            abort=True,
+        )
+
+    typer.echo(f"Restoring database (schema: {schema}) from {input_file}...")
+
+    manifest = asyncio.run(_run_restore(config.database_url, input_file, schema))
+
+    total_rows = sum(t["rows"] for t in manifest["tables"].values())
+    typer.echo(f"Restored {total_rows} rows across {len(BACKUP_TABLES)} tables")
+    typer.echo("Restore complete")
+
+
+async def _run_migration(db_url: str, schema: str = "public") -> None:
+    """Resolve database URL and run migrations."""
+    from ..migrations import run_migrations
+
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+    run_migrations(resolved_url, schema=schema)
+
+
+@app.command(name="run-db-migration")
+def run_db_migration(
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema to run migrations on"),
+):
+    """Run database migrations to the latest version."""
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    typer.echo(f"Running database migrations (schema: {schema})...")
+
+    asyncio.run(_run_migration(config.database_url, schema))
+
+    typer.echo("Database migrations completed successfully")
+
+
+async def _decommission_worker(db_url: str, worker_id: str, schema: str = "public") -> int:
+    """Release all tasks owned by a worker, setting them back to pending status."""
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+
+    conn = await asyncpg.connect(resolved_url)
+    try:
+        table = _fq_table("async_operations", schema)
+        result = await conn.fetch(
+            f"""
+            UPDATE {table}
+            SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
+            WHERE worker_id = $1 AND status = 'processing'
+            RETURNING operation_id
+            """,
+            worker_id,
+        )
+        return len(result)
+    finally:
+        await conn.close()
+
+
+@app.command(name="decommission-worker")
+def decommission_worker(
+    worker_id: str = typer.Argument(..., help="Worker ID to decommission"),
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema"),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"),
+):
+    """Release all tasks owned by a worker (sets status back to pending).
+
+    Use this command when a worker has crashed or been removed without graceful shutdown.
+    All tasks that were being processed by the worker will be released back to the queue
+    so other workers can pick them up.
+    """
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    if not yes:
+        typer.confirm(
+            f"This will release all tasks owned by worker '{worker_id}' back to pending. Continue?",
+            abort=True,
+        )
+
+    typer.echo(f"Decommissioning worker '{worker_id}' (schema: {schema})...")
+
+    count = asyncio.run(_decommission_worker(config.database_url, worker_id, schema))
+
+    if count > 0:
+        typer.echo(f"Released {count} task(s) from worker '{worker_id}'")
+    else:
+        typer.echo(f"No tasks found for worker '{worker_id}'")
+
+
+def main():
+    app()
+
+
+if __name__ == "__main__":
+    main()

hindsight-api/hindsight_api/alembic/env.py

@@ -0,0 +1,166 @@
+"""
+Alembic environment configuration for SQLAlchemy with pgvector.
+Uses synchronous psycopg2 driver for migrations to avoid pgbouncer issues.
+"""
+
+import logging
+import os
+from pathlib import Path
+
+from alembic import context
+from dotenv import load_dotenv
+from sqlalchemy import engine_from_config, pool
+
+# Import your models here
+from hindsight_api.models import Base
+
+
+# Load environment variables based on HINDSIGHT_API_DATABASE_URL env var or default to local
+def load_env():
+    """Load environment variables from .env"""
+    # Check if HINDSIGHT_API_DATABASE_URL is already set (e.g., by CI/CD)
+    if os.getenv("HINDSIGHT_API_DATABASE_URL"):
+        return
+
+    # Look for .env file in the parent directory (root of the workspace)
+    root_dir = Path(__file__).parent.parent.parent
+    env_file = root_dir / ".env"
+
+    if env_file.exists():
+        load_dotenv(env_file)
+
+
+load_env()
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Note: We don't call fileConfig() here to avoid overriding the application's logging configuration.
+# Alembic will use the existing logging configuration from the application.
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def get_database_url() -> str:
+    """
+    Get and process the database URL from config or environment.
+
+    Returns the URL with the correct driver (psycopg2) for migrations.
+    """
+    # Get database URL from config (set programmatically) or environment
+    database_url = config.get_main_option("sqlalchemy.url")
+    if not database_url:
+        database_url = os.getenv("HINDSIGHT_API_DATABASE_URL")
+        if not database_url:
+            raise ValueError(
+                "Database URL not found. "
+                "Set HINDSIGHT_API_DATABASE_URL environment variable or pass database_url to run_migrations()."
+            )
+
+    # For migrations, use psycopg2 (sync driver) to avoid pgbouncer prepared statement issues
+    if database_url.startswith("postgresql+asyncpg://"):
+        database_url = database_url.replace("postgresql+asyncpg://", "postgresql://", 1)
+    elif database_url.startswith("postgres+asyncpg://"):
+        database_url = database_url.replace("postgres+asyncpg://", "postgresql://", 1)
+
+    # Update config with processed URL for engine_from_config to use
+    config.set_main_option("sqlalchemy.url", database_url)
+
+    return database_url
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    logging.info("running offline")
+    database_url = get_database_url()
+
+    context.configure(
+        url=database_url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode with synchronous engine."""
+    from sqlalchemy import event, text
+
+    get_database_url()  # Process and set the database URL in config
+
+    # Check if we're targeting a specific schema (for multi-tenant isolation)
+    target_schema = config.get_main_option("target_schema")
+
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    # Add event listener to ensure connection is in read-write mode
+    # This is needed for Supabase which may start connections in read-only mode
+    @event.listens_for(connectable, "connect")
+    def set_read_write_mode(dbapi_connection, connection_record):
+        cursor = dbapi_connection.cursor()
+        cursor.execute("SET SESSION CHARACTERISTICS AS TRANSACTION READ WRITE")
+        # If targeting a specific schema, set search_path
+        # Include public in search_path for access to shared extensions (pgvector)
+        if target_schema:
+            cursor.execute(f'CREATE SCHEMA IF NOT EXISTS "{target_schema}"')
+            cursor.execute(f'SET search_path TO "{target_schema}", public')
+        cursor.close()
+
+    with connectable.connect() as connection:
+        # Also explicitly set read-write mode on this connection
+        connection.execute(text("SET SESSION CHARACTERISTICS AS TRANSACTION READ WRITE"))
+
+        # If targeting a specific schema, set search_path
+        # Include public in search_path for access to shared extensions (pgvector)
+        if target_schema:
+            connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{target_schema}"'))
+            connection.execute(text(f'SET search_path TO "{target_schema}", public'))
+
+        connection.commit()  # Commit the SET command
+
+        # Configure context with version_table_schema if using a specific schema
+        context_opts = {
+            "connection": connection,
+            "target_metadata": target_metadata,
+        }
+        if target_schema:
+            context_opts["version_table_schema"] = target_schema
+
+        context.configure(**context_opts)
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+        # Explicit commit to ensure changes are persisted (especially for Supabase)
+        connection.commit()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

hindsight-api/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py

@@ -0,0 +1,374 @@
+"""initial_schema
+
+Revision ID: 5a366d414dce
+Revises:
+Create Date: 2025-11-27 11:54:19.228030
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from pgvector.sqlalchemy import Vector
+from sqlalchemy import text
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "5a366d414dce"
+down_revision: str | Sequence[str] | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Upgrade schema - create all tables from scratch."""
+
+    # Note: pgvector extension is installed globally BEFORE migrations run
+    # See migrations.py:run_migrations() - this ensures the extension is available
+    # to all schemas, not just the one being migrated
+
+    # We keep this here as a fallback for backwards compatibility
+    # This may fail if user lacks permissions, which is fine if extension already exists
+    try:
+        op.execute("CREATE EXTENSION IF NOT EXISTS vector")
+    except Exception:
+        # Extension might already exist or user lacks permissions - verify it exists
+        conn = op.get_bind()
+        result = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).fetchone()
+        if not result:
+            # Extension truly doesn't exist - re-raise the error
+            raise
+
+    # Create banks table
+    op.create_table(
+        "banks",
+        sa.Column("bank_id", sa.Text(), nullable=False),
+        sa.Column("name", sa.Text(), nullable=True),
+        sa.Column(
+            "personality",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default=sa.text("'{}'::jsonb"),
+            nullable=False,
+        ),
+        sa.Column("background", sa.Text(), nullable=True),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.PrimaryKeyConstraint("bank_id", name=op.f("pk_banks")),
+    )
+
+    # Create documents table
+    op.create_table(
+        "documents",
+        sa.Column("id", sa.Text(), nullable=False),
+        sa.Column("bank_id", sa.Text(), nullable=False),
+        sa.Column("original_text", sa.Text(), nullable=True),
+        sa.Column("content_hash", sa.Text(), nullable=True),
+        sa.Column(
+            "metadata", postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False
+        ),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.PrimaryKeyConstraint("id", "bank_id", name=op.f("pk_documents")),
+    )
+    op.create_index("idx_documents_bank_id", "documents", ["bank_id"])
+    op.create_index("idx_documents_content_hash", "documents", ["content_hash"])
+
+    # Create async_operations table
+    op.create_table(
+        "async_operations",
+        sa.Column(
+            "operation_id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False
+        ),
+        sa.Column("bank_id", sa.Text(), nullable=False),
+        sa.Column("operation_type", sa.Text(), nullable=False),
+        sa.Column("status", sa.Text(), server_default="pending", nullable=False),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("completed_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+        sa.Column(
+            "result_metadata",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default=sa.text("'{}'::jsonb"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("operation_id", name=op.f("pk_async_operations")),
+        sa.CheckConstraint(
+            "status IN ('pending', 'processing', 'completed', 'failed')", name="async_operations_status_check"
+        ),
+    )
+    op.create_index("idx_async_operations_bank_id", "async_operations", ["bank_id"])
+    op.create_index("idx_async_operations_status", "async_operations", ["status"])
+    op.create_index("idx_async_operations_bank_status", "async_operations", ["bank_id", "status"])
+
+    # Create entities table
+    op.create_table(
+        "entities",
+        sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False),
+        sa.Column("canonical_name", sa.Text(), nullable=False),
+        sa.Column("bank_id", sa.Text(), nullable=False),
+        sa.Column(
+            "metadata", postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False
+        ),
+        sa.Column("first_seen", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("last_seen", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("mention_count", sa.Integer(), server_default="1", nullable=False),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_entities")),
+    )
+    op.create_index("idx_entities_bank_id", "entities", ["bank_id"])
+    op.create_index("idx_entities_canonical_name", "entities", ["canonical_name"])
+    op.create_index("idx_entities_bank_name", "entities", ["bank_id", "canonical_name"])
+    # Create unique index on (bank_id, LOWER(canonical_name)) for entity resolution
+    op.execute("CREATE UNIQUE INDEX idx_entities_bank_lower_name ON entities (bank_id, LOWER(canonical_name))")
+
+    # Create memory_units table
+    op.create_table(
+        "memory_units",
+        sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False),
+        sa.Column("bank_id", sa.Text(), nullable=False),
+        sa.Column("document_id", sa.Text(), nullable=True),
+        sa.Column("text", sa.Text(), nullable=False),
+        sa.Column("embedding", Vector(384), nullable=True),
+        sa.Column("context", sa.Text(), nullable=True),
+        sa.Column("event_date", postgresql.TIMESTAMP(timezone=True), nullable=False),
+        sa.Column("occurred_start", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("occurred_end", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("mentioned_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("fact_type", sa.Text(), server_default="world", nullable=False),
+        sa.Column("confidence_score", sa.Float(), nullable=True),
+        sa.Column("access_count", sa.Integer(), server_default="0", nullable=False),
+        sa.Column(
+            "metadata", postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False
+        ),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id", "bank_id"],
+            ["documents.id", "documents.bank_id"],
+            name="memory_units_document_fkey",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_memory_units")),
+        sa.CheckConstraint(
+            "fact_type IN ('world', 'bank', 'opinion', 'observation')", name="memory_units_fact_type_check"
+        ),
+        sa.CheckConstraint(
+            "confidence_score IS NULL OR (confidence_score >= 0.0 AND confidence_score <= 1.0)",
+            name="memory_units_confidence_range_check",
+        ),
+        sa.CheckConstraint(
+            "(fact_type = 'opinion' AND confidence_score IS NOT NULL) OR "
+            "(fact_type = 'observation') OR "
+            "(fact_type NOT IN ('opinion', 'observation') AND confidence_score IS NULL)",
+            name="confidence_score_fact_type_check",
+        ),
+    )
+
+    # Add search_vector column for full-text search
+    op.execute("""
+        ALTER TABLE memory_units
+        ADD COLUMN search_vector tsvector
+        GENERATED ALWAYS AS (to_tsvector('english', COALESCE(text, '') || ' ' || COALESCE(context, ''))) STORED
+    """)
+
+    op.create_index("idx_memory_units_bank_id", "memory_units", ["bank_id"])
+    op.create_index("idx_memory_units_document_id", "memory_units", ["document_id"])
+    op.create_index("idx_memory_units_event_date", "memory_units", [sa.text("event_date DESC")])
+    op.create_index("idx_memory_units_bank_date", "memory_units", ["bank_id", sa.text("event_date DESC")])
+    op.create_index("idx_memory_units_access_count", "memory_units", [sa.text("access_count DESC")])
+    op.create_index("idx_memory_units_fact_type", "memory_units", ["fact_type"])
+    op.create_index("idx_memory_units_bank_fact_type", "memory_units", ["bank_id", "fact_type"])
+    op.create_index(
+        "idx_memory_units_bank_type_date", "memory_units", ["bank_id", "fact_type", sa.text("event_date DESC")]
+    )
+    op.create_index(
+        "idx_memory_units_opinion_confidence",
+        "memory_units",
+        ["bank_id", sa.text("confidence_score DESC")],
+        postgresql_where=sa.text("fact_type = 'opinion'"),
+    )
+    op.create_index(
+        "idx_memory_units_opinion_date",
+        "memory_units",
+        ["bank_id", sa.text("event_date DESC")],
+        postgresql_where=sa.text("fact_type = 'opinion'"),
+    )
+    op.create_index(
+        "idx_memory_units_observation_date",
+        "memory_units",
+        ["bank_id", sa.text("event_date DESC")],
+        postgresql_where=sa.text("fact_type = 'observation'"),
+    )
+    op.create_index(
+        "idx_memory_units_embedding",
+        "memory_units",
+        ["embedding"],
+        postgresql_using="hnsw",
+        postgresql_ops={"embedding": "vector_cosine_ops"},
+    )
+
+    # Create BM25 full-text search index on search_vector
+    op.execute("""
+        CREATE INDEX idx_memory_units_text_search ON memory_units
+        USING gin(search_vector)
+    """)
+
+    op.execute("""
+        CREATE MATERIALIZED VIEW memory_units_bm25 AS
+        SELECT
+            id,
+            bank_id,
+            text,
+            to_tsvector('english', text) AS text_vector,
+            log(1.0 + length(text)::float / (SELECT avg(length(text)) FROM memory_units)) AS doc_length_factor
+        FROM memory_units
+    """)
+
+    op.create_index("idx_memory_units_bm25_bank", "memory_units_bm25", ["bank_id"])
+    op.create_index("idx_memory_units_bm25_text_vector", "memory_units_bm25", ["text_vector"], postgresql_using="gin")
+
+    # Create entity_cooccurrences table
+    op.create_table(
+        "entity_cooccurrences",
+        sa.Column("entity_id_1", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("entity_id_2", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("cooccurrence_count", sa.Integer(), server_default="1", nullable=False),
+        sa.Column(
+            "last_cooccurred", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["entity_id_1"],
+            ["entities.id"],
+            name=op.f("fk_entity_cooccurrences_entity_id_1_entities"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["entity_id_2"],
+            ["entities.id"],
+            name=op.f("fk_entity_cooccurrences_entity_id_2_entities"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("entity_id_1", "entity_id_2", name=op.f("pk_entity_cooccurrences")),
+        sa.CheckConstraint("entity_id_1 < entity_id_2", name="entity_cooccurrence_order_check"),
+    )
+    op.create_index("idx_entity_cooccurrences_entity1", "entity_cooccurrences", ["entity_id_1"])
+    op.create_index("idx_entity_cooccurrences_entity2", "entity_cooccurrences", ["entity_id_2"])
+    op.create_index("idx_entity_cooccurrences_count", "entity_cooccurrences", [sa.text("cooccurrence_count DESC")])
+
+    # Create memory_links table
+    op.create_table(
+        "memory_links",
+        sa.Column("from_unit_id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("to_unit_id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("link_type", sa.Text(), nullable=False),
+        sa.Column("entity_id", postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column("weight", sa.Float(), server_default="1.0", nullable=False),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["entity_id"], ["entities.id"], name=op.f("fk_memory_links_entity_id_entities"), ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["from_unit_id"],
+            ["memory_units.id"],
+            name=op.f("fk_memory_links_from_unit_id_memory_units"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["to_unit_id"],
+            ["memory_units.id"],
+            name=op.f("fk_memory_links_to_unit_id_memory_units"),
+            ondelete="CASCADE",
+        ),
+        sa.CheckConstraint(
+            "link_type IN ('temporal', 'semantic', 'entity', 'causes', 'caused_by', 'enables', 'prevents')",
+            name="memory_links_link_type_check",
+        ),
+        sa.CheckConstraint("weight >= 0.0 AND weight <= 1.0", name="memory_links_weight_check"),
+    )
+    # Create unique constraint using COALESCE for nullable entity_id
+    op.execute(
+        "CREATE UNIQUE INDEX idx_memory_links_unique ON memory_links (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid))"
+    )
+    op.create_index("idx_memory_links_from_unit", "memory_links", ["from_unit_id"])
+    op.create_index("idx_memory_links_to_unit", "memory_links", ["to_unit_id"])
+    op.create_index("idx_memory_links_entity", "memory_links", ["entity_id"])
+    op.create_index("idx_memory_links_link_type", "memory_links", ["link_type"])
+
+    # Create unit_entities table
+    op.create_table(
+        "unit_entities",
+        sa.Column("unit_id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("entity_id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["entity_id"], ["entities.id"], name=op.f("fk_unit_entities_entity_id_entities"), ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["unit_id"], ["memory_units.id"], name=op.f("fk_unit_entities_unit_id_memory_units"), ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("unit_id", "entity_id", name=op.f("pk_unit_entities")),
+    )
+    op.create_index("idx_unit_entities_unit", "unit_entities", ["unit_id"])
+    op.create_index("idx_unit_entities_entity", "unit_entities", ["entity_id"])
+
+
+def downgrade() -> None:
+    """Downgrade schema - drop all tables."""
+
+    # Drop tables in reverse dependency order
+    op.drop_index("idx_unit_entities_entity", table_name="unit_entities")
+    op.drop_index("idx_unit_entities_unit", table_name="unit_entities")
+    op.drop_table("unit_entities")
+
+    op.drop_index("idx_memory_links_link_type", table_name="memory_links")
+    op.drop_index("idx_memory_links_entity", table_name="memory_links")
+    op.drop_index("idx_memory_links_to_unit", table_name="memory_links")
+    op.drop_index("idx_memory_links_from_unit", table_name="memory_links")
+    op.execute("DROP INDEX IF EXISTS idx_memory_links_unique")
+    op.drop_table("memory_links")
+
+    op.drop_index("idx_entity_cooccurrences_count", table_name="entity_cooccurrences")
+    op.drop_index("idx_entity_cooccurrences_entity2", table_name="entity_cooccurrences")
+    op.drop_index("idx_entity_cooccurrences_entity1", table_name="entity_cooccurrences")
+    op.drop_table("entity_cooccurrences")
+
+    # Drop BM25 materialized view and index
+    op.drop_index("idx_memory_units_bm25_text_vector", table_name="memory_units_bm25")
+    op.drop_index("idx_memory_units_bm25_bank", table_name="memory_units_bm25")
+    op.execute("DROP MATERIALIZED VIEW IF EXISTS memory_units_bm25")
+
+    op.drop_index("idx_memory_units_embedding", table_name="memory_units")
+    op.drop_index("idx_memory_units_observation_date", table_name="memory_units")
+    op.drop_index("idx_memory_units_opinion_date", table_name="memory_units")
+    op.drop_index("idx_memory_units_opinion_confidence", table_name="memory_units")
+    op.drop_index("idx_memory_units_bank_type_date", table_name="memory_units")
+    op.drop_index("idx_memory_units_bank_fact_type", table_name="memory_units")
+    op.drop_index("idx_memory_units_fact_type", table_name="memory_units")
+    op.drop_index("idx_memory_units_access_count", table_name="memory_units")
+    op.drop_index("idx_memory_units_bank_date", table_name="memory_units")
+    op.drop_index("idx_memory_units_event_date", table_name="memory_units")
+    op.drop_index("idx_memory_units_document_id", table_name="memory_units")
+    op.drop_index("idx_memory_units_bank_id", table_name="memory_units")
+    op.execute("DROP INDEX IF EXISTS idx_memory_units_text_search")
+    op.drop_table("memory_units")
+
+    op.execute("DROP INDEX IF EXISTS idx_entities_bank_lower_name")
+    op.drop_index("idx_entities_bank_name", table_name="entities")
+    op.drop_index("idx_entities_canonical_name", table_name="entities")
+    op.drop_index("idx_entities_bank_id", table_name="entities")
+    op.drop_table("entities")
+
+    op.drop_index("idx_async_operations_bank_status", table_name="async_operations")
+    op.drop_index("idx_async_operations_status", table_name="async_operations")
+    op.drop_index("idx_async_operations_bank_id", table_name="async_operations")
+    op.drop_table("async_operations")
+
+    op.drop_index("idx_documents_content_hash", table_name="documents")
+    op.drop_index("idx_documents_bank_id", table_name="documents")
+    op.drop_table("documents")
+
+    op.drop_table("banks")
+
+    # Drop extensions (optional - comment out if you want to keep them)
+    # op.execute('DROP EXTENSION IF EXISTS vector')
+    # op.execute('DROP EXTENSION IF EXISTS "uuid-ossp"')

hindsight-api/hindsight_api/alembic/versions/b7c4d8e9f1a2_add_chunks_table.py

@@ -0,0 +1,70 @@
+"""add_chunks_table
+
+Revision ID: b7c4d8e9f1a2
+Revises: 5a366d414dce
+Create Date: 2025-11-28 00:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "b7c4d8e9f1a2"
+down_revision: str | Sequence[str] | None = "5a366d414dce"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Add chunks table and link memory_units to chunks."""
+
+    # Create chunks table with single text PK (bank_id_document_id_chunk_index)
+    op.create_table(
+        "chunks",
+        sa.Column("chunk_id", sa.Text(), nullable=False),
+        sa.Column("document_id", sa.Text(), nullable=False),
+        sa.Column("bank_id", sa.Text(), nullable=False),
+        sa.Column("chunk_index", sa.Integer(), nullable=False),
+        sa.Column("chunk_text", sa.Text(), nullable=False),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id", "bank_id"],
+            ["documents.id", "documents.bank_id"],
+            name="chunks_document_fkey",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("chunk_id", name=op.f("pk_chunks")),
+    )
+
+    # Add indexes for efficient queries
+    op.create_index("idx_chunks_document_id", "chunks", ["document_id"])
+    op.create_index("idx_chunks_bank_id", "chunks", ["bank_id"])
+
+    # Add chunk_id column to memory_units (nullable, as existing records won't have chunks)
+    op.add_column("memory_units", sa.Column("chunk_id", sa.Text(), nullable=True))
+
+    # Add foreign key constraint to chunks table
+    op.create_foreign_key(
+        "memory_units_chunk_fkey", "memory_units", "chunks", ["chunk_id"], ["chunk_id"], ondelete="SET NULL"
+    )
+
+    # Add index on chunk_id for efficient lookups
+    op.create_index("idx_memory_units_chunk_id", "memory_units", ["chunk_id"])
+
+
+def downgrade() -> None:
+    """Remove chunks table and chunk_id from memory_units."""
+
+    # Drop index and foreign key from memory_units
+    op.drop_index("idx_memory_units_chunk_id", table_name="memory_units")
+    op.drop_constraint("memory_units_chunk_fkey", "memory_units", type_="foreignkey")
+    op.drop_column("memory_units", "chunk_id")
+
+    # Drop chunks table indexes and table
+    op.drop_index("idx_chunks_bank_id", table_name="chunks")
+    op.drop_index("idx_chunks_document_id", table_name="chunks")
+    op.drop_table("chunks")

hindsight-api/hindsight_api/alembic/versions/c8e5f2a3b4d1_add_retain_params_to_documents.py

@@ -0,0 +1,39 @@
+"""add_retain_params_to_documents
+
+Revision ID: c8e5f2a3b4d1
+Revises: b7c4d8e9f1a2
+Create Date: 2025-12-02 00:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "c8e5f2a3b4d1"
+down_revision: str | Sequence[str] | None = "b7c4d8e9f1a2"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Add retain_params JSONB column to documents table."""
+
+    # Add retain_params column to store parameters passed during retain
+    op.add_column("documents", sa.Column("retain_params", postgresql.JSONB(), nullable=True))
+
+    # Add index for efficient queries on retain_params
+    op.create_index("idx_documents_retain_params", "documents", ["retain_params"], postgresql_using="gin")
+
+
+def downgrade() -> None:
+    """Remove retain_params column from documents table."""
+
+    # Drop index
+    op.drop_index("idx_documents_retain_params", table_name="documents")
+
+    # Drop column
+    op.drop_column("documents", "retain_params")

hindsight-api/hindsight_api/alembic/versions/d9f6a3b4c5e2_rename_bank_to_interactions.py

@@ -0,0 +1,53 @@
+"""Rename fact_type 'bank' to 'experience'
+
+Revision ID: d9f6a3b4c5e2
+Revises: c8e5f2a3b4d1
+Create Date: 2024-12-04 15:00:00.000000
+
+"""
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision = "d9f6a3b4c5e2"
+down_revision = "c8e5f2a3b4d1"
+branch_labels = None
+depends_on = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (e.g., 'tenant_x.' or '' for public)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade():
+    schema = _get_schema_prefix()
+
+    # Drop old check constraint FIRST (before updating data)
+    op.drop_constraint("memory_units_fact_type_check", "memory_units", type_="check")
+
+    # Update existing 'bank' values to 'experience'
+    op.execute(f"UPDATE {schema}memory_units SET fact_type = 'experience' WHERE fact_type = 'bank'")
+    # Also update any 'interactions' values (in case of partial migration)
+    op.execute(f"UPDATE {schema}memory_units SET fact_type = 'experience' WHERE fact_type = 'interactions'")
+
+    # Create new check constraint with 'experience' instead of 'bank'
+    op.create_check_constraint(
+        "memory_units_fact_type_check", "memory_units", "fact_type IN ('world', 'experience', 'opinion', 'observation')"
+    )
+
+
+def downgrade():
+    schema = _get_schema_prefix()
+
+    # Drop new check constraint FIRST
+    op.drop_constraint("memory_units_fact_type_check", "memory_units", type_="check")
+
+    # Update 'experience' back to 'bank'
+    op.execute(f"UPDATE {schema}memory_units SET fact_type = 'bank' WHERE fact_type = 'experience'")
+
+    # Recreate old check constraint
+    op.create_check_constraint(
+        "memory_units_fact_type_check", "memory_units", "fact_type IN ('world', 'bank', 'opinion', 'observation')"
+    )

hindsight-api/hindsight_api/alembic/versions/e0a1b2c3d4e5_disposition_to_3_traits.py

@@ -0,0 +1,111 @@
+"""disposition_to_3_traits
+
+Revision ID: e0a1b2c3d4e5
+Revises: rename_personality
+Create Date: 2024-12-08
+
+Migrate disposition traits from Big Five (openness, conscientiousness, extraversion,
+agreeableness, neuroticism, bias_strength with 0-1 float values) to the new 3-trait
+system (skepticism, literalism, empathy with 1-5 integer values).
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "e0a1b2c3d4e5"
+down_revision: str | Sequence[str] | None = "rename_personality"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (e.g., 'tenant_x.' or '' for public)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def _get_target_schema() -> str:
+    """Get the target schema name (tenant schema or 'public')."""
+    schema = context.config.get_main_option("target_schema")
+    return schema if schema else "public"
+
+
+def upgrade() -> None:
+    """Convert Big Five disposition to 3-trait disposition."""
+    conn = op.get_bind()
+    schema = _get_schema_prefix()
+    target_schema = _get_target_schema()
+
+    # Check if disposition column exists (should have been created by previous migration)
+    result = conn.execute(
+        sa.text("""
+        SELECT column_name
+        FROM information_schema.columns
+        WHERE table_schema = :schema AND table_name = 'banks' AND column_name = 'disposition'
+    """),
+        {"schema": target_schema},
+    )
+    if not result.fetchone():
+        # Column doesn't exist yet (shouldn't happen but be safe)
+        return
+
+    # Update all existing banks to use the new disposition format
+    # Convert from old format to new format with reasonable mappings:
+    # - skepticism: derived from inverse of agreeableness (skeptical people are less agreeable)
+    # - literalism: derived from conscientiousness (detail-oriented people are more literal)
+    # - empathy: derived from agreeableness + inverse of neuroticism
+    # Default all to 3 (neutral) for simplicity
+    conn.execute(
+        sa.text(f"""
+        UPDATE {schema}banks
+        SET disposition = '{{"skepticism": 3, "literalism": 3, "empathy": 3}}'::jsonb
+        WHERE disposition IS NOT NULL
+    """)
+    )
+
+    # Update the default for new banks
+    conn.execute(
+        sa.text(f"""
+        ALTER TABLE {schema}banks
+        ALTER COLUMN disposition SET DEFAULT '{{"skepticism": 3, "literalism": 3, "empathy": 3}}'::jsonb
+    """)
+    )
+
+
+def downgrade() -> None:
+    """Convert back to Big Five disposition."""
+    conn = op.get_bind()
+    schema = _get_schema_prefix()
+    target_schema = _get_target_schema()
+
+    # Check if disposition column exists
+    result = conn.execute(
+        sa.text("""
+        SELECT column_name
+        FROM information_schema.columns
+        WHERE table_schema = :schema AND table_name = 'banks' AND column_name = 'disposition'
+    """),
+        {"schema": target_schema},
+    )
+    if not result.fetchone():
+        return
+
+    # Revert to Big Five format with default values
+    conn.execute(
+        sa.text(f"""
+        UPDATE {schema}banks
+        SET disposition = '{{"openness": 0.5, "conscientiousness": 0.5, "extraversion": 0.5, "agreeableness": 0.5, "neuroticism": 0.5, "bias_strength": 0.5}}'::jsonb
+        WHERE disposition IS NOT NULL
+    """)
+    )
+
+    # Update the default for new banks
+    conn.execute(
+        sa.text(f"""
+        ALTER TABLE {schema}banks
+        ALTER COLUMN disposition SET DEFAULT '{{"openness": 0.5, "conscientiousness": 0.5, "extraversion": 0.5, "agreeableness": 0.5, "neuroticism": 0.5, "bias_strength": 0.5}}'::jsonb
+    """)
+    )

hindsight-api/hindsight_api/alembic/versions/f1a2b3c4d5e6_add_memory_links_composite_index.py

@@ -0,0 +1,44 @@
+"""add_memory_links_from_type_weight_index
+
+Revision ID: f1a2b3c4d5e6
+Revises: e0a1b2c3d4e5
+Create Date: 2025-01-12
+
+Add composite index on memory_links (from_unit_id, link_type, weight DESC)
+to optimize MPFP graph traversal queries that need top-k edges per type.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "f1a2b3c4d5e6"
+down_revision: str | Sequence[str] | None = "e0a1b2c3d4e5"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (e.g., 'tenant_x.' or '' for public)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add composite index for efficient MPFP edge loading."""
+    schema = _get_schema_prefix()
+    # Create composite index for efficient top-k per (from_node, link_type) queries
+    # This enables LATERAL joins to use index-only scans with early termination
+    # Note: Not using CONCURRENTLY here as it requires running outside a transaction
+    # For production with large tables, consider running this manually with CONCURRENTLY
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS idx_memory_links_from_type_weight "
+        f"ON {schema}memory_links(from_unit_id, link_type, weight DESC)"
+    )
+
+
+def downgrade() -> None:
+    """Remove the composite index."""
+    schema = _get_schema_prefix()
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_links_from_type_weight")

hindsight-api/hindsight_api/alembic/versions/g2a3b4c5d6e7_add_tags_column.py

@@ -0,0 +1,48 @@
+"""add_tags_column
+
+Revision ID: g2a3b4c5d6e7
+Revises: f1a2b3c4d5e6
+Create Date: 2025-01-13
+
+Add tags column to memory_units and documents tables for visibility scoping.
+Tags enable filtering memories by scope (e.g., user IDs, session IDs) during recall/reflect.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "g2a3b4c5d6e7"
+down_revision: str | Sequence[str] | None = "f1a2b3c4d5e6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (e.g., 'tenant_x.' or '' for public)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add tags column to memory_units and documents tables."""
+    schema = _get_schema_prefix()
+
+    # Add tags column to memory_units table
+    op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS tags VARCHAR[] NOT NULL DEFAULT '{{}}'")
+
+    # Create GIN index for efficient array containment queries (tags && ARRAY['x'])
+    op.execute(f"CREATE INDEX IF NOT EXISTS idx_memory_units_tags ON {schema}memory_units USING GIN (tags)")
+
+    # Add tags column to documents table for document-level tags
+    op.execute(f"ALTER TABLE {schema}documents ADD COLUMN IF NOT EXISTS tags VARCHAR[] NOT NULL DEFAULT '{{}}'")
+
+
+def downgrade() -> None:
+    """Remove tags columns and index."""
+    schema = _get_schema_prefix()
+
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_tags")
+    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS tags")
+    op.execute(f"ALTER TABLE {schema}documents DROP COLUMN IF EXISTS tags")

hindsight-api/hindsight_api/alembic/versions/h3c4d5e6f7g8_mental_models_v4.py

@@ -0,0 +1,112 @@
+"""mental_models_v4
+
+Revision ID: h3c4d5e6f7g8
+Revises: g2a3b4c5d6e7
+Create Date: 2026-01-08 00:00:00.000000
+
+This migration implements the v4 mental models system:
+1. Deletes existing observation memory_units (observations now in mental models)
+2. Adds mission column to banks (replacing background)
+3. Creates mental_models table with final schema
+
+Mental models can reference entities when an entity is "promoted" to a mental model.
+Summary content is stored as JSONB observations with per-observation fact attribution.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "h3c4d5e6f7g8"
+down_revision: str | Sequence[str] | None = "g2a3b4c5d6e7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Apply mental models v4 changes."""
+    schema = _get_schema_prefix()
+
+    # Step 1: Delete observation memory_units (cascades to unit_entities links)
+    # Observations are now handled through mental models, not memory_units
+    op.execute(f"DELETE FROM {schema}memory_units WHERE fact_type = 'observation'")
+
+    # Step 2: Drop observation-specific index (if it exists)
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_observation_date")
+
+    # Step 3: Add mission column to banks (replacing background)
+    op.execute(f"ALTER TABLE {schema}banks ADD COLUMN IF NOT EXISTS mission TEXT")
+
+    # Migrate: copy background to mission if background column exists
+    # Use DO block to check column existence first (idempotent for re-runs)
+    schema_name = context.config.get_main_option("target_schema") or "public"
+    op.execute(f"""
+        DO $$
+        BEGIN
+            IF EXISTS (
+                SELECT 1 FROM information_schema.columns
+                WHERE table_schema = '{schema_name}' AND table_name = 'banks' AND column_name = 'background'
+            ) THEN
+                UPDATE {schema}banks
+                SET mission = background
+                WHERE mission IS NULL;
+            END IF;
+        END $$;
+    """)
+
+    # Remove background column (replaced by mission)
+    op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS background")
+
+    # Step 4: Create mental_models table with final v4 schema (if not exists)
+    op.execute(f"""
+        CREATE TABLE IF NOT EXISTS {schema}mental_models (
+            id VARCHAR(64) NOT NULL,
+            bank_id VARCHAR(64) NOT NULL,
+            subtype VARCHAR(32) NOT NULL,
+            name VARCHAR(256) NOT NULL,
+            description TEXT NOT NULL,
+            entity_id UUID,
+            observations JSONB DEFAULT '{{"observations": []}}'::jsonb,
+            links VARCHAR[],
+            tags VARCHAR[] DEFAULT '{{}}',
+            last_updated TIMESTAMP WITH TIME ZONE,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+            PRIMARY KEY (id, bank_id),
+            FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE,
+            FOREIGN KEY (entity_id) REFERENCES {schema}entities(id) ON DELETE SET NULL,
+            CONSTRAINT ck_mental_models_subtype CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
+        )
+    """)
+
+    # Step 5: Create indexes for efficient queries (if not exist)
+    op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_bank_id ON {schema}mental_models(bank_id)")
+    op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_subtype ON {schema}mental_models(bank_id, subtype)")
+    op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_entity_id ON {schema}mental_models(entity_id)")
+    # GIN index for efficient tags array filtering
+    op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_tags ON {schema}mental_models USING GIN(tags)")
+
+
+def downgrade() -> None:
+    """Revert mental models v4 changes."""
+    schema = _get_schema_prefix()
+
+    # Drop mental_models table (cascades to indexes)
+    op.execute(f"DROP TABLE IF EXISTS {schema}mental_models CASCADE")
+
+    # Add back background column to banks
+    op.execute(f"ALTER TABLE {schema}banks ADD COLUMN IF NOT EXISTS background TEXT")
+
+    # Migrate mission back to background
+    op.execute(f"UPDATE {schema}banks SET background = mission WHERE background IS NULL")
+
+    # Remove mission column
+    op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS mission")
+
+    # Note: Cannot restore deleted observations - they are lost on downgrade

hindsight-api/hindsight_api/alembic/versions/i4d5e6f7g8h9_delete_opinions.py

@@ -0,0 +1,41 @@
+"""delete_opinions
+
+Revision ID: i4d5e6f7g8h9
+Revises: h3c4d5e6f7g8
+Create Date: 2026-01-15 00:00:00.000000
+
+This migration removes opinion facts from memory_units.
+Opinions are no longer a separate fact type - they are now represented
+through mental model observations with confidence scores.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "i4d5e6f7g8h9"
+down_revision: str | Sequence[str] | None = "h3c4d5e6f7g8"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Delete opinion memory_units."""
+    schema = _get_schema_prefix()
+
+    # Delete opinion memory_units (cascades to unit_entities links)
+    # Opinions are now handled through mental model observations
+    op.execute(f"DELETE FROM {schema}memory_units WHERE fact_type = 'opinion'")
+
+
+def downgrade() -> None:
+    """Cannot restore deleted opinions."""
+    # Note: Cannot restore deleted opinions - they are lost on downgrade
+    pass

hindsight-api/hindsight_api/alembic/versions/j5e6f7g8h9i0_mental_model_versions.py

@@ -0,0 +1,95 @@
+"""mental_model_versions
+
+Revision ID: j5e6f7g8h9i0
+Revises: i4d5e6f7g8h9
+Create Date: 2026-01-16 00:00:00.000000
+
+This migration adds versioning support for mental models:
+1. Creates mental_model_versions table to store observation snapshots
+2. Adds version column to mental_models for tracking current version
+
+This enables changelog/diff functionality for mental model observations.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "j5e6f7g8h9i0"
+down_revision: str | Sequence[str] | None = "i4d5e6f7g8h9"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Create mental_model_versions table and add version tracking."""
+    schema = _get_schema_prefix()
+
+    # Create mental_model_versions table for storing observation snapshots
+    op.execute(f"""
+        CREATE TABLE {schema}mental_model_versions (
+            id SERIAL PRIMARY KEY,
+            mental_model_id VARCHAR(64) NOT NULL,
+            bank_id VARCHAR(64) NOT NULL,
+            version INT NOT NULL,
+            observations JSONB NOT NULL DEFAULT '{{"observations": []}}'::jsonb,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+            FOREIGN KEY (mental_model_id, bank_id)
+                REFERENCES {schema}mental_models(id, bank_id) ON DELETE CASCADE,
+            UNIQUE (mental_model_id, bank_id, version)
+        )
+    """)
+
+    # Index for efficient version queries (get latest, list versions)
+    op.execute(f"""
+        CREATE INDEX idx_mental_model_versions_lookup
+        ON {schema}mental_model_versions(mental_model_id, bank_id, version DESC)
+    """)
+
+    # Add version column to mental_models to track current version
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD COLUMN IF NOT EXISTS version INT NOT NULL DEFAULT 0
+    """)
+
+    # Migrate existing mental models: create version 1 for any that have observations
+    op.execute(f"""
+        INSERT INTO {schema}mental_model_versions (mental_model_id, bank_id, version, observations, created_at)
+        SELECT id, bank_id, 1, observations, COALESCE(last_updated, created_at)
+        FROM {schema}mental_models
+        WHERE observations IS NOT NULL
+          AND observations != '{{"observations": []}}'::jsonb
+          AND (observations->'observations') IS NOT NULL
+          AND jsonb_array_length(observations->'observations') > 0
+    """)
+
+    # Update version to 1 for migrated mental models
+    op.execute(f"""
+        UPDATE {schema}mental_models
+        SET version = 1
+        WHERE observations IS NOT NULL
+          AND observations != '{{"observations": []}}'::jsonb
+          AND (observations->'observations') IS NOT NULL
+          AND jsonb_array_length(observations->'observations') > 0
+    """)
+
+
+def downgrade() -> None:
+    """Remove mental_model_versions table and version column."""
+    schema = _get_schema_prefix()
+
+    # Drop index
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_mental_model_versions_lookup")
+
+    # Drop versions table
+    op.execute(f"DROP TABLE IF EXISTS {schema}mental_model_versions")
+
+    # Remove version column from mental_models
+    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS version")

hindsight-api/hindsight_api/alembic/versions/k6f7g8h9i0j1_add_directive_subtype.py

@@ -0,0 +1,58 @@
+"""add_directive_subtype
+
+Revision ID: k6f7g8h9i0j1
+Revises: j5e6f7g8h9i0
+Create Date: 2026-01-16 00:00:00.000000
+
+This migration adds 'directive' to the mental_models subtype constraint.
+Directives are hard rules with user-provided observations that the reflect agent must follow.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "k6f7g8h9i0j1"
+down_revision: str | Sequence[str] | None = "j5e6f7g8h9i0"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add 'directive' to mental_models subtype constraint."""
+    schema = _get_schema_prefix()
+
+    # Drop existing constraint
+    op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
+
+    # Create new constraint with 'directive' added
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD CONSTRAINT ck_mental_models_subtype
+        CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned', 'directive'))
+    """)
+
+
+def downgrade() -> None:
+    """Remove 'directive' from mental_models subtype constraint."""
+    schema = _get_schema_prefix()
+
+    # First delete any directives (cannot downgrade if they exist)
+    op.execute(f"DELETE FROM {schema}mental_models WHERE subtype = 'directive'")
+
+    # Drop constraint with directive
+    op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
+
+    # Recreate original constraint without directive
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD CONSTRAINT ck_mental_models_subtype
+        CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
+    """)

hindsight-api/hindsight_api/alembic/versions/l7g8h9i0j1k2_add_worker_columns.py

@@ -0,0 +1,109 @@
+"""add_worker_columns
+
+Revision ID: l7g8h9i0j1k2
+Revises: k6f7g8h9i0j1
+Create Date: 2026-01-19 00:00:00.000000
+
+This migration adds columns to async_operations for distributed worker support:
+- worker_id: ID of the worker that claimed the task
+- claimed_at: When the task was claimed
+- retry_count: Number of retry attempts
+- task_payload: The serialized task dictionary
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import context, op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "l7g8h9i0j1k2"
+down_revision: str | Sequence[str] | None = "k6f7g8h9i0j1"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add worker columns to async_operations."""
+    schema = _get_schema_prefix()
+
+    # Add worker_id column (ID of worker that claimed the task)
+    op.add_column(
+        "async_operations",
+        sa.Column("worker_id", sa.Text(), nullable=True),
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+
+    # Add claimed_at column (when task was claimed by worker)
+    op.add_column(
+        "async_operations",
+        sa.Column("claimed_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+
+    # Add retry_count column (number of retry attempts)
+    op.add_column(
+        "async_operations",
+        sa.Column("retry_count", sa.Integer(), server_default="0", nullable=False),
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+
+    # Add task_payload column (serialized task dictionary)
+    op.add_column(
+        "async_operations",
+        sa.Column(
+            "task_payload",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+        ),
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+
+    # Add index for efficient worker polling (pending tasks ordered by creation time)
+    op.execute(
+        f"CREATE INDEX idx_async_operations_pending_claim ON {schema}async_operations (status, created_at) "
+        f"WHERE status = 'pending' AND task_payload IS NOT NULL"
+    )
+
+    # Add index for finding tasks by worker_id (for decommissioning)
+    op.execute(
+        f"CREATE INDEX idx_async_operations_worker_id ON {schema}async_operations (worker_id) WHERE worker_id IS NOT NULL"
+    )
+
+
+def downgrade() -> None:
+    """Remove worker columns from async_operations."""
+    schema = _get_schema_prefix()
+
+    # Drop indexes
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_pending_claim")
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_worker_id")
+
+    # Drop columns
+    op.drop_column(
+        "async_operations",
+        "task_payload",
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+    op.drop_column(
+        "async_operations",
+        "retry_count",
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+    op.drop_column(
+        "async_operations",
+        "claimed_at",
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+    op.drop_column(
+        "async_operations",
+        "worker_id",
+        schema=context.config.get_main_option("target_schema") or None,
+    )

hindsight-api/hindsight_api/alembic/versions/m8h9i0j1k2l3_mental_model_id_to_text.py

@@ -0,0 +1,41 @@
+"""mental_model_id_to_text
+
+Revision ID: m8h9i0j1k2l3
+Revises: l7g8h9i0j1k2
+Create Date: 2026-01-19 00:00:00.000000
+
+This migration changes the mental_models.id column from VARCHAR(64) to TEXT
+to support longer model IDs (e.g., entity names that exceed 64 characters).
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "m8h9i0j1k2l3"
+down_revision: str | Sequence[str] | None = "l7g8h9i0j1k2"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Change mental_models.id from VARCHAR(64) to TEXT."""
+    schema = _get_schema_prefix()
+
+    # Alter the id column type from VARCHAR(64) to TEXT
+    op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE TEXT")
+
+
+def downgrade() -> None:
+    """Revert mental_models.id from TEXT to VARCHAR(64)."""
+    schema = _get_schema_prefix()
+
+    # Note: This may fail if any id values exceed 64 characters
+    op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE VARCHAR(64)")

hindsight-api/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py

@@ -0,0 +1,134 @@
+"""learnings_and_pinned_reflections
+
+Revision ID: n9i0j1k2l3m4
+Revises: m8h9i0j1k2l3
+Create Date: 2026-01-21 00:00:00.000000
+
+This migration:
+1. Creates the 'learnings' table for automatic bottom-up consolidation
+2. Creates the 'pinned_reflections' table for user-curated living documents
+3. Adds consolidation tracking columns to the 'banks' table
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "n9i0j1k2l3m4"
+down_revision: str | Sequence[str] | None = "m8h9i0j1k2l3"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Create learnings and pinned_reflections tables."""
+    schema = _get_schema_prefix()
+
+    # 1. Create learnings table
+    op.execute(f"""
+        CREATE TABLE {schema}learnings (
+            id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            bank_id VARCHAR(64) NOT NULL,
+            text TEXT NOT NULL,
+            proof_count INT NOT NULL DEFAULT 1,
+            history JSONB DEFAULT '[]'::jsonb,
+            mission_context VARCHAR(64),
+            pre_mission_change BOOLEAN DEFAULT FALSE,
+            embedding vector(384),
+            tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+            updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
+        )
+    """)
+
+    # Add foreign key constraint
+    op.execute(f"""
+        ALTER TABLE {schema}learnings
+        ADD CONSTRAINT fk_learnings_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+    # Indexes for learnings
+    op.execute(f"CREATE INDEX idx_learnings_bank_id ON {schema}learnings(bank_id)")
+    op.execute(f"""
+        CREATE INDEX idx_learnings_embedding ON {schema}learnings
+        USING hnsw (embedding vector_cosine_ops)
+    """)
+    op.execute(f"CREATE INDEX idx_learnings_tags ON {schema}learnings USING GIN(tags)")
+
+    # Full-text search for learnings
+    op.execute(f"""
+        ALTER TABLE {schema}learnings ADD COLUMN search_vector tsvector
+        GENERATED ALWAYS AS (to_tsvector('english', text)) STORED
+    """)
+    op.execute(f"CREATE INDEX idx_learnings_text_search ON {schema}learnings USING gin(search_vector)")
+
+    # 2. Create pinned_reflections table
+    op.execute(f"""
+        CREATE TABLE {schema}pinned_reflections (
+            id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            bank_id VARCHAR(64) NOT NULL,
+            name VARCHAR(256) NOT NULL,
+            source_query TEXT NOT NULL,
+            content TEXT NOT NULL,
+            embedding vector(384),
+            tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
+            last_refreshed_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
+        )
+    """)
+
+    # Add foreign key constraint
+    op.execute(f"""
+        ALTER TABLE {schema}pinned_reflections
+        ADD CONSTRAINT fk_pinned_reflections_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+    # Indexes for pinned_reflections
+    op.execute(f"CREATE INDEX idx_pinned_reflections_bank_id ON {schema}pinned_reflections(bank_id)")
+    op.execute(f"""
+        CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
+        USING hnsw (embedding vector_cosine_ops)
+    """)
+    op.execute(f"CREATE INDEX idx_pinned_reflections_tags ON {schema}pinned_reflections USING GIN(tags)")
+
+    # Full-text search for pinned_reflections
+    op.execute(f"""
+        ALTER TABLE {schema}pinned_reflections ADD COLUMN search_vector tsvector
+        GENERATED ALWAYS AS (to_tsvector('english', COALESCE(name, '') || ' ' || content)) STORED
+    """)
+    op.execute(f"""
+        CREATE INDEX idx_pinned_reflections_text_search ON {schema}pinned_reflections
+        USING gin(search_vector)
+    """)
+
+    # 3. Add consolidation tracking columns to banks table
+    op.execute(f"""
+        ALTER TABLE {schema}banks
+        ADD COLUMN IF NOT EXISTS last_consolidated_at TIMESTAMP WITH TIME ZONE
+    """)
+    op.execute(f"""
+        ALTER TABLE {schema}banks
+        ADD COLUMN IF NOT EXISTS mission_changed_at TIMESTAMP WITH TIME ZONE
+    """)
+
+
+def downgrade() -> None:
+    """Drop learnings and pinned_reflections tables."""
+    schema = _get_schema_prefix()
+
+    # Drop tables
+    op.execute(f"DROP TABLE IF EXISTS {schema}learnings CASCADE")
+    op.execute(f"DROP TABLE IF EXISTS {schema}pinned_reflections CASCADE")
+
+    # Remove columns from banks
+    op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS last_consolidated_at")
+    op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS mission_changed_at")

hindsight-api/hindsight_api/alembic/versions/o0j1k2l3m4n5_migrate_mental_models_data.py

@@ -0,0 +1,113 @@
+"""migrate_mental_models_data
+
+Revision ID: o0j1k2l3m4n5
+Revises: n9i0j1k2l3m4
+Create Date: 2026-01-21 00:00:00.000000
+
+This migration:
+1. Migrates existing 'pinned' mental models to the new 'pinned_reflections' table
+2. Migrates existing 'learned' mental models to the new 'learnings' table
+3. Deletes non-directive mental models (structural, emergent, pinned, learned)
+4. Drops the mental_model_versions table (no longer used)
+5. Adds a CHECK constraint that only 'directive' subtype is allowed
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "o0j1k2l3m4n5"
+down_revision: str | Sequence[str] | None = "n9i0j1k2l3m4"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Migrate data and clean up old mental models."""
+    schema = _get_schema_prefix()
+
+    # 1. Migrate 'pinned' mental models to pinned_reflections
+    # For pinned models, the first observation's content becomes the pinned reflection content
+    op.execute(f"""
+        INSERT INTO {schema}pinned_reflections (bank_id, name, source_query, content, tags, created_at)
+        SELECT
+            bank_id,
+            name,
+            description AS source_query,
+            COALESCE(
+                observations->'observations'->0->>'content',
+                description,
+                ''
+            ) AS content,
+            tags,
+            created_at
+        FROM {schema}mental_models
+        WHERE subtype = 'pinned'
+        ON CONFLICT DO NOTHING
+    """)
+
+    # 2. Migrate 'learned' mental models to learnings
+    # Each observation in a learned model becomes a separate learning
+    op.execute(f"""
+        INSERT INTO {schema}learnings (bank_id, text, proof_count, tags, created_at)
+        SELECT
+            mm.bank_id,
+            obs->>'content' AS text,
+            GREATEST(1, COALESCE(jsonb_array_length(obs->'evidence'), 1)) AS proof_count,
+            mm.tags,
+            mm.created_at
+        FROM {schema}mental_models mm,
+        LATERAL jsonb_array_elements(mm.observations->'observations') AS obs
+        WHERE mm.subtype = 'learned'
+          AND obs->>'content' IS NOT NULL
+          AND obs->>'content' != ''
+        ON CONFLICT DO NOTHING
+    """)
+
+    # 3. Delete all non-directive mental models (they've been migrated or are obsolete)
+    op.execute(f"""
+        DELETE FROM {schema}mental_models
+        WHERE subtype != 'directive'
+    """)
+
+    # 4. Drop the mental_model_versions table (no longer used)
+    op.execute(f"DROP TABLE IF EXISTS {schema}mental_model_versions CASCADE")
+
+    # 5. Drop old constraints and add new one that only allows 'directive'
+    op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD CONSTRAINT ck_mental_models_subtype CHECK (subtype = 'directive')
+    """)
+
+
+def downgrade() -> None:
+    """Reverse the migration (data migration is one-way, so this just removes constraints)."""
+    schema = _get_schema_prefix()
+
+    # Remove the directive-only constraint
+    op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
+
+    # Re-create mental_model_versions table
+    op.execute(f"""
+        CREATE TABLE IF NOT EXISTS {schema}mental_model_versions (
+            id SERIAL PRIMARY KEY,
+            bank_id VARCHAR(64) NOT NULL,
+            model_id VARCHAR(128) NOT NULL,
+            version INT NOT NULL,
+            observations JSONB NOT NULL,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
+        )
+    """)
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS idx_mm_versions_lookup ON {schema}mental_model_versions(bank_id, model_id, version DESC)"
+    )
+
+    # Note: Data migration cannot be reversed - pinned_reflections and learnings data remains

hindsight-api/hindsight_api/alembic/versions/p1k2l3m4n5o6_new_knowledge_architecture.py

@@ -0,0 +1,194 @@
+"""new_knowledge_architecture
+
+Revision ID: p1k2l3m4n5o6
+Revises: o0j1k2l3m4n5
+Create Date: 2026-01-21 00:00:00.000000
+
+This migration implements the new knowledge architecture:
+1. Drops the 'learnings' table (mental models are now in memory_units)
+2. Renames 'pinned_reflections' to 'reflections'
+3. Drops the 'mental_models' table completely
+4. Creates 'directives' table for hard rules
+5. Adds mental model support columns to 'memory_units' (proof_count, source_memory_ids, history)
+
+The new architecture:
+- Directives: Hard rules in their own table
+- Mental Models: Stored in memory_units with fact_type='mental_model'
+- Reflections: User-curated documents (renamed from pinned_reflections)
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "p1k2l3m4n5o6"
+down_revision: str | Sequence[str] | None = "o0j1k2l3m4n5"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Implement new knowledge architecture."""
+    schema = _get_schema_prefix()
+
+    # 1. Drop the learnings table (mental models will be in memory_units)
+    op.execute(f"DROP TABLE IF EXISTS {schema}learnings CASCADE")
+
+    # 2. Rename pinned_reflections to reflections
+    op.execute(f"ALTER TABLE IF EXISTS {schema}pinned_reflections RENAME TO reflections")
+
+    # Rename indexes for reflections
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_bank_id RENAME TO idx_reflections_bank_id")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_embedding RENAME TO idx_reflections_embedding")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_tags RENAME TO idx_reflections_tags")
+    op.execute(
+        f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_text_search RENAME TO idx_reflections_text_search"
+    )
+
+    # Rename foreign key constraint
+    op.execute(f"""
+        ALTER TABLE {schema}reflections
+        DROP CONSTRAINT IF EXISTS fk_pinned_reflections_bank_id
+    """)
+    op.execute(f"""
+        ALTER TABLE {schema}reflections
+        ADD CONSTRAINT fk_reflections_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+    # 3. Drop the mental_models table completely
+    op.execute(f"DROP TABLE IF EXISTS {schema}mental_models CASCADE")
+
+    # 4. Create directives table
+    op.execute(f"""
+        CREATE TABLE {schema}directives (
+            id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            bank_id VARCHAR(64) NOT NULL,
+            name VARCHAR(256) NOT NULL,
+            content TEXT NOT NULL,
+            priority INT NOT NULL DEFAULT 0,
+            is_active BOOLEAN NOT NULL DEFAULT TRUE,
+            tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+            updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
+        )
+    """)
+
+    # Add foreign key and indexes for directives
+    op.execute(f"""
+        ALTER TABLE {schema}directives
+        ADD CONSTRAINT fk_directives_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+    op.execute(f"CREATE INDEX idx_directives_bank_id ON {schema}directives(bank_id)")
+    op.execute(f"CREATE INDEX idx_directives_bank_active ON {schema}directives(bank_id, is_active)")
+    op.execute(f"CREATE INDEX idx_directives_tags ON {schema}directives USING GIN(tags)")
+
+    # 5. Add mental model support columns to memory_units
+    # proof_count: Number of memories that support this mental model
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD COLUMN IF NOT EXISTS proof_count INT DEFAULT 1
+    """)
+
+    # source_memory_ids: Array of memory IDs that consolidated into this mental model
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD COLUMN IF NOT EXISTS source_memory_ids UUID[] DEFAULT ARRAY[]::UUID[]
+    """)
+
+    # history: JSONB array tracking changes to mental models
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD COLUMN IF NOT EXISTS history JSONB DEFAULT '[]'::jsonb
+    """)
+
+    # Add index for finding mental models
+    op.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_memory_units_mental_models
+        ON {schema}memory_units(bank_id, fact_type)
+        WHERE fact_type = 'mental_model'
+    """)
+
+    # 6. Update fact_type check constraint to include 'mental_model'
+    op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD CONSTRAINT memory_units_fact_type_check
+        CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation', 'mental_model'))
+    """)
+
+
+def downgrade() -> None:
+    """Reverse the migration."""
+    schema = _get_schema_prefix()
+
+    # Restore original fact_type check constraint (without 'mental_model')
+    op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD CONSTRAINT memory_units_fact_type_check
+        CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation'))
+    """)
+
+    # Drop mental model columns from memory_units
+    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS proof_count")
+    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS source_memory_ids")
+    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS history")
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_mental_models")
+
+    # Drop directives table
+    op.execute(f"DROP TABLE IF EXISTS {schema}directives CASCADE")
+
+    # Rename reflections back to pinned_reflections
+    op.execute(f"ALTER TABLE IF EXISTS {schema}reflections RENAME TO pinned_reflections")
+
+    # Restore indexes
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_bank_id RENAME TO idx_pinned_reflections_bank_id")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_embedding RENAME TO idx_pinned_reflections_embedding")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_tags RENAME TO idx_pinned_reflections_tags")
+    op.execute(
+        f"ALTER INDEX IF EXISTS {schema}idx_reflections_text_search RENAME TO idx_pinned_reflections_text_search"
+    )
+
+    # Restore foreign key
+    op.execute(f"""
+        ALTER TABLE {schema}pinned_reflections
+        DROP CONSTRAINT IF EXISTS fk_reflections_bank_id
+    """)
+    op.execute(f"""
+        ALTER TABLE {schema}pinned_reflections
+        ADD CONSTRAINT fk_pinned_reflections_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+    # Re-create learnings table
+    op.execute(f"""
+        CREATE TABLE IF NOT EXISTS {schema}learnings (
+            id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            bank_id VARCHAR(64) NOT NULL,
+            text TEXT NOT NULL,
+            proof_count INT NOT NULL DEFAULT 1,
+            history JSONB DEFAULT '[]'::jsonb,
+            mission_context VARCHAR(64),
+            pre_mission_change BOOLEAN DEFAULT FALSE,
+            embedding vector(384),
+            tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+            updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
+        )
+    """)
+    op.execute(f"""
+        ALTER TABLE {schema}learnings
+        ADD CONSTRAINT fk_learnings_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+    # Note: mental_models table recreation is complex and would need separate handling

hindsight-api/hindsight_api/alembic/versions/q2l3m4n5o6p7_fix_mental_model_fact_type.py

@@ -0,0 +1,50 @@
+"""fix_mental_model_fact_type
+
+Revision ID: q2l3m4n5o6p7
+Revises: p1k2l3m4n5o6
+Create Date: 2026-01-21 13:30:00.000000
+
+Fix the fact_type check constraint to include 'mental_model'.
+This is a fix for p1k2l3m4n5o6 which should have included this change.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "q2l3m4n5o6p7"
+down_revision: str | Sequence[str] | None = "p1k2l3m4n5o6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add 'mental_model' to the fact_type check constraint."""
+    schema = _get_schema_prefix()
+
+    # Drop the old constraint and add the new one with mental_model included
+    op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD CONSTRAINT memory_units_fact_type_check
+        CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation', 'mental_model'))
+    """)
+
+
+def downgrade() -> None:
+    """Remove 'mental_model' from the fact_type check constraint."""
+    schema = _get_schema_prefix()
+
+    op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD CONSTRAINT memory_units_fact_type_check
+        CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation'))
+    """)

hindsight-api/hindsight_api/alembic/versions/r3m4n5o6p7q8_add_reflect_response_to_reflections.py

@@ -0,0 +1,47 @@
+"""Add reflect_response JSONB column to reflections
+
+Revision ID: r3m4n5o6p7q8
+Revises: q2l3m4n5o6p7
+Create Date: 2026-01-21
+
+This migration adds a reflect_response JSONB column to store the full
+reflect API response payload, including based_on facts and trace data.
+
+Note: Table was renamed from pinned_reflections to reflections in p1k2l3m4n5o6.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "r3m4n5o6p7q8"
+down_revision: str | Sequence[str] | None = "q2l3m4n5o6p7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add reflect_response JSONB column to reflections."""
+    schema = _get_schema_prefix()
+
+    # Add reflect_response column to store the full reflect API response
+    op.execute(f"""
+        ALTER TABLE {schema}reflections
+        ADD COLUMN IF NOT EXISTS reflect_response JSONB
+    """)
+
+
+def downgrade() -> None:
+    """Remove reflect_response column from reflections."""
+    schema = _get_schema_prefix()
+
+    op.execute(f"""
+        ALTER TABLE {schema}reflections
+        DROP COLUMN IF EXISTS reflect_response
+    """)

hindsight-api/hindsight_api/alembic/versions/rename_personality_to_disposition.py

@@ -0,0 +1,85 @@
+"""rename_personality_to_disposition
+
+Revision ID: rename_personality
+Revises: d9f6a3b4c5e2
+Create Date: 2024-12-04
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import context, op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "rename_personality"
+down_revision: str | Sequence[str] | None = "d9f6a3b4c5e2"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_target_schema() -> str:
+    """Get the target schema name (tenant schema or 'public')."""
+    schema = context.config.get_main_option("target_schema")
+    return schema if schema else "public"
+
+
+def upgrade() -> None:
+    """Rename personality column to disposition in banks table (if it exists)."""
+    conn = op.get_bind()
+    target_schema = _get_target_schema()
+
+    # Check if 'personality' column exists (old database)
+    result = conn.execute(
+        sa.text("""
+        SELECT column_name
+        FROM information_schema.columns
+        WHERE table_schema = :schema AND table_name = 'banks' AND column_name = 'personality'
+    """),
+        {"schema": target_schema},
+    )
+    has_personality = result.fetchone() is not None
+
+    # Check if 'disposition' column exists (new database)
+    result = conn.execute(
+        sa.text("""
+        SELECT column_name
+        FROM information_schema.columns
+        WHERE table_schema = :schema AND table_name = 'banks' AND column_name = 'disposition'
+    """),
+        {"schema": target_schema},
+    )
+    has_disposition = result.fetchone() is not None
+
+    if has_personality and not has_disposition:
+        # Old database: rename personality -> disposition
+        op.alter_column("banks", "personality", new_column_name="disposition")
+    elif not has_personality and not has_disposition:
+        # Neither exists (shouldn't happen, but be safe): add disposition column
+        op.add_column(
+            "banks",
+            sa.Column(
+                "disposition",
+                postgresql.JSONB(astext_type=sa.Text()),
+                server_default=sa.text("'{}'::jsonb"),
+                nullable=False,
+            ),
+        )
+    # else: disposition already exists, nothing to do
+
+
+def downgrade() -> None:
+    """Revert disposition column back to personality."""
+    conn = op.get_bind()
+    target_schema = _get_target_schema()
+    result = conn.execute(
+        sa.text("""
+        SELECT column_name
+        FROM information_schema.columns
+        WHERE table_schema = :schema AND table_name = 'banks' AND column_name = 'disposition'
+    """),
+        {"schema": target_schema},
+    )
+    if result.fetchone():
+        op.alter_column("banks", "disposition", new_column_name="personality")

hindsight-api/hindsight_api/alembic/versions/s4n5o6p7q8r9_add_consolidated_at_to_memory_units.py

@@ -0,0 +1,53 @@
+"""Add consolidated_at column to memory_units for incremental consolidation tracking.
+
+This allows consolidation to track progress at the memory level rather than
+using a bank-level watermark. If consolidation crashes, already-processed
+memories won't be reprocessed.
+
+Revision ID: s4n5o6p7q8r9
+Revises: r3m4n5o6p7q8
+Create Date: 2025-01-22
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "s4n5o6p7q8r9"
+down_revision: str | Sequence[str] | None = "r3m4n5o6p7q8"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+
+    # Add consolidated_at column to memory_units
+    op.execute(
+        f"""
+        ALTER TABLE {schema}memory_units
+        ADD COLUMN IF NOT EXISTS consolidated_at TIMESTAMPTZ DEFAULT NULL
+        """
+    )
+
+    # Create index for efficient querying of unconsolidated memories
+    op.execute(
+        f"""
+        CREATE INDEX IF NOT EXISTS idx_memory_units_unconsolidated
+        ON {schema}memory_units (bank_id, created_at)
+        WHERE consolidated_at IS NULL AND fact_type IN ('experience', 'world')
+        """
+    )
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_unconsolidated")
+    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS consolidated_at")

hindsight-api/hindsight_api/alembic/versions/t5o6p7q8r9s0_rename_mental_models_to_observations.py

@@ -0,0 +1,134 @@
+"""Rename mental_model fact_type to observation and reflections table to mental_models
+
+Revision ID: t5o6p7q8r9s0
+Revises: s4n5o6p7q8r9
+Create Date: 2026-01-26
+
+This migration implements the terminology rename:
+1. mental_model (fact_type in memory_units) -> observation
+2. reflections table -> mental_models table
+
+The new terminology:
+- Observations: Consolidated knowledge synthesized from facts (was mental_model)
+- Mental Models: Stored reflect responses (was reflections)
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "t5o6p7q8r9s0"
+down_revision: str | Sequence[str] | None = "s4n5o6p7q8r9"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Rename mental_model -> observation and reflections -> mental_models."""
+    schema = _get_schema_prefix()
+
+    # 1. Update fact_type values: mental_model -> observation
+    op.execute(f"""
+        UPDATE {schema}memory_units
+        SET fact_type = 'observation'
+        WHERE fact_type = 'mental_model'
+    """)
+
+    # 2. Update the CHECK constraint - remove mental_model, keep observation
+    op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD CONSTRAINT memory_units_fact_type_check
+        CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation'))
+    """)
+
+    # 3. Rename the index for observations (was for mental_models)
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_mental_models")
+    op.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_memory_units_observations
+        ON {schema}memory_units(bank_id, fact_type)
+        WHERE fact_type = 'observation'
+    """)
+
+    # 4. Update the unconsolidated index to not filter by fact_type since observations
+    # are now the consolidated type
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_unconsolidated")
+    op.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_memory_units_unconsolidated
+        ON {schema}memory_units (bank_id, created_at)
+        WHERE consolidated_at IS NULL AND fact_type IN ('experience', 'world')
+    """)
+
+    # 5. Rename reflections table to mental_models
+    op.execute(f"ALTER TABLE IF EXISTS {schema}reflections RENAME TO mental_models")
+
+    # 6. Rename indexes for mental_models (was reflections)
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_bank_id RENAME TO idx_mental_models_bank_id")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_embedding RENAME TO idx_mental_models_embedding")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_tags RENAME TO idx_mental_models_tags")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_text_search RENAME TO idx_mental_models_text_search")
+
+    # 7. Rename foreign key constraint
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        DROP CONSTRAINT IF EXISTS fk_reflections_bank_id
+    """)
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD CONSTRAINT fk_mental_models_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+
+def downgrade() -> None:
+    """Reverse: observation -> mental_model and mental_models -> reflections."""
+    schema = _get_schema_prefix()
+
+    # 1. Rename mental_models table back to reflections
+    op.execute(f"ALTER TABLE IF EXISTS {schema}mental_models RENAME TO reflections")
+
+    # 2. Rename indexes back
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_mental_models_bank_id RENAME TO idx_reflections_bank_id")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_mental_models_embedding RENAME TO idx_reflections_embedding")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_mental_models_tags RENAME TO idx_reflections_tags")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_mental_models_text_search RENAME TO idx_reflections_text_search")
+
+    # 3. Rename foreign key back
+    op.execute(f"""
+        ALTER TABLE {schema}reflections
+        DROP CONSTRAINT IF EXISTS fk_mental_models_bank_id
+    """)
+    op.execute(f"""
+        ALTER TABLE {schema}reflections
+        ADD CONSTRAINT fk_reflections_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+    # 4. Update fact_type values: observation -> mental_model
+    op.execute(f"""
+        UPDATE {schema}memory_units
+        SET fact_type = 'mental_model'
+        WHERE fact_type = 'observation'
+    """)
+
+    # 5. Update the CHECK constraint back
+    op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD CONSTRAINT memory_units_fact_type_check
+        CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation', 'mental_model'))
+    """)
+
+    # 6. Rename index back
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_observations")
+    op.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_memory_units_mental_models
+        ON {schema}memory_units(bank_id, fact_type)
+        WHERE fact_type = 'mental_model'
+    """)

hindsight-api/hindsight_api/alembic/versions/u6p7q8r9s0t1_mental_models_text_id.py

@@ -0,0 +1,41 @@
+"""Change mental_models.id from UUID to TEXT
+
+Revision ID: u6p7q8r9s0t1
+Revises: t5o6p7q8r9s0
+Create Date: 2026-01-27
+
+This migration changes the mental_models.id column from UUID to TEXT
+to support user-defined text identifiers like 'team-communication' instead of UUIDs.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "u6p7q8r9s0t1"
+down_revision: str | Sequence[str] | None = "t5o6p7q8r9s0"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Change mental_models.id from UUID to TEXT."""
+    schema = _get_schema_prefix()
+
+    # Change the id column type from UUID to TEXT
+    # Existing UUIDs will be converted to their string representation
+    op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE TEXT USING id::TEXT")
+
+
+def downgrade() -> None:
+    """Revert mental_models.id from TEXT to UUID."""
+    schema = _get_schema_prefix()
+
+    # Note: This will fail if any id values are not valid UUIDs
+    op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE UUID USING id::UUID")

hindsight-api/hindsight_api/alembic/versions/v7q8r9s0t1u2_add_max_tokens_to_mental_models.py

@@ -0,0 +1,50 @@
+"""Add max_tokens and trigger columns to mental_models
+
+Revision ID: v7q8r9s0t1u2
+Revises: u6p7q8r9s0t1
+Create Date: 2026-01-27
+
+This migration adds:
+- max_tokens column: token limit for content generation during refresh
+- trigger column: JSONB for trigger settings (e.g., refresh_after_consolidation)
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "v7q8r9s0t1u2"
+down_revision: str | Sequence[str] | None = "u6p7q8r9s0t1"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add max_tokens and trigger columns to mental_models."""
+    schema = _get_schema_prefix()
+
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD COLUMN IF NOT EXISTS max_tokens INT NOT NULL DEFAULT 2048
+    """)
+
+    # trigger column stores trigger settings as JSONB
+    # Default: refresh_after_consolidation = false (not "real time")
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD COLUMN IF NOT EXISTS trigger JSONB NOT NULL DEFAULT '{{"refresh_after_consolidation": false}}'::jsonb
+    """)
+
+
+def downgrade() -> None:
+    """Remove max_tokens and trigger columns from mental_models."""
+    schema = _get_schema_prefix()
+
+    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS max_tokens")
+    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS trigger")

hindsight-api/hindsight_api/alembic/versions/w8r9s0t1u2v3_fix_mental_models_pk_isolation.py

@@ -0,0 +1,60 @@
+"""Fix mental_models primary key to be scoped per bank
+
+Revision ID: w8r9s0t1u2v3
+Revises: v7q8r9s0t1u2
+Create Date: 2026-02-05
+
+This migration fixes a critical bank isolation bug where mental_models.id was
+globally unique across all banks instead of being scoped per bank. This caused
+conflicts when different banks tried to use the same custom ID.
+
+CRITICAL FIX: Changes primary key from (id) to (bank_id, id) to ensure proper isolation.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "w8r9s0t1u2v3"
+down_revision: str | Sequence[str] | None = "v7q8r9s0t1u2"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Change mental_models primary key from (id) to (bank_id, id) for proper bank isolation."""
+    schema = _get_schema_prefix()
+
+    # Drop the old primary key constraint (just id)
+    # Note: The constraint might be named differently on different DBs
+    # Try both old names (pinned_reflections_pkey from original, mental_models_pkey from rename)
+    op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS pinned_reflections_pkey")
+    op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS mental_models_pkey")
+
+    # Create the new composite primary key (bank_id, id)
+    # This ensures IDs are scoped per bank, not globally
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD CONSTRAINT mental_models_pkey PRIMARY KEY (bank_id, id)
+    """)
+
+
+def downgrade() -> None:
+    """Revert mental_models primary key from (bank_id, id) to (id)."""
+    schema = _get_schema_prefix()
+
+    # Drop the composite primary key
+    op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS mental_models_pkey")
+
+    # Restore the old primary key (just id)
+    # WARNING: This downgrade will fail if there are duplicate IDs across banks
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD CONSTRAINT mental_models_pkey PRIMARY KEY (id)
+    """)

hindsight-api/hindsight_api/api/__init__.py

@@ -3,8 +3,10 @@ Unified API module for Hindsight.
 
 Provides both HTTP REST API and MCP (Model Context Protocol) server.
 """
+
 import logging
-from typing import Optional
+from contextlib import asynccontextmanager
+
 from fastapi import FastAPI
 
 from hindsight_api import MemoryEngine
@@ -17,18 +19,17 @@ def create_app(
     http_api_enabled: bool = True,
     mcp_api_enabled: bool = False,
     mcp_mount_path: str = "/mcp",
-    run_migrations: bool = True,
-    initialize_memory: bool = True
+    initialize_memory: bool = True,
 ) -> FastAPI:
     """
     Create and configure the unified Hindsight API application.
 
     Args:
-        memory: MemoryEngine instance (already initialized with required parameters)
+        memory: MemoryEngine instance (already initialized with required parameters).
+                Migrations are controlled by the MemoryEngine's run_migrations parameter.
         http_api_enabled: Whether to enable HTTP REST API endpoints (default: True)
         mcp_api_enabled: Whether to enable MCP server (default: False)
         mcp_mount_path: Path to mount MCP server (default: /mcp)
-        run_migrations: Whether to run database migrations on startup (default: True)
         initialize_memory: Whether to initialize memory system on startup (default: True)
 
     Returns:
@@ -44,51 +45,81 @@ def create_app(
         # Both HTTP and MCP
         app = create_app(memory, mcp_api_enabled=True)
     """
+    mcp_servers = None
+
+    # Create MCP servers first if enabled (we need their lifespans for chaining)
+    if mcp_api_enabled:
+        try:
+            from .mcp import MCPMiddleware, create_mcp_servers
+
+            mcp_servers = create_mcp_servers(memory=memory)
+        except ImportError as e:
+            logger.error(f"MCP server requested but dependencies not available: {e}")
+            logger.error("Install with: pip install hindsight-api[mcp]")
+            raise
 
     # Import and create HTTP API if enabled
     if http_api_enabled:
         from .http import create_app as create_http_app
-        app = create_http_app(
-            memory=memory,
-            run_migrations=run_migrations,
-            initialize_memory=initialize_memory
-        )
+
+        app = create_http_app(memory=memory, initialize_memory=initialize_memory)
         logger.info("HTTP REST API enabled")
     else:
         # Create minimal FastAPI app
         app = FastAPI(title="Hindsight API", version="0.0.7")
         logger.info("HTTP REST API disabled")
 
-    # Mount MCP server if enabled
-    if mcp_api_enabled:
-        try:
-            from .mcp import create_mcp_server
+    # Add MCP middleware and chain its lifespan if enabled
+    if mcp_servers is not None:
+        multi_bank_server, single_bank_server, multi_bank_starlette_app, single_bank_starlette_app = mcp_servers
 
-            # Create MCP server with shared memory instance
-            mcp_server = create_mcp_server(memory=memory)
+        # Store the original lifespan
+        original_lifespan = app.router.lifespan_context
 
-            # Mount at specified path using http_app (modern non-SSE alternative)
-            app.mount(mcp_mount_path, mcp_server.http_app())
-            logger.info(f"MCP server enabled at {mcp_mount_path}")
-        except ImportError as e:
-            logger.error(f"MCP server requested but dependencies not available: {e}")
-            logger.error("Install with: pip install hindsight-api[mcp]")
-            raise
+        @asynccontextmanager
+        async def chained_lifespan(app_instance: FastAPI):
+            """Chain both MCP lifespans with the main app lifespan."""
+            # Start both MCP lifespans (multi-bank and single-bank)
+            async with multi_bank_starlette_app.router.lifespan_context(multi_bank_starlette_app):
+                async with single_bank_starlette_app.router.lifespan_context(single_bank_starlette_app):
+                    logger.info("MCP lifespans started (multi-bank and single-bank)")
+                    # Then start the original app lifespan
+                    async with original_lifespan(app_instance):
+                        yield
+                logger.info("MCP lifespans stopped")
+
+        # Replace the app's lifespan with the chained version
+        app.router.lifespan_context = chained_lifespan
+
+        # Add MCP as a wrapping middleware — intercepts /mcp* requests directly,
+        # passes everything else through to the FastAPI app. No Starlette Mount
+        # means no 307 redirect for /mcp (no trailing slash).
+        app.add_middleware(
+            MCPMiddleware,
+            memory=memory,
+            prefix=mcp_mount_path,
+            multi_bank_app=multi_bank_starlette_app,
+            single_bank_app=single_bank_starlette_app,
+            multi_bank_server=multi_bank_server,
+            single_bank_server=single_bank_server,
+        )
+
+        logger.info(f"MCP server enabled at {mcp_mount_path}/")
 
     return app
 
 
 # Re-export commonly used items for backwards compatibility
 from .http import (
-    RecallRequest,
-    RecallResult,
-    RecallResponse,
+    CreateBankRequest,
+    DispositionTraits,
     MemoryItem,
-    RetainRequest,
+    RecallRequest,
+    RecallResponse,
+    RecallResult,
     ReflectRequest,
     ReflectResponse,
-    CreateBankRequest,
-    PersonalityTraits,
+    RetainRequest,
 )
 
 __all__ = [
@@ -101,5 +132,5 @@ __all__ = [
     "ReflectRequest",
     "ReflectResponse",
     "CreateBankRequest",
-    "PersonalityTraits",
+    "DispositionTraits",
 ]

hindsight-api/hindsight_api/api/mcp.py

@@ -1,157 +1,347 @@
-"""Hindsight MCP Server implementation using FastMCP."""
+"""Hindsight MCP Server implementation using FastMCP (HTTP transport)."""
 
 import json
 import logging
 import os
+from contextvars import ContextVar
 
 from fastmcp import FastMCP
+
 from hindsight_api import MemoryEngine
+from hindsight_api.engine.memory_engine import _current_schema
+from hindsight_api.extensions import MCPExtension, load_extension
+from hindsight_api.extensions.tenant import AuthenticationError
+from hindsight_api.mcp_tools import MCPToolsConfig, register_mcp_tools
+from hindsight_api.models import RequestContext
 
 # Configure logging from HINDSIGHT_API_LOG_LEVEL environment variable
 _log_level_str = os.environ.get("HINDSIGHT_API_LOG_LEVEL", "info").lower()
-_log_level_map = {"critical": logging.CRITICAL, "error": logging.ERROR, "warning": logging.WARNING,
-                  "info": logging.INFO, "debug": logging.DEBUG, "trace": logging.DEBUG}
+_log_level_map = {
+    "critical": logging.CRITICAL,
+    "error": logging.ERROR,
+    "warning": logging.WARNING,
+    "info": logging.INFO,
+    "debug": logging.DEBUG,
+    "trace": logging.DEBUG,
+}
 logging.basicConfig(
     level=_log_level_map.get(_log_level_str, logging.INFO),
-    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"
+    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
 )
 logger = logging.getLogger(__name__)
 
+# Default bank_id from environment variable
+DEFAULT_BANK_ID = os.environ.get("HINDSIGHT_MCP_BANK_ID", "default")
 
-def create_mcp_server(memory: MemoryEngine) -> FastMCP:
+# Legacy MCP authentication token (for backwards compatibility)
+# If set, this token is checked first before TenantExtension auth
+MCP_AUTH_TOKEN = os.environ.get("HINDSIGHT_API_MCP_AUTH_TOKEN")
+
+# Context variable to hold the current bank_id
+_current_bank_id: ContextVar[str | None] = ContextVar("current_bank_id", default=None)
+
+# Context variable to hold the current API key (for tenant auth propagation)
+_current_api_key: ContextVar[str | None] = ContextVar("current_api_key", default=None)
+
+# Context variables for tenant_id and api_key_id (set by authenticate, used by usage metering)
+_current_tenant_id: ContextVar[str | None] = ContextVar("current_tenant_id", default=None)
+_current_api_key_id: ContextVar[str | None] = ContextVar("current_api_key_id", default=None)
+
+
+def get_current_bank_id() -> str | None:
+    """Get the current bank_id from context."""
+    return _current_bank_id.get()
+
+
+def get_current_api_key() -> str | None:
+    """Get the current API key from context."""
+    return _current_api_key.get()
+
+
+def get_current_tenant_id() -> str | None:
+    """Get the current tenant_id from context."""
+    return _current_tenant_id.get()
+
+
+def get_current_api_key_id() -> str | None:
+    """Get the current api_key_id from context."""
+    return _current_api_key_id.get()
+
+
+def create_mcp_server(memory: MemoryEngine, multi_bank: bool = True) -> FastMCP:
     """
     Create and configure the Hindsight MCP server.
 
     Args:
         memory: MemoryEngine instance (required)
+        multi_bank: If True, expose all tools with bank_id parameters (default).
+                   If False, only expose bank-scoped tools without bank_id parameters.
 
     Returns:
-        Configured FastMCP server instance
+        Configured FastMCP server instance with stateless_http enabled
     """
-    # Create FastMCP server
-    mcp = FastMCP("hindsight-mcp-server")
+    # Use stateless_http=True for Claude Code compatibility
+    mcp = FastMCP("hindsight-mcp-server", stateless_http=True)
 
-    @mcp.tool()
-    async def hindsight_put(bank_id: str, content: str, context: str, explanation: str = "") -> str:
-        """
-        **CRITICAL: Store important user information to long-term memory.**
+    # Configure and register tools using shared module
+    config = MCPToolsConfig(
+        bank_id_resolver=get_current_bank_id,
+        api_key_resolver=get_current_api_key,  # Propagate API key for tenant auth
+        tenant_id_resolver=get_current_tenant_id,  # Propagate tenant_id for usage metering
+        api_key_id_resolver=get_current_api_key_id,  # Propagate api_key_id for usage metering
+        include_bank_id_param=multi_bank,
+        tools=None
+        if multi_bank
+        else {
+            "retain",
+            "recall",
+            "reflect",
+            "list_mental_models",
+            "get_mental_model",
+            "create_mental_model",
+            "update_mental_model",
+            "delete_mental_model",
+            "refresh_mental_model",
+        },  # Scoped tools for single-bank mode (excludes bank management: list_banks, create_bank)
+        retain_fire_and_forget=False,  # HTTP MCP supports sync/async modes
+    )
 
-        **⚠️ PER-USER TOOL - REQUIRES USER IDENTIFICATION:**
-        - This tool is STRICTLY per-user. Each user MUST have a unique `bank_id`.
-        - ONLY use this tool if you have a valid user identifier (user ID, email, session ID, etc.) to map to `bank_id`.
-        - DO NOT use this tool if you cannot identify the specific user.
-        - DO NOT share memories between different users - each user's memories are isolated by their `bank_id`.
-        - If you don't have a user identifier, DO NOT use this tool at all.
+    register_mcp_tools(mcp, memory, config)
 
-        Use this tool PROACTIVELY whenever the user shares:
-        - Personal facts, preferences, or interests (e.g., "I love hiking", "I'm a vegetarian")
-        - Important events or milestones (e.g., "I got promoted", "My birthday is June 15")
-        - User history, experiences, or background (e.g., "I used to work at Google", "I studied CS at MIT")
-        - Decisions, opinions, or stated preferences (e.g., "I prefer Python over JavaScript")
-        - Goals, plans, or future intentions (e.g., "I'm planning to visit Japan next year")
-        - Relationships or people mentioned (e.g., "My manager Sarah", "My wife Alice")
-        - Work context, projects, or responsibilities
-        - Any other information the user would want remembered for future conversations
-
-        **When to use**: Immediately after user shares personal information. Don't ask permission - just store it naturally.
-
-        **Context guidelines**: Use descriptive contexts like "personal_preferences", "work_history", "family", "hobbies",
-        "career_goals", "project_details", etc. This helps organize and retrieve related memories later.
-
-        Args:
-            bank_id: **REQUIRED** - The unique, persistent identifier for this specific user (e.g., user_id, email, session_id).
-                     This MUST be consistent across all interactions with the same user.
-                     Example: "user_12345", "[email protected]", "session_abc123"
-            content: The fact/memory to store (be specific and include relevant details)
-            context: Categorize the memory (e.g., 'personal_preferences', 'work_history', 'hobbies', 'family')
-            explanation: Optional explanation for why this memory is being stored
-        """
-        try:
-            # Log explanation if provided
-            if explanation:
-                pass  # Explanation provided
-
-            # Store memory using put_batch_async
-            await memory.put_batch_async(
-                bank_id=bank_id,
-                contents=[{"content": content, "context": context}]
-            )
-            return f"Fact stored successfully"
-        except Exception as e:
-            logger.error(f"Error storing fact: {e}", exc_info=True)
-            return f"Error: {str(e)}"
-
-    @mcp.tool()
-    async def hindsight_search(bank_id: str, query: str, max_tokens: int = 4096, explanation: str = "") -> str:
-        """
-        **CRITICAL: Search user's memory to provide personalized, context-aware responses.**
-
-        **⚠️ PER-USER TOOL - REQUIRES USER IDENTIFICATION:**
-        - This tool is STRICTLY per-user. Each user MUST have a unique `bank_id`.
-        - ONLY use this tool if you have a valid user identifier (user ID, email, session ID, etc.) to map to `bank_id`.
-        - DO NOT use this tool if you cannot identify the specific user.
-        - DO NOT search across multiple users - each user's memories are isolated by their `bank_id`.
-        - If you don't have a user identifier, DO NOT use this tool at all.
-
-        Use this tool PROACTIVELY at the start of conversations or when making recommendations to:
-        - Check user's preferences before making suggestions (e.g., "what foods does the user like?")
-        - Recall user's history to provide continuity (e.g., "what projects has the user worked on?")
-        - Remember user's goals and context (e.g., "what is the user trying to accomplish?")
-        - Avoid repeating information or asking questions you should already know
-        - Personalize responses based on user's background, interests, and past interactions
-        - Reference past conversations or events the user mentioned
-
-        **When to use**:
-        - Start of conversation: Search for relevant context about the user
-        - Before recommendations: Check user preferences and past experiences
-        - When user asks about something they may have mentioned before
-        - To provide continuity across conversations
-
-        **Search tips**: Use natural language queries like "user's programming language preferences",
-        "user's work experience", "user's dietary restrictions", "what does the user know about X?"
-
-        Args:
-            bank_id: **REQUIRED** - The unique, persistent identifier for this specific user (e.g., user_id, email, session_id).
-                     This MUST be consistent across all interactions with the same user.
-                     Example: "user_12345", "[email protected]", "session_abc123"
-            query: Natural language search query to find relevant memories
-            max_tokens: Maximum tokens for search context (default: 4096)
-            explanation: Optional explanation for why this search is being performed
-        """
-        try:
-            # Log all parameters for debugging
-            logger.info(f"hindsight_search called with: query={query!r}, max_tokens={max_tokens}, explanation={explanation!r}")
-
-            # Log explanation if provided
-            if explanation:
-                pass  # Explanation provided
-
-            # Search using recall_async
-            from hindsight_api.engine.memory_engine import Budget
-            search_result = await memory.recall_async(
-                bank_id=bank_id,
-                query=query,
-                fact_type=["world", "bank", "opinion"],  # Search all fact types
-                max_tokens=max_tokens,
-                budget=Budget.LOW
-            )
-
-            # Convert results to dict format
-            results = [
-                {
-                    "id": fact.id,
-                    "text": fact.text,
-                    "type": fact.fact_type,
-                    "context": fact.context,
-                    "event_date": fact.event_date,  # Already a string from the database
-                    "document_id": fact.document_id
-                }
-                for fact in search_result.results
-            ]
-
-            return json.dumps({"results": results}, indent=2)
-        except Exception as e:
-            logger.error(f"Error searching: {e}", exc_info=True)
-            return json.dumps({"error": str(e), "results": []})
+    # Load and register additional tools from MCP extension if configured
+    mcp_extension = load_extension("MCP", MCPExtension)
+    if mcp_extension:
+        logger.info(f"Loading MCP extension: {mcp_extension.__class__.__name__}")
+        mcp_extension.register_tools(mcp, memory)
 
     return mcp
+
+
+class MCPMiddleware:
+    """ASGI middleware that intercepts MCP requests and routes to appropriate MCP server.
+
+    This middleware wraps the main FastAPI app and intercepts requests matching the
+    configured prefix (default: /mcp). Non-MCP requests pass through to the inner app.
+
+    Authentication:
+        1. If HINDSIGHT_API_MCP_AUTH_TOKEN is set (legacy), validates against that token
+        2. Otherwise, uses TenantExtension.authenticate_mcp() from the MemoryEngine
+           - DefaultTenantExtension: no auth required (local dev)
+           - ApiKeyTenantExtension: validates against env var
+
+    Two modes based on URL structure:
+
+    1. Multi-bank mode (for /mcp/ root endpoint):
+       - Exposes all tools: retain, recall, reflect, list_banks, create_bank
+       - All tools include optional bank_id parameter for cross-bank operations
+       - Bank ID from: X-Bank-Id header or HINDSIGHT_MCP_BANK_ID env var
+
+    2. Single-bank mode (for /mcp/{bank_id}/ endpoints):
+       - Exposes bank-scoped tools only: retain, recall, reflect
+       - No bank_id parameter (comes from URL)
+       - No bank management tools (list_banks, create_bank)
+       - Recommended for agent isolation
+
+    Examples:
+        # Single-bank mode (recommended for agent isolation)
+        claude mcp add --transport http my-agent http://localhost:8888/mcp/my-agent-bank/ \\
+            --header "Authorization: Bearer <token>"
+
+        # Multi-bank mode (for cross-bank operations)
+        claude mcp add --transport http hindsight http://localhost:8888/mcp \\
+            --header "X-Bank-Id: my-bank" --header "Authorization: Bearer <token>"
+    """
+
+    def __init__(
+        self,
+        app,
+        memory: MemoryEngine,
+        prefix: str = "/mcp",
+        multi_bank_app=None,
+        single_bank_app=None,
+        multi_bank_server=None,
+        single_bank_server=None,
+    ):
+        self.app = app
+        self.prefix = prefix
+        self.memory = memory
+        self.tenant_extension = memory._tenant_extension
+
+        if multi_bank_app and single_bank_app:
+            # Pre-created servers (used when called via add_middleware from create_app)
+            self.multi_bank_app = multi_bank_app
+            self.single_bank_app = single_bank_app
+            self.multi_bank_server = multi_bank_server
+            self.single_bank_server = single_bank_server
+        else:
+            # Create servers internally (for direct construction / tests)
+            self.multi_bank_server = create_mcp_server(memory, multi_bank=True)
+            self.multi_bank_app = self.multi_bank_server.http_app(path="/")
+            self.single_bank_server = create_mcp_server(memory, multi_bank=False)
+            self.single_bank_app = self.single_bank_server.http_app(path="/")
+
+    def _get_header(self, scope: dict, name: str) -> str | None:
+        """Extract a header value from ASGI scope."""
+        name_lower = name.lower().encode()
+        for header_name, header_value in scope.get("headers", []):
+            if header_name.lower() == name_lower:
+                return header_value.decode()
+        return None
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        path = scope.get("path", "")
+
+        # Check if this is an MCP request (matches prefix)
+        if not (path == self.prefix or path.startswith(self.prefix + "/")):
+            # Not an MCP request — pass through to the inner app
+            await self.app(scope, receive, send)
+            return
+
+        # Strip prefix from path
+        path = path[len(self.prefix) :] or "/"
+
+        # Extract auth token from header (for tenant auth propagation)
+        auth_header = self._get_header(scope, "Authorization")
+        auth_token: str | None = None
+        if auth_header:
+            # Support both "Bearer <token>" and direct token
+            auth_token = auth_header[7:].strip() if auth_header.startswith("Bearer ") else auth_header.strip()
+
+        # Authenticate: check legacy MCP_AUTH_TOKEN first, then TenantExtension
+        tenant_context = None
+        auth_tenant_id: str | None = None
+        auth_api_key_id: str | None = None
+        if MCP_AUTH_TOKEN:
+            # Legacy authentication mode - validate against static token
+            if not auth_token:
+                await self._send_error(send, 401, "Authorization header required")
+                return
+            if auth_token != MCP_AUTH_TOKEN:
+                await self._send_error(send, 401, "Invalid authentication token")
+                return
+            # Legacy mode doesn't use tenant schemas
+            tenant_context = None
+        else:
+            # Use TenantExtension.authenticate_mcp() for auth
+            try:
+                auth_context = RequestContext(api_key=auth_token)
+                tenant_context = await self.tenant_extension.authenticate_mcp(auth_context)
+                # Capture tenant_id and api_key_id set by authenticate() for usage metering
+                auth_tenant_id = auth_context.tenant_id
+                auth_api_key_id = auth_context.api_key_id
+            except AuthenticationError as e:
+                await self._send_error(send, 401, str(e))
+                return
+
+        # Set schema from tenant context so downstream DB queries use the correct schema
+        schema_token = (
+            _current_schema.set(tenant_context.schema_name) if tenant_context and tenant_context.schema_name else None
+        )
+
+        # Try to get bank_id from header first (for Claude Code compatibility)
+        bank_id = self._get_header(scope, "X-Bank-Id")
+        bank_id_from_path = False
+
+        # If no header, try to extract from path: /{bank_id}/...
+        new_path = path
+        if not bank_id and path.startswith("/") and len(path) > 1:
+            parts = path[1:].split("/", 1)
+            if parts[0]:
+                # First segment looks like a bank_id
+                bank_id = parts[0]
+                bank_id_from_path = True
+                new_path = "/" + parts[1] if len(parts) > 1 else "/"
+
+        # Fall back to default bank_id
+        if not bank_id:
+            bank_id = DEFAULT_BANK_ID
+            logger.debug(f"Using default bank_id: {bank_id}")
+
+        # Select the appropriate MCP app based on how bank_id was provided:
+        # - Path-based bank_id → single-bank app (no bank_id param, scoped tools)
+        # - Header/env bank_id → multi-bank app (bank_id param, all tools)
+        target_app = self.single_bank_app if bank_id_from_path else self.multi_bank_app
+
+        # Set bank_id, api_key, tenant_id, and api_key_id context
+        bank_id_token = _current_bank_id.set(bank_id)
+        # Store the auth token for tenant extension to validate
+        api_key_token = _current_api_key.set(auth_token) if auth_token else None
+        # Store tenant_id and api_key_id from authentication for usage metering
+        tenant_id_token = _current_tenant_id.set(auth_tenant_id) if auth_tenant_id else None
+        api_key_id_token = _current_api_key_id.set(auth_api_key_id) if auth_api_key_id else None
+        try:
+            new_scope = scope.copy()
+            new_scope["path"] = new_path
+            # Clear root_path since we're passing directly to the app
+            new_scope["root_path"] = ""
+
+            # Wrap send to rewrite the SSE endpoint URL to include bank_id if using path-based routing.
+            # Only rewrite SSE (text/event-stream) responses to avoid corrupting tool results
+            # that might contain the literal string "data: /messages".
+            is_sse_response = False
+
+            async def send_wrapper(message):
+                nonlocal is_sse_response
+                if message["type"] == "http.response.start":
+                    for header_name, header_value in message.get("headers", []):
+                        if header_name == b"content-type" and b"text/event-stream" in header_value:
+                            is_sse_response = True
+                            break
+                if message["type"] == "http.response.body" and bank_id_from_path and is_sse_response:
+                    body = message.get("body", b"")
+                    if body and b"/messages" in body:
+                        # Rewrite /messages to /{bank_id}/messages in SSE endpoint event
+                        body = body.replace(b"data: /messages", f"data: /{bank_id}/messages".encode())
+                        message = {**message, "body": body}
+                await send(message)
+
+            await target_app(new_scope, receive, send_wrapper)
+        finally:
+            _current_bank_id.reset(bank_id_token)
+            if api_key_token is not None:
+                _current_api_key.reset(api_key_token)
+            if tenant_id_token is not None:
+                _current_tenant_id.reset(tenant_id_token)
+            if api_key_id_token is not None:
+                _current_api_key_id.reset(api_key_id_token)
+            if schema_token is not None:
+                _current_schema.reset(schema_token)
+
+    async def _send_error(self, send, status: int, message: str):
+        """Send an error response."""
+        body = json.dumps({"error": message}).encode()
+        await send(
+            {
+                "type": "http.response.start",
+                "status": status,
+                "headers": [(b"content-type", b"application/json")],
+            }
+        )
+        await send(
+            {
+                "type": "http.response.body",
+                "body": body,
+            }
+        )
+
+
+def create_mcp_servers(memory: MemoryEngine):
+    """Create multi-bank and single-bank MCP servers and their Starlette apps.
+
+    Returns the servers and apps separately so lifespans can be chained before
+    the middleware wraps the main app.
+
+    Returns:
+        Tuple of (multi_bank_server, single_bank_server, multi_bank_app, single_bank_app)
+    """
+    multi_bank_server = create_mcp_server(memory, multi_bank=True)
+    multi_bank_app = multi_bank_server.http_app(path="/")
+
+    single_bank_server = create_mcp_server(memory, multi_bank=False)
+    single_bank_app = single_bank_server.http_app(path="/")
+
+    return multi_bank_server, single_bank_server, multi_bank_app, single_bank_app

hindsight-api/hindsight_api/banner.py

@@ -0,0 +1,101 @@
+"""
+Banner display for Hindsight API startup.
+
+Shows the logo and tagline with gradient colors.
+"""
+
+from .utils import mask_network_location
+
+# Gradient colors: #0074d9 -> #009296
+GRADIENT_START = (0, 116, 217)  # #0074d9
+GRADIENT_END = (0, 146, 150)  # #009296
+
+# Pre-generated logo (generated by test-logo.py)
+LOGO = """\
+  \033[38;2;9;127;184m\u2584\033[0m\033[48;2;8;130;178m\033[38;2;5;133;186m\u2584\033[0m       \033[48;2;10;143;160m\033[38;2;10;143;165m\u2584\033[0m\033[38;2;7;140;156m\u2584\033[0m
+ \033[38;2;8;125;192m\u2584\033[0m \033[38;2;3;132;191m\u2580\033[0m\033[38;2;2;133;192m\u2584\033[0m \033[38;2;3;132;180m\u2584\033[0m\033[38;2;1;137;184m\u2584\033[0m\033[38;2;3;133;174m\u2584\033[0m \033[38;2;3;142;176m\u2584\033[0m\033[38;2;4;142;169m\u2580\033[0m \033[38;2;10;144;164m\u2584\033[0m
+\033[38;2;6;121;195m\u2580\033[0m\033[38;2;5;128;203m\u2580\033[0m\033[48;2;5;124;195m\033[38;2;3;125;200m\u2584\033[0m\033[38;2;2;126;196m\u2584\033[0m\033[48;2;3;128;188m\033[38;2;1;131;196m\u2584\033[0m\033[48;2;0;152;219m\033[38;2;2;131;191m\u2584\033[0m\033[38;2;1;141;196m\u2580\033[0m\033[38;2;1;135;183m\u2580\033[0m\033[38;2;1;148;198m\u2580\033[0m\033[48;2;1;156;202m\033[38;2;2;135;180m\u2584\033[0m\033[48;2;4;134;169m\033[38;2;1;137;177m\u2584\033[0m\033[38;2;3;138;173m\u2584\033[0m\033[48;2;6;137;165m\033[38;2;2;140;170m\u2584\033[0m\033[38;2;7;144;169m\u2580\033[0m\033[38;2;7;139;158m\u2580\033[0m
+   \033[48;2;2;128;202m\033[38;2;2;124;201m\u2584\033[0m\033[48;2;1;130;201m\033[38;2;0;135;212m\u2584\033[0m\033[38;2;2;128;196m\u2584\033[0m \033[48;2;2;142;204m\033[38;2;7;138;199m\u2584\033[0m \033[38;2;1;135;186m\u2584\033[0m\033[48;2;1;142;186m\033[38;2;2;144;194m\u2584\033[0m\033[48;2;3;138;176m\033[38;2;2;134;176m\u2584\033[0m
+ \033[48;2;8;118;200m\033[38;2;8;121;209m\u2584\033[0m\033[38;2;3;121;203m\u2580\033[0m \033[38;2;3;122;192m\u2580\033[0m\033[38;2;1;138;216m\u2580\033[0m\033[48;2;0;138;210m\033[38;2;3;128;198m\u2584\033[0m\033[48;2;0;126;188m\033[38;2;2;131;198m\u2584\033[0m\033[48;2;0;142;205m\033[38;2;3;132;193m\u2584\033[0m\033[38;2;1;140;196m\u2580\033[0m  \033[38;2;4;134;175m\u2580\033[0m\033[48;2;13;135;167m\033[38;2;8;136;174m\u2584\033[0m """
+
+
+def _interpolate_color(start: tuple, end: tuple, t: float) -> tuple:
+    """Interpolate between two RGB colors."""
+    return (
+        int(start[0] + (end[0] - start[0]) * t),
+        int(start[1] + (end[1] - start[1]) * t),
+        int(start[2] + (end[2] - start[2]) * t),
+    )
+
+
+def gradient_text(text: str, start: tuple = GRADIENT_START, end: tuple = GRADIENT_END) -> str:
+    """Render text with a gradient color effect."""
+    result = []
+    length = len(text)
+    for i, char in enumerate(text):
+        if char == " ":
+            result.append(" ")
+        else:
+            t = i / max(length - 1, 1)
+            r, g, b = _interpolate_color(start, end, t)
+            result.append(f"\033[38;2;{r};{g};{b}m{char}")
+    result.append("\033[0m")
+    return "".join(result)
+
+
+def print_banner():
+    """Print the Hindsight startup banner."""
+    print(LOGO)
+    tagline = gradient_text("Hindsight: Agent Memory That Works Like Human Memory")
+    print(f"\n  {tagline}\n")
+
+
+def color(text: str, t: float = 0.0) -> str:
+    """Color text using gradient position (0.0 = start, 1.0 = end)."""
+    r, g, b = _interpolate_color(GRADIENT_START, GRADIENT_END, t)
+    return f"\033[38;2;{r};{g};{b}m{text}\033[0m"
+
+
+def color_start(text: str) -> str:
+    """Color text with gradient start color (#0074d9)."""
+    return color(text, 0.0)
+
+
+def color_end(text: str) -> str:
+    """Color text with gradient end color (#009296)."""
+    return color(text, 1.0)
+
+
+def color_mid(text: str) -> str:
+    """Color text with gradient middle color."""
+    return color(text, 0.5)
+
+
+def dim(text: str) -> str:
+    """Dim/gray text."""
+    return f"\033[38;2;128;128;128m{text}\033[0m"
+
+
+def print_startup_info(
+    host: str,
+    port: int,
+    database_url: str,
+    llm_provider: str,
+    llm_model: str,
+    embeddings_provider: str,
+    reranker_provider: str,
+    mcp_enabled: bool = False,
+    version: str | None = None,
+):
+    """Print styled startup information."""
+    print(color_start("Starting Hindsight API..."))
+    if version:
+        print(f"  {dim('Version:')} {color(f'v{version}', 0.1)}")
+    print(f"  {dim('URL:')} {color(f'http://{host}:{port}', 0.2)}")
+    print(f"  {dim('Database:')} {color(mask_network_location(database_url), 0.4)}")
+    print(f"  {dim('LLM:')} {color(f'{llm_provider} / {llm_model}', 0.6)}")
+    print(f"  {dim('Embeddings:')} {color(embeddings_provider, 0.8)}")
+    print(f"  {dim('Reranker:')} {color(reranker_provider, 1.0)}")
+    if mcp_enabled:
+        print(f"  {dim('MCP:')} {color_end('enabled at /mcp')}")
+    print()

hindsight-api/hindsight_api/config.py

@@ -0,0 +1,819 @@
+"""
+Centralized configuration for Hindsight API.
+
+All environment variables and their defaults are defined here.
+"""
+
+import json
+import logging
+import os
+import sys
+from dataclasses import dataclass
+from datetime import datetime, timezone
+
+from dotenv import find_dotenv, load_dotenv
+
+# Load .env file, searching current and parent directories (overrides existing env vars)
+load_dotenv(find_dotenv(usecwd=True), override=True)
+
+logger = logging.getLogger(__name__)
+
+# Environment variable names
+ENV_DATABASE_URL = "HINDSIGHT_API_DATABASE_URL"
+ENV_DATABASE_SCHEMA = "HINDSIGHT_API_DATABASE_SCHEMA"
+ENV_LLM_PROVIDER = "HINDSIGHT_API_LLM_PROVIDER"
+ENV_LLM_API_KEY = "HINDSIGHT_API_LLM_API_KEY"
+ENV_LLM_MODEL = "HINDSIGHT_API_LLM_MODEL"
+ENV_LLM_BASE_URL = "HINDSIGHT_API_LLM_BASE_URL"
+ENV_LLM_MAX_CONCURRENT = "HINDSIGHT_API_LLM_MAX_CONCURRENT"
+ENV_LLM_MAX_RETRIES = "HINDSIGHT_API_LLM_MAX_RETRIES"
+ENV_LLM_INITIAL_BACKOFF = "HINDSIGHT_API_LLM_INITIAL_BACKOFF"
+ENV_LLM_MAX_BACKOFF = "HINDSIGHT_API_LLM_MAX_BACKOFF"
+ENV_LLM_TIMEOUT = "HINDSIGHT_API_LLM_TIMEOUT"
+ENV_LLM_GROQ_SERVICE_TIER = "HINDSIGHT_API_LLM_GROQ_SERVICE_TIER"
+
+# Per-operation LLM configuration (optional, falls back to global LLM config)
+ENV_RETAIN_LLM_PROVIDER = "HINDSIGHT_API_RETAIN_LLM_PROVIDER"
+ENV_RETAIN_LLM_API_KEY = "HINDSIGHT_API_RETAIN_LLM_API_KEY"
+ENV_RETAIN_LLM_MODEL = "HINDSIGHT_API_RETAIN_LLM_MODEL"
+ENV_RETAIN_LLM_BASE_URL = "HINDSIGHT_API_RETAIN_LLM_BASE_URL"
+ENV_RETAIN_LLM_MAX_CONCURRENT = "HINDSIGHT_API_RETAIN_LLM_MAX_CONCURRENT"
+ENV_RETAIN_LLM_MAX_RETRIES = "HINDSIGHT_API_RETAIN_LLM_MAX_RETRIES"
+ENV_RETAIN_LLM_INITIAL_BACKOFF = "HINDSIGHT_API_RETAIN_LLM_INITIAL_BACKOFF"
+ENV_RETAIN_LLM_MAX_BACKOFF = "HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF"
+ENV_RETAIN_LLM_TIMEOUT = "HINDSIGHT_API_RETAIN_LLM_TIMEOUT"
+
+ENV_REFLECT_LLM_PROVIDER = "HINDSIGHT_API_REFLECT_LLM_PROVIDER"
+ENV_REFLECT_LLM_API_KEY = "HINDSIGHT_API_REFLECT_LLM_API_KEY"
+ENV_REFLECT_LLM_MODEL = "HINDSIGHT_API_REFLECT_LLM_MODEL"
+ENV_REFLECT_LLM_BASE_URL = "HINDSIGHT_API_REFLECT_LLM_BASE_URL"
+ENV_REFLECT_LLM_MAX_CONCURRENT = "HINDSIGHT_API_REFLECT_LLM_MAX_CONCURRENT"
+ENV_REFLECT_LLM_MAX_RETRIES = "HINDSIGHT_API_REFLECT_LLM_MAX_RETRIES"
+ENV_REFLECT_LLM_INITIAL_BACKOFF = "HINDSIGHT_API_REFLECT_LLM_INITIAL_BACKOFF"
+ENV_REFLECT_LLM_MAX_BACKOFF = "HINDSIGHT_API_REFLECT_LLM_MAX_BACKOFF"
+ENV_REFLECT_LLM_TIMEOUT = "HINDSIGHT_API_REFLECT_LLM_TIMEOUT"
+
+ENV_CONSOLIDATION_LLM_PROVIDER = "HINDSIGHT_API_CONSOLIDATION_LLM_PROVIDER"
+ENV_CONSOLIDATION_LLM_API_KEY = "HINDSIGHT_API_CONSOLIDATION_LLM_API_KEY"
+ENV_CONSOLIDATION_LLM_MODEL = "HINDSIGHT_API_CONSOLIDATION_LLM_MODEL"
+ENV_CONSOLIDATION_LLM_BASE_URL = "HINDSIGHT_API_CONSOLIDATION_LLM_BASE_URL"
+ENV_CONSOLIDATION_LLM_MAX_CONCURRENT = "HINDSIGHT_API_CONSOLIDATION_LLM_MAX_CONCURRENT"
+ENV_CONSOLIDATION_LLM_MAX_RETRIES = "HINDSIGHT_API_CONSOLIDATION_LLM_MAX_RETRIES"
+ENV_CONSOLIDATION_LLM_INITIAL_BACKOFF = "HINDSIGHT_API_CONSOLIDATION_LLM_INITIAL_BACKOFF"
+ENV_CONSOLIDATION_LLM_MAX_BACKOFF = "HINDSIGHT_API_CONSOLIDATION_LLM_MAX_BACKOFF"
+ENV_CONSOLIDATION_LLM_TIMEOUT = "HINDSIGHT_API_CONSOLIDATION_LLM_TIMEOUT"
+
+ENV_EMBEDDINGS_PROVIDER = "HINDSIGHT_API_EMBEDDINGS_PROVIDER"
+ENV_EMBEDDINGS_LOCAL_MODEL = "HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL"
+ENV_EMBEDDINGS_LOCAL_FORCE_CPU = "HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU"
+ENV_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE = "HINDSIGHT_API_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE"
+ENV_EMBEDDINGS_TEI_URL = "HINDSIGHT_API_EMBEDDINGS_TEI_URL"
+ENV_EMBEDDINGS_OPENAI_API_KEY = "HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY"
+ENV_EMBEDDINGS_OPENAI_MODEL = "HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL"
+ENV_EMBEDDINGS_OPENAI_BASE_URL = "HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL"
+
+# Cohere configuration (separate for embeddings and reranker)
+ENV_EMBEDDINGS_COHERE_API_KEY = "HINDSIGHT_API_EMBEDDINGS_COHERE_API_KEY"
+ENV_EMBEDDINGS_COHERE_MODEL = "HINDSIGHT_API_EMBEDDINGS_COHERE_MODEL"
+ENV_EMBEDDINGS_COHERE_BASE_URL = "HINDSIGHT_API_EMBEDDINGS_COHERE_BASE_URL"
+ENV_RERANKER_COHERE_API_KEY = "HINDSIGHT_API_RERANKER_COHERE_API_KEY"
+ENV_RERANKER_COHERE_MODEL = "HINDSIGHT_API_RERANKER_COHERE_MODEL"
+ENV_RERANKER_COHERE_BASE_URL = "HINDSIGHT_API_RERANKER_COHERE_BASE_URL"
+
+# Deprecated: Legacy shared Cohere API key (for backward compatibility)
+ENV_COHERE_API_KEY = "HINDSIGHT_API_COHERE_API_KEY"
+
+# LiteLLM configuration (separate for embeddings and reranker)
+ENV_EMBEDDINGS_LITELLM_API_BASE = "HINDSIGHT_API_EMBEDDINGS_LITELLM_API_BASE"
+ENV_EMBEDDINGS_LITELLM_API_KEY = "HINDSIGHT_API_EMBEDDINGS_LITELLM_API_KEY"
+ENV_EMBEDDINGS_LITELLM_MODEL = "HINDSIGHT_API_EMBEDDINGS_LITELLM_MODEL"
+ENV_RERANKER_LITELLM_API_BASE = "HINDSIGHT_API_RERANKER_LITELLM_API_BASE"
+ENV_RERANKER_LITELLM_API_KEY = "HINDSIGHT_API_RERANKER_LITELLM_API_KEY"
+ENV_RERANKER_LITELLM_MODEL = "HINDSIGHT_API_RERANKER_LITELLM_MODEL"
+
+# Deprecated: Legacy shared LiteLLM config (for backward compatibility)
+ENV_LITELLM_API_BASE = "HINDSIGHT_API_LITELLM_API_BASE"
+ENV_LITELLM_API_KEY = "HINDSIGHT_API_LITELLM_API_KEY"
+
+ENV_RERANKER_PROVIDER = "HINDSIGHT_API_RERANKER_PROVIDER"
+ENV_RERANKER_LOCAL_MODEL = "HINDSIGHT_API_RERANKER_LOCAL_MODEL"
+ENV_RERANKER_LOCAL_FORCE_CPU = "HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU"
+ENV_RERANKER_LOCAL_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_LOCAL_MAX_CONCURRENT"
+ENV_RERANKER_LOCAL_TRUST_REMOTE_CODE = "HINDSIGHT_API_RERANKER_LOCAL_TRUST_REMOTE_CODE"
+ENV_RERANKER_TEI_URL = "HINDSIGHT_API_RERANKER_TEI_URL"
+ENV_RERANKER_TEI_BATCH_SIZE = "HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE"
+ENV_RERANKER_TEI_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT"
+ENV_RERANKER_MAX_CANDIDATES = "HINDSIGHT_API_RERANKER_MAX_CANDIDATES"
+ENV_RERANKER_FLASHRANK_MODEL = "HINDSIGHT_API_RERANKER_FLASHRANK_MODEL"
+ENV_RERANKER_FLASHRANK_CACHE_DIR = "HINDSIGHT_API_RERANKER_FLASHRANK_CACHE_DIR"
+
+ENV_HOST = "HINDSIGHT_API_HOST"
+ENV_PORT = "HINDSIGHT_API_PORT"
+ENV_LOG_LEVEL = "HINDSIGHT_API_LOG_LEVEL"
+ENV_LOG_FORMAT = "HINDSIGHT_API_LOG_FORMAT"
+ENV_WORKERS = "HINDSIGHT_API_WORKERS"
+ENV_MCP_ENABLED = "HINDSIGHT_API_MCP_ENABLED"
+ENV_GRAPH_RETRIEVER = "HINDSIGHT_API_GRAPH_RETRIEVER"
+ENV_MPFP_TOP_K_NEIGHBORS = "HINDSIGHT_API_MPFP_TOP_K_NEIGHBORS"
+ENV_RECALL_MAX_CONCURRENT = "HINDSIGHT_API_RECALL_MAX_CONCURRENT"
+ENV_RECALL_CONNECTION_BUDGET = "HINDSIGHT_API_RECALL_CONNECTION_BUDGET"
+ENV_MCP_LOCAL_BANK_ID = "HINDSIGHT_API_MCP_LOCAL_BANK_ID"
+ENV_MCP_INSTRUCTIONS = "HINDSIGHT_API_MCP_INSTRUCTIONS"
+ENV_MENTAL_MODEL_REFRESH_CONCURRENCY = "HINDSIGHT_API_MENTAL_MODEL_REFRESH_CONCURRENCY"
+
+# OpenTelemetry tracing configuration
+ENV_OTEL_TRACES_ENABLED = "HINDSIGHT_API_OTEL_TRACES_ENABLED"
+ENV_OTEL_EXPORTER_OTLP_ENDPOINT = "HINDSIGHT_API_OTEL_EXPORTER_OTLP_ENDPOINT"
+ENV_OTEL_EXPORTER_OTLP_HEADERS = "HINDSIGHT_API_OTEL_EXPORTER_OTLP_HEADERS"
+ENV_OTEL_SERVICE_NAME = "HINDSIGHT_API_OTEL_SERVICE_NAME"
+ENV_OTEL_DEPLOYMENT_ENVIRONMENT = "HINDSIGHT_API_OTEL_DEPLOYMENT_ENVIRONMENT"
+
+# Vertex AI configuration
+ENV_LLM_VERTEXAI_PROJECT_ID = "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID"
+ENV_LLM_VERTEXAI_REGION = "HINDSIGHT_API_LLM_VERTEXAI_REGION"
+ENV_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY = "HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY"
+
+# Retain settings
+ENV_RETAIN_MAX_COMPLETION_TOKENS = "HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS"
+ENV_RETAIN_CHUNK_SIZE = "HINDSIGHT_API_RETAIN_CHUNK_SIZE"
+ENV_RETAIN_EXTRACT_CAUSAL_LINKS = "HINDSIGHT_API_RETAIN_EXTRACT_CAUSAL_LINKS"
+ENV_RETAIN_EXTRACTION_MODE = "HINDSIGHT_API_RETAIN_EXTRACTION_MODE"
+ENV_RETAIN_CUSTOM_INSTRUCTIONS = "HINDSIGHT_API_RETAIN_CUSTOM_INSTRUCTIONS"
+
+# Observations settings (consolidated knowledge from facts)
+ENV_ENABLE_OBSERVATIONS = "HINDSIGHT_API_ENABLE_OBSERVATIONS"
+ENV_CONSOLIDATION_BATCH_SIZE = "HINDSIGHT_API_CONSOLIDATION_BATCH_SIZE"
+ENV_CONSOLIDATION_MAX_TOKENS = "HINDSIGHT_API_CONSOLIDATION_MAX_TOKENS"
+
+# Optimization flags
+ENV_SKIP_LLM_VERIFICATION = "HINDSIGHT_API_SKIP_LLM_VERIFICATION"
+ENV_LAZY_RERANKER = "HINDSIGHT_API_LAZY_RERANKER"
+
+# Database migrations
+ENV_RUN_MIGRATIONS_ON_STARTUP = "HINDSIGHT_API_RUN_MIGRATIONS_ON_STARTUP"
+
+# Database connection pool
+ENV_DB_POOL_MIN_SIZE = "HINDSIGHT_API_DB_POOL_MIN_SIZE"
+ENV_DB_POOL_MAX_SIZE = "HINDSIGHT_API_DB_POOL_MAX_SIZE"
+ENV_DB_COMMAND_TIMEOUT = "HINDSIGHT_API_DB_COMMAND_TIMEOUT"
+ENV_DB_ACQUIRE_TIMEOUT = "HINDSIGHT_API_DB_ACQUIRE_TIMEOUT"
+
+# Worker configuration (distributed task processing)
+ENV_WORKER_ENABLED = "HINDSIGHT_API_WORKER_ENABLED"
+ENV_WORKER_ID = "HINDSIGHT_API_WORKER_ID"
+ENV_WORKER_POLL_INTERVAL_MS = "HINDSIGHT_API_WORKER_POLL_INTERVAL_MS"
+ENV_WORKER_MAX_RETRIES = "HINDSIGHT_API_WORKER_MAX_RETRIES"
+ENV_WORKER_HTTP_PORT = "HINDSIGHT_API_WORKER_HTTP_PORT"
+ENV_WORKER_MAX_SLOTS = "HINDSIGHT_API_WORKER_MAX_SLOTS"
+ENV_WORKER_CONSOLIDATION_MAX_SLOTS = "HINDSIGHT_API_WORKER_CONSOLIDATION_MAX_SLOTS"
+
+# Reflect agent settings
+ENV_REFLECT_MAX_ITERATIONS = "HINDSIGHT_API_REFLECT_MAX_ITERATIONS"
+
+# Default values
+DEFAULT_DATABASE_URL = "pg0"
+DEFAULT_DATABASE_SCHEMA = "public"
+DEFAULT_LLM_PROVIDER = "openai"
+
+# Provider-specific default models
+PROVIDER_DEFAULT_MODELS = {
+    "openai": "o3-mini",
+    "anthropic": "claude-haiku-4-5-20251001",
+    "gemini": "gemini-2.5-flash",
+    "groq": "openai/gpt-oss-120b",
+    "ollama": "gemma3:12b",
+    "lmstudio": "local-model",
+    "vertexai": "gemini-2.0-flash-001",
+    "openai-codex": "gpt-5.2-codex",
+    "claude-code": "claude-sonnet-4-5-20250929",
+    "mock": "mock-model",
+}
+DEFAULT_LLM_MODEL = "o3-mini"  # Fallback if provider not in table
+DEFAULT_LLM_MAX_CONCURRENT = 32
+DEFAULT_LLM_MAX_RETRIES = 10  # Max retry attempts for LLM API calls
+DEFAULT_LLM_INITIAL_BACKOFF = 1.0  # Initial backoff in seconds for retry exponential backoff
+DEFAULT_LLM_MAX_BACKOFF = 60.0  # Max backoff cap in seconds for retry exponential backoff
+DEFAULT_LLM_TIMEOUT = 120.0  # seconds
+
+# Vertex AI defaults
+DEFAULT_LLM_VERTEXAI_PROJECT_ID = None  # Required for Vertex AI
+DEFAULT_LLM_VERTEXAI_REGION = "us-central1"
+DEFAULT_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY = None  # Optional, uses ADC if not set
+
+DEFAULT_EMBEDDINGS_PROVIDER = "local"
+DEFAULT_EMBEDDINGS_LOCAL_MODEL = "BAAI/bge-small-en-v1.5"
+DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU = False  # Force CPU mode for local embeddings (avoids MPS/XPC issues on macOS)
+DEFAULT_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE = False  # Security: disabled by default, required for some models
+DEFAULT_EMBEDDINGS_OPENAI_MODEL = "text-embedding-3-small"
+DEFAULT_EMBEDDING_DIMENSION = 384
+
+DEFAULT_RERANKER_PROVIDER = "local"
+DEFAULT_RERANKER_LOCAL_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"
+DEFAULT_RERANKER_LOCAL_FORCE_CPU = False  # Force CPU mode for local reranker (avoids MPS/XPC issues on macOS)
+DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT = 4  # Limit concurrent CPU-bound reranking to prevent thrashing
+DEFAULT_RERANKER_LOCAL_TRUST_REMOTE_CODE = (
+    False  # Security: disabled by default, required for some models like jina-reranker-v2
+)
+DEFAULT_RERANKER_TEI_BATCH_SIZE = 128
+DEFAULT_RERANKER_TEI_MAX_CONCURRENT = 8
+DEFAULT_RERANKER_MAX_CANDIDATES = 300
+DEFAULT_RERANKER_FLASHRANK_MODEL = "ms-marco-MiniLM-L-12-v2"  # Best balance of speed and quality
+DEFAULT_RERANKER_FLASHRANK_CACHE_DIR = None  # Use default cache directory
+
+DEFAULT_EMBEDDINGS_COHERE_MODEL = "embed-english-v3.0"
+DEFAULT_RERANKER_COHERE_MODEL = "rerank-english-v3.0"
+
+# LiteLLM defaults
+DEFAULT_LITELLM_API_BASE = "http://localhost:4000"
+DEFAULT_EMBEDDINGS_LITELLM_MODEL = "text-embedding-3-small"
+DEFAULT_RERANKER_LITELLM_MODEL = "cohere/rerank-english-v3.0"
+
+DEFAULT_HOST = "0.0.0.0"
+DEFAULT_PORT = 8888
+DEFAULT_LOG_LEVEL = "info"
+DEFAULT_LOG_FORMAT = "text"  # Options: "text", "json"
+DEFAULT_WORKERS = 1
+DEFAULT_MCP_ENABLED = True
+DEFAULT_GRAPH_RETRIEVER = "link_expansion"  # Options: "link_expansion", "mpfp", "bfs"
+DEFAULT_MPFP_TOP_K_NEIGHBORS = 20  # Fan-out limit per node in MPFP graph traversal
+DEFAULT_RECALL_MAX_CONCURRENT = 32  # Max concurrent recall operations per worker
+DEFAULT_RECALL_CONNECTION_BUDGET = 4  # Max concurrent DB connections per recall operation
+DEFAULT_MCP_LOCAL_BANK_ID = "mcp"
+DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY = 8  # Max concurrent mental model refreshes
+
+# Retain settings
+DEFAULT_RETAIN_MAX_COMPLETION_TOKENS = 64000  # Max tokens for fact extraction LLM call
+DEFAULT_RETAIN_CHUNK_SIZE = 3000  # Max chars per chunk for fact extraction
+DEFAULT_RETAIN_EXTRACT_CAUSAL_LINKS = True  # Extract causal links between facts
+DEFAULT_RETAIN_EXTRACTION_MODE = "concise"  # Extraction mode: "concise", "verbose", or "custom"
+RETAIN_EXTRACTION_MODES = ("concise", "verbose", "custom")  # Allowed extraction modes
+DEFAULT_RETAIN_CUSTOM_INSTRUCTIONS = None  # Custom extraction guidelines (only used when mode="custom")
+
+# Observations defaults (consolidated knowledge from facts)
+DEFAULT_ENABLE_OBSERVATIONS = True  # Observations enabled by default
+DEFAULT_CONSOLIDATION_BATCH_SIZE = 50  # Memories to load per batch (internal memory optimization)
+DEFAULT_CONSOLIDATION_MAX_TOKENS = 1024  # Max tokens for recall when finding related observations
+
+# Database migrations
+DEFAULT_RUN_MIGRATIONS_ON_STARTUP = True
+
+# Database connection pool
+DEFAULT_DB_POOL_MIN_SIZE = 5
+DEFAULT_DB_POOL_MAX_SIZE = 100
+DEFAULT_DB_COMMAND_TIMEOUT = 60  # seconds
+DEFAULT_DB_ACQUIRE_TIMEOUT = 30  # seconds
+
+# Worker configuration (distributed task processing)
+DEFAULT_WORKER_ENABLED = True  # API runs worker by default (standalone mode)
+DEFAULT_WORKER_ID = None  # Will use hostname if not specified
+DEFAULT_WORKER_POLL_INTERVAL_MS = 500  # Poll database every 500ms
+DEFAULT_WORKER_MAX_RETRIES = 3  # Max retries before marking task failed
+DEFAULT_WORKER_HTTP_PORT = 8889  # HTTP port for worker metrics/health
+DEFAULT_WORKER_MAX_SLOTS = 10  # Total concurrent tasks per worker
+DEFAULT_WORKER_CONSOLIDATION_MAX_SLOTS = 2  # Max concurrent consolidation tasks per worker
+
+# Reflect agent settings
+DEFAULT_REFLECT_MAX_ITERATIONS = 10  # Max tool call iterations before forcing response
+
+# OpenTelemetry tracing configuration
+DEFAULT_OTEL_TRACES_ENABLED = False  # Disabled by default for backward compatibility
+DEFAULT_OTEL_SERVICE_NAME = "hindsight-api"
+DEFAULT_OTEL_DEPLOYMENT_ENVIRONMENT = "development"
+
+# Default MCP tool descriptions (can be customized via env vars)
+DEFAULT_MCP_RETAIN_DESCRIPTION = """Store important information to long-term memory.
+
+Use this tool PROACTIVELY whenever the user shares:
+- Personal facts, preferences, or interests
+- Important events or milestones
+- User history, experiences, or background
+- Decisions, opinions, or stated preferences
+- Goals, plans, or future intentions
+- Relationships or people mentioned
+- Work context, projects, or responsibilities"""
+
+DEFAULT_MCP_RECALL_DESCRIPTION = """Search memories to provide personalized, context-aware responses.
+
+Use this tool PROACTIVELY to:
+- Check user's preferences before making suggestions
+- Recall user's history to provide continuity
+- Remember user's goals and context
+- Personalize responses based on past interactions"""
+
+# Default embedding dimension (used by initial migration, adjusted at runtime)
+EMBEDDING_DIMENSION = DEFAULT_EMBEDDING_DIMENSION
+
+
+class JsonFormatter(logging.Formatter):
+    """JSON formatter for structured logging.
+
+    Outputs logs in JSON format with a 'severity' field that cloud logging
+    systems (GCP, AWS CloudWatch, etc.) can parse to correctly categorize log levels.
+    """
+
+    SEVERITY_MAP = {
+        logging.DEBUG: "DEBUG",
+        logging.INFO: "INFO",
+        logging.WARNING: "WARNING",
+        logging.ERROR: "ERROR",
+        logging.CRITICAL: "CRITICAL",
+    }
+
+    def format(self, record: logging.LogRecord) -> str:
+        log_entry = {
+            "severity": self.SEVERITY_MAP.get(record.levelno, "DEFAULT"),
+            "message": record.getMessage(),
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "logger": record.name,
+        }
+
+        # Add exception info if present
+        if record.exc_info:
+            log_entry["exception"] = self.formatException(record.exc_info)
+
+        return json.dumps(log_entry)
+
+
+def _validate_extraction_mode(mode: str) -> str:
+    """Validate and normalize extraction mode."""
+    mode_lower = mode.lower()
+    if mode_lower not in RETAIN_EXTRACTION_MODES:
+        logger.warning(
+            f"Invalid extraction mode '{mode}', must be one of {RETAIN_EXTRACTION_MODES}. "
+            f"Defaulting to '{DEFAULT_RETAIN_EXTRACTION_MODE}'."
+        )
+        return DEFAULT_RETAIN_EXTRACTION_MODE
+    return mode_lower
+
+
+def _get_default_model_for_provider(provider: str) -> str:
+    """Get the default model for a given provider."""
+    return PROVIDER_DEFAULT_MODELS.get(provider.lower(), DEFAULT_LLM_MODEL)
+
+
+@dataclass
+class HindsightConfig:
+    """Configuration container for Hindsight API."""
+
+    # Database
+    database_url: str
+    database_schema: str
+
+    # LLM (default, used as fallback for per-operation config)
+    llm_provider: str
+    llm_api_key: str | None
+    llm_model: str
+    llm_base_url: str | None
+    llm_max_concurrent: int
+    llm_max_retries: int
+    llm_initial_backoff: float
+    llm_max_backoff: float
+    llm_timeout: float
+
+    # Vertex AI configuration
+    llm_vertexai_project_id: str | None
+    llm_vertexai_region: str
+    llm_vertexai_service_account_key: str | None
+
+    # Per-operation LLM configuration (None = use default LLM config)
+    retain_llm_provider: str | None
+    retain_llm_api_key: str | None
+    retain_llm_model: str | None
+    retain_llm_base_url: str | None
+    retain_llm_max_concurrent: int | None
+    retain_llm_max_retries: int | None
+    retain_llm_initial_backoff: float | None
+    retain_llm_max_backoff: float | None
+    retain_llm_timeout: float | None
+
+    reflect_llm_provider: str | None
+    reflect_llm_api_key: str | None
+    reflect_llm_model: str | None
+    reflect_llm_base_url: str | None
+    reflect_llm_max_concurrent: int | None
+    reflect_llm_max_retries: int | None
+    reflect_llm_initial_backoff: float | None
+    reflect_llm_max_backoff: float | None
+    reflect_llm_timeout: float | None
+
+    consolidation_llm_provider: str | None
+    consolidation_llm_api_key: str | None
+    consolidation_llm_model: str | None
+    consolidation_llm_base_url: str | None
+    consolidation_llm_max_concurrent: int | None
+    consolidation_llm_max_retries: int | None
+    consolidation_llm_initial_backoff: float | None
+    consolidation_llm_max_backoff: float | None
+    consolidation_llm_timeout: float | None
+
+    # Embeddings
+    embeddings_provider: str
+    embeddings_local_model: str
+    embeddings_local_force_cpu: bool
+    embeddings_local_trust_remote_code: bool
+    embeddings_tei_url: str | None
+    embeddings_openai_base_url: str | None
+    embeddings_cohere_api_key: str | None
+    embeddings_cohere_model: str
+    embeddings_cohere_base_url: str | None
+    embeddings_litellm_api_base: str
+    embeddings_litellm_api_key: str | None
+    embeddings_litellm_model: str
+
+    # Reranker
+    reranker_provider: str
+    reranker_local_model: str
+    reranker_local_force_cpu: bool
+    reranker_local_max_concurrent: int
+    reranker_local_trust_remote_code: bool
+    reranker_tei_url: str | None
+    reranker_tei_batch_size: int
+    reranker_tei_max_concurrent: int
+    reranker_max_candidates: int
+    reranker_cohere_api_key: str | None
+    reranker_cohere_model: str
+    reranker_cohere_base_url: str | None
+    reranker_litellm_api_base: str
+    reranker_litellm_api_key: str | None
+    reranker_litellm_model: str
+
+    # Server
+    host: str
+    port: int
+    log_level: str
+    log_format: str
+    mcp_enabled: bool
+
+    # Recall
+    graph_retriever: str
+    mpfp_top_k_neighbors: int
+    recall_max_concurrent: int
+    recall_connection_budget: int
+    mental_model_refresh_concurrency: int
+
+    # Retain settings
+    retain_max_completion_tokens: int
+    retain_chunk_size: int
+    retain_extract_causal_links: bool
+    retain_extraction_mode: str
+    retain_custom_instructions: str | None
+
+    # Observations settings (consolidated knowledge from facts)
+    enable_observations: bool
+    consolidation_batch_size: int
+    consolidation_max_tokens: int
+
+    # Optimization flags
+    skip_llm_verification: bool
+    lazy_reranker: bool
+
+    # Database migrations
+    run_migrations_on_startup: bool
+
+    # Database connection pool
+    db_pool_min_size: int
+    db_pool_max_size: int
+    db_command_timeout: int
+    db_acquire_timeout: int
+
+    # Worker configuration (distributed task processing)
+    worker_enabled: bool
+    worker_id: str | None
+    worker_poll_interval_ms: int
+    worker_max_retries: int
+    worker_http_port: int
+    worker_max_slots: int
+    worker_consolidation_max_slots: int
+
+    # Reflect agent settings
+    reflect_max_iterations: int
+
+    # OpenTelemetry tracing configuration
+    otel_traces_enabled: bool
+    otel_exporter_otlp_endpoint: str | None
+    otel_exporter_otlp_headers: str | None
+    otel_service_name: str
+    otel_deployment_environment: str
+
+    def validate(self) -> None:
+        """Validate configuration values and raise errors for invalid combinations."""
+        # RETAIN_MAX_COMPLETION_TOKENS must be greater than RETAIN_CHUNK_SIZE
+        # to ensure the LLM has enough output capacity to extract facts from chunks
+        if self.retain_max_completion_tokens <= self.retain_chunk_size:
+            raise ValueError(
+                f"Invalid configuration: HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS "
+                f"({self.retain_max_completion_tokens}) must be greater than "
+                f"HINDSIGHT_API_RETAIN_CHUNK_SIZE ({self.retain_chunk_size}). "
+                f"\n\nYou have two options to fix this:"
+                f"\n  1. Increase HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS to a value > {self.retain_chunk_size}"
+                f"\n  2. Use a model that supports at least {self.retain_max_completion_tokens} output tokens"
+                f"\n     (current model: {self.retain_llm_model or self.llm_model}, "
+                f"provider: {self.retain_llm_provider or self.llm_provider})"
+            )
+
+    @classmethod
+    def from_env(cls) -> "HindsightConfig":
+        """Create configuration from environment variables."""
+        # Get provider first to determine default model
+        llm_provider = os.getenv(ENV_LLM_PROVIDER, DEFAULT_LLM_PROVIDER)
+        llm_model = os.getenv(ENV_LLM_MODEL) or _get_default_model_for_provider(llm_provider)
+
+        config = cls(
+            # Database
+            database_url=os.getenv(ENV_DATABASE_URL, DEFAULT_DATABASE_URL),
+            database_schema=os.getenv(ENV_DATABASE_SCHEMA, DEFAULT_DATABASE_SCHEMA),
+            # LLM
+            llm_provider=llm_provider,
+            llm_api_key=os.getenv(ENV_LLM_API_KEY),
+            llm_model=llm_model,
+            llm_base_url=os.getenv(ENV_LLM_BASE_URL) or None,
+            llm_max_concurrent=int(os.getenv(ENV_LLM_MAX_CONCURRENT, str(DEFAULT_LLM_MAX_CONCURRENT))),
+            llm_max_retries=int(os.getenv(ENV_LLM_MAX_RETRIES, str(DEFAULT_LLM_MAX_RETRIES))),
+            llm_initial_backoff=float(os.getenv(ENV_LLM_INITIAL_BACKOFF, str(DEFAULT_LLM_INITIAL_BACKOFF))),
+            llm_max_backoff=float(os.getenv(ENV_LLM_MAX_BACKOFF, str(DEFAULT_LLM_MAX_BACKOFF))),
+            llm_timeout=float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT))),
+            # Vertex AI
+            llm_vertexai_project_id=os.getenv(ENV_LLM_VERTEXAI_PROJECT_ID) or DEFAULT_LLM_VERTEXAI_PROJECT_ID,
+            llm_vertexai_region=os.getenv(ENV_LLM_VERTEXAI_REGION, DEFAULT_LLM_VERTEXAI_REGION),
+            llm_vertexai_service_account_key=os.getenv(ENV_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY)
+            or DEFAULT_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY,
+            # Per-operation LLM config (None = use default)
+            retain_llm_provider=os.getenv(ENV_RETAIN_LLM_PROVIDER) or None,
+            retain_llm_api_key=os.getenv(ENV_RETAIN_LLM_API_KEY) or None,
+            retain_llm_model=os.getenv(ENV_RETAIN_LLM_MODEL)
+            or (
+                _get_default_model_for_provider(os.getenv(ENV_RETAIN_LLM_PROVIDER))
+                if os.getenv(ENV_RETAIN_LLM_PROVIDER)
+                else None
+            ),
+            retain_llm_base_url=os.getenv(ENV_RETAIN_LLM_BASE_URL) or None,
+            retain_llm_max_concurrent=int(os.getenv(ENV_RETAIN_LLM_MAX_CONCURRENT))
+            if os.getenv(ENV_RETAIN_LLM_MAX_CONCURRENT)
+            else None,
+            retain_llm_max_retries=int(os.getenv(ENV_RETAIN_LLM_MAX_RETRIES))
+            if os.getenv(ENV_RETAIN_LLM_MAX_RETRIES)
+            else None,
+            retain_llm_initial_backoff=float(os.getenv(ENV_RETAIN_LLM_INITIAL_BACKOFF))
+            if os.getenv(ENV_RETAIN_LLM_INITIAL_BACKOFF)
+            else None,
+            retain_llm_max_backoff=float(os.getenv(ENV_RETAIN_LLM_MAX_BACKOFF))
+            if os.getenv(ENV_RETAIN_LLM_MAX_BACKOFF)
+            else None,
+            retain_llm_timeout=float(os.getenv(ENV_RETAIN_LLM_TIMEOUT)) if os.getenv(ENV_RETAIN_LLM_TIMEOUT) else None,
+            reflect_llm_provider=os.getenv(ENV_REFLECT_LLM_PROVIDER) or None,
+            reflect_llm_api_key=os.getenv(ENV_REFLECT_LLM_API_KEY) or None,
+            reflect_llm_model=os.getenv(ENV_REFLECT_LLM_MODEL)
+            or (
+                _get_default_model_for_provider(os.getenv(ENV_REFLECT_LLM_PROVIDER))
+                if os.getenv(ENV_REFLECT_LLM_PROVIDER)
+                else None
+            ),
+            reflect_llm_base_url=os.getenv(ENV_REFLECT_LLM_BASE_URL) or None,
+            reflect_llm_max_concurrent=int(os.getenv(ENV_REFLECT_LLM_MAX_CONCURRENT))
+            if os.getenv(ENV_REFLECT_LLM_MAX_CONCURRENT)
+            else None,
+            reflect_llm_max_retries=int(os.getenv(ENV_REFLECT_LLM_MAX_RETRIES))
+            if os.getenv(ENV_REFLECT_LLM_MAX_RETRIES)
+            else None,
+            reflect_llm_initial_backoff=float(os.getenv(ENV_REFLECT_LLM_INITIAL_BACKOFF))
+            if os.getenv(ENV_REFLECT_LLM_INITIAL_BACKOFF)
+            else None,
+            reflect_llm_max_backoff=float(os.getenv(ENV_REFLECT_LLM_MAX_BACKOFF))
+            if os.getenv(ENV_REFLECT_LLM_MAX_BACKOFF)
+            else None,
+            reflect_llm_timeout=float(os.getenv(ENV_REFLECT_LLM_TIMEOUT))
+            if os.getenv(ENV_REFLECT_LLM_TIMEOUT)
+            else None,
+            consolidation_llm_provider=os.getenv(ENV_CONSOLIDATION_LLM_PROVIDER) or None,
+            consolidation_llm_api_key=os.getenv(ENV_CONSOLIDATION_LLM_API_KEY) or None,
+            consolidation_llm_model=os.getenv(ENV_CONSOLIDATION_LLM_MODEL)
+            or (
+                _get_default_model_for_provider(os.getenv(ENV_CONSOLIDATION_LLM_PROVIDER))
+                if os.getenv(ENV_CONSOLIDATION_LLM_PROVIDER)
+                else None
+            ),
+            consolidation_llm_base_url=os.getenv(ENV_CONSOLIDATION_LLM_BASE_URL) or None,
+            consolidation_llm_max_concurrent=int(os.getenv(ENV_CONSOLIDATION_LLM_MAX_CONCURRENT))
+            if os.getenv(ENV_CONSOLIDATION_LLM_MAX_CONCURRENT)
+            else None,
+            consolidation_llm_max_retries=int(os.getenv(ENV_CONSOLIDATION_LLM_MAX_RETRIES))
+            if os.getenv(ENV_CONSOLIDATION_LLM_MAX_RETRIES)
+            else None,
+            consolidation_llm_initial_backoff=float(os.getenv(ENV_CONSOLIDATION_LLM_INITIAL_BACKOFF))
+            if os.getenv(ENV_CONSOLIDATION_LLM_INITIAL_BACKOFF)
+            else None,
+            consolidation_llm_max_backoff=float(os.getenv(ENV_CONSOLIDATION_LLM_MAX_BACKOFF))
+            if os.getenv(ENV_CONSOLIDATION_LLM_MAX_BACKOFF)
+            else None,
+            consolidation_llm_timeout=float(os.getenv(ENV_CONSOLIDATION_LLM_TIMEOUT))
+            if os.getenv(ENV_CONSOLIDATION_LLM_TIMEOUT)
+            else None,
+            # Embeddings
+            embeddings_provider=os.getenv(ENV_EMBEDDINGS_PROVIDER, DEFAULT_EMBEDDINGS_PROVIDER),
+            embeddings_local_model=os.getenv(ENV_EMBEDDINGS_LOCAL_MODEL, DEFAULT_EMBEDDINGS_LOCAL_MODEL),
+            embeddings_local_force_cpu=os.getenv(
+                ENV_EMBEDDINGS_LOCAL_FORCE_CPU, str(DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU)
+            ).lower()
+            in ("true", "1"),
+            embeddings_local_trust_remote_code=os.getenv(
+                ENV_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE, str(DEFAULT_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE)
+            ).lower()
+            in ("true", "1"),
+            embeddings_tei_url=os.getenv(ENV_EMBEDDINGS_TEI_URL),
+            embeddings_openai_base_url=os.getenv(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None,
+            # Cohere embeddings (with backward-compatible fallback to shared API key)
+            embeddings_cohere_api_key=os.getenv(ENV_EMBEDDINGS_COHERE_API_KEY) or os.getenv(ENV_COHERE_API_KEY),
+            embeddings_cohere_model=os.getenv(ENV_EMBEDDINGS_COHERE_MODEL, DEFAULT_EMBEDDINGS_COHERE_MODEL),
+            embeddings_cohere_base_url=os.getenv(ENV_EMBEDDINGS_COHERE_BASE_URL) or None,
+            # LiteLLM embeddings (with backward-compatible fallback to shared config)
+            embeddings_litellm_api_base=os.getenv(ENV_EMBEDDINGS_LITELLM_API_BASE)
+            or os.getenv(ENV_LITELLM_API_BASE, DEFAULT_LITELLM_API_BASE),
+            embeddings_litellm_api_key=os.getenv(ENV_EMBEDDINGS_LITELLM_API_KEY) or os.getenv(ENV_LITELLM_API_KEY),
+            embeddings_litellm_model=os.getenv(ENV_EMBEDDINGS_LITELLM_MODEL, DEFAULT_EMBEDDINGS_LITELLM_MODEL),
+            # Reranker
+            reranker_provider=os.getenv(ENV_RERANKER_PROVIDER, DEFAULT_RERANKER_PROVIDER),
+            reranker_local_model=os.getenv(ENV_RERANKER_LOCAL_MODEL, DEFAULT_RERANKER_LOCAL_MODEL),
+            reranker_local_force_cpu=os.getenv(
+                ENV_RERANKER_LOCAL_FORCE_CPU, str(DEFAULT_RERANKER_LOCAL_FORCE_CPU)
+            ).lower()
+            in ("true", "1"),
+            reranker_local_max_concurrent=int(
+                os.getenv(ENV_RERANKER_LOCAL_MAX_CONCURRENT, str(DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT))
+            ),
+            reranker_local_trust_remote_code=os.getenv(
+                ENV_RERANKER_LOCAL_TRUST_REMOTE_CODE, str(DEFAULT_RERANKER_LOCAL_TRUST_REMOTE_CODE)
+            ).lower()
+            in ("true", "1"),
+            reranker_tei_url=os.getenv(ENV_RERANKER_TEI_URL),
+            reranker_tei_batch_size=int(os.getenv(ENV_RERANKER_TEI_BATCH_SIZE, str(DEFAULT_RERANKER_TEI_BATCH_SIZE))),
+            reranker_tei_max_concurrent=int(
+                os.getenv(ENV_RERANKER_TEI_MAX_CONCURRENT, str(DEFAULT_RERANKER_TEI_MAX_CONCURRENT))
+            ),
+            reranker_max_candidates=int(os.getenv(ENV_RERANKER_MAX_CANDIDATES, str(DEFAULT_RERANKER_MAX_CANDIDATES))),
+            # Cohere reranker (with backward-compatible fallback to shared API key)
+            reranker_cohere_api_key=os.getenv(ENV_RERANKER_COHERE_API_KEY) or os.getenv(ENV_COHERE_API_KEY),
+            reranker_cohere_model=os.getenv(ENV_RERANKER_COHERE_MODEL, DEFAULT_RERANKER_COHERE_MODEL),
+            reranker_cohere_base_url=os.getenv(ENV_RERANKER_COHERE_BASE_URL) or None,
+            # LiteLLM reranker (with backward-compatible fallback to shared config)
+            reranker_litellm_api_base=os.getenv(ENV_RERANKER_LITELLM_API_BASE)
+            or os.getenv(ENV_LITELLM_API_BASE, DEFAULT_LITELLM_API_BASE),
+            reranker_litellm_api_key=os.getenv(ENV_RERANKER_LITELLM_API_KEY) or os.getenv(ENV_LITELLM_API_KEY),
+            reranker_litellm_model=os.getenv(ENV_RERANKER_LITELLM_MODEL, DEFAULT_RERANKER_LITELLM_MODEL),
+            # Server
+            host=os.getenv(ENV_HOST, DEFAULT_HOST),
+            port=int(os.getenv(ENV_PORT, DEFAULT_PORT)),
+            log_level=os.getenv(ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL),
+            log_format=os.getenv(ENV_LOG_FORMAT, DEFAULT_LOG_FORMAT).lower(),
+            mcp_enabled=os.getenv(ENV_MCP_ENABLED, str(DEFAULT_MCP_ENABLED)).lower() == "true",
+            # Recall
+            graph_retriever=os.getenv(ENV_GRAPH_RETRIEVER, DEFAULT_GRAPH_RETRIEVER),
+            mpfp_top_k_neighbors=int(os.getenv(ENV_MPFP_TOP_K_NEIGHBORS, str(DEFAULT_MPFP_TOP_K_NEIGHBORS))),
+            recall_max_concurrent=int(os.getenv(ENV_RECALL_MAX_CONCURRENT, str(DEFAULT_RECALL_MAX_CONCURRENT))),
+            recall_connection_budget=int(
+                os.getenv(ENV_RECALL_CONNECTION_BUDGET, str(DEFAULT_RECALL_CONNECTION_BUDGET))
+            ),
+            mental_model_refresh_concurrency=int(
+                os.getenv(ENV_MENTAL_MODEL_REFRESH_CONCURRENCY, str(DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY))
+            ),
+            # Optimization flags
+            skip_llm_verification=os.getenv(ENV_SKIP_LLM_VERIFICATION, "false").lower() == "true",
+            lazy_reranker=os.getenv(ENV_LAZY_RERANKER, "false").lower() == "true",
+            # Retain settings
+            retain_max_completion_tokens=int(
+                os.getenv(ENV_RETAIN_MAX_COMPLETION_TOKENS, str(DEFAULT_RETAIN_MAX_COMPLETION_TOKENS))
+            ),
+            retain_chunk_size=int(os.getenv(ENV_RETAIN_CHUNK_SIZE, str(DEFAULT_RETAIN_CHUNK_SIZE))),
+            retain_extract_causal_links=os.getenv(
+                ENV_RETAIN_EXTRACT_CAUSAL_LINKS, str(DEFAULT_RETAIN_EXTRACT_CAUSAL_LINKS)
+            ).lower()
+            == "true",
+            retain_extraction_mode=_validate_extraction_mode(
+                os.getenv(ENV_RETAIN_EXTRACTION_MODE, DEFAULT_RETAIN_EXTRACTION_MODE)
+            ),
+            retain_custom_instructions=os.getenv(ENV_RETAIN_CUSTOM_INSTRUCTIONS) or DEFAULT_RETAIN_CUSTOM_INSTRUCTIONS,
+            # Observations settings (consolidated knowledge from facts)
+            enable_observations=os.getenv(ENV_ENABLE_OBSERVATIONS, str(DEFAULT_ENABLE_OBSERVATIONS)).lower() == "true",
+            consolidation_batch_size=int(
+                os.getenv(ENV_CONSOLIDATION_BATCH_SIZE, str(DEFAULT_CONSOLIDATION_BATCH_SIZE))
+            ),
+            consolidation_max_tokens=int(
+                os.getenv(ENV_CONSOLIDATION_MAX_TOKENS, str(DEFAULT_CONSOLIDATION_MAX_TOKENS))
+            ),
+            # Database migrations
+            run_migrations_on_startup=os.getenv(ENV_RUN_MIGRATIONS_ON_STARTUP, "true").lower() == "true",
+            # Database connection pool
+            db_pool_min_size=int(os.getenv(ENV_DB_POOL_MIN_SIZE, str(DEFAULT_DB_POOL_MIN_SIZE))),
+            db_pool_max_size=int(os.getenv(ENV_DB_POOL_MAX_SIZE, str(DEFAULT_DB_POOL_MAX_SIZE))),
+            db_command_timeout=int(os.getenv(ENV_DB_COMMAND_TIMEOUT, str(DEFAULT_DB_COMMAND_TIMEOUT))),
+            db_acquire_timeout=int(os.getenv(ENV_DB_ACQUIRE_TIMEOUT, str(DEFAULT_DB_ACQUIRE_TIMEOUT))),
+            # Worker configuration
+            worker_enabled=os.getenv(ENV_WORKER_ENABLED, str(DEFAULT_WORKER_ENABLED)).lower() == "true",
+            worker_id=os.getenv(ENV_WORKER_ID) or DEFAULT_WORKER_ID,
+            worker_poll_interval_ms=int(os.getenv(ENV_WORKER_POLL_INTERVAL_MS, str(DEFAULT_WORKER_POLL_INTERVAL_MS))),
+            worker_max_retries=int(os.getenv(ENV_WORKER_MAX_RETRIES, str(DEFAULT_WORKER_MAX_RETRIES))),
+            worker_http_port=int(os.getenv(ENV_WORKER_HTTP_PORT, str(DEFAULT_WORKER_HTTP_PORT))),
+            worker_max_slots=int(os.getenv(ENV_WORKER_MAX_SLOTS, str(DEFAULT_WORKER_MAX_SLOTS))),
+            worker_consolidation_max_slots=int(
+                os.getenv(ENV_WORKER_CONSOLIDATION_MAX_SLOTS, str(DEFAULT_WORKER_CONSOLIDATION_MAX_SLOTS))
+            ),
+            # Reflect agent settings
+            reflect_max_iterations=int(os.getenv(ENV_REFLECT_MAX_ITERATIONS, str(DEFAULT_REFLECT_MAX_ITERATIONS))),
+            # OpenTelemetry tracing configuration
+            otel_traces_enabled=os.getenv(ENV_OTEL_TRACES_ENABLED, str(DEFAULT_OTEL_TRACES_ENABLED)).lower()
+            in ("true", "1", "yes"),
+            otel_exporter_otlp_endpoint=os.getenv(ENV_OTEL_EXPORTER_OTLP_ENDPOINT) or None,
+            otel_exporter_otlp_headers=os.getenv(ENV_OTEL_EXPORTER_OTLP_HEADERS) or None,
+            otel_service_name=os.getenv(ENV_OTEL_SERVICE_NAME, DEFAULT_OTEL_SERVICE_NAME),
+            otel_deployment_environment=os.getenv(ENV_OTEL_DEPLOYMENT_ENVIRONMENT, DEFAULT_OTEL_DEPLOYMENT_ENVIRONMENT),
+        )
+        config.validate()
+        return config
+
+    def get_llm_base_url(self) -> str:
+        """Get the LLM base URL, with provider-specific defaults."""
+        if self.llm_base_url:
+            return self.llm_base_url
+
+        provider = self.llm_provider.lower()
+        if provider == "groq":
+            return "https://api.groq.com/openai/v1"
+        elif provider == "ollama":
+            return "http://localhost:11434/v1"
+        elif provider == "lmstudio":
+            return "http://localhost:1234/v1"
+        else:
+            return ""
+
+    def get_python_log_level(self) -> int:
+        """Get the Python logging level from the configured log level string."""
+        log_level_map = {
+            "critical": logging.CRITICAL,
+            "error": logging.ERROR,
+            "warning": logging.WARNING,
+            "info": logging.INFO,
+            "debug": logging.DEBUG,
+            "trace": logging.DEBUG,  # Python doesn't have TRACE, use DEBUG
+        }
+        return log_level_map.get(self.log_level.lower(), logging.INFO)
+
+    def configure_logging(self) -> None:
+        """Configure Python logging based on the log level and format.
+
+        When log_format is "json", outputs structured JSON logs with a severity
+        field that GCP Cloud Logging can parse for proper log level categorization.
+        """
+        root_logger = logging.getLogger()
+        root_logger.setLevel(self.get_python_log_level())
+
+        # Remove existing handlers
+        for handler in root_logger.handlers[:]:
+            root_logger.removeHandler(handler)
+
+        # Create handler writing to stdout (GCP treats stderr as ERROR)
+        handler = logging.StreamHandler(sys.stdout)
+        handler.setLevel(self.get_python_log_level())
+
+        if self.log_format == "json":
+            handler.setFormatter(JsonFormatter())
+        else:
+            handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s"))
+
+        root_logger.addHandler(handler)
+
+    def log_config(self) -> None:
+        """Log the current configuration (without sensitive values)."""
+        logger.info(f"Database: {self.database_url} (schema: {self.database_schema})")
+        logger.info(f"LLM: provider={self.llm_provider}, model={self.llm_model}")
+        if self.retain_llm_provider or self.retain_llm_model:
+            retain_provider = self.retain_llm_provider or self.llm_provider
+            retain_model = self.retain_llm_model or self.llm_model
+            logger.info(f"LLM (retain): provider={retain_provider}, model={retain_model}")
+        if self.reflect_llm_provider or self.reflect_llm_model:
+            reflect_provider = self.reflect_llm_provider or self.llm_provider
+            reflect_model = self.reflect_llm_model or self.llm_model
+            logger.info(f"LLM (reflect): provider={reflect_provider}, model={reflect_model}")
+        if self.consolidation_llm_provider or self.consolidation_llm_model:
+            consolidation_provider = self.consolidation_llm_provider or self.llm_provider
+            consolidation_model = self.consolidation_llm_model or self.llm_model
+            logger.info(f"LLM (consolidation): provider={consolidation_provider}, model={consolidation_model}")
+        logger.info(f"Embeddings: provider={self.embeddings_provider}")
+        logger.info(f"Reranker: provider={self.reranker_provider}")
+        logger.info(f"Graph retriever: {self.graph_retriever}")
+
+
+# Cached config instance
+_config_cache: HindsightConfig | None = None
+
+
+def get_config() -> HindsightConfig:
+    """Get the cached configuration, loading from environment on first call."""
+    global _config_cache
+    if _config_cache is None:
+        _config_cache = HindsightConfig.from_env()
+    return _config_cache
+
+
+def clear_config_cache() -> None:
+    """Clear the config cache. Useful for testing or reloading config."""
+    global _config_cache
+    _config_cache = None

hindsight-api/hindsight_api/daemon.py

@@ -0,0 +1,113 @@
+"""
+Daemon mode support for Hindsight API.
+
+Provides idle timeout for running as a background daemon.
+"""
+
+import asyncio
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Default daemon configuration
+DEFAULT_DAEMON_PORT = 8888
+DEFAULT_IDLE_TIMEOUT = 0  # 0 = no auto-exit (hindsight-embed passes its own timeout)
+
+# Allow override via environment variable for profile-specific logs
+DAEMON_LOG_PATH = Path(os.getenv("HINDSIGHT_API_DAEMON_LOG", str(Path.home() / ".hindsight" / "daemon.log")))
+
+
+class IdleTimeoutMiddleware:
+    """ASGI middleware that tracks activity and exits after idle timeout."""
+
+    def __init__(self, app, idle_timeout: int = DEFAULT_IDLE_TIMEOUT):
+        self.app = app
+        self.idle_timeout = idle_timeout
+        self.last_activity = time.time()
+        self._checker_task = None
+
+    async def __call__(self, scope, receive, send):
+        # Update activity timestamp on each request
+        self.last_activity = time.time()
+        await self.app(scope, receive, send)
+
+    def start_idle_checker(self):
+        """Start the background task that checks for idle timeout."""
+        self._checker_task = asyncio.create_task(self._check_idle())
+
+    async def _check_idle(self):
+        """Background task that exits the process after idle timeout."""
+        # If idle_timeout is 0, don't auto-exit
+        if self.idle_timeout <= 0:
+            return
+
+        while True:
+            await asyncio.sleep(30)  # Check every 30 seconds
+            idle_time = time.time() - self.last_activity
+            if idle_time > self.idle_timeout:
+                logger.info(f"Idle timeout reached ({self.idle_timeout}s), shutting down daemon")
+                # Give a moment for any in-flight requests
+                await asyncio.sleep(1)
+                # Send SIGTERM to ourselves to trigger graceful shutdown
+                import signal
+
+                os.kill(os.getpid(), signal.SIGTERM)
+
+
+def daemonize():
+    """
+    Fork the current process into a background daemon.
+
+    Uses double-fork technique to properly detach from terminal.
+    """
+    # First fork - detach from parent
+    try:
+        pid = os.fork()
+        if pid > 0:
+            sys.exit(0)
+    except OSError as e:
+        sys.stderr.write(f"fork #1 failed: {e}\n")
+        sys.exit(1)
+
+    # Decouple from parent environment
+    os.chdir("/")
+    os.setsid()
+    os.umask(0)
+
+    # Second fork - prevent zombie
+    pid = os.fork()
+    if pid > 0:
+        sys.exit(0)
+
+    # Redirect standard file descriptors to log file
+    DAEMON_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
+
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    # Redirect stdin to /dev/null
+    with open("/dev/null", "r") as devnull:
+        os.dup2(devnull.fileno(), sys.stdin.fileno())
+
+    # Redirect stdout/stderr to log file
+    log_fd = open(DAEMON_LOG_PATH, "a")
+    os.dup2(log_fd.fileno(), sys.stdout.fileno())
+    os.dup2(log_fd.fileno(), sys.stderr.fileno())
+
+
+def check_daemon_running(port: int = DEFAULT_DAEMON_PORT) -> bool:
+    """Check if a daemon is running and responsive on the given port."""
+    import socket
+
+    try:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.settimeout(1)
+        result = sock.connect_ex(("127.0.0.1", port))
+        sock.close()
+        return result == 0
+    except Exception:
+        return False

hindsight-api/hindsight_api/engine/__init__.py

@@ -7,29 +7,40 @@ This package contains all the implementation details of the memory engine:
 - Supporting modules: embeddings, cross_encoder, entity_resolver, etc.
 """
 
-from .memory_engine import MemoryEngine
+from .cross_encoder import CrossEncoderModel, LocalSTCrossEncoder, RemoteTEICrossEncoder
 from .db_utils import acquire_with_retry
-from .embeddings import Embeddings, SentenceTransformersEmbeddings
+from .embeddings import Embeddings, LocalSTEmbeddings, RemoteTEIEmbeddings
+from .llm_wrapper import LLMConfig
+from .memory_engine import (
+    MemoryEngine,
+    UnqualifiedTableError,
+    fq_table,
+    get_current_schema,
+    validate_sql_schema,
+)
+from .response_models import MemoryFact, RecallResult, ReflectResult
 from .search.trace import (
-    SearchTrace,
-    QueryInfo,
     EntryPoint,
-    NodeVisit,
-    WeightComponents,
     LinkInfo,
+    NodeVisit,
     PruningDecision,
-    SearchSummary,
+    QueryInfo,
     SearchPhaseMetrics,
+    SearchSummary,
+    SearchTrace,
+    WeightComponents,
 )
 from .search.tracer import SearchTracer
-from .llm_wrapper import LLMConfig
-from .response_models import RecallResult, ReflectResult, MemoryFact
 
 __all__ = [
     "MemoryEngine",
     "acquire_with_retry",
     "Embeddings",
-    "SentenceTransformersEmbeddings",
+    "LocalSTEmbeddings",
+    "RemoteTEIEmbeddings",
+    "CrossEncoderModel",
+    "LocalSTCrossEncoder",
+    "RemoteTEICrossEncoder",
     "SearchTrace",
     "SearchTracer",
     "QueryInfo",
@@ -44,4 +55,9 @@ __all__ = [
     "RecallResult",
     "ReflectResult",
     "MemoryFact",
+    # Schema safety utilities
+    "fq_table",
+    "get_current_schema",
+    "validate_sql_schema",
+    "UnqualifiedTableError",
 ]

hindsight-api/hindsight_api/engine/consolidation/__init__.py

@@ -0,0 +1,5 @@
+"""Consolidation engine for automatic learning creation from memories."""
+
+from .consolidator import run_consolidation_job
+
+__all__ = ["run_consolidation_job"]

hindsight-api/hindsight_api/engine/consolidation/prompts.py

@@ -0,0 +1,85 @@
+"""Prompts for the consolidation engine."""
+
+CONSOLIDATION_SYSTEM_PROMPT = """You are a memory consolidation system. Your job is to convert facts into durable knowledge (observations) and merge with existing knowledge when appropriate.
+
+You must output ONLY valid JSON with no markdown code blocks or additional text. However, the "text" field within each observation should use markdown formatting (headers, lists, bold, etc.) for clarity and readability.
+
+## EXTRACT DURABLE KNOWLEDGE, NOT EPHEMERAL STATE
+Facts often describe events or actions. Extract the DURABLE KNOWLEDGE implied by the fact, not the transient state.
+
+Examples of extracting durable knowledge:
+- "User moved to Room 203" -> "Room 203 exists" (location exists, not where user is now)
+- "User visited Acme Corp at Room 105" -> "Acme Corp is located in Room 105"
+- "User took the elevator to floor 3" -> "Floor 3 is accessible by elevator"
+- "User met Sarah at the lobby" -> "Sarah can be found at the lobby"
+
+DO NOT track current user position/state as knowledge - that changes constantly.
+DO track permanent facts learned from the user's actions.
+
+## PRESERVE SPECIFIC DETAILS
+Keep names, locations, numbers, and other specifics. Do NOT:
+- Abstract into general principles
+- Generate business insights
+- Make knowledge generic
+
+GOOD examples:
+- Fact: "John likes pizza" -> "John likes pizza"
+- Fact: "Alice works at Google" -> "Alice works at Google"
+
+BAD examples:
+- "John likes pizza" -> "Understanding dietary preferences helps..." (TOO ABSTRACT)
+- "User is at Room 203" -> "User is currently at Room 203" (EPHEMERAL STATE)
+
+## MERGE RULES (when comparing to existing observations):
+1. REDUNDANT: Same information worded differently → update existing
+2. CONTRADICTION: Opposite information about same topic → update with temporal markers showing change
+   Example: "Alex used to love pizza but now hates it" OR "Alex's pizza preference changed from love to hate"
+3. UPDATE: New state replacing old state → update showing the transition with "used to", "now", "changed from X to Y"
+
+## CRITICAL RULES:
+- NEVER merge facts about DIFFERENT people
+- NEVER merge unrelated topics (food preferences vs work vs hobbies)
+- When merging contradictions, the "text" field MUST capture BOTH states with temporal markers:
+  * Use "used to X, now Y" OR "changed from X to Y" OR "X but now Y"
+  * DO NOT just state the new fact - you MUST show the change
+- Keep observations focused on ONE specific topic per person
+- The "text" field MUST contain durable knowledge, not ephemeral state
+- Do NOT include "tags" in output - tags are handled automatically"""
+
+CONSOLIDATION_USER_PROMPT = """Analyze this new fact and consolidate into knowledge.
+{mission_section}
+NEW FACT: {fact_text}
+
+EXISTING OBSERVATIONS (JSON array with source memories and dates):
+{observations_text}
+
+Each observation includes:
+- id: unique identifier for updating
+- text: the observation content
+- proof_count: number of supporting memories
+- tags: visibility scope (handled automatically)
+- created_at/updated_at: when observation was created/modified
+- occurred_start/occurred_end: temporal range of source facts
+- source_memories: array of supporting facts with their text and dates
+
+Instructions:
+1. Extract DURABLE KNOWLEDGE from the new fact (not ephemeral state)
+2. Review source_memories in existing observations to understand evidence
+3. Check dates to detect contradictions or updates
+4. Compare with observations:
+   - Same topic → UPDATE with learning_id
+   - New topic → CREATE new observation
+   - Purely ephemeral → return []
+
+Output JSON array of actions (the "text" field should use markdown formatting for structure):
+[
+  {{"action": "update", "learning_id": "uuid-from-observations", "text": "## Updated Knowledge\n\n**Key point**: details here\n\n- Supporting detail 1\n- Supporting detail 2", "reason": "..."}},
+  {{"action": "create", "text": "## New Durable Knowledge\n\nDescription with **emphasis** and proper structure", "reason": "..."}}
+]
+
+Return [] if fact contains no durable knowledge.
+
+IMPORTANT: Format the "text" field with markdown for better readability:
+- Use headers, lists, bold/italic, tables where appropriate
+- CRITICAL: Add blank lines before and after block elements (tables, code blocks, lists)
+- Ensure proper spacing for markdown to render correctly"""

hindsight-api/hindsight_api/engine/cross_encoder.py

@@ -2,10 +2,45 @@
 Cross-encoder abstraction for reranking.
 
 Provides an interface for reranking with different backends.
+
+Configuration via environment variables - see hindsight_api.config for all env var names.
 """
-from abc import ABC, abstractmethod
-from typing import List, Tuple
+
+import asyncio
 import logging
+import os
+import warnings
+from abc import ABC, abstractmethod
+from concurrent.futures import ThreadPoolExecutor
+
+import httpx
+
+from ..config import (
+    DEFAULT_LITELLM_API_BASE,
+    DEFAULT_RERANKER_COHERE_MODEL,
+    DEFAULT_RERANKER_FLASHRANK_CACHE_DIR,
+    DEFAULT_RERANKER_FLASHRANK_MODEL,
+    DEFAULT_RERANKER_LITELLM_MODEL,
+    DEFAULT_RERANKER_LOCAL_FORCE_CPU,
+    DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT,
+    DEFAULT_RERANKER_LOCAL_MODEL,
+    DEFAULT_RERANKER_LOCAL_TRUST_REMOTE_CODE,
+    DEFAULT_RERANKER_PROVIDER,
+    DEFAULT_RERANKER_TEI_BATCH_SIZE,
+    DEFAULT_RERANKER_TEI_MAX_CONCURRENT,
+    ENV_RERANKER_COHERE_API_KEY,
+    ENV_RERANKER_COHERE_MODEL,
+    ENV_RERANKER_FLASHRANK_CACHE_DIR,
+    ENV_RERANKER_FLASHRANK_MODEL,
+    ENV_RERANKER_LOCAL_FORCE_CPU,
+    ENV_RERANKER_LOCAL_MAX_CONCURRENT,
+    ENV_RERANKER_LOCAL_MODEL,
+    ENV_RERANKER_LOCAL_TRUST_REMOTE_CODE,
+    ENV_RERANKER_PROVIDER,
+    ENV_RERANKER_TEI_BATCH_SIZE,
+    ENV_RERANKER_TEI_MAX_CONCURRENT,
+    ENV_RERANKER_TEI_URL,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -17,18 +52,24 @@ class CrossEncoderModel(ABC):
     Cross-encoders take query-document pairs and return relevance scores.
     """
 
+    @property
     @abstractmethod
-    def load(self) -> None:
-        """
-        Load the cross-encoder model.
+    def provider_name(self) -> str:
+        """Return a human-readable name for this provider (e.g., 'local', 'tei')."""
+        pass
 
-        This should be called during initialization to load the model
+    @abstractmethod
+    async def initialize(self) -> None:
+        """
+        Initialize the cross-encoder model asynchronously.
+
+        This should be called during startup to load/connect to the model
         and avoid cold start latency on first predict() call.
         """
         pass
 
     @abstractmethod
-    def predict(self, pairs: List[Tuple[str, str]]) -> List[float]:
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
         """
         Score query-document pairs for relevance.
 
@@ -41,31 +82,57 @@ class CrossEncoderModel(ABC):
         pass
 
 
-class SentenceTransformersCrossEncoder(CrossEncoderModel):
+class LocalSTCrossEncoder(CrossEncoderModel):
     """
-    Cross-encoder implementation using SentenceTransformers.
+    Local cross-encoder implementation using SentenceTransformers.
 
-    Call load() during initialization to load the model and avoid cold starts.
+    Call initialize() during startup to load the model and avoid cold starts.
 
     Default model is cross-encoder/ms-marco-MiniLM-L-6-v2:
     - Fast inference (~80ms for 100 pairs on CPU)
     - Small model (80MB)
     - Trained for passage re-ranking
+
+    Uses a dedicated thread pool to limit concurrent CPU-bound work.
     """
 
-    def __init__(self, model_name: str = "cross-encoder/ms-marco-MiniLM-L-6-v2"):
+    # Shared executor across all instances (one model loaded anyway)
+    _executor: ThreadPoolExecutor | None = None
+    _max_concurrent: int = 4  # Limit concurrent CPU-bound reranking calls
+
+    def __init__(
+        self,
+        model_name: str | None = None,
+        max_concurrent: int = 4,
+        force_cpu: bool = False,
+        trust_remote_code: bool = False,
+    ):
         """
-        Initialize SentenceTransformers cross-encoder.
+        Initialize local SentenceTransformers cross-encoder.
 
         Args:
             model_name: Name of the CrossEncoder model to use.
                        Default: cross-encoder/ms-marco-MiniLM-L-6-v2
+            max_concurrent: Maximum concurrent reranking calls (default: 2).
+                           Higher values may cause CPU thrashing under load.
+            force_cpu: Force CPU mode (avoids MPS/XPC issues on macOS in daemon mode).
+                      Default: False
+            trust_remote_code: Allow loading models with custom code (security risk).
+                              Required for some models like jina-reranker-v2-base-multilingual.
+                              Default: False (disabled for security)
         """
-        self.model_name = model_name
+        self.model_name = model_name or DEFAULT_RERANKER_LOCAL_MODEL
+        self.force_cpu = force_cpu
+        self.trust_remote_code = trust_remote_code
         self._model = None
+        LocalSTCrossEncoder._max_concurrent = max_concurrent
 
-    def load(self) -> None:
-        """Load the cross-encoder model."""
+    @property
+    def provider_name(self) -> str:
+        return "local"
+
+    async def initialize(self) -> None:
+        """Load the cross-encoder model and initialize the executor."""
         if self._model is not None:
             return
 
@@ -73,18 +140,82 @@ class SentenceTransformersCrossEncoder(CrossEncoderModel):
             from sentence_transformers import CrossEncoder
         except ImportError:
             raise ImportError(
-                "sentence-transformers is required for SentenceTransformersCrossEncoder. "
+                "sentence-transformers is required for LocalSTCrossEncoder. "
                 "Install it with: pip install sentence-transformers"
             )
 
-        logger.info(f"Loading cross-encoder model: {self.model_name}...")
-        self._model = CrossEncoder(self.model_name)
-        logger.info("Cross-encoder model loaded")
+        logger.info(f"Reranker: initializing local provider with model {self.model_name}")
 
-    def predict(self, pairs: List[Tuple[str, str]]) -> List[float]:
+        # Determine device based on hardware availability.
+        # We always set low_cpu_mem_usage=False to prevent lazy loading (meta tensors)
+        # which can cause issues when accelerate is installed but no GPU is available.
+        # Note: We do NOT use device_map because CrossEncoder internally calls .to(device)
+        # after loading, which conflicts with accelerate's device_map handling.
+        import torch
+
+        # Force CPU mode if configured (used in daemon mode to avoid MPS/XPC issues on macOS)
+        if self.force_cpu:
+            device = "cpu"
+            logger.info("Reranker: forcing CPU mode (HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU=1)")
+        else:
+            # Check for GPU (CUDA) or Apple Silicon (MPS)
+            # Wrap in try-except to gracefully handle any device detection issues
+            # (e.g., in CI environments or when PyTorch is built without GPU support)
+            device = "cpu"  # Default to CPU
+            try:
+                has_gpu = torch.cuda.is_available() or (
+                    hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
+                )
+                if has_gpu:
+                    device = None  # Let sentence-transformers auto-detect GPU/MPS
+            except Exception as e:
+                logger.warning(f"Failed to detect GPU/MPS, falling back to CPU: {e}")
+
+        # Suppress verbose transformers warnings during model loading
+        # This suppresses the "UNEXPECTED" warnings from CrossEncoder which are harmless
+        # but look alarming to users (e.g., "embeddings.position_ids | UNEXPECTED")
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", category=UserWarning)
+            warnings.filterwarnings("ignore", message=".*was not found in model state dict.*")
+            warnings.filterwarnings("ignore", message=".*UNEXPECTED.*")
+
+            # Also suppress transformers library logging temporarily
+            transformers_logger = logging.getLogger("transformers")
+            original_level = transformers_logger.level
+            transformers_logger.setLevel(logging.ERROR)
+
+            try:
+                self._model = CrossEncoder(
+                    self.model_name,
+                    device=device,
+                    model_kwargs={"low_cpu_mem_usage": False},
+                    trust_remote_code=self.trust_remote_code,
+                )
+            finally:
+                # Restore original logging level
+                transformers_logger.setLevel(original_level)
+
+        # Initialize shared executor (limited workers naturally limits concurrency)
+        if LocalSTCrossEncoder._executor is None:
+            LocalSTCrossEncoder._executor = ThreadPoolExecutor(
+                max_workers=LocalSTCrossEncoder._max_concurrent,
+                thread_name_prefix="reranker",
+            )
+            logger.info(f"Reranker: local provider initialized (max_concurrent={LocalSTCrossEncoder._max_concurrent})")
+        else:
+            logger.info("Reranker: local provider initialized (using existing executor)")
+
+    def _predict_sync(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """Synchronous prediction wrapper for thread pool execution."""
+        scores = self._model.predict(pairs, show_progress_bar=False)
+        return scores.tolist() if hasattr(scores, "tolist") else list(scores)
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
         """
         Score query-document pairs for relevance.
 
+        Uses a dedicated thread pool with limited workers to prevent CPU thrashing.
+
         Args:
             pairs: List of (query, document) tuples to score
 
@@ -92,6 +223,663 @@ class SentenceTransformersCrossEncoder(CrossEncoderModel):
             List of relevance scores (raw logits from the model)
         """
         if self._model is None:
-            self.load()
-        scores = self._model.predict(pairs, show_progress_bar=False)
-        return scores.tolist() if hasattr(scores, 'tolist') else list(scores)
+            raise RuntimeError("Reranker not initialized. Call initialize() first.")
+
+        # Use dedicated executor - limited workers naturally limits concurrency
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(
+            LocalSTCrossEncoder._executor,
+            self._predict_sync,
+            pairs,
+        )
+
+
+class RemoteTEICrossEncoder(CrossEncoderModel):
+    """
+    Remote cross-encoder implementation using HuggingFace Text Embeddings Inference (TEI) HTTP API.
+
+    TEI supports reranking via the /rerank endpoint.
+    See: https://github.com/huggingface/text-embeddings-inference
+
+    Note: The TEI server must be running a cross-encoder/reranker model.
+
+    Requests are made in parallel with configurable batch size and max concurrency (backpressure).
+    Uses a GLOBAL semaphore to limit concurrent requests across ALL recall operations.
+    """
+
+    # Global semaphore shared across all instances and calls to prevent thundering herd
+    _global_semaphore: asyncio.Semaphore | None = None
+    _global_max_concurrent: int = DEFAULT_RERANKER_TEI_MAX_CONCURRENT
+
+    def __init__(
+        self,
+        base_url: str,
+        timeout: float = 30.0,
+        batch_size: int = DEFAULT_RERANKER_TEI_BATCH_SIZE,
+        max_concurrent: int = DEFAULT_RERANKER_TEI_MAX_CONCURRENT,
+        max_retries: int = 3,
+        retry_delay: float = 0.5,
+    ):
+        """
+        Initialize remote TEI cross-encoder client.
+
+        Args:
+            base_url: Base URL of the TEI server (e.g., "http://localhost:8080")
+            timeout: Request timeout in seconds (default: 30.0)
+            batch_size: Maximum batch size for rerank requests (default: 128)
+            max_concurrent: Maximum concurrent requests for backpressure (default: 8).
+                           This is a GLOBAL limit across all parallel recall operations.
+            max_retries: Maximum number of retries for failed requests (default: 3)
+            retry_delay: Initial delay between retries in seconds, doubles each retry (default: 0.5)
+        """
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self.batch_size = batch_size
+        self.max_concurrent = max_concurrent
+        self.max_retries = max_retries
+        self.retry_delay = retry_delay
+        self._async_client: httpx.AsyncClient | None = None
+        self._model_id: str | None = None
+
+        # Update global semaphore if max_concurrent changed
+        if (
+            RemoteTEICrossEncoder._global_semaphore is None
+            or RemoteTEICrossEncoder._global_max_concurrent != max_concurrent
+        ):
+            RemoteTEICrossEncoder._global_max_concurrent = max_concurrent
+            RemoteTEICrossEncoder._global_semaphore = asyncio.Semaphore(max_concurrent)
+
+    @property
+    def provider_name(self) -> str:
+        return "tei"
+
+    async def _async_request_with_retry(
+        self,
+        client: httpx.AsyncClient,
+        semaphore: asyncio.Semaphore,
+        method: str,
+        url: str,
+        **kwargs,
+    ) -> httpx.Response:
+        """Make an async HTTP request with automatic retries on transient errors and semaphore for backpressure."""
+        last_error = None
+        delay = self.retry_delay
+
+        async with semaphore:
+            for attempt in range(self.max_retries + 1):
+                try:
+                    if method == "GET":
+                        response = await client.get(url, **kwargs)
+                    else:
+                        response = await client.post(url, **kwargs)
+                    response.raise_for_status()
+                    return response
+                except (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout) as e:
+                    last_error = e
+                    if attempt < self.max_retries:
+                        logger.warning(
+                            f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. "
+                            f"Retrying in {delay}s..."
+                        )
+                        await asyncio.sleep(delay)
+                        delay *= 2  # Exponential backoff
+                except httpx.HTTPStatusError as e:
+                    # Retry on 5xx server errors
+                    if e.response.status_code >= 500 and attempt < self.max_retries:
+                        last_error = e
+                        logger.warning(
+                            f"TEI server error (attempt {attempt + 1}/{self.max_retries + 1}): {e}. "
+                            f"Retrying in {delay}s..."
+                        )
+                        await asyncio.sleep(delay)
+                        delay *= 2
+                    else:
+                        raise
+
+        raise last_error
+
+    async def initialize(self) -> None:
+        """Initialize the HTTP client and verify server connectivity."""
+        if self._async_client is not None:
+            return
+
+        logger.info(
+            f"Reranker: initializing TEI provider at {self.base_url} "
+            f"(batch_size={self.batch_size}, max_concurrent={self.max_concurrent})"
+        )
+        self._async_client = httpx.AsyncClient(timeout=self.timeout)
+
+        # Verify server is reachable and get model info
+        # Use a temporary semaphore for initialization
+        init_semaphore = asyncio.Semaphore(1)
+        try:
+            response = await self._async_request_with_retry(
+                self._async_client, init_semaphore, "GET", f"{self.base_url}/info"
+            )
+            info = response.json()
+            self._model_id = info.get("model_id", "unknown")
+            logger.info(f"Reranker: TEI provider initialized (model: {self._model_id})")
+        except httpx.HTTPError as e:
+            self._async_client = None
+            raise RuntimeError(f"Failed to connect to TEI server at {self.base_url}: {e}")
+
+    async def _rerank_query_group(
+        self,
+        client: httpx.AsyncClient,
+        semaphore: asyncio.Semaphore,
+        query: str,
+        texts: list[str],
+    ) -> list[tuple[int, float]]:
+        """Rerank a single query group and return list of (original_index, score) tuples."""
+        try:
+            response = await self._async_request_with_retry(
+                client,
+                semaphore,
+                "POST",
+                f"{self.base_url}/rerank",
+                json={
+                    "query": query,
+                    "texts": texts,
+                    "return_text": False,
+                },
+            )
+            results = response.json()
+            # TEI returns results sorted by score descending, with original index
+            return [(result["index"], result["score"]) for result in results]
+        except httpx.HTTPError as e:
+            raise RuntimeError(f"TEI rerank request failed: {e}")
+
+    async def _predict_async(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """Async implementation of predict that runs requests in parallel with backpressure."""
+        if not pairs:
+            return []
+
+        # Group all pairs by query
+        query_groups: dict[str, list[tuple[int, str]]] = {}
+        for idx, (query, text) in enumerate(pairs):
+            if query not in query_groups:
+                query_groups[query] = []
+            query_groups[query].append((idx, text))
+
+        # Split each query group into batches
+        tasks_info: list[tuple[str, list[int], list[str]]] = []  # (query, indices, texts)
+        for query, indexed_texts in query_groups.items():
+            indices = [idx for idx, _ in indexed_texts]
+            texts = [text for _, text in indexed_texts]
+
+            # Split into batches
+            for i in range(0, len(texts), self.batch_size):
+                batch_indices = indices[i : i + self.batch_size]
+                batch_texts = texts[i : i + self.batch_size]
+                tasks_info.append((query, batch_indices, batch_texts))
+
+        # Run all requests in parallel with GLOBAL semaphore for backpressure
+        # This ensures max_concurrent is respected across ALL parallel recall operations
+        all_scores = [0.0] * len(pairs)
+        semaphore = RemoteTEICrossEncoder._global_semaphore
+
+        tasks = [
+            self._rerank_query_group(self._async_client, semaphore, query, texts) for query, _, texts in tasks_info
+        ]
+        results = await asyncio.gather(*tasks)
+
+        # Map scores back to original positions
+        for (_, indices, _), result_scores in zip(tasks_info, results):
+            for original_idx_in_batch, score in result_scores:
+                global_idx = indices[original_idx_in_batch]
+                all_scores[global_idx] = score
+
+        return all_scores
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """
+        Score query-document pairs using the remote TEI reranker.
+
+        Requests are made in parallel with configurable backpressure.
+
+        Args:
+            pairs: List of (query, document) tuples to score
+
+        Returns:
+            List of relevance scores
+        """
+        if self._async_client is None:
+            raise RuntimeError("Reranker not initialized. Call initialize() first.")
+
+        return await self._predict_async(pairs)
+
+
+class CohereCrossEncoder(CrossEncoderModel):
+    """
+    Cohere cross-encoder implementation using the Cohere Rerank API.
+
+    Supports rerank-english-v3.0 and rerank-multilingual-v3.0 models.
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        model: str = DEFAULT_RERANKER_COHERE_MODEL,
+        base_url: str | None = None,
+        timeout: float = 60.0,
+    ):
+        """
+        Initialize Cohere cross-encoder client.
+
+        Args:
+            api_key: Cohere API key
+            model: Cohere rerank model name (default: rerank-english-v3.0)
+            base_url: Custom base URL for Cohere-compatible API (e.g., Azure-hosted endpoint)
+            timeout: Request timeout in seconds (default: 60.0)
+        """
+        self.api_key = api_key
+        self.model = model
+        self.base_url = base_url
+        self.timeout = timeout
+        self._client = None
+
+    @property
+    def provider_name(self) -> str:
+        return "cohere"
+
+    async def initialize(self) -> None:
+        """Initialize the Cohere client."""
+        if self._client is not None:
+            return
+
+        try:
+            import cohere
+        except ImportError:
+            raise ImportError("cohere is required for CohereCrossEncoder. Install it with: pip install cohere")
+
+        base_url_msg = f" at {self.base_url}" if self.base_url else ""
+        logger.info(f"Reranker: initializing Cohere provider with model {self.model}{base_url_msg}")
+
+        # Build client kwargs, only including base_url if set (for Azure or custom endpoints)
+        client_kwargs = {"api_key": self.api_key, "timeout": self.timeout}
+        if self.base_url:
+            client_kwargs["base_url"] = self.base_url
+        self._client = cohere.Client(**client_kwargs)
+        logger.info("Reranker: Cohere provider initialized")
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """
+        Score query-document pairs using the Cohere Rerank API.
+
+        Args:
+            pairs: List of (query, document) tuples to score
+
+        Returns:
+            List of relevance scores
+        """
+        if self._client is None:
+            raise RuntimeError("Reranker not initialized. Call initialize() first.")
+
+        if not pairs:
+            return []
+
+        # Run sync Cohere API calls in thread pool
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, self._predict_sync, pairs)
+
+    def _predict_sync(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """Synchronous predict implementation for Cohere API."""
+        # Group pairs by query for efficient batching
+        # Cohere rerank expects one query with multiple documents
+        query_groups: dict[str, list[tuple[int, str]]] = {}
+        for idx, (query, text) in enumerate(pairs):
+            if query not in query_groups:
+                query_groups[query] = []
+            query_groups[query].append((idx, text))
+
+        all_scores = [0.0] * len(pairs)
+
+        for query, indexed_texts in query_groups.items():
+            texts = [text for _, text in indexed_texts]
+            indices = [idx for idx, _ in indexed_texts]
+
+            response = self._client.rerank(
+                query=query,
+                documents=texts,
+                model=self.model,
+                return_documents=False,
+            )
+
+            # Map scores back to original positions
+            for result in response.results:
+                original_idx = result.index
+                score = result.relevance_score
+                all_scores[indices[original_idx]] = score
+
+        return all_scores
+
+
+class RRFPassthroughCrossEncoder(CrossEncoderModel):
+    """
+    Passthrough cross-encoder that preserves RRF scores without neural reranking.
+
+    This is useful for:
+    - Testing retrieval quality without reranking overhead
+    - Deployments where reranking latency is unacceptable
+    - Debugging to isolate retrieval vs reranking issues
+    """
+
+    def __init__(self):
+        """Initialize RRF passthrough cross-encoder."""
+        pass
+
+    @property
+    def provider_name(self) -> str:
+        return "rrf"
+
+    async def initialize(self) -> None:
+        """No initialization needed."""
+        logger.info("Reranker: RRF passthrough provider initialized (neural reranking disabled)")
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """
+        Return neutral scores - actual ranking uses RRF scores from retrieval.
+
+        Args:
+            pairs: List of (query, document) tuples (ignored)
+
+        Returns:
+            List of 0.5 scores (neutral, lets RRF scores dominate)
+        """
+        # Return neutral scores so RRF ranking is preserved
+        return [0.5] * len(pairs)
+
+
+class FlashRankCrossEncoder(CrossEncoderModel):
+    """
+    FlashRank cross-encoder implementation.
+
+    FlashRank is an ultra-lite reranking library that runs on CPU without
+    requiring PyTorch or Transformers. It's ideal for serverless deployments
+    with minimal cold-start overhead.
+
+    Available models:
+    - ms-marco-TinyBERT-L-2-v2: Fastest, ~4MB
+    - ms-marco-MiniLM-L-12-v2: Best quality, ~34MB (default)
+    - rank-T5-flan: Best zero-shot, ~110MB
+    - ms-marco-MultiBERT-L-12: Multi-lingual, ~150MB
+    """
+
+    # Shared executor for CPU-bound reranking
+    _executor: ThreadPoolExecutor | None = None
+    _max_concurrent: int = 4
+
+    def __init__(
+        self,
+        model_name: str | None = None,
+        cache_dir: str | None = None,
+        max_length: int = 512,
+        max_concurrent: int = 4,
+    ):
+        """
+        Initialize FlashRank cross-encoder.
+
+        Args:
+            model_name: FlashRank model name. Default: ms-marco-MiniLM-L-12-v2
+            cache_dir: Directory to cache downloaded models. Default: system cache
+            max_length: Maximum sequence length for reranking. Default: 512
+            max_concurrent: Maximum concurrent reranking calls. Default: 4
+        """
+        self.model_name = model_name or DEFAULT_RERANKER_FLASHRANK_MODEL
+        self.cache_dir = cache_dir or DEFAULT_RERANKER_FLASHRANK_CACHE_DIR
+        self.max_length = max_length
+        self._ranker = None
+        FlashRankCrossEncoder._max_concurrent = max_concurrent
+
+    @property
+    def provider_name(self) -> str:
+        return "flashrank"
+
+    async def initialize(self) -> None:
+        """Load the FlashRank model."""
+        if self._ranker is not None:
+            return
+
+        try:
+            from flashrank import Ranker
+        except ImportError:
+            raise ImportError("flashrank is required for FlashRankCrossEncoder. Install it with: pip install flashrank")
+
+        logger.info(f"Reranker: initializing FlashRank provider with model {self.model_name}")
+
+        # Initialize ranker with optional cache directory
+        ranker_kwargs = {"model_name": self.model_name, "max_length": self.max_length}
+        if self.cache_dir:
+            ranker_kwargs["cache_dir"] = self.cache_dir
+
+        self._ranker = Ranker(**ranker_kwargs)
+
+        # Initialize shared executor
+        if FlashRankCrossEncoder._executor is None:
+            FlashRankCrossEncoder._executor = ThreadPoolExecutor(
+                max_workers=FlashRankCrossEncoder._max_concurrent,
+                thread_name_prefix="flashrank",
+            )
+            logger.info(
+                f"Reranker: FlashRank provider initialized (max_concurrent={FlashRankCrossEncoder._max_concurrent})"
+            )
+        else:
+            logger.info("Reranker: FlashRank provider initialized (using existing executor)")
+
+    def _predict_sync(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """Synchronous predict - processes each query group."""
+        from flashrank import RerankRequest
+
+        if not pairs:
+            return []
+
+        # Group pairs by query
+        query_groups: dict[str, list[tuple[int, str]]] = {}
+        for idx, (query, text) in enumerate(pairs):
+            if query not in query_groups:
+                query_groups[query] = []
+            query_groups[query].append((idx, text))
+
+        all_scores = [0.0] * len(pairs)
+
+        for query, indexed_texts in query_groups.items():
+            # Build passages list for FlashRank
+            passages = [{"id": i, "text": text} for i, (_, text) in enumerate(indexed_texts)]
+            global_indices = [idx for idx, _ in indexed_texts]
+
+            # Create rerank request
+            request = RerankRequest(query=query, passages=passages)
+            results = self._ranker.rerank(request)
+
+            # Map scores back to original positions
+            for result in results:
+                local_idx = result["id"]
+                score = result["score"]
+                global_idx = global_indices[local_idx]
+                all_scores[global_idx] = score
+
+        return all_scores
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """
+        Score query-document pairs using FlashRank.
+
+        Args:
+            pairs: List of (query, document) tuples to score
+
+        Returns:
+            List of relevance scores (higher = more relevant)
+        """
+        if self._ranker is None:
+            raise RuntimeError("Reranker not initialized. Call initialize() first.")
+
+        # Run in thread pool to avoid blocking event loop
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(FlashRankCrossEncoder._executor, self._predict_sync, pairs)
+
+
+class LiteLLMCrossEncoder(CrossEncoderModel):
+    """
+    LiteLLM cross-encoder implementation using LiteLLM proxy's /rerank endpoint.
+
+    LiteLLM provides a unified interface for multiple reranking providers via
+    the Cohere-compatible /rerank endpoint.
+    See: https://docs.litellm.ai/docs/rerank
+
+    Supported providers via LiteLLM:
+    - Cohere (rerank-english-v3.0, etc.) - prefix with cohere/
+    - Together AI - prefix with together_ai/
+    - Azure AI - prefix with azure_ai/
+    - Jina AI - prefix with jina_ai/
+    - AWS Bedrock - prefix with bedrock/
+    - Voyage AI - prefix with voyage/
+    """
+
+    def __init__(
+        self,
+        api_base: str = DEFAULT_LITELLM_API_BASE,
+        api_key: str | None = None,
+        model: str = DEFAULT_RERANKER_LITELLM_MODEL,
+        timeout: float = 60.0,
+    ):
+        """
+        Initialize LiteLLM cross-encoder client.
+
+        Args:
+            api_base: Base URL of the LiteLLM proxy (default: http://localhost:4000)
+            api_key: API key for the LiteLLM proxy (optional, depends on proxy config)
+            model: Reranking model name (default: cohere/rerank-english-v3.0)
+                   Use provider prefix (e.g., cohere/, together_ai/, voyage/)
+            timeout: Request timeout in seconds (default: 60.0)
+        """
+        self.api_base = api_base.rstrip("/")
+        self.api_key = api_key
+        self.model = model
+        self.timeout = timeout
+        self._async_client: httpx.AsyncClient | None = None
+
+    @property
+    def provider_name(self) -> str:
+        return "litellm"
+
+    async def initialize(self) -> None:
+        """Initialize the async HTTP client."""
+        if self._async_client is not None:
+            return
+
+        logger.info(f"Reranker: initializing LiteLLM provider at {self.api_base} with model {self.model}")
+
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+
+        self._async_client = httpx.AsyncClient(timeout=self.timeout, headers=headers)
+        logger.info("Reranker: LiteLLM provider initialized")
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """
+        Score query-document pairs using the LiteLLM proxy's /rerank endpoint.
+
+        Args:
+            pairs: List of (query, document) tuples to score
+
+        Returns:
+            List of relevance scores
+        """
+        if self._async_client is None:
+            raise RuntimeError("Reranker not initialized. Call initialize() first.")
+
+        if not pairs:
+            return []
+
+        # Group pairs by query (LiteLLM rerank expects one query with multiple documents)
+        query_groups: dict[str, list[tuple[int, str]]] = {}
+        for idx, (query, text) in enumerate(pairs):
+            if query not in query_groups:
+                query_groups[query] = []
+            query_groups[query].append((idx, text))
+
+        all_scores = [0.0] * len(pairs)
+
+        for query, indexed_texts in query_groups.items():
+            texts = [text for _, text in indexed_texts]
+            indices = [idx for idx, _ in indexed_texts]
+
+            # LiteLLM /rerank follows Cohere API format
+            response = await self._async_client.post(
+                f"{self.api_base}/rerank",
+                json={
+                    "model": self.model,
+                    "query": query,
+                    "documents": texts,
+                    "top_n": len(texts),  # Return all scores
+                },
+            )
+            response.raise_for_status()
+            result = response.json()
+
+            # Map scores back to original positions
+            # Response format: {"results": [{"index": 0, "relevance_score": 0.9}, ...]}
+            for item in result.get("results", []):
+                original_idx = item["index"]
+                score = item.get("relevance_score", item.get("score", 0.0))
+                all_scores[indices[original_idx]] = score
+
+        return all_scores
+
+
+def create_cross_encoder_from_env() -> CrossEncoderModel:
+    """
+    Create a CrossEncoderModel instance based on configuration.
+
+    Reads configuration via get_config() to ensure consistency across the codebase.
+
+    Returns:
+        Configured CrossEncoderModel instance
+    """
+    from ..config import get_config
+
+    config = get_config()
+    provider = config.reranker_provider.lower()
+
+    if provider == "tei":
+        url = config.reranker_tei_url
+        if not url:
+            raise ValueError(f"{ENV_RERANKER_TEI_URL} is required when {ENV_RERANKER_PROVIDER} is 'tei'")
+        return RemoteTEICrossEncoder(
+            base_url=url,
+            batch_size=config.reranker_tei_batch_size,
+            max_concurrent=config.reranker_tei_max_concurrent,
+        )
+    elif provider == "local":
+        return LocalSTCrossEncoder(
+            model_name=config.reranker_local_model,
+            max_concurrent=config.reranker_local_max_concurrent,
+            force_cpu=config.reranker_local_force_cpu,
+            trust_remote_code=config.reranker_local_trust_remote_code,
+        )
+    elif provider == "cohere":
+        api_key = config.reranker_cohere_api_key
+        if not api_key:
+            raise ValueError(f"{ENV_RERANKER_COHERE_API_KEY} is required when {ENV_RERANKER_PROVIDER} is 'cohere'")
+        return CohereCrossEncoder(
+            api_key=api_key,
+            model=config.reranker_cohere_model,
+            base_url=config.reranker_cohere_base_url,
+        )
+    elif provider == "flashrank":
+        model = os.environ.get(ENV_RERANKER_FLASHRANK_MODEL, DEFAULT_RERANKER_FLASHRANK_MODEL)
+        cache_dir = os.environ.get(ENV_RERANKER_FLASHRANK_CACHE_DIR, DEFAULT_RERANKER_FLASHRANK_CACHE_DIR)
+        return FlashRankCrossEncoder(model_name=model, cache_dir=cache_dir)
+    elif provider == "litellm":
+        return LiteLLMCrossEncoder(
+            api_base=config.reranker_litellm_api_base,
+            api_key=config.reranker_litellm_api_key,
+            model=config.reranker_litellm_model,
+        )
+    elif provider == "rrf":
+        return RRFPassthroughCrossEncoder()
+    else:
+        raise ValueError(
+            f"Unknown reranker provider: {provider}. Supported: 'local', 'tei', 'cohere', 'flashrank', 'litellm', 'rrf'"
+        )

hindsight-api/hindsight_api/engine/db_budget.py

@@ -0,0 +1,284 @@
+"""
+Database connection budget management.
+
+Limits concurrent database connections per operation to prevent
+a single operation (e.g., recall with parallel queries) from
+exhausting the connection pool.
+"""
+
+import asyncio
+import logging
+import uuid
+from contextlib import asynccontextmanager
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, AsyncIterator
+
+if TYPE_CHECKING:
+    import asyncpg
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class OperationBudget:
+    """
+    Tracks connection budget for a single operation.
+
+    Each operation gets a semaphore limiting its concurrent connections.
+    """
+
+    operation_id: str
+    max_connections: int
+    semaphore: asyncio.Semaphore = field(init=False)
+    active_count: int = field(default=0, init=False)
+
+    def __post_init__(self):
+        self.semaphore = asyncio.Semaphore(self.max_connections)
+
+
+class ConnectionBudgetManager:
+    """
+    Manages per-operation connection budgets.
+
+    Usage:
+        manager = ConnectionBudgetManager(default_budget=4)
+
+        # Start an operation
+        async with manager.operation(max_connections=2) as op:
+            # Acquire connections within the budget
+            async with op.acquire(pool) as conn:
+                await conn.fetch(...)
+
+            # Multiple connections respect the budget
+            async with op.acquire(pool) as conn1, op.acquire(pool) as conn2:
+                # At most 2 concurrent connections for this operation
+                ...
+    """
+
+    def __init__(self, default_budget: int = 4):
+        """
+        Initialize the budget manager.
+
+        Args:
+            default_budget: Default max connections per operation
+        """
+        self.default_budget = default_budget
+        self._operations: dict[str, OperationBudget] = {}
+        self._lock = asyncio.Lock()
+
+    @asynccontextmanager
+    async def operation(
+        self,
+        max_connections: int | None = None,
+        operation_id: str | None = None,
+    ) -> AsyncIterator["BudgetedOperation"]:
+        """
+        Create a budgeted operation context.
+
+        Args:
+            max_connections: Max concurrent connections for this operation.
+                           Defaults to manager's default_budget.
+            operation_id: Optional custom operation ID. Auto-generated if not provided.
+
+        Yields:
+            BudgetedOperation context for acquiring connections
+        """
+        op_id = operation_id or f"op-{uuid.uuid4().hex[:12]}"
+        budget = max_connections or self.default_budget
+
+        async with self._lock:
+            if op_id in self._operations:
+                raise ValueError(f"Operation {op_id} already exists")
+            self._operations[op_id] = OperationBudget(op_id, budget)
+
+        try:
+            yield BudgetedOperation(self, op_id)
+        finally:
+            async with self._lock:
+                self._operations.pop(op_id, None)
+
+    def _get_budget(self, operation_id: str) -> OperationBudget:
+        """Get budget for an operation (internal use)."""
+        budget = self._operations.get(operation_id)
+        if not budget:
+            raise ValueError(f"Operation {operation_id} not found")
+        return budget
+
+
+class BudgetedOperation:
+    """
+    A single operation with connection budget.
+
+    Provides methods to acquire connections within the budget.
+    """
+
+    def __init__(self, manager: ConnectionBudgetManager, operation_id: str):
+        self._manager = manager
+        self.operation_id = operation_id
+
+    @property
+    def budget(self) -> OperationBudget:
+        """Get the budget for this operation."""
+        return self._manager._get_budget(self.operation_id)
+
+    @asynccontextmanager
+    async def acquire(self, pool: "asyncpg.Pool") -> AsyncIterator["asyncpg.Connection"]:
+        """
+        Acquire a connection within the operation's budget.
+
+        Blocks if the operation has reached its connection limit.
+
+        Args:
+            pool: asyncpg connection pool
+
+        Yields:
+            Database connection
+        """
+        budget = self.budget
+        async with budget.semaphore:
+            budget.active_count += 1
+            conn = await pool.acquire()
+            try:
+                yield conn
+            finally:
+                budget.active_count -= 1
+                await pool.release(conn)
+
+    def wrap_pool(self, pool: "asyncpg.Pool") -> "BudgetedPool":
+        """
+        Wrap a pool with this operation's budget.
+
+        The returned BudgetedPool can be passed to functions expecting a pool,
+        and all acquire() calls will be limited by this operation's budget.
+
+        Args:
+            pool: asyncpg connection pool to wrap
+
+        Returns:
+            BudgetedPool that limits connections to this operation's budget
+        """
+        return BudgetedPool(pool, self)
+
+    async def acquire_many(
+        self,
+        pool: "asyncpg.Pool",
+        count: int,
+    ) -> AsyncIterator[list["asyncpg.Connection"]]:
+        """
+        Acquire multiple connections within the budget.
+
+        Note: This acquires connections sequentially to respect the budget.
+        For parallel acquisition, use multiple acquire() calls with asyncio.gather().
+
+        Args:
+            pool: asyncpg connection pool
+            count: Number of connections to acquire
+
+        Yields:
+            List of database connections
+        """
+        connections = []
+        try:
+            for _ in range(count):
+                conn = await pool.acquire()
+                connections.append(conn)
+            yield connections
+        finally:
+            for conn in connections:
+                await pool.release(conn)
+
+
+# Global default manager instance
+_default_manager: ConnectionBudgetManager | None = None
+
+
+def get_budget_manager(default_budget: int = 4) -> ConnectionBudgetManager:
+    """
+    Get or create the global budget manager.
+
+    Args:
+        default_budget: Default max connections per operation
+
+    Returns:
+        Global ConnectionBudgetManager instance
+    """
+    global _default_manager
+    if _default_manager is None:
+        _default_manager = ConnectionBudgetManager(default_budget=default_budget)
+    return _default_manager
+
+
+@asynccontextmanager
+async def budgeted_operation(
+    max_connections: int | None = None,
+    operation_id: str | None = None,
+    default_budget: int = 4,
+) -> AsyncIterator[BudgetedOperation]:
+    """
+    Convenience function to create a budgeted operation.
+
+    Args:
+        max_connections: Max concurrent connections for this operation
+        operation_id: Optional custom operation ID
+        default_budget: Default budget if manager not yet created
+
+    Yields:
+        BudgetedOperation context
+
+    Example:
+        async with budgeted_operation(max_connections=2) as op:
+            async with op.acquire(pool) as conn:
+                await conn.fetch(...)
+    """
+    manager = get_budget_manager(default_budget)
+    async with manager.operation(max_connections, operation_id) as op:
+        yield op
+
+
+class BudgetedPool:
+    """
+    A pool wrapper that limits concurrent connection acquisitions.
+
+    This can be passed to functions expecting a pool, and acquire()
+    calls will be limited by the budget semaphore.
+
+    Usage:
+        async with budgeted_operation(max_connections=4) as op:
+            budgeted_pool = op.wrap_pool(pool)
+            # Pass budgeted_pool to functions that expect a pool
+            await some_function(budgeted_pool, ...)
+    """
+
+    def __init__(self, pool: "asyncpg.Pool", operation: BudgetedOperation):
+        self._pool = pool
+        self._operation = operation
+
+    async def acquire(self) -> "asyncpg.Connection":
+        """
+        Acquire a connection within the budget.
+
+        Note: Caller must release the connection when done.
+        Prefer using as context manager via acquire_with_retry or op.acquire().
+        """
+        budget = self._operation.budget
+        await budget.semaphore.acquire()
+        budget.active_count += 1
+        try:
+            return await self._pool.acquire()
+        except Exception:
+            budget.active_count -= 1
+            budget.semaphore.release()
+            raise
+
+    async def release(self, conn: "asyncpg.Connection") -> None:
+        """Release a connection back to the pool."""
+        budget = self._operation.budget
+        try:
+            await self._pool.release(conn)
+        finally:
+            budget.active_count -= 1
+            budget.semaphore.release()
+
+    def __getattr__(self, name):
+        """Proxy other attributes to the underlying pool."""
+        return getattr(self._pool, name)

hindsight-api/hindsight_api/engine/db_utils.py

@@ -1,9 +1,11 @@
 """
 Database utility functions for connection management with retry logic.
 """
+
 import asyncio
 import logging
 from contextlib import asynccontextmanager
+
 import asyncpg
 
 logger = logging.getLogger(__name__)
@@ -54,16 +56,14 @@ async def retry_with_backoff(
         except retryable_exceptions as e:
             last_exception = e
             if attempt < max_retries:
-                delay = min(base_delay * (2 ** attempt), max_delay)
+                delay = min(base_delay * (2**attempt), max_delay)
                 logger.warning(
                     f"Database operation failed (attempt {attempt + 1}/{max_retries + 1}): {e}. "
                     f"Retrying in {delay:.1f}s..."
                 )
                 await asyncio.sleep(delay)
             else:
-                logger.error(
-                    f"Database operation failed after {max_retries + 1} attempts: {e}"
-                )
+                logger.error(f"Database operation failed after {max_retries + 1} attempts: {e}")
     raise last_exception
 
 
@@ -83,10 +83,22 @@ async def acquire_with_retry(pool: asyncpg.Pool, max_retries: int = DEFAULT_MAX_
     Yields:
         An asyncpg connection
     """
+    import time
+
+    start = time.time()
+
     async def acquire():
         return await pool.acquire()
 
     conn = await retry_with_backoff(acquire, max_retries=max_retries)
+    acquire_time = time.time() - start
+
+    # Log slow connection acquisitions (indicates pool contention)
+    if acquire_time > 0.05:  # 50ms threshold
+        pool_size = pool.get_size()
+        pool_free = pool.get_idle_size()
+        logger.warning(f"[DB POOL] Slow acquire: {acquire_time:.3f}s | size={pool_size}, idle={pool_free}")
+
     try:
         yield conn
     finally:

hindsight-api/hindsight_api/engine/directives/__init__.py

@@ -0,0 +1,5 @@
+"""Directives module for hard rules injected into prompts."""
+
+from .models import Directive
+
+__all__ = ["Directive"]

hindsight-api/hindsight_api/engine/directives/models.py

@@ -0,0 +1,37 @@
+"""Pydantic models for directives."""
+
+from datetime import datetime, timezone
+from uuid import UUID
+
+from pydantic import BaseModel, Field
+
+
+class Directive(BaseModel):
+    """A directive is a hard rule injected into prompts.
+
+    Directives are user-defined rules that guide agent behavior. Unlike mental models
+    which are automatically consolidated from memories, directives are explicit
+    instructions that are always included in relevant prompts.
+
+    Examples:
+    - "Always respond in formal English"
+    - "Never share personal data with third parties"
+    - "Prefer conservative investment recommendations"
+    """
+
+    id: UUID = Field(description="Unique identifier")
+    bank_id: str = Field(description="Bank this directive belongs to")
+    name: str = Field(description="Human-readable name")
+    content: str = Field(description="The directive text to inject into prompts")
+    priority: int = Field(default=0, description="Higher priority directives are injected first")
+    is_active: bool = Field(default=True, description="Whether this directive is currently active")
+    tags: list[str] = Field(default_factory=list, description="Tags for filtering")
+    created_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc), description="When this directive was created"
+    )
+    updated_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc), description="When this directive was last updated"
+    )
+
+    class Config:
+        from_attributes = True

hindsight-api/hindsight_api/engine/embeddings.py

@@ -3,74 +3,125 @@ Embeddings abstraction for the memory system.
 
 Provides an interface for generating embeddings with different backends.
 
-IMPORTANT: All embeddings must produce 384-dimensional vectors to match
-the database schema (pgvector column defined as vector(384)).
+The embedding dimension is auto-detected from the model at initialization.
+The database schema is automatically adjusted to match the model's dimension.
+
+Configuration via environment variables - see hindsight_api.config for all env var names.
 """
-from abc import ABC, abstractmethod
-from typing import List
+
 import logging
+import os
+import warnings
+from abc import ABC, abstractmethod
+
+import httpx
+
+from ..config import (
+    DEFAULT_EMBEDDINGS_COHERE_MODEL,
+    DEFAULT_EMBEDDINGS_LITELLM_MODEL,
+    DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU,
+    DEFAULT_EMBEDDINGS_LOCAL_MODEL,
+    DEFAULT_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE,
+    DEFAULT_EMBEDDINGS_OPENAI_MODEL,
+    DEFAULT_EMBEDDINGS_PROVIDER,
+    DEFAULT_LITELLM_API_BASE,
+    ENV_EMBEDDINGS_COHERE_API_KEY,
+    ENV_EMBEDDINGS_LOCAL_FORCE_CPU,
+    ENV_EMBEDDINGS_LOCAL_MODEL,
+    ENV_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE,
+    ENV_EMBEDDINGS_OPENAI_API_KEY,
+    ENV_EMBEDDINGS_OPENAI_BASE_URL,
+    ENV_EMBEDDINGS_OPENAI_MODEL,
+    ENV_EMBEDDINGS_PROVIDER,
+    ENV_EMBEDDINGS_TEI_URL,
+    ENV_LLM_API_KEY,
+)
 
 logger = logging.getLogger(__name__)
 
-# Fixed embedding dimension required by database schema
-EMBEDDING_DIMENSION = 384
-
 
 class Embeddings(ABC):
     """
     Abstract base class for embedding generation.
 
-    All implementations MUST generate 384-dimensional embeddings to match
-    the database schema.
+    The embedding dimension is determined by the model and detected at initialization.
+    The database schema is automatically adjusted to match the model's dimension.
     """
 
+    @property
     @abstractmethod
-    def load(self) -> None:
-        """
-        Load the embedding model.
+    def provider_name(self) -> str:
+        """Return a human-readable name for this provider (e.g., 'local', 'tei')."""
+        pass
 
-        This should be called during initialization to load the model
+    @property
+    @abstractmethod
+    def dimension(self) -> int:
+        """Return the embedding dimension produced by this model."""
+        pass
+
+    @abstractmethod
+    async def initialize(self) -> None:
+        """
+        Initialize the embedding model asynchronously.
+
+        This should be called during startup to load/connect to the model
         and avoid cold start latency on first encode() call.
         """
         pass
 
     @abstractmethod
-    def encode(self, texts: List[str]) -> List[List[float]]:
+    def encode(self, texts: list[str]) -> list[list[float]]:
         """
-        Generate 384-dimensional embeddings for a list of texts.
+        Generate embeddings for a list of texts.
 
         Args:
             texts: List of text strings to encode
 
         Returns:
-            List of 384-dimensional embedding vectors (each is a list of floats)
+            List of embedding vectors (each is a list of floats)
         """
         pass
 
 
-class SentenceTransformersEmbeddings(Embeddings):
+class LocalSTEmbeddings(Embeddings):
     """
-    Embeddings implementation using SentenceTransformers.
+    Local embeddings implementation using SentenceTransformers.
 
-    Call load() during initialization to load the model and avoid cold starts.
-
-    Default model is BAAI/bge-small-en-v1.5 which produces 384-dimensional
-    embeddings matching the database schema.
+    Call initialize() during startup to load the model and avoid cold starts.
+    The embedding dimension is auto-detected from the model.
     """
 
-    def __init__(self, model_name: str = "BAAI/bge-small-en-v1.5"):
+    def __init__(self, model_name: str | None = None, force_cpu: bool = False, trust_remote_code: bool = False):
         """
-        Initialize SentenceTransformers embeddings.
+        Initialize local SentenceTransformers embeddings.
 
         Args:
             model_name: Name of the SentenceTransformer model to use.
-                       Must produce 384-dimensional embeddings.
                        Default: BAAI/bge-small-en-v1.5
+            force_cpu: Force CPU mode (avoids MPS/XPC issues on macOS in daemon mode).
+                      Default: False
+            trust_remote_code: Allow loading models with custom code (security risk).
+                              Required for some models with custom architectures.
+                              Default: False (disabled for security)
         """
-        self.model_name = model_name
+        self.model_name = model_name or DEFAULT_EMBEDDINGS_LOCAL_MODEL
+        self.force_cpu = force_cpu
+        self.trust_remote_code = trust_remote_code
         self._model = None
+        self._dimension: int | None = None
 
-    def load(self) -> None:
+    @property
+    def provider_name(self) -> str:
+        return "local"
+
+    @property
+    def dimension(self) -> int:
+        if self._dimension is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+        return self._dimension
+
+    async def initialize(self) -> None:
         """Load the embedding model."""
         if self._model is not None:
             return
@@ -79,35 +130,648 @@ class SentenceTransformersEmbeddings(Embeddings):
             from sentence_transformers import SentenceTransformer
         except ImportError:
             raise ImportError(
-                "sentence-transformers is required for SentenceTransformersEmbeddings. "
+                "sentence-transformers is required for LocalSTEmbeddings. "
                 "Install it with: pip install sentence-transformers"
             )
 
-        logger.info(f"Loading embedding model: {self.model_name}...")
-        self._model = SentenceTransformer(self.model_name)
+        logger.info(f"Embeddings: initializing local provider with model {self.model_name}")
 
-        # Validate dimension matches database schema
-        model_dim = self._model.get_sentence_embedding_dimension()
-        if model_dim != EMBEDDING_DIMENSION:
-            raise ValueError(
-                f"Model {self.model_name} produces {model_dim}-dimensional embeddings, "
-                f"but database schema requires {EMBEDDING_DIMENSION} dimensions. "
-                f"Use a model that produces {EMBEDDING_DIMENSION}-dimensional embeddings."
-            )
+        # Determine device based on hardware availability.
+        # We always set low_cpu_mem_usage=False to prevent lazy loading (meta tensors)
+        # which can cause issues when accelerate is installed but no GPU is available.
+        import torch
 
-        logger.info(f"Model loaded (embedding dim: {model_dim})")
+        # Force CPU mode if configured (used in daemon mode to avoid MPS/XPC issues on macOS)
+        if self.force_cpu:
+            device = "cpu"
+            logger.info("Embeddings: forcing CPU mode")
+        else:
+            # Check for GPU (CUDA) or Apple Silicon (MPS)
+            # Wrap in try-except to gracefully handle any device detection issues
+            # (e.g., in CI environments or when PyTorch is built without GPU support)
+            device = "cpu"  # Default to CPU
+            try:
+                has_gpu = torch.cuda.is_available() or (
+                    hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
+                )
+                if has_gpu:
+                    device = None  # Let sentence-transformers auto-detect GPU/MPS
+            except Exception as e:
+                logger.warning(f"Failed to detect GPU/MPS, falling back to CPU: {e}")
 
-    def encode(self, texts: List[str]) -> List[List[float]]:
+        # Suppress verbose transformers warnings during model loading
+        # This suppresses the "UNEXPECTED" warnings from BertModel which are harmless
+        # but look alarming to users (e.g., "embeddings.position_ids | UNEXPECTED")
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", category=UserWarning)
+            warnings.filterwarnings("ignore", message=".*was not found in model state dict.*")
+            warnings.filterwarnings("ignore", message=".*UNEXPECTED.*")
+
+            # Also suppress transformers library logging temporarily
+            transformers_logger = logging.getLogger("transformers")
+            original_level = transformers_logger.level
+            transformers_logger.setLevel(logging.ERROR)
+
+            try:
+                self._model = SentenceTransformer(
+                    self.model_name,
+                    device=device,
+                    model_kwargs={"low_cpu_mem_usage": False},
+                    trust_remote_code=self.trust_remote_code,
+                )
+            finally:
+                # Restore original logging level
+                transformers_logger.setLevel(original_level)
+
+        self._dimension = self._model.get_sentence_embedding_dimension()
+        logger.info(f"Embeddings: local provider initialized (dim: {self._dimension})")
+
+    def encode(self, texts: list[str]) -> list[list[float]]:
         """
-        Generate 384-dimensional embeddings for a list of texts.
+        Generate embeddings for a list of texts.
 
         Args:
             texts: List of text strings to encode
 
         Returns:
-            List of 384-dimensional embedding vectors
+            List of embedding vectors
         """
         if self._model is None:
-            self.load()
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+
         embeddings = self._model.encode(texts, convert_to_numpy=True, show_progress_bar=False)
         return [emb.tolist() for emb in embeddings]
+
+
+class RemoteTEIEmbeddings(Embeddings):
+    """
+    Remote embeddings implementation using HuggingFace Text Embeddings Inference (TEI) HTTP API.
+
+    TEI provides a high-performance inference server for embedding models.
+    See: https://github.com/huggingface/text-embeddings-inference
+
+    The embedding dimension is auto-detected from the server at initialization.
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        timeout: float = 30.0,
+        batch_size: int = 32,
+        max_retries: int = 3,
+        retry_delay: float = 0.5,
+    ):
+        """
+        Initialize remote TEI embeddings client.
+
+        Args:
+            base_url: Base URL of the TEI server (e.g., "http://localhost:8080")
+            timeout: Request timeout in seconds (default: 30.0)
+            batch_size: Maximum batch size for embedding requests (default: 32)
+            max_retries: Maximum number of retries for failed requests (default: 3)
+            retry_delay: Initial delay between retries in seconds, doubles each retry (default: 0.5)
+        """
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self.batch_size = batch_size
+        self.max_retries = max_retries
+        self.retry_delay = retry_delay
+        self._client: httpx.Client | None = None
+        self._model_id: str | None = None
+        self._dimension: int | None = None
+
+    @property
+    def provider_name(self) -> str:
+        return "tei"
+
+    @property
+    def dimension(self) -> int:
+        if self._dimension is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+        return self._dimension
+
+    def _request_with_retry(self, method: str, url: str, **kwargs) -> httpx.Response:
+        """Make an HTTP request with automatic retries on transient errors."""
+        import time
+
+        last_error = None
+        delay = self.retry_delay
+
+        for attempt in range(self.max_retries + 1):
+            try:
+                if method == "GET":
+                    response = self._client.get(url, **kwargs)
+                else:
+                    response = self._client.post(url, **kwargs)
+                response.raise_for_status()
+                return response
+            except (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout) as e:
+                last_error = e
+                if attempt < self.max_retries:
+                    logger.warning(
+                        f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. Retrying in {delay}s..."
+                    )
+                    time.sleep(delay)
+                    delay *= 2  # Exponential backoff
+            except httpx.HTTPStatusError as e:
+                # Retry on 5xx server errors
+                if e.response.status_code >= 500 and attempt < self.max_retries:
+                    last_error = e
+                    logger.warning(
+                        f"TEI server error (attempt {attempt + 1}/{self.max_retries + 1}): {e}. Retrying in {delay}s..."
+                    )
+                    time.sleep(delay)
+                    delay *= 2
+                else:
+                    raise
+
+        raise last_error
+
+    async def initialize(self) -> None:
+        """Initialize the HTTP client and verify server connectivity."""
+        if self._client is not None:
+            return
+
+        logger.info(f"Embeddings: initializing TEI provider at {self.base_url}")
+        self._client = httpx.Client(timeout=self.timeout)
+
+        # Verify server is reachable and get model info
+        try:
+            response = self._request_with_retry("GET", f"{self.base_url}/info")
+            info = response.json()
+            self._model_id = info.get("model_id", "unknown")
+
+            # Get dimension from server info or by doing a test embedding
+            if "max_input_length" in info and "model_dtype" in info:
+                # Try to get dimension from info endpoint (some TEI versions expose it)
+                # If not available, do a test embedding
+                pass
+
+            # Do a test embedding to detect dimension
+            test_response = self._request_with_retry(
+                "POST",
+                f"{self.base_url}/embed",
+                json={"inputs": ["test"]},
+            )
+            test_embeddings = test_response.json()
+            if test_embeddings and len(test_embeddings) > 0:
+                self._dimension = len(test_embeddings[0])
+
+            logger.info(f"Embeddings: TEI provider initialized (model: {self._model_id}, dim: {self._dimension})")
+        except httpx.HTTPError as e:
+            raise RuntimeError(f"Failed to connect to TEI server at {self.base_url}: {e}")
+
+    def encode(self, texts: list[str]) -> list[list[float]]:
+        """
+        Generate embeddings using the remote TEI server.
+
+        Args:
+            texts: List of text strings to encode
+
+        Returns:
+            List of embedding vectors
+        """
+        if self._client is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+
+        if not texts:
+            return []
+
+        all_embeddings = []
+
+        # Process in batches
+        for i in range(0, len(texts), self.batch_size):
+            batch = texts[i : i + self.batch_size]
+
+            try:
+                response = self._request_with_retry(
+                    "POST",
+                    f"{self.base_url}/embed",
+                    json={"inputs": batch},
+                )
+                batch_embeddings = response.json()
+                all_embeddings.extend(batch_embeddings)
+            except httpx.HTTPError as e:
+                raise RuntimeError(f"TEI embedding request failed: {e}")
+
+        return all_embeddings
+
+
+class OpenAIEmbeddings(Embeddings):
+    """
+    OpenAI embeddings implementation using the OpenAI API.
+
+    Supports text-embedding-3-small (1536 dims), text-embedding-3-large (3072 dims),
+    and text-embedding-ada-002 (1536 dims, legacy).
+
+    The embedding dimension is auto-detected from the model at initialization.
+    """
+
+    # Known dimensions for OpenAI embedding models
+    MODEL_DIMENSIONS = {
+        "text-embedding-3-small": 1536,
+        "text-embedding-3-large": 3072,
+        "text-embedding-ada-002": 1536,
+    }
+
+    def __init__(
+        self,
+        api_key: str,
+        model: str = DEFAULT_EMBEDDINGS_OPENAI_MODEL,
+        base_url: str | None = None,
+        batch_size: int = 100,
+        max_retries: int = 3,
+    ):
+        """
+        Initialize OpenAI embeddings client.
+
+        Args:
+            api_key: OpenAI API key
+            model: OpenAI embedding model name (default: text-embedding-3-small)
+            base_url: Custom base URL for OpenAI-compatible API (e.g., Azure OpenAI endpoint)
+            batch_size: Maximum batch size for embedding requests (default: 100)
+            max_retries: Maximum number of retries for failed requests (default: 3)
+        """
+        self.api_key = api_key
+        self.model = model
+        self.base_url = base_url
+        self.batch_size = batch_size
+        self.max_retries = max_retries
+        self._client = None
+        self._dimension: int | None = None
+
+    @property
+    def provider_name(self) -> str:
+        return "openai"
+
+    @property
+    def dimension(self) -> int:
+        if self._dimension is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+        return self._dimension
+
+    async def initialize(self) -> None:
+        """Initialize the OpenAI client and detect dimension."""
+        if self._client is not None:
+            return
+
+        try:
+            from openai import OpenAI
+        except ImportError:
+            raise ImportError("openai is required for OpenAIEmbeddings. Install it with: pip install openai")
+
+        base_url_msg = f" at {self.base_url}" if self.base_url else ""
+        logger.info(f"Embeddings: initializing OpenAI provider with model {self.model}{base_url_msg}")
+
+        # Build client kwargs, only including base_url if set (for Azure or custom endpoints)
+        client_kwargs = {"api_key": self.api_key, "max_retries": self.max_retries}
+        if self.base_url:
+            client_kwargs["base_url"] = self.base_url
+        self._client = OpenAI(**client_kwargs)
+
+        # Try to get dimension from known models, otherwise do a test embedding
+        if self.model in self.MODEL_DIMENSIONS:
+            self._dimension = self.MODEL_DIMENSIONS[self.model]
+        else:
+            # Do a test embedding to detect dimension
+            response = self._client.embeddings.create(
+                model=self.model,
+                input=["test"],
+            )
+            if response.data:
+                self._dimension = len(response.data[0].embedding)
+
+        logger.info(f"Embeddings: OpenAI provider initialized (model: {self.model}, dim: {self._dimension})")
+
+    def encode(self, texts: list[str]) -> list[list[float]]:
+        """
+        Generate embeddings using the OpenAI API.
+
+        Args:
+            texts: List of text strings to encode
+
+        Returns:
+            List of embedding vectors
+        """
+        if self._client is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+
+        if not texts:
+            return []
+
+        all_embeddings = []
+
+        # Process in batches
+        for i in range(0, len(texts), self.batch_size):
+            batch = texts[i : i + self.batch_size]
+
+            response = self._client.embeddings.create(
+                model=self.model,
+                input=batch,
+            )
+
+            # Sort by index to ensure correct order
+            batch_embeddings = sorted(response.data, key=lambda x: x.index)
+            all_embeddings.extend([e.embedding for e in batch_embeddings])
+
+        return all_embeddings
+
+
+class CohereEmbeddings(Embeddings):
+    """
+    Cohere embeddings implementation using the Cohere API.
+
+    Supports embed-english-v3.0 (1024 dims) and embed-multilingual-v3.0 (1024 dims).
+
+    The embedding dimension is auto-detected from the model at initialization.
+    """
+
+    # Known dimensions for Cohere embedding models
+    MODEL_DIMENSIONS = {
+        "embed-english-v3.0": 1024,
+        "embed-multilingual-v3.0": 1024,
+        "embed-english-light-v3.0": 384,
+        "embed-multilingual-light-v3.0": 384,
+        "embed-english-v2.0": 4096,
+        "embed-multilingual-v2.0": 768,
+    }
+
+    def __init__(
+        self,
+        api_key: str,
+        model: str = DEFAULT_EMBEDDINGS_COHERE_MODEL,
+        base_url: str | None = None,
+        batch_size: int = 96,
+        timeout: float = 60.0,
+        input_type: str = "search_document",
+    ):
+        """
+        Initialize Cohere embeddings client.
+
+        Args:
+            api_key: Cohere API key
+            model: Cohere embedding model name (default: embed-english-v3.0)
+            base_url: Custom base URL for Cohere-compatible API (e.g., Azure-hosted endpoint)
+            batch_size: Maximum batch size for embedding requests (default: 96, Cohere's limit)
+            timeout: Request timeout in seconds (default: 60.0)
+            input_type: Input type for embeddings (default: search_document).
+                       Options: search_document, search_query, classification, clustering
+        """
+        self.api_key = api_key
+        self.model = model
+        self.base_url = base_url
+        self.batch_size = batch_size
+        self.timeout = timeout
+        self.input_type = input_type
+        self._client = None
+        self._dimension: int | None = None
+
+    @property
+    def provider_name(self) -> str:
+        return "cohere"
+
+    @property
+    def dimension(self) -> int:
+        if self._dimension is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+        return self._dimension
+
+    async def initialize(self) -> None:
+        """Initialize the Cohere client and detect dimension."""
+        if self._client is not None:
+            return
+
+        try:
+            import cohere
+        except ImportError:
+            raise ImportError("cohere is required for CohereEmbeddings. Install it with: pip install cohere")
+
+        base_url_msg = f" at {self.base_url}" if self.base_url else ""
+        logger.info(f"Embeddings: initializing Cohere provider with model {self.model}{base_url_msg}")
+
+        # Build client kwargs, only including base_url if set (for Azure or custom endpoints)
+        client_kwargs = {"api_key": self.api_key, "timeout": self.timeout}
+        if self.base_url:
+            client_kwargs["base_url"] = self.base_url
+        self._client = cohere.Client(**client_kwargs)
+
+        # Try to get dimension from known models, otherwise do a test embedding
+        if self.model in self.MODEL_DIMENSIONS:
+            self._dimension = self.MODEL_DIMENSIONS[self.model]
+        else:
+            # Do a test embedding to detect dimension
+            response = self._client.embed(
+                texts=["test"],
+                model=self.model,
+                input_type=self.input_type,
+            )
+            if response.embeddings and isinstance(response.embeddings, list):
+                self._dimension = len(response.embeddings[0])
+
+        logger.info(f"Embeddings: Cohere provider initialized (model: {self.model}, dim: {self._dimension})")
+
+    def encode(self, texts: list[str]) -> list[list[float]]:
+        """
+        Generate embeddings using the Cohere API.
+
+        Args:
+            texts: List of text strings to encode
+
+        Returns:
+            List of embedding vectors
+        """
+        if self._client is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+
+        if not texts:
+            return []
+
+        all_embeddings = []
+
+        # Process in batches
+        for i in range(0, len(texts), self.batch_size):
+            batch = texts[i : i + self.batch_size]
+
+            response = self._client.embed(
+                texts=batch,
+                model=self.model,
+                input_type=self.input_type,
+            )
+
+            all_embeddings.extend(response.embeddings)
+
+        return all_embeddings
+
+
+class LiteLLMEmbeddings(Embeddings):
+    """
+    LiteLLM embeddings implementation using LiteLLM proxy's /embeddings endpoint.
+
+    LiteLLM provides a unified interface for multiple embedding providers.
+    The proxy exposes an OpenAI-compatible /embeddings endpoint.
+    See: https://docs.litellm.ai/docs/embedding/supported_embedding
+
+    Supported providers via LiteLLM:
+    - OpenAI (text-embedding-3-small, text-embedding-ada-002, etc.)
+    - Cohere (embed-english-v3.0, etc.) - prefix with cohere/
+    - Vertex AI (textembedding-gecko, etc.) - prefix with vertex_ai/
+    - HuggingFace, Mistral, Voyage AI, etc.
+
+    The embedding dimension is auto-detected from the model at initialization.
+    """
+
+    def __init__(
+        self,
+        api_base: str = DEFAULT_LITELLM_API_BASE,
+        api_key: str | None = None,
+        model: str = DEFAULT_EMBEDDINGS_LITELLM_MODEL,
+        batch_size: int = 100,
+        timeout: float = 60.0,
+    ):
+        """
+        Initialize LiteLLM embeddings client.
+
+        Args:
+            api_base: Base URL of the LiteLLM proxy (default: http://localhost:4000)
+            api_key: API key for the LiteLLM proxy (optional, depends on proxy config)
+            model: Embedding model name (default: text-embedding-3-small)
+                   Use provider prefix for non-OpenAI models (e.g., cohere/embed-english-v3.0)
+            batch_size: Maximum batch size for embedding requests (default: 100)
+            timeout: Request timeout in seconds (default: 60.0)
+        """
+        self.api_base = api_base.rstrip("/")
+        self.api_key = api_key
+        self.model = model
+        self.batch_size = batch_size
+        self.timeout = timeout
+        self._client: httpx.Client | None = None
+        self._dimension: int | None = None
+
+    @property
+    def provider_name(self) -> str:
+        return "litellm"
+
+    @property
+    def dimension(self) -> int:
+        if self._dimension is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+        return self._dimension
+
+    async def initialize(self) -> None:
+        """Initialize the HTTP client and detect embedding dimension."""
+        if self._client is not None:
+            return
+
+        logger.info(f"Embeddings: initializing LiteLLM provider at {self.api_base} with model {self.model}")
+
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+
+        self._client = httpx.Client(timeout=self.timeout, headers=headers)
+
+        # Do a test embedding to detect dimension
+        try:
+            response = self._client.post(
+                f"{self.api_base}/embeddings",
+                json={"model": self.model, "input": ["test"]},
+            )
+            response.raise_for_status()
+            result = response.json()
+            if result.get("data") and len(result["data"]) > 0:
+                self._dimension = len(result["data"][0]["embedding"])
+            logger.info(f"Embeddings: LiteLLM provider initialized (model: {self.model}, dim: {self._dimension})")
+        except httpx.HTTPError as e:
+            raise RuntimeError(f"Failed to connect to LiteLLM proxy at {self.api_base}: {e}")
+
+    def encode(self, texts: list[str]) -> list[list[float]]:
+        """
+        Generate embeddings using the LiteLLM proxy.
+
+        Args:
+            texts: List of text strings to encode
+
+        Returns:
+            List of embedding vectors
+        """
+        if self._client is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+
+        if not texts:
+            return []
+
+        all_embeddings = []
+
+        # Process in batches
+        for i in range(0, len(texts), self.batch_size):
+            batch = texts[i : i + self.batch_size]
+
+            response = self._client.post(
+                f"{self.api_base}/embeddings",
+                json={"model": self.model, "input": batch},
+            )
+            response.raise_for_status()
+            result = response.json()
+
+            # Sort by index to ensure correct order
+            batch_embeddings = sorted(result["data"], key=lambda x: x["index"])
+            all_embeddings.extend([e["embedding"] for e in batch_embeddings])
+
+        return all_embeddings
+
+
+def create_embeddings_from_env() -> Embeddings:
+    """
+    Create an Embeddings instance based on configuration.
+
+    Reads configuration via get_config() to ensure consistency across the codebase.
+
+    Returns:
+        Configured Embeddings instance
+    """
+    from ..config import get_config
+
+    config = get_config()
+    provider = config.embeddings_provider.lower()
+
+    if provider == "tei":
+        url = config.embeddings_tei_url
+        if not url:
+            raise ValueError(f"{ENV_EMBEDDINGS_TEI_URL} is required when {ENV_EMBEDDINGS_PROVIDER} is 'tei'")
+        return RemoteTEIEmbeddings(base_url=url)
+    elif provider == "local":
+        return LocalSTEmbeddings(
+            model_name=config.embeddings_local_model,
+            force_cpu=config.embeddings_local_force_cpu,
+            trust_remote_code=config.embeddings_local_trust_remote_code,
+        )
+    elif provider == "openai":
+        # Use dedicated embeddings API key, or fall back to LLM API key
+        api_key = os.environ.get(ENV_EMBEDDINGS_OPENAI_API_KEY) or os.environ.get(ENV_LLM_API_KEY)
+        if not api_key:
+            raise ValueError(
+                f"{ENV_EMBEDDINGS_OPENAI_API_KEY} or {ENV_LLM_API_KEY} is required "
+                f"when {ENV_EMBEDDINGS_PROVIDER} is 'openai'"
+            )
+        model = os.environ.get(ENV_EMBEDDINGS_OPENAI_MODEL, DEFAULT_EMBEDDINGS_OPENAI_MODEL)
+        base_url = os.environ.get(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None
+        return OpenAIEmbeddings(api_key=api_key, model=model, base_url=base_url)
+    elif provider == "cohere":
+        api_key = config.embeddings_cohere_api_key
+        if not api_key:
+            raise ValueError(f"{ENV_EMBEDDINGS_COHERE_API_KEY} is required when {ENV_EMBEDDINGS_PROVIDER} is 'cohere'")
+        return CohereEmbeddings(
+            api_key=api_key,
+            model=config.embeddings_cohere_model,
+            base_url=config.embeddings_cohere_base_url,
+        )
+    elif provider == "litellm":
+        return LiteLLMEmbeddings(
+            api_base=config.embeddings_litellm_api_base,
+            api_key=config.embeddings_litellm_api_key,
+            model=config.embeddings_litellm_model,
+        )
+    else:
+        raise ValueError(
+            f"Unknown embeddings provider: {provider}. Supported: 'local', 'tei', 'openai', 'cohere', 'litellm'"
+        )

hindsight-api/hindsight_api/engine/entity_resolver.py

@@ -4,12 +4,14 @@ Entity extraction and resolution for memory system.
 Uses spaCy for entity extraction and implements resolution logic
 to disambiguate entities across memory units.
 """
-import asyncpg
-from typing import List, Dict, Optional, Set, Any
-from difflib import SequenceMatcher
-from datetime import datetime, timezone
-from .db_utils import acquire_with_retry
 
+from datetime import UTC, datetime
+from difflib import SequenceMatcher
+
+import asyncpg
+
+from .db_utils import acquire_with_retry
+from .memory_engine import fq_table
 
 # Load spaCy model (singleton)
 _nlp = None
@@ -32,11 +34,11 @@ class EntityResolver:
     async def resolve_entities_batch(
         self,
         bank_id: str,
-        entities_data: List[Dict],
+        entities_data: list[dict],
         context: str,
         unit_event_date,
         conn=None,
-    ) -> List[str]:
+    ) -> list[str]:
         """
         Resolve multiple entities in batch (MUCH faster than sequential).
 
@@ -62,36 +64,38 @@ class EntityResolver:
         else:
             return await self._resolve_entities_batch_impl(conn, bank_id, entities_data, context, unit_event_date)
 
-    async def _resolve_entities_batch_impl(self, conn, bank_id: str, entities_data: List[Dict], context: str, unit_event_date) -> List[str]:
+    async def _resolve_entities_batch_impl(
+        self, conn, bank_id: str, entities_data: list[dict], context: str, unit_event_date
+    ) -> list[str]:
         # Query ALL candidates for this bank
         all_entities = await conn.fetch(
-            """
+            f"""
             SELECT canonical_name, id, metadata, last_seen, mention_count
-            FROM entities
+            FROM {fq_table("entities")}
             WHERE bank_id = $1
             """,
-            bank_id
+            bank_id,
         )
 
         # Build entity ID to name mapping for co-occurrence lookups
-        entity_id_to_name = {row['id']: row['canonical_name'].lower() for row in all_entities}
+        entity_id_to_name = {row["id"]: row["canonical_name"].lower() for row in all_entities}
 
         # Query ALL co-occurrences for this bank's entities in one query
         # This builds a map of entity_id -> set of co-occurring entity names
         all_cooccurrences = await conn.fetch(
-            """
+            f"""
             SELECT ec.entity_id_1, ec.entity_id_2, ec.cooccurrence_count
-            FROM entity_cooccurrences ec
-            WHERE ec.entity_id_1 IN (SELECT id FROM entities WHERE bank_id = $1)
-               OR ec.entity_id_2 IN (SELECT id FROM entities WHERE bank_id = $1)
+            FROM {fq_table("entity_cooccurrences")} ec
+            WHERE ec.entity_id_1 IN (SELECT id FROM {fq_table("entities")} WHERE bank_id = $1)
+               OR ec.entity_id_2 IN (SELECT id FROM {fq_table("entities")} WHERE bank_id = $1)
             """,
-            bank_id
+            bank_id,
         )
 
         # Build co-occurrence map: entity_id -> set of co-occurring entity names (lowercase)
-        cooccurrence_map: Dict[str, Set[str]] = {}
+        cooccurrence_map: dict[str, set[str]] = {}
         for row in all_cooccurrences:
-            eid1, eid2 = row['entity_id_1'], row['entity_id_2']
+            eid1, eid2 = row["entity_id_1"], row["entity_id_2"]
             # Add both directions
             if eid1 not in cooccurrence_map:
                 cooccurrence_map[eid1] = set()
@@ -105,56 +109,56 @@ class EntityResolver:
 
         # Build candidate map for each entity text
         all_candidates = {}  # Maps entity_text -> list of candidates
-        entity_texts = list(set(e['text'] for e in entities_data))
+        entity_texts = list(set(e["text"] for e in entities_data))
 
         for entity_text in entity_texts:
             matching = []
             entity_text_lower = entity_text.lower()
             for row in all_entities:
-                canonical_name = row['canonical_name']
-                ent_id = row['id']
-                metadata = row['metadata']
-                last_seen = row['last_seen']
-                mention_count = row['mention_count']
+                canonical_name = row["canonical_name"]
+                ent_id = row["id"]
+                metadata = row["metadata"]
+                last_seen = row["last_seen"]
+                mention_count = row["mention_count"]
                 canonical_lower = canonical_name.lower()
                 # Match if exact or substring match
-                if (entity_text_lower == canonical_lower or
-                    entity_text_lower in canonical_lower or
-                    canonical_lower in entity_text_lower):
+                if (
+                    entity_text_lower == canonical_lower
+                    or entity_text_lower in canonical_lower
+                    or canonical_lower in entity_text_lower
+                ):
                     matching.append((ent_id, canonical_name, metadata, last_seen, mention_count))
             all_candidates[entity_text] = matching
 
         # Resolve each entity using pre-fetched candidates
         entity_ids = [None] * len(entities_data)
-        entities_to_update = []  # (entity_id, unit_event_date)
-        entities_to_create = []  # (idx, entity_data)
+        entities_to_update = []  # (entity_id, event_date)
+        entities_to_create = []  # (idx, entity_data, event_date)
 
         for idx, entity_data in enumerate(entities_data):
-            entity_text = entity_data['text']
-            nearby_entities = entity_data.get('nearby_entities', [])
+            entity_text = entity_data["text"]
+            nearby_entities = entity_data.get("nearby_entities", [])
+            # Use per-entity date if available, otherwise fall back to batch-level date
+            entity_event_date = entity_data.get("event_date", unit_event_date)
 
             candidates = all_candidates.get(entity_text, [])
 
             if not candidates:
                 # Will create new entity
-                entities_to_create.append((idx, entity_data))
+                entities_to_create.append((idx, entity_data, entity_event_date))
                 continue
 
             # Score candidates
             best_candidate = None
             best_score = 0.0
 
-            nearby_entity_set = {e['text'].lower() for e in nearby_entities if e['text'] != entity_text}
+            nearby_entity_set = {e["text"].lower() for e in nearby_entities if e["text"] != entity_text}
 
             for candidate_id, canonical_name, metadata, last_seen, mention_count in candidates:
                 score = 0.0
 
                 # 1. Name similarity (0-0.5)
-                name_similarity = SequenceMatcher(
-                    None,
-                    entity_text.lower(),
-                    canonical_name.lower()
-                ).ratio()
+                name_similarity = SequenceMatcher(None, entity_text.lower(), canonical_name.lower()).ratio()
                 score += name_similarity * 0.5
 
                 # 2. Co-occurring entities (0-0.3)
@@ -165,10 +169,12 @@ class EntityResolver:
                     score += co_entity_score * 0.3
 
                 # 3. Temporal proximity (0-0.2)
-                if last_seen:
+                if last_seen and entity_event_date:
                     # Normalize timezone awareness for comparison
-                    event_date_utc = unit_event_date if unit_event_date.tzinfo else unit_event_date.replace(tzinfo=timezone.utc)
-                    last_seen_utc = last_seen if last_seen.tzinfo else last_seen.replace(tzinfo=timezone.utc)
+                    event_date_utc = (
+                        entity_event_date if entity_event_date.tzinfo else entity_event_date.replace(tzinfo=UTC)
+                    )
+                    last_seen_utc = last_seen if last_seen.tzinfo else last_seen.replace(tzinfo=UTC)
                     days_diff = abs((event_date_utc - last_seen_utc).total_seconds() / 86400)
                     if days_diff < 7:
                         temporal_score = max(0, 1.0 - (days_diff / 7))
@@ -183,45 +189,73 @@ class EntityResolver:
 
             if best_score > threshold:
                 entity_ids[idx] = best_candidate
-                entities_to_update.append((best_candidate, unit_event_date))
+                entities_to_update.append((best_candidate, entity_event_date))
             else:
-                entities_to_create.append((idx, entity_data))
+                entities_to_create.append((idx, entity_data, entity_event_date))
 
         # Batch update existing entities
         if entities_to_update:
             await conn.executemany(
-                """
-                UPDATE entities SET
+                f"""
+                UPDATE {fq_table("entities")} SET
                     mention_count = mention_count + 1,
                     last_seen = $2
                 WHERE id = $1::uuid
                 """,
-                entities_to_update
+                entities_to_update,
             )
 
-        # Create new entities using INSERT ... ON CONFLICT to handle race conditions
-        # This ensures that if two concurrent transactions try to create the same entity,
-        # only one succeeds and the other gets the existing ID
+        # Batch create new entities using COPY + INSERT for maximum speed
+        # This handles duplicates via ON CONFLICT and returns all IDs
         if entities_to_create:
-            for idx, entity_data in entities_to_create:
-                # Use INSERT ... ON CONFLICT to atomically get-or-create
-                # The unique index is on (bank_id, LOWER(canonical_name))
-                row = await conn.fetchrow(
-                    """
-                    INSERT INTO entities (bank_id, canonical_name, first_seen, last_seen, mention_count)
-                    VALUES ($1, $2, $3, $4, 1)
-                    ON CONFLICT (bank_id, LOWER(canonical_name))
-                    DO UPDATE SET
-                        mention_count = entities.mention_count + 1,
-                        last_seen = EXCLUDED.last_seen
-                    RETURNING id
-                    """,
-                    bank_id,
-                    entity_data['text'],
-                    unit_event_date,
-                    unit_event_date
-                )
-                entity_ids[idx] = row['id']
+            # Group entities by canonical name (lowercase) to handle duplicates within batch
+            # For duplicates, we only insert once and reuse the ID, but track the count
+            unique_entities = {}  # lowercase_name -> (entity_data, event_date, [indices])
+            for idx, entity_data, event_date in entities_to_create:
+                name_lower = entity_data["text"].lower()
+                if name_lower not in unique_entities:
+                    unique_entities[name_lower] = (entity_data, event_date, [idx])
+                else:
+                    # Same entity appears multiple times - add index to list
+                    unique_entities[name_lower][2].append(idx)
+
+            # Batch insert unique entities and get their IDs
+            # Use a single query with unnest for speed
+            entity_names = []
+            entity_dates = []
+            entity_counts = []  # Track how many times each entity appears in this batch
+            indices_map = []  # Maps result index -> list of original indices
+
+            for name_lower, (entity_data, event_date, indices) in unique_entities.items():
+                entity_names.append(entity_data["text"])
+                entity_dates.append(event_date)
+                entity_counts.append(len(indices))  # Count of occurrences in this batch
+                indices_map.append(indices)
+
+            # Batch INSERT ... ON CONFLICT with RETURNING
+            # Uses the batch count for mention_count instead of always 1
+            rows = await conn.fetch(
+                f"""
+                INSERT INTO {fq_table("entities")} (bank_id, canonical_name, first_seen, last_seen, mention_count)
+                SELECT $1, name, event_date, event_date, cnt
+                FROM unnest($2::text[], $3::timestamptz[], $4::int[]) AS t(name, event_date, cnt)
+                ON CONFLICT (bank_id, LOWER(canonical_name))
+                DO UPDATE SET
+                    mention_count = {fq_table("entities")}.mention_count + EXCLUDED.mention_count,
+                    last_seen = EXCLUDED.last_seen
+                RETURNING id
+                """,
+                bank_id,
+                entity_names,
+                entity_dates,
+                entity_counts,
+            )
+
+            # Map returned IDs back to original indices
+            for result_idx, row in enumerate(rows):
+                entity_id = row["id"]
+                for original_idx in indices_map[result_idx]:
+                    entity_ids[original_idx] = entity_id
 
         return entity_ids
 
@@ -230,7 +264,7 @@ class EntityResolver:
         bank_id: str,
         entity_text: str,
         context: str,
-        nearby_entities: List[Dict],
+        nearby_entities: list[dict],
         unit_event_date,
     ) -> str:
         """
@@ -249,9 +283,9 @@ class EntityResolver:
         async with acquire_with_retry(self.pool) as conn:
             # Find candidate entities with similar name
             candidates = await conn.fetch(
-                """
+                f"""
                 SELECT id, canonical_name, metadata, last_seen
-                FROM entities
+                FROM {fq_table("entities")}
                 WHERE bank_id = $1
                   AND (
                     canonical_name ILIKE $2
@@ -260,14 +294,14 @@ class EntityResolver:
                   )
                 ORDER BY mention_count DESC
                 """,
-                bank_id, entity_text, f"%{entity_text}%"
+                bank_id,
+                entity_text,
+                f"%{entity_text}%",
             )
 
             if not candidates:
                 # New entity - create it
-                return await self._create_entity(
-                    conn, bank_id, entity_text, unit_event_date
-                )
+                return await self._create_entity(conn, bank_id, entity_text, unit_event_date)
 
             # Score candidates based on:
             # 1. Name similarity
@@ -279,31 +313,27 @@ class EntityResolver:
             best_score = 0.0
             best_name_similarity = 0.0
 
-            nearby_entity_set = {e['text'].lower() for e in nearby_entities if e['text'] != entity_text}
+            nearby_entity_set = {e["text"].lower() for e in nearby_entities if e["text"] != entity_text}
 
             for row in candidates:
-                candidate_id = row['id']
-                canonical_name = row['canonical_name']
-                metadata = row['metadata']
-                last_seen = row['last_seen']
+                candidate_id = row["id"]
+                canonical_name = row["canonical_name"]
+                metadata = row["metadata"]
+                last_seen = row["last_seen"]
                 score = 0.0
 
                 # 1. Name similarity (0-1)
-                name_similarity = SequenceMatcher(
-                    None,
-                    entity_text.lower(),
-                    canonical_name.lower()
-                ).ratio()
+                name_similarity = SequenceMatcher(None, entity_text.lower(), canonical_name.lower()).ratio()
                 score += name_similarity * 0.5
 
                 # 2. Co-occurring entities (0-0.5)
                 # Get entities that co-occurred with this candidate before
                 # Use the materialized co-occurrence cache for fast lookup
                 co_entity_rows = await conn.fetch(
-                    """
+                    f"""
                     SELECT e.canonical_name, ec.cooccurrence_count
-                    FROM entity_cooccurrences ec
-                    JOIN entities e ON (
+                    FROM {fq_table("entity_cooccurrences")} ec
+                    JOIN {fq_table("entities")} e ON (
                         CASE
                             WHEN ec.entity_id_1 = $1 THEN ec.entity_id_2
                             WHEN ec.entity_id_2 = $1 THEN ec.entity_id_1
@@ -311,9 +341,9 @@ class EntityResolver:
                     )
                     WHERE ec.entity_id_1 = $1 OR ec.entity_id_2 = $1
                     """,
-                    candidate_id
+                    candidate_id,
                 )
-                co_entities = {r['canonical_name'].lower() for r in co_entity_rows}
+                co_entities = {r["canonical_name"].lower() for r in co_entity_rows}
 
                 # Check overlap with nearby entities
                 overlap = len(nearby_entity_set & co_entities)
@@ -339,20 +369,19 @@ class EntityResolver:
             if best_score > threshold:
                 # Update entity
                 await conn.execute(
-                    """
-                    UPDATE entities
+                    f"""
+                    UPDATE {fq_table("entities")}
                     SET mention_count = mention_count + 1,
                         last_seen = $1
                     WHERE id = $2
                     """,
-                    unit_event_date, best_candidate
+                    unit_event_date,
+                    best_candidate,
                 )
                 return best_candidate
             else:
                 # Not confident - create new entity
-                return await self._create_entity(
-                    conn, bank_id, entity_text, unit_event_date
-                )
+                return await self._create_entity(conn, bank_id, entity_text, unit_event_date)
 
     async def _create_entity(
         self,
@@ -377,16 +406,19 @@ class EntityResolver:
             Entity ID
         """
         entity_id = await conn.fetchval(
-            """
-            INSERT INTO entities (bank_id, canonical_name, first_seen, last_seen, mention_count)
+            f"""
+            INSERT INTO {fq_table("entities")} (bank_id, canonical_name, first_seen, last_seen, mention_count)
             VALUES ($1, $2, $3, $4, 1)
             ON CONFLICT (bank_id, LOWER(canonical_name))
             DO UPDATE SET
-                mention_count = entities.mention_count + 1,
+                mention_count = {fq_table("entities")}.mention_count + 1,
                 last_seen = EXCLUDED.last_seen
             RETURNING id
             """,
-            bank_id, entity_text, event_date, event_date
+            bank_id,
+            entity_text,
+            event_date,
+            event_date,
         )
         return entity_id
 
@@ -402,25 +434,27 @@ class EntityResolver:
         async with acquire_with_retry(self.pool) as conn:
             # Insert unit-entity link
             await conn.execute(
-                """
-                INSERT INTO unit_entities (unit_id, entity_id)
+                f"""
+                INSERT INTO {fq_table("unit_entities")} (unit_id, entity_id)
                 VALUES ($1, $2)
                 ON CONFLICT DO NOTHING
                 """,
-                unit_id, entity_id
+                unit_id,
+                entity_id,
             )
 
             # Update co-occurrence cache: find other entities in this unit
             rows = await conn.fetch(
-                """
+                f"""
                 SELECT entity_id
-                FROM unit_entities
+                FROM {fq_table("unit_entities")}
                 WHERE unit_id = $1 AND entity_id != $2
                 """,
-                unit_id, entity_id
+                unit_id,
+                entity_id,
             )
 
-            other_entities = [row['entity_id'] for row in rows]
+            other_entities = [row["entity_id"] for row in rows]
 
             # Update co-occurrences for each pair
             for other_entity_id in other_entities:
@@ -442,18 +476,19 @@ class EntityResolver:
             entity_id_1, entity_id_2 = entity_id_2, entity_id_1
 
         await conn.execute(
-            """
-            INSERT INTO entity_cooccurrences (entity_id_1, entity_id_2, cooccurrence_count, last_cooccurred)
+            f"""
+            INSERT INTO {fq_table("entity_cooccurrences")} (entity_id_1, entity_id_2, cooccurrence_count, last_cooccurred)
             VALUES ($1, $2, 1, NOW())
             ON CONFLICT (entity_id_1, entity_id_2)
             DO UPDATE SET
-                cooccurrence_count = entity_cooccurrences.cooccurrence_count + 1,
+                cooccurrence_count = {fq_table("entity_cooccurrences")}.cooccurrence_count + 1,
                 last_cooccurred = NOW()
             """,
-            entity_id_1, entity_id_2
+            entity_id_1,
+            entity_id_2,
         )
 
-    async def link_units_to_entities_batch(self, unit_entity_pairs: List[tuple[str, str]], conn=None):
+    async def link_units_to_entities_batch(self, unit_entity_pairs: list[tuple[str, str]], conn=None):
         """
         Link multiple memory units to entities in batch (MUCH faster than sequential).
 
@@ -472,15 +507,15 @@ class EntityResolver:
         else:
             return await self._link_units_to_entities_batch_impl(conn, unit_entity_pairs)
 
-    async def _link_units_to_entities_batch_impl(self, conn, unit_entity_pairs: List[tuple[str, str]]):
+    async def _link_units_to_entities_batch_impl(self, conn, unit_entity_pairs: list[tuple[str, str]]):
         # Batch insert all unit-entity links
         await conn.executemany(
-            """
-            INSERT INTO unit_entities (unit_id, entity_id)
+            f"""
+            INSERT INTO {fq_table("unit_entities")} (unit_id, entity_id)
             VALUES ($1, $2)
             ON CONFLICT DO NOTHING
             """,
-            unit_entity_pairs
+            unit_entity_pairs,
         )
 
         # Build map of unit -> entities for co-occurrence calculation
@@ -497,7 +532,7 @@ class EntityResolver:
             entity_list = list(entity_ids)  # Convert set to list for iteration
             # For each pair of entities in this unit, create co-occurrence
             for i, entity_id_1 in enumerate(entity_list):
-                for entity_id_2 in entity_list[i+1:]:
+                for entity_id_2 in entity_list[i + 1 :]:
                     # Skip if same entity (shouldn't happen with set, but be safe)
                     if entity_id_1 == entity_id_2:
                         continue
@@ -508,20 +543,20 @@ class EntityResolver:
 
         # Batch update co-occurrences
         if cooccurrence_pairs:
-            now = datetime.now(timezone.utc)
+            now = datetime.now(UTC)
             await conn.executemany(
-                """
-                INSERT INTO entity_cooccurrences (entity_id_1, entity_id_2, cooccurrence_count, last_cooccurred)
+                f"""
+                INSERT INTO {fq_table("entity_cooccurrences")} (entity_id_1, entity_id_2, cooccurrence_count, last_cooccurred)
                 VALUES ($1, $2, $3, $4)
                 ON CONFLICT (entity_id_1, entity_id_2)
                 DO UPDATE SET
-                    cooccurrence_count = entity_cooccurrences.cooccurrence_count + 1,
+                    cooccurrence_count = {fq_table("entity_cooccurrences")}.cooccurrence_count + 1,
                     last_cooccurred = EXCLUDED.last_cooccurred
                 """,
-                [(e1, e2, 1, now) for e1, e2 in cooccurrence_pairs]
+                [(e1, e2, 1, now) for e1, e2 in cooccurrence_pairs],
             )
 
-    async def get_units_by_entity(self, entity_id: str, limit: int = 100) -> List[str]:
+    async def get_units_by_entity(self, entity_id: str, limit: int = 100) -> list[str]:
         """
         Get all units that mention an entity.
 
@@ -534,22 +569,23 @@ class EntityResolver:
         """
         async with acquire_with_retry(self.pool) as conn:
             rows = await conn.fetch(
-                """
+                f"""
                 SELECT unit_id
-                FROM unit_entities
+                FROM {fq_table("unit_entities")}
                 WHERE entity_id = $1
                 ORDER BY unit_id
                 LIMIT $2
                 """,
-                entity_id, limit
+                entity_id,
+                limit,
             )
-            return [row['unit_id'] for row in rows]
+            return [row["unit_id"] for row in rows]
 
     async def get_entity_by_text(
         self,
         bank_id: str,
         entity_text: str,
-    ) -> Optional[str]:
+    ) -> str | None:
         """
         Find an entity by text (for query resolution).
 
@@ -562,14 +598,15 @@ class EntityResolver:
         """
         async with acquire_with_retry(self.pool) as conn:
             row = await conn.fetchrow(
-                """
-                SELECT id FROM entities
+                f"""
+                SELECT id FROM {fq_table("entities")}
                 WHERE bank_id = $1
                   AND canonical_name ILIKE $2
                 ORDER BY mention_count DESC
                 LIMIT 1
                 """,
-                bank_id, entity_text
+                bank_id,
+                entity_text,
             )
 
-            return row['id'] if row else None
+            return row["id"] if row else None

hindsight-api/hindsight_api/engine/interface.py

@@ -0,0 +1,576 @@
+"""Abstract interface for MemoryEngine public methods.
+
+This module defines the public API that HTTP endpoints and extensions should use
+to interact with the memory system. All methods require a RequestContext for
+authentication when a TenantExtension is configured.
+"""
+
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from hindsight_api.engine.memory_engine import Budget
+    from hindsight_api.engine.response_models import RecallResult, ReflectResult
+    from hindsight_api.models import RequestContext
+
+
+class MemoryEngineInterface(ABC):
+    """
+    Abstract interface for the Memory Engine.
+
+    This defines the public API that should be used by HTTP endpoints and extensions.
+    All methods require a RequestContext for authentication.
+    """
+
+    # =========================================================================
+    # Health & Status
+    # =========================================================================
+
+    @abstractmethod
+    async def health_check(self) -> dict:
+        """
+        Check the health of the memory system.
+
+        Returns:
+            Dict with 'status' key ('healthy' or 'unhealthy') and additional info.
+        """
+        ...
+
+    # =========================================================================
+    # Core Memory Operations
+    # =========================================================================
+
+    @abstractmethod
+    async def retain_batch_async(
+        self,
+        bank_id: str,
+        contents: list[dict[str, Any]],
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        Retain a batch of memory items.
+
+        Args:
+            bank_id: The memory bank ID.
+            contents: List of content dicts with 'content', optional 'event_date',
+                     'context', 'metadata', 'document_id'.
+            request_context: Request context for authentication.
+
+        Returns:
+            Dict with processing results.
+        """
+        ...
+
+    @abstractmethod
+    async def recall_async(
+        self,
+        bank_id: str,
+        query: str,
+        *,
+        budget: "Budget | None" = None,
+        max_tokens: int = 4096,
+        enable_trace: bool = False,
+        fact_type: list[str] | None = None,
+        question_date: datetime | None = None,
+        include_entities: bool = False,
+        max_entity_tokens: int = 500,
+        include_chunks: bool = False,
+        max_chunk_tokens: int = 8192,
+        request_context: "RequestContext",
+    ) -> "RecallResult":
+        """
+        Recall memories relevant to a query.
+
+        Args:
+            bank_id: The memory bank ID.
+            query: The search query.
+            budget: Search budget (LOW, MID, HIGH).
+            max_tokens: Maximum tokens in response.
+            enable_trace: Include trace information.
+            fact_type: Filter by fact types.
+            question_date: Context date for temporal relevance.
+            include_entities: Include entity observations.
+            max_entity_tokens: Max tokens for entity observations.
+            include_chunks: Include raw chunks.
+            max_chunk_tokens: Max tokens for chunks.
+            request_context: Request context for authentication.
+
+        Returns:
+            RecallResult with matching memories.
+        """
+        ...
+
+    @abstractmethod
+    async def reflect_async(
+        self,
+        bank_id: str,
+        query: str,
+        *,
+        budget: "Budget | None" = None,
+        context: str | None = None,
+        max_tokens: int = 4096,
+        response_schema: dict | None = None,
+        request_context: "RequestContext",
+    ) -> "ReflectResult":
+        """
+        Reflect on a query and generate a thoughtful response.
+
+        Args:
+            bank_id: The memory bank ID.
+            query: The question to reflect on.
+            budget: Search budget for retrieving context.
+            context: Additional context for the reflection.
+            max_tokens: Maximum tokens for the response.
+            response_schema: Optional JSON Schema for structured output.
+            request_context: Request context for authentication.
+
+        Returns:
+            ReflectResult with generated response and supporting facts.
+        """
+        ...
+
+    # =========================================================================
+    # Bank Management
+    # =========================================================================
+
+    @abstractmethod
+    async def list_banks(
+        self,
+        *,
+        request_context: "RequestContext",
+    ) -> list[dict[str, Any]]:
+        """
+        List all memory banks.
+
+        Args:
+            request_context: Request context for authentication.
+
+        Returns:
+            List of bank info dicts.
+        """
+        ...
+
+    @abstractmethod
+    async def get_bank_profile(
+        self,
+        bank_id: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        Get bank profile including disposition and mission.
+
+        Args:
+            bank_id: The memory bank ID.
+            request_context: Request context for authentication.
+
+        Returns:
+            Bank profile dict with bank_id, name, disposition, and mission.
+        """
+        ...
+
+    @abstractmethod
+    async def update_bank_disposition(
+        self,
+        bank_id: str,
+        disposition: dict[str, int],
+        *,
+        request_context: "RequestContext",
+    ) -> None:
+        """
+        Update bank disposition traits.
+
+        Args:
+            bank_id: The memory bank ID.
+            disposition: Dict with trait values.
+            request_context: Request context for authentication.
+        """
+        ...
+
+    @abstractmethod
+    async def merge_bank_mission(
+        self,
+        bank_id: str,
+        new_info: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        Merge new mission information into bank profile.
+
+        Args:
+            bank_id: The memory bank ID.
+            new_info: New mission information to merge.
+            request_context: Request context for authentication.
+
+        Returns:
+            Updated mission info.
+        """
+        ...
+
+    @abstractmethod
+    async def set_bank_mission(
+        self,
+        bank_id: str,
+        mission: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        Set the bank's mission (replaces existing).
+
+        Args:
+            bank_id: The memory bank ID.
+            mission: The mission text.
+            request_context: Request context for authentication.
+
+        Returns:
+            Dict with bank_id and mission.
+        """
+        ...
+
+    @abstractmethod
+    async def delete_bank(
+        self,
+        bank_id: str,
+        *,
+        fact_type: str | None = None,
+        request_context: "RequestContext",
+    ) -> dict[str, int]:
+        """
+        Delete a bank or its memories.
+
+        Args:
+            bank_id: The memory bank ID.
+            fact_type: If specified, only delete memories of this type.
+            request_context: Request context for authentication.
+
+        Returns:
+            Dict with deletion counts.
+        """
+        ...
+
+    # =========================================================================
+    # Memory Units
+    # =========================================================================
+
+    @abstractmethod
+    async def list_memory_units(
+        self,
+        bank_id: str,
+        *,
+        fact_type: str | None = None,
+        search_query: str | None = None,
+        limit: int = 100,
+        offset: int = 0,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        List memory units with pagination.
+
+        Args:
+            bank_id: The memory bank ID.
+            fact_type: Filter by fact type.
+            search_query: Full-text search query.
+            limit: Maximum results.
+            offset: Pagination offset.
+            request_context: Request context for authentication.
+
+        Returns:
+            Dict with 'items', 'total', 'limit', 'offset'.
+        """
+        ...
+
+    @abstractmethod
+    async def delete_memory_unit(
+        self,
+        unit_id: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        Delete a specific memory unit.
+
+        Args:
+            unit_id: The memory unit ID.
+            request_context: Request context for authentication.
+
+        Returns:
+            Deletion result.
+        """
+        ...
+
+    @abstractmethod
+    async def get_graph_data(
+        self,
+        bank_id: str,
+        *,
+        fact_type: str | None = None,
+        limit: int = 1000,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        Get graph data for visualization.
+
+        Args:
+            bank_id: The memory bank ID.
+            fact_type: Filter by fact type.
+            limit: Maximum number of items to return (default: 1000).
+            request_context: Request context for authentication.
+
+        Returns:
+            Dict with nodes, edges, table_rows, total_units, limit.
+        """
+        ...
+
+    # =========================================================================
+    # Documents
+    # =========================================================================
+
+    @abstractmethod
+    async def list_documents(
+        self,
+        bank_id: str,
+        *,
+        search_query: str | None = None,
+        limit: int = 100,
+        offset: int = 0,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        List documents with pagination.
+
+        Args:
+            bank_id: The memory bank ID.
+            search_query: Search query.
+            limit: Maximum results.
+            offset: Pagination offset.
+            request_context: Request context for authentication.
+
+        Returns:
+            Dict with 'items', 'total', 'limit', 'offset'.
+        """
+        ...
+
+    @abstractmethod
+    async def get_document(
+        self,
+        document_id: str,
+        bank_id: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any] | None:
+        """
+        Get a specific document.
+
+        Args:
+            document_id: The document ID.
+            bank_id: The memory bank ID.
+            request_context: Request context for authentication.
+
+        Returns:
+            Document dict or None if not found.
+        """
+        ...
+
+    @abstractmethod
+    async def delete_document(
+        self,
+        document_id: str,
+        bank_id: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, int]:
+        """
+        Delete a document and its memory units.
+
+        Args:
+            document_id: The document ID.
+            bank_id: The memory bank ID.
+            request_context: Request context for authentication.
+
+        Returns:
+            Dict with deletion counts.
+        """
+        ...
+
+    @abstractmethod
+    async def get_chunk(
+        self,
+        chunk_id: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any] | None:
+        """
+        Get a specific chunk.
+
+        Args:
+            chunk_id: The chunk ID.
+            request_context: Request context for authentication.
+
+        Returns:
+            Chunk dict or None if not found.
+        """
+        ...
+
+    # =========================================================================
+    # Entities
+    # =========================================================================
+
+    @abstractmethod
+    async def list_entities(
+        self,
+        bank_id: str,
+        *,
+        limit: int = 100,
+        offset: int = 0,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        List entities for a bank with pagination.
+
+        Args:
+            bank_id: The memory bank ID.
+            limit: Maximum results.
+            offset: Offset for pagination.
+            request_context: Request context for authentication.
+
+        Returns:
+            Dict with items, total, limit, offset.
+        """
+        ...
+
+    # =========================================================================
+    # Statistics & Operations
+    # =========================================================================
+
+    @abstractmethod
+    async def get_bank_stats(
+        self,
+        bank_id: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        Get statistics about memory nodes and links for a bank.
+
+        Args:
+            bank_id: The memory bank ID.
+            request_context: Request context for authentication.
+
+        Returns:
+            Dict with node_counts, link_counts, link_counts_by_fact_type,
+            link_breakdown, and operations stats.
+        """
+        ...
+
+    @abstractmethod
+    async def get_entity(
+        self,
+        bank_id: str,
+        entity_id: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any] | None:
+        """
+        Get entity details including metadata and observations.
+
+        Args:
+            bank_id: The memory bank ID.
+            entity_id: The entity ID.
+            request_context: Request context for authentication.
+
+        Returns:
+            Entity dict with id, canonical_name, mention_count, first_seen,
+            last_seen, metadata, and observations. None if not found.
+        """
+        ...
+
+    @abstractmethod
+    async def list_operations(
+        self,
+        bank_id: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        List async operations for a bank.
+
+        Args:
+            bank_id: The memory bank ID.
+            request_context: Request context for authentication.
+
+        Returns:
+            Dict with 'total' (int) and 'operations' (list of operation dicts).
+        """
+        ...
+
+    @abstractmethod
+    async def cancel_operation(
+        self,
+        bank_id: str,
+        operation_id: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        Cancel a pending async operation.
+
+        Args:
+            bank_id: The memory bank ID.
+            operation_id: The operation ID to cancel.
+            request_context: Request context for authentication.
+
+        Returns:
+            Dict with success status and message.
+
+        Raises:
+            ValueError: If operation not found.
+        """
+        ...
+
+    @abstractmethod
+    async def update_bank(
+        self,
+        bank_id: str,
+        *,
+        name: str | None = None,
+        mission: str | None = None,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        Update bank name and/or mission.
+
+        Args:
+            bank_id: The memory bank ID.
+            name: New bank name (optional).
+            mission: New mission text (optional, replaces existing).
+            request_context: Request context for authentication.
+
+        Returns:
+            Updated bank profile dict.
+        """
+        ...
+
+    @abstractmethod
+    async def submit_async_retain(
+        self,
+        bank_id: str,
+        contents: list[dict[str, Any]],
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        Submit a batch retain operation to run asynchronously.
+
+        Args:
+            bank_id: The memory bank ID.
+            contents: List of content dicts to retain.
+            request_context: Request context for authentication.
+
+        Returns:
+            Dict with operation_id and items_count.
+        """
+        ...

hindsight-api/hindsight_api/engine/llm_interface.py

@@ -0,0 +1,146 @@
+"""
+Abstract interface for LLM providers.
+
+This module defines the interface that all LLM providers must implement,
+enabling support for multiple LLM backends (OpenAI, Anthropic, Gemini, Codex, etc.)
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from .response_models import LLMToolCallResult, TokenUsage
+
+
+class LLMInterface(ABC):
+    """
+    Abstract interface for LLM providers.
+
+    All LLM provider implementations must inherit from this class and implement
+    the required methods.
+    """
+
+    def __init__(
+        self,
+        provider: str,
+        api_key: str,
+        base_url: str,
+        model: str,
+        reasoning_effort: str = "low",
+        **kwargs: Any,
+    ):
+        """
+        Initialize LLM provider.
+
+        Args:
+            provider: Provider name (e.g., "openai", "codex", "anthropic", "gemini").
+            api_key: API key or authentication token.
+            base_url: Base URL for the API.
+            model: Model name.
+            reasoning_effort: Reasoning effort level for supported providers.
+            **kwargs: Additional provider-specific parameters.
+        """
+        self.provider = provider.lower()
+        self.api_key = api_key
+        self.base_url = base_url
+        self.model = model
+        self.reasoning_effort = reasoning_effort
+
+    @abstractmethod
+    async def verify_connection(self) -> None:
+        """
+        Verify that the LLM provider is configured correctly by making a simple test call.
+
+        Raises:
+            RuntimeError: If the connection test fails.
+        """
+        pass
+
+    @abstractmethod
+    async def call(
+        self,
+        messages: list[dict[str, str]],
+        response_format: Any | None = None,
+        max_completion_tokens: int | None = None,
+        temperature: float | None = None,
+        scope: str = "memory",
+        max_retries: int = 10,
+        initial_backoff: float = 1.0,
+        max_backoff: float = 60.0,
+        skip_validation: bool = False,
+        strict_schema: bool = False,
+        return_usage: bool = False,
+    ) -> Any:
+        """
+        Make an LLM API call with retry logic.
+
+        Args:
+            messages: List of message dicts with 'role' and 'content'.
+            response_format: Optional Pydantic model for structured output.
+            max_completion_tokens: Maximum tokens in response.
+            temperature: Sampling temperature (0.0-2.0).
+            scope: Scope identifier for tracking.
+            max_retries: Maximum retry attempts.
+            initial_backoff: Initial backoff time in seconds.
+            max_backoff: Maximum backoff time in seconds.
+            skip_validation: Return raw JSON without Pydantic validation.
+            strict_schema: Use strict JSON schema enforcement (OpenAI only).
+            return_usage: If True, return tuple (result, TokenUsage) instead of just result.
+
+        Returns:
+            If return_usage=False: Parsed response if response_format is provided, otherwise text content.
+            If return_usage=True: Tuple of (result, TokenUsage) with token counts.
+
+        Raises:
+            OutputTooLongError: If output exceeds token limits.
+            Exception: Re-raises API errors after retries exhausted.
+        """
+        pass
+
+    @abstractmethod
+    async def call_with_tools(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]],
+        max_completion_tokens: int | None = None,
+        temperature: float | None = None,
+        scope: str = "tools",
+        max_retries: int = 5,
+        initial_backoff: float = 1.0,
+        max_backoff: float = 30.0,
+        tool_choice: str | dict[str, Any] = "auto",
+    ) -> LLMToolCallResult:
+        """
+        Make an LLM API call with tool/function calling support.
+
+        Args:
+            messages: List of message dicts. Can include tool results with role='tool'.
+            tools: List of tool definitions in OpenAI format.
+            max_completion_tokens: Maximum tokens in response.
+            temperature: Sampling temperature (0.0-2.0).
+            scope: Scope identifier for tracking.
+            max_retries: Maximum retry attempts.
+            initial_backoff: Initial backoff time in seconds.
+            max_backoff: Maximum backoff time in seconds.
+            tool_choice: How to choose tools - "auto", "none", "required", or specific function.
+
+        Returns:
+            LLMToolCallResult with content and/or tool_calls.
+        """
+        pass
+
+    @abstractmethod
+    async def cleanup(self) -> None:
+        """Clean up resources (close connections, etc.)."""
+        pass
+
+
+class OutputTooLongError(Exception):
+    """
+    Bridge exception raised when LLM output exceeds token limits.
+
+    This wraps provider-specific errors (e.g., OpenAI's LengthFinishReasonError)
+    to allow callers to handle output length issues without depending on
+    provider-specific implementations.
+    """
+
+    pass