fix: remove unused PluginState import from tools.ts

- Rename CI job from build-opencode-integration to test-opencode-integration
  to match naming convention for integrations that run tests
- Fix tsconfig module resolution to Node16 (consistent with other integrations)
- Extract shared makeConfig test helper to avoid duplication across 3 test files

.claude-plugin/marketplace.json

@@ -0,0 +1,15 @@
+{
+  "$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
+  "name": "hindsight",
+  "description": "Official Hindsight integrations for Claude Code",
+  "owner": {
+    "name": "vectorize-io"
+  },
+  "plugins": [
+    {
+      "name": "hindsight-memory",
+      "description": "Automatic long-term memory for Claude Code via Hindsight",
+      "source": "./hindsight-integrations/claude-code"
+    }
+  ]
+}

.claude/skills/code-review/SKILL.md

@@ -0,0 +1,196 @@
+---
+name: code-review
+description: Review changed code against project standards. Checks for missing tests, dead code, type safety, lint issues, and coding conventions. Run after completing any implementation work.
+user_invocable: true
+---
+
+# Code Review
+
+Review all changed code against the project's quality standards and coding conventions.
+
+## Code Standards
+
+Read and internalize these standards before writing code. The review steps below verify compliance.
+
+### Python Style
+- Python 3.11+, type hints required
+- Async throughout (asyncpg, async FastAPI)
+- Pydantic models for request/response
+- Ruff for linting (line-length 120)
+- No Python files at project root - maintain clean directory structure
+- **Never use multi-item tuple return values** — not even for internal/private functions. Always use a dataclass or Pydantic model. No exceptions, no "it's just two values" shortcuts. If a function returns more than one value, define a named type for it.
+
+### Type Safety with Pydantic Models
+**NEVER use raw `dict` types for structured data** — this applies to all code, including internal helpers and private functions. If the dict has known keys, it must be a dataclass or Pydantic model:
+- Use Pydantic `BaseModel` for all data structures passed between functions
+- Use `@dataclass` for lightweight internal data containers when Pydantic validation isn't needed
+- Add `@field_validator` for type coercion (e.g., ensuring datetimes are timezone-aware)
+- Avoid `dict.get()` patterns - use typed model attributes instead
+- Parse external data (JSON, API responses) into Pydantic models at the boundary
+- This catches type errors at parse time, not deep in business logic
+- The only acceptable `dict` usage is for truly dynamic/unknown keys (e.g., arbitrary metadata, JSON blobs with no fixed schema)
+
+```python
+# BAD - error-prone dict access
+def process(data: dict) -> str:
+    return data.get("name", "")  # No validation, silent failures
+
+# GOOD - typed and validated
+class UserData(BaseModel):
+    name: str
+    created_at: datetime
+
+    @field_validator("created_at", mode="before")
+    @classmethod
+    def ensure_tz_aware(cls, v):
+        if isinstance(v, str):
+            v = datetime.fromisoformat(v.replace("Z", "+00:00"))
+        if v.tzinfo is None:
+            return v.replace(tzinfo=timezone.utc)
+        return v
+
+def process(data: UserData) -> str:
+    return data.name  # Type-safe, validated at construction
+```
+
+### TypeScript Style
+- Next.js App Router for control plane
+- Tailwind CSS with shadcn/ui components
+
+### Code Comments
+- **Always comment non-trivial technical decisions** with the reasoning behind the choice. If someone would ask "why is it done this way?", there should be a comment.
+- **Keep comments up to date with history** — when changing an approach, update the comment to explain what was tried before and why it was changed. Comments serve as a tracker of previous implementations that likely had problems.
+- Don't comment obvious code — only where the "why" isn't self-evident from the code itself.
+
+```python
+# BAD - no context for future readers
+results = await asyncio.gather(*tasks, return_exceptions=True)
+
+# GOOD - explains the non-obvious choice
+# Use return_exceptions=True to avoid cancelling sibling tasks on failure.
+# Previously we used TaskGroup but it cancelled all tasks when one failed,
+# causing partial writes that left orphaned entity links (see #412).
+results = await asyncio.gather(*tasks, return_exceptions=True)
+```
+
+### Branch Hygiene
+- **Always start new feature branches from `origin/main`** — rebase to ensure a clean base.
+- **Only include commits relevant to the PR/branch/feature** — no unrelated changes. If the branch contains commits that don't belong, they must be removed before merging.
+
+### General Principles
+- Don't add features, refactor code, or make "improvements" beyond what was asked
+- Don't add unnecessary error handling for impossible scenarios
+- Don't create helpers or abstractions for one-time operations
+- No backwards-compatibility hacks (unused vars, re-exports, "removed" comments)
+- Three similar lines of code is better than a premature abstraction
+
+## Review Steps
+
+### 1. Check branch hygiene
+
+- Run `git log --oneline main..HEAD` to list all commits on the branch.
+- Verify every commit is relevant to the feature/PR. Flag any unrelated commits.
+- Check the branch is based on a recent `origin/main` (no stale base).
+
+### 2. Identify changed files
+
+Run `git diff --name-only HEAD` (unstaged) and `git diff --cached --name-only` (staged) to get all changed files. If there are no local changes, diff against the base branch using `git diff main...HEAD --name-only` and `git diff main...HEAD` to review all commits on the current branch.
+
+### 3. Run linters
+
+```bash
+./scripts/hooks/lint.sh
+```
+
+Report any failures. Do NOT fix them yourself — just report.
+
+### 4. Check for dead code
+
+For each changed Python file, check for:
+- Unused imports (Ruff should catch these, but verify)
+- Functions/methods/classes that were added but are never called from anywhere
+- Variables assigned but never read
+- Commented-out code blocks that should be removed
+
+For each changed TypeScript file, check for:
+- Unused imports
+- Unused variables or functions
+- Commented-out code
+
+### 5. Check type safety (Python)
+
+For each changed Python file, check for violations:
+- **No raw `dict` for structured data** — must use Pydantic model or dataclass, even for internal/private functions (only exception: truly dynamic/unknown keys)
+- **No multi-item tuple returns** — must use dataclass or Pydantic model, even for internal/private functions (no exceptions)
+- **Missing type hints** on function parameters and return types
+- **Missing `@field_validator`** for datetime fields that should be timezone-aware
+
+### 6. Check for missing tests
+
+For each new or significantly changed function/endpoint/class:
+- Check if there is a corresponding test addition or update
+- New API endpoints MUST have integration tests
+- New utility functions MUST have unit tests
+- Bug fixes SHOULD have a regression test
+
+Flag any new logic that lacks test coverage.
+
+### 7. Check API consistency
+
+If any files in `hindsight-api-slim/hindsight_api/api/` were changed:
+- Were the OpenAPI specs regenerated? (`./scripts/generate-openapi.sh`)
+- Were the client SDKs regenerated? (`./scripts/generate-clients.sh`)
+- Were the control plane proxy routes updated? (`hindsight-control-plane/src/app/api/`)
+
+### 8. Check code comments
+
+For each non-trivial change:
+- **New non-obvious logic** — is there a comment explaining the reasoning?
+- **Changed approach** — does the comment include what was done before and why it changed?
+- **Stale comments** — do existing comments near the changed code still accurately describe the behavior?
+
+### 9. Check integration completeness
+
+If any files in `hindsight-integrations/` were added or changed, verify:
+- **Tests exist** — the integration must have tests that simulate/exercise the external framework (not just pure unit tests of helpers). Check for a `tests/` directory with meaningful test files.
+- **CI job exists** — check `.github/workflows/test.yml` for a corresponding `test-<name>-integration` job. If missing, flag it.
+- **Release process** — check that the integration name is in the `VALID_INTEGRATIONS` array in `scripts/release-integration.sh`. If missing, flag it.
+- **Code standards** — the integration code must follow all Python style rules (type hints, no raw dicts, no tuple returns, etc.).
+
+### 10. Review against other coding standards
+
+Check the diff for violations of the standards listed above:
+- Python files at project root (not allowed)
+- Missing async patterns (should be async throughout)
+- Pydantic models for request/response
+- Line length > 120 chars
+- New features/code beyond what was asked (over-engineering)
+- Unnecessary error handling for impossible scenarios
+- Premature abstractions or speculative helpers
+- Backwards-compatibility hacks (unused vars, re-exports, "removed" comments)
+
+### 11. Report findings
+
+Present a clear summary organized by severity:
+
+**Must fix** — issues that will break CI or violate hard project rules:
+- Unrelated commits on the branch
+- Lint failures
+- Missing type hints on public functions
+- Raw dict usage for structured data (including internal code)
+- Multi-item tuple returns (including internal code)
+- Missing tests for new endpoints
+- New integration missing tests, CI job, or release-integration.sh entry
+
+**Should fix** — issues that hurt code quality:
+- Dead code / unused imports missed by linter
+- Missing tests for non-trivial utility functions
+- Over-engineering beyond the task scope
+
+**Note** — observations that may or may not need action:
+- API changes that might need client regeneration
+- Patterns that deviate from nearby code style
+
+For each finding, include the file path, line number, and a brief explanation.
+
+Do NOT auto-fix any issues. Report all findings and let the user decide what to address. If there are no findings, confirm the code looks good.

.env.example

@@ -2,14 +2,84 @@
 # Copy this file to .env and fill in your values
 
 # LLM Configuration (Required)
+# Supported providers: openai, groq, ollama, gemini, anthropic, lmstudio, vertexai, minimax, volcano
+HINDSIGHT_API_LLM_PROVIDER=openai
 HINDSIGHT_API_LLM_API_KEY=your-api-key-here
 HINDSIGHT_API_LLM_MODEL=gpt-4o-mini
 HINDSIGHT_API_LLM_BASE_URL=https://api.openai.com/v1
 
+# Example: Anthropic Claude configuration
+# HINDSIGHT_API_LLM_PROVIDER=anthropic
+# HINDSIGHT_API_LLM_API_KEY=your-anthropic-api-key
+# HINDSIGHT_API_LLM_MODEL=claude-sonnet-4-20250514
+
+# Example: Google Vertex AI configuration
+# HINDSIGHT_API_LLM_PROVIDER=vertexai
+# HINDSIGHT_API_LLM_MODEL=google/gemini-2.0-flash-001
+# HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=your-gcp-project-id
+# HINDSIGHT_API_LLM_VERTEXAI_REGION=us-central1
+# HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY=/path/to/service-account-key.json  # Optional, uses ADC if not set
+
+# Example: MiniMax configuration (1M context window)
+# HINDSIGHT_API_LLM_PROVIDER=minimax
+# HINDSIGHT_API_LLM_API_KEY=your-minimax-api-key
+# HINDSIGHT_API_LLM_MODEL=MiniMax-M2.7
+
+# Example: LM Studio local configuration (Qwen 2.5 32B recommended)
+# HINDSIGHT_API_LLM_PROVIDER=lmstudio
+# HINDSIGHT_API_LLM_API_KEY=lmstudio
+# HINDSIGHT_API_LLM_BASE_URL=http://localhost:1234/v1
+# HINDSIGHT_API_LLM_MODEL=qwen2.5-32b-instruct
+
 # API Configuration (Optional)
 HINDSIGHT_API_HOST=0.0.0.0
 HINDSIGHT_API_PORT=8888
 HINDSIGHT_API_LOG_LEVEL=info
 
+# Base Path / Reverse Proxy Support (Optional)
+# Set these when deploying behind a reverse proxy with path-based routing
+# Example: To deploy at example.com/hindsight/, set both to "/hindsight"
+# HINDSIGHT_API_BASE_PATH=/hindsight
+# NEXT_PUBLIC_BASE_PATH=/hindsight
+
 # Database (Optional - uses embedded pg0 by default)
 # HINDSIGHT_API_DATABASE_URL=postgresql://user:pass@host:5432/db
+# HINDSIGHT_API_MIGRATION_DATABASE_URL=  # Direct PostgreSQL URL for migrations (bypasses PgBouncer). Falls back to DATABASE_URL.
+# HINDSIGHT_API_DATABASE_SCHEMA=public  # PostgreSQL schema name (default: public)
+
+# Vector Extension (Optional - uses pgvector by default)
+# Options: "pgvector" (default), "vchord", "pgvectorscale" (DiskANN)
+# HINDSIGHT_API_VECTOR_EXTENSION=pgvector
+# For Azure PostgreSQL with DiskANN:
+# HINDSIGHT_API_VECTOR_EXTENSION=pgvectorscale  # Auto-detects pg_diskann on Azure
+
+# Embeddings Configuration (Optional - uses local by default)
+# Provider: "local" (default) or "tei" (HuggingFace Text Embeddings Inference)
+# HINDSIGHT_API_EMBEDDINGS_PROVIDER=local
+# For local provider:
+# HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL=BAAI/bge-small-en-v1.5
+# For TEI provider:
+# HINDSIGHT_API_EMBEDDINGS_TEI_URL=http://localhost:8080
+
+# Reranker Configuration (Optional - uses local by default)
+# Provider: "local" (default) or "tei" (HuggingFace Text Embeddings Inference)
+# HINDSIGHT_API_RERANKER_PROVIDER=local
+# For local provider:
+# HINDSIGHT_API_RERANKER_LOCAL_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2
+# For TEI provider:
+# HINDSIGHT_API_RERANKER_TEI_URL=http://localhost:8081
+
+# Observability & Tracing (Optional - disabled by default)
+# Enable OpenTelemetry tracing for LLM calls (GenAI semantic conventions)
+# HINDSIGHT_API_OTEL_TRACES_ENABLED=true
+#
+# Local development with Grafana LGTM stack (recommended - see scripts/dev/grafana/README.md)
+# HINDSIGHT_API_OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+#
+# Cloud backends (Grafana Cloud, Langfuse, DataDog, etc.)
+# HINDSIGHT_API_OTEL_EXPORTER_OTLP_ENDPOINT=https://your-backend-url
+# HINDSIGHT_API_OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer your-token"
+#
+# Custom service name and environment (optional, defaults: hindsight-api, development)
+# HINDSIGHT_API_OTEL_SERVICE_NAME=hindsight-production
+# HINDSIGHT_API_OTEL_DEPLOYMENT_ENVIRONMENT=production

.githooks/pre-commit

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Pre-commit hook - runs all scripts in scripts/hooks/
+
+set -e
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+HOOKS_DIR="$REPO_ROOT/scripts/hooks"
+
+if [ ! -d "$HOOKS_DIR" ]; then
+    exit 0
+fi
+
+echo ""
+echo "=== Running pre-commit hooks ==="
+echo ""
+
+# Run all executable scripts in hooks directory
+for hook in "$HOOKS_DIR"/*.sh; do
+    if [ -x "$hook" ]; then
+        echo "[hook] $(basename "$hook")"
+        (cd "$REPO_ROOT" && "$hook")
+    fi
+done
+
+echo ""
+echo "=== Pre-commit hooks completed ==="
+echo ""

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,71 @@
+name: Bug Report
+description: Report a bug or unexpected behavior
+labels: ["bug", "triage"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to report a bug! Please fill out the sections below.
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Bug Description
+      description: A clear and concise description of the bug
+      placeholder: What happened?
+    validations:
+      required: true
+
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to Reproduce
+      description: Steps to reproduce the behavior
+      placeholder: |
+        1. Configure '...'
+        2. Call '...'
+        3. See error
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected Behavior
+      description: What did you expect to happen?
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual
+    attributes:
+      label: Actual Behavior
+      description: What actually happened?
+    validations:
+      required: true
+
+  - type: input
+    id: version
+    attributes:
+      label: Version
+      description: What version are you using?
+      placeholder: e.g., 0.1.0 or commit hash
+    validations:
+      required: false
+
+  - type: dropdown
+    id: llm-provider
+    attributes:
+      label: LLM Provider
+      description: Which LLM provider are you using?
+      options:
+        - OpenAI
+        - Anthropic
+        - Gemini
+        - Groq
+        - Ollama
+        - LM Studio
+        - Other
+    validations:
+      required: false
+

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Questions & Help
+    url: https://github.com/vectorize-io/hindsight/discussions/categories/q-a
+    about: Please ask questions and get help in Discussions instead of opening an issue.
+  - name: Ideas & Feedback
+    url: https://github.com/vectorize-io/hindsight/discussions/categories/ideas
+    about: Share ideas or give feedback in Discussions.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,82 @@
+name: Feature Request
+description: Suggest a new feature or enhancement
+labels: ["enhancement", "triage"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for suggesting a feature! Please describe what you'd like to see added.
+
+  - type: textarea
+    id: use-case
+    attributes:
+      label: Use Case
+      description: Describe your specific use case. What are you building? What's your goal?
+      placeholder: |
+        I'm building an AI agent that needs to...
+        My application handles...
+    validations:
+      required: true
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem Statement
+      description: What problem are you facing? What's missing or difficult today?
+      placeholder: Currently I have to... which causes...
+    validations:
+      required: true
+
+  - type: textarea
+    id: benefit
+    attributes:
+      label: How This Feature Would Help
+      description: Explain how this feature would improve your workflow or solve your problem
+      placeholder: With this feature, I would be able to...
+    validations:
+      required: true
+
+  - type: textarea
+    id: solution
+    attributes:
+      label: Proposed Solution
+      description: Describe your ideal solution (optional - we may have ideas too!)
+      placeholder: It would be great if Hindsight could...
+    validations:
+      required: false
+
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives Considered
+      description: Have you considered any alternative solutions or workarounds?
+    validations:
+      required: false
+
+  - type: dropdown
+    id: priority
+    attributes:
+      label: Priority
+      description: How important is this feature to you?
+      options:
+        - Nice to have
+        - Important - affects my workflow
+        - Critical - blocking my use case
+    validations:
+      required: true
+
+  - type: textarea
+    id: additional
+    attributes:
+      label: Additional Context
+      description: Any other context, mockups, or examples?
+    validations:
+      required: false
+
+  - type: checkboxes
+    id: checklist
+    attributes:
+      label: Checklist
+      options:
+        - label: I would be willing to contribute this feature
+          required: false

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/deploy-docs.yml

@@ -20,19 +20,21 @@ concurrency:
 jobs:
   build:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: hindsight-docs
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
+      - uses: actions/checkout@v6
+      - uses: actions/setup-node@v6
         with:
           node-version: 20
           cache: npm
-          cache-dependency-path: hindsight-docs/package-lock.json
-      - run: npm ci
-      - run: npm run build
-      - uses: actions/upload-pages-artifact@v3
+          cache-dependency-path: package-lock.json
+      - uses: astral-sh/setup-uv@v7
+      - run: npm ci --workspace=hindsight-docs
+      - run: uv run generate-llms-full
+      - run: npm run build --workspace=hindsight-docs
+        env:
+          UMAMI_URL: https://analytics.hindsight.vectorize.io
+          UMAMI_WEBSITE_ID: ${{ secrets.UMAMI_WEBSITE_ID }}
+      - uses: actions/upload-pages-artifact@v4
         with:
           path: hindsight-docs/build
   deploy:
@@ -42,5 +44,5 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
-      - uses: actions/deploy-pages@v4
+      - uses: actions/deploy-pages@v5
         id: deployment

.github/workflows/release-integration.yml

@@ -0,0 +1,111 @@
+name: Release Integration
+
+on:
+  push:
+    tags:
+      - 'integrations/**'
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # for PyPI trusted publishing
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Extract integration info
+      id: info
+      run: |
+        # refs/tags/integrations/litellm/v0.1.0 → integration=litellm, version=0.1.0
+        TAG="${GITHUB_REF#refs/tags/}"
+        INTEGRATION=$(echo "$TAG" | cut -d'/' -f2)
+        VERSION=$(echo "$TAG" | cut -d'/' -f3 | sed 's/^v//')
+        echo "integration=$INTEGRATION" >> $GITHUB_OUTPUT
+        echo "version=$VERSION" >> $GITHUB_OUTPUT
+        echo "tag=$TAG" >> $GITHUB_OUTPUT
+        echo "Integration: $INTEGRATION, Version: $VERSION"
+
+    - name: Detect integration type
+      id: type
+      run: |
+        if [ -f "hindsight-integrations/${{ steps.info.outputs.integration }}/pyproject.toml" ]; then
+          echo "type=python" >> $GITHUB_OUTPUT
+        elif [ -f "hindsight-integrations/${{ steps.info.outputs.integration }}/package.json" ]; then
+          echo "type=typescript" >> $GITHUB_OUTPUT
+        else
+          echo "type=plugin" >> $GITHUB_OUTPUT
+        fi
+
+    # ── Python integrations (litellm, pydantic-ai, crewai) ──────────────────
+
+    - name: Install uv
+      if: steps.type.outputs.type == 'python'
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+
+    - name: Set up Python
+      if: steps.type.outputs.type == 'python'
+      uses: actions/setup-python@v6
+      with:
+        python-version-file: ".python-version"
+
+    - name: Build Python package
+      if: steps.type.outputs.type == 'python'
+      working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
+      run: uv build --out-dir dist
+
+    - name: Publish Python package to PyPI
+      if: steps.type.outputs.type == 'python'
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: ./hindsight-integrations/${{ steps.info.outputs.integration }}/dist
+        skip-existing: true
+
+    # ── TypeScript integrations (ai-sdk, chat, openclaw) ────────────────────
+
+    # ── Plugin integrations (claude-code) — no package to publish ───────────
+
+    - name: Plugin release
+      if: steps.type.outputs.type == 'plugin'
+      run: |
+        echo "Plugin integration ${{ steps.info.outputs.integration }} v${{ steps.info.outputs.version }} — no package to publish."
+        echo "Users install via: claude plugin marketplace add vectorize-io/hindsight --sparse hindsight-integrations"
+
+    # ── TypeScript integrations (ai-sdk, chat, openclaw) ────────────────────
+
+    - name: Set up Node.js
+      if: steps.type.outputs.type == 'typescript'
+      uses: actions/setup-node@v6
+      with:
+        node-version: '22'
+        registry-url: 'https://registry.npmjs.org'
+
+    - name: Install dependencies
+      if: steps.type.outputs.type == 'typescript'
+      working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
+      run: npm ci
+
+    - name: Build TypeScript package
+      if: steps.type.outputs.type == 'typescript'
+      working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
+      run: npm run build
+
+    - name: Publish TypeScript package to npm
+      if: steps.type.outputs.type == 'typescript'
+      working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
+      run: |
+        set +e
+        OUTPUT=$(npm publish --access public 2>&1)
+        EXIT_CODE=$?
+        echo "$OUTPUT"
+        if [ $EXIT_CODE -ne 0 ]; then
+          if echo "$OUTPUT" | grep -q "cannot publish over"; then
+            echo "Package version already published, skipping..."
+            exit 0
+          fi
+          exit $EXIT_CODE
+        fi
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/release.yml

@@ -13,15 +13,15 @@ jobs:
       id-token: write
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
       with:
         enable-cache: true
 
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version-file: ".python-version"
 
@@ -30,21 +30,39 @@ jobs:
       working-directory: ./hindsight-clients/python
       run: uv build --out-dir dist
 
+    - name: Build hindsight-api-slim
+      working-directory: ./hindsight-api-slim
+      run: uv build --out-dir dist
+
     - name: Build hindsight-api
       working-directory: ./hindsight-api
       run: uv build --out-dir dist
 
     - name: Build hindsight-all
-      working-directory: ./hindsight
+      working-directory: ./hindsight-all
       run: uv build --out-dir dist
 
-    # Publish in order (client and api first, then hindsight-all which depends on them)
+    - name: Build hindsight-all-slim
+      working-directory: ./hindsight-all-slim
+      run: uv build --out-dir dist
+
+    - name: Build hindsight-embed
+      working-directory: ./hindsight-embed
+      run: uv build --out-dir dist
+
+    # Publish in order (client and api-slim first, then api/all wrappers which depend on them)
     - name: Publish hindsight-client to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         packages-dir: ./hindsight-clients/python/dist
         skip-existing: true
 
+    - name: Publish hindsight-api-slim to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: ./hindsight-api-slim/dist
+        skip-existing: true
+
     - name: Publish hindsight-api to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
@@ -54,18 +72,33 @@ jobs:
     - name: Publish hindsight-all to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
-        packages-dir: ./hindsight/dist
+        packages-dir: ./hindsight-all/dist
+        skip-existing: true
+
+    - name: Publish hindsight-all-slim to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: ./hindsight-all-slim/dist
+        skip-existing: true
+
+    - name: Publish hindsight-embed to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: ./hindsight-embed/dist
         skip-existing: true
 
     # Upload artifacts for GitHub release
     - name: Upload artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: python-packages
         path: |
           hindsight-clients/python/dist/*
+          hindsight-api-slim/dist/*
           hindsight-api/dist/*
-          hindsight/dist/*
+          hindsight-all/dist/*
+          hindsight-all-slim/dist/*
+          hindsight-embed/dist/*
         retention-days: 1
 
   release-typescript-client:
@@ -73,25 +106,36 @@ jobs:
     environment: npm
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Set up Node.js
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
       with:
         node-version: '20'
         registry-url: 'https://registry.npmjs.org'
+        cache: 'npm'
+        cache-dependency-path: package-lock.json
 
     - name: Install dependencies
-      working-directory: ./hindsight-clients/typescript
-      run: npm ci
+      run: npm ci --workspace=hindsight-clients/typescript
 
     - name: Build
-      working-directory: ./hindsight-clients/typescript
-      run: npm run build
+      run: npm run build --workspace=hindsight-clients/typescript
 
     - name: Publish to npm
       working-directory: ./hindsight-clients/typescript
-      run: npm publish --access public
+      run: |
+        set +e
+        OUTPUT=$(npm publish --access public 2>&1)
+        EXIT_CODE=$?
+        echo "$OUTPUT"
+        if [ $EXIT_CODE -ne 0 ]; then
+          if echo "$OUTPUT" | grep -q "cannot publish over"; then
+            echo "Package version already published, skipping..."
+            exit 0
+          fi
+          exit $EXIT_CODE
+        fi
       env:
         NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
 
@@ -100,12 +144,74 @@ jobs:
       run: npm pack
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: typescript-client
         path: hindsight-clients/typescript/*.tgz
         retention-days: 1
 
+  release-control-plane:
+    runs-on: ubuntu-latest
+    environment: npm
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v6
+      with:
+        node-version: '20'
+        registry-url: 'https://registry.npmjs.org'
+        cache: 'npm'
+        cache-dependency-path: package-lock.json
+
+    - name: Install dependencies
+      run: npm ci
+
+    - name: Build TypeScript client (dependency)
+      run: npm run build --workspace=hindsight-clients/typescript
+
+    - name: Fix platform-specific native modules
+      run: |
+        # npm ci installs from lockfile which may have wrong platform binaries
+        # Delete hoisted native modules and reinstall for current platform
+        rm -rf node_modules/lightningcss node_modules/@tailwindcss
+        npm install lightningcss @tailwindcss/postcss @tailwindcss/node
+
+    - name: Build
+      run: npm run build --workspace=hindsight-control-plane
+
+    - name: Verify standalone build
+      run: test -f hindsight-control-plane/standalone/server.js || (echo 'standalone/server.js missing - build failed' && exit 1)
+
+    - name: Publish to npm
+      working-directory: ./hindsight-control-plane
+      run: |
+        set +e
+        OUTPUT=$(npm publish --access public --ignore-scripts 2>&1)
+        EXIT_CODE=$?
+        echo "$OUTPUT"
+        if [ $EXIT_CODE -ne 0 ]; then
+          if echo "$OUTPUT" | grep -q "cannot publish over"; then
+            echo "Package version already published, skipping..."
+            exit 0
+          fi
+          exit $EXIT_CODE
+        fi
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+    - name: Pack for GitHub release
+      working-directory: ./hindsight-control-plane
+      run: npm pack
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v7
+      with:
+        name: control-plane
+        path: hindsight-control-plane/*.tgz
+        retention-days: 1
+
   release-rust-cli:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -123,9 +229,13 @@ jobs:
             target: aarch64-apple-darwin
             artifact_name: hindsight
             asset_name: hindsight-darwin-arm64
+          - os: ubuntu-24.04-arm
+            target: aarch64-unknown-linux-gnu
+            artifact_name: hindsight
+            asset_name: hindsight-linux-arm64
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Install Rust
       uses: dtolnay/rust-toolchain@stable
@@ -143,13 +253,14 @@ jobs:
         chmod +x artifacts/${{ matrix.asset_name }}
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: rust-cli-${{ matrix.asset_name }}
         path: artifacts/${{ matrix.asset_name }}
         retention-days: 1
 
   release-docker-images:
+    name: Release Docker (${{ matrix.image_name }}${{ matrix.tag_suffix }})
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -159,18 +270,36 @@ jobs:
         include:
           - target: api-only
             image_name: hindsight-api
+            tag_suffix: ""
+            build_args: ""
+          - target: api-only
+            image_name: hindsight-api
+            tag_suffix: "-slim"
+            build_args: |
+              INCLUDE_LOCAL_MODELS=false
+              PRELOAD_ML_MODELS=false
           - target: cp-only
             image_name: hindsight-control-plane
+            tag_suffix: ""
+            build_args: ""
           - target: standalone
             image_name: hindsight
+            tag_suffix: ""
+            build_args: ""
+          - target: standalone
+            image_name: hindsight
+            tag_suffix: "-slim"
+            build_args: |
+              INCLUDE_LOCAL_MODELS=false
+              PRELOAD_ML_MODELS=false
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Free Disk Space
       uses: jlumbroso/free-disk-space@main
       with:
-        tool-cache: false
+        tool-cache: true
         android: true
         dotnet: true
         haskell: true
@@ -179,13 +308,13 @@ jobs:
         swap-storage: true
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
+      uses: docker/setup-qemu-action@v4
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@v4
 
     - name: Log in to GitHub Container Registry
-      uses: docker/login-action@v3
+      uses: docker/login-action@v4
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
@@ -195,23 +324,49 @@ jobs:
       id: get_version
       run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
 
-    - name: Extract metadata
+    - name: Extract metadata for release tags
       id: meta
-      uses: docker/metadata-action@v5
+      uses: docker/metadata-action@v6
       with:
         images: ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}
+        flavor: |
+          latest=auto
+          suffix=${{ matrix.tag_suffix }}
         tags: |
           type=semver,pattern={{version}},value=${{ steps.get_version.outputs.VERSION }}
           type=semver,pattern={{major}}.{{minor}},value=${{ steps.get_version.outputs.VERSION }}
           type=semver,pattern={{major}},value=${{ steps.get_version.outputs.VERSION }}
           type=raw,value=latest
 
-    - name: Build and push
-      uses: docker/build-push-action@v6
+    # TODO: Re-enable smoke test when disk space issue is resolved
+    # # Step 1: Build for local testing (single platform, no push)
+    # # This creates an identical image to what will be released, just for one platform
+    # - name: Build image for testing
+    #   uses: docker/build-push-action@v7
+    #   with:
+    #     context: .
+    #     file: docker/standalone/Dockerfile
+    #     target: ${{ matrix.target }}
+    #     push: false
+    #     load: true
+    #     tags: ${{ matrix.image_name }}:test
+    #     cache-from: type=gha
+    #     cache-to: type=gha,mode=max
+
+    # # Step 2: Test the image before pushing anything
+    # - name: Smoke test - verify container starts
+    #   env:
+    #     GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
+    #   run: ./docker/test-image.sh "${{ matrix.image_name }}:test" "${{ matrix.target }}"
+
+    # Build multi-platform and push to release tags
+    - name: Build and push release images
+      uses: docker/build-push-action@v7
       with:
         context: .
         file: docker/standalone/Dockerfile
         target: ${{ matrix.target }}
+        build-args: ${{ matrix.build_args }}
         push: true
         platforms: linux/amd64,linux/arm64
         tags: ${{ steps.meta.outputs.tags }}
@@ -219,23 +374,32 @@ jobs:
 
   release-helm-chart:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Install Helm
-      uses: azure/setup-helm@v4
+      uses: azure/setup-helm@v5
       with:
         version: 'latest'
 
+    - name: Log in to GHCR
+      run: echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin
+
     - name: Lint Helm chart
       run: helm lint helm/hindsight
 
     - name: Package Helm chart
       run: helm package helm/hindsight --destination ./helm-packages
 
+    - name: Push to GHCR OCI
+      run: helm push helm-packages/*.tgz oci://ghcr.io/${{ github.repository_owner }}/charts
+
     - name: Upload artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: helm-chart
         path: helm-packages/*.tgz
@@ -243,81 +407,86 @@ jobs:
 
   create-github-release:
     runs-on: ubuntu-latest
-    needs: [release-python-packages, release-typescript-client, release-rust-cli, release-docker-images, release-helm-chart]
+    needs: [release-python-packages, release-typescript-client, release-control-plane, release-rust-cli, release-docker-images, release-helm-chart]
     permissions:
       contents: write
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Extract version from tag
       id: get_version
       run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
 
-    - name: Download all artifacts
-      uses: actions/download-artifact@v4
+    - name: Download Python packages
+      uses: actions/download-artifact@v8
       with:
-        path: ./artifacts
+        name: python-packages
+        path: ./artifacts/python-packages
+
+    - name: Download TypeScript client
+      uses: actions/download-artifact@v8
+      with:
+        name: typescript-client
+        path: ./artifacts/typescript-client
+
+    - name: Download Control Plane
+      uses: actions/download-artifact@v8
+      with:
+        name: control-plane
+        path: ./artifacts/control-plane
+
+    - name: Download Rust CLI (Linux)
+      uses: actions/download-artifact@v8
+      with:
+        name: rust-cli-hindsight-linux-amd64
+        path: ./artifacts/rust-cli-linux
+
+    - name: Download Rust CLI (macOS Intel)
+      uses: actions/download-artifact@v8
+      with:
+        name: rust-cli-hindsight-darwin-amd64
+        path: ./artifacts/rust-cli-darwin-amd64
+
+    - name: Download Rust CLI (macOS ARM)
+      uses: actions/download-artifact@v8
+      with:
+        name: rust-cli-hindsight-darwin-arm64
+        path: ./artifacts/rust-cli-darwin-arm64
+
+    - name: Download Helm chart
+      uses: actions/download-artifact@v8
+      with:
+        name: helm-chart
+        path: ./artifacts/helm-chart
 
     - name: Prepare release assets
       run: |
         mkdir -p release-assets
         # Python packages
         cp artifacts/python-packages/hindsight-clients/python/dist/* release-assets/ || true
+        cp artifacts/python-packages/hindsight-api-slim/dist/* release-assets/ || true
         cp artifacts/python-packages/hindsight-api/dist/* release-assets/ || true
-        cp artifacts/python-packages/hindsight/dist/* release-assets/ || true
+        cp artifacts/python-packages/hindsight-all/dist/* release-assets/ || true
+        cp artifacts/python-packages/hindsight-all-slim/dist/* release-assets/ || true
+        cp artifacts/python-packages/hindsight-embed/dist/* release-assets/ || true
         # TypeScript client
         cp artifacts/typescript-client/*.tgz release-assets/ || true
+        # Control Plane
+        cp artifacts/control-plane/*.tgz release-assets/ || true
         # Rust CLI binaries
-        cp artifacts/rust-cli-hindsight-linux-amd64/hindsight-linux-amd64 release-assets/ || true
-        cp artifacts/rust-cli-hindsight-darwin-amd64/hindsight-darwin-amd64 release-assets/ || true
-        cp artifacts/rust-cli-hindsight-darwin-arm64/hindsight-darwin-arm64 release-assets/ || true
+        cp artifacts/rust-cli-linux/hindsight-linux-amd64 release-assets/ || true
+        cp artifacts/rust-cli-darwin-amd64/hindsight-darwin-amd64 release-assets/ || true
+        cp artifacts/rust-cli-darwin-arm64/hindsight-darwin-arm64 release-assets/ || true
         # Helm chart
         cp artifacts/helm-chart/*.tgz release-assets/ || true
         ls -la release-assets/
 
-    - name: Generate release notes
-      run: |
-        cat << 'EOF' > release-notes.md
-        ## Quick Start
-
-        ```bash
-        docker run -p 8888:8888 -p 9999:9999 \
-          -e HINDSIGHT_API_LLM_PROVIDER=openai \
-          -e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \
-          -e HINDSIGHT_API_LLM_MODEL=gpt-4o-mini \
-          ghcr.io/${{ github.repository_owner }}/hindsight:${{ steps.get_version.outputs.VERSION }}
-        ```
-
-        ## Docker Images
-        - `ghcr.io/${{ github.repository_owner }}/hindsight:${{ steps.get_version.outputs.VERSION }}` - Standalone (recommended)
-        - `ghcr.io/${{ github.repository_owner }}/hindsight-api:${{ steps.get_version.outputs.VERSION }}` - API only
-        - `ghcr.io/${{ github.repository_owner }}/hindsight-control-plane:${{ steps.get_version.outputs.VERSION }}` - Web UI only
-
-        ## Python
-        ```bash
-        pip install hindsight-all  # or hindsight-api, hindsight-client
-        ```
-
-        ## TypeScript/JavaScript
-        ```bash
-        npm install @vectorize-io/hindsight-client
-        ```
-
-        ## CLI
-        Download the appropriate binary from the release assets below.
-
-        ## Helm
-        ```bash
-        helm install hindsight oci://ghcr.io/${{ github.repository_owner }}/charts/hindsight --version ${{ steps.get_version.outputs.VERSION }}
-        ```
-        EOF
-
     - name: Create GitHub Release
       uses: softprops/action-gh-release@v2
       with:
         files: release-assets/*
-        body_path: release-notes.md
+        generate_release_notes: true
         draft: false
         prerelease: false
       env:

.gitignore

@@ -5,12 +5,18 @@ build/
 dist/
 wheels/
 *.egg-info
-
+.mcp.json
+.osgrep
 # Virtual environments
 .venv
 
-# Environment variables
+# Node
+node_modules/
+
+# Environment variables and local config
 .env
+docker-compose.yml
+docker-compose.override.yml
 
 # IDE
 .idea/
@@ -21,6 +27,10 @@ wheels/
 # NLTK data (will be downloaded automatically)
 nltk_data/
 
+# Monitoring stack (Prometheus/Grafana binaries and data)
+.monitoring/
+.pgbouncer/
+
 # Large benchmark datasets (will be downloaded automatically)
 **/longmemeval_s_cleaned.json
 
@@ -29,6 +39,22 @@ logs/
 
 .DS_Store
 
+# Generated docs files
+hindsight-docs/static/llms-full.txt
+
 
 hindsight-dev/benchmarks/locomo/results/
-hindsight-dev/benchmarks/longmemeval/results/
+hindsight-dev/benchmarks/longmemeval/results/
+hindsight-dev/benchmarks/consolidation/results/
+hindsight-dev/benchmarks/perf/results/
+benchmarks/results/
+hindsight-cli/target
+hindsight-clients/rust/target
+.claude/*
+!.claude/skills/
+whats-next.md
+TASK.md
+# Changelog is now tracked in hindsight-docs/src/pages/changelog.md
+# CHANGELOG.md
+
+blog-post*

AGENTS.md

@@ -0,0 +1,3 @@
+# AGENTS.md
+
+See [CLAUDE.md](./CLAUDE.md) for project documentation and coding conventions.

CLAUDE.md

@@ -0,0 +1,305 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+Hindsight is an agent memory system that provides long-term memory for AI agents using biomimetic data structures. Memories are organized as:
+- **World facts**: General knowledge ("The sky is blue")
+- **Experience facts**: Personal experiences ("I visited Paris in 2023")
+- **Mental models**: Consolidated knowledge synthesized from facts ("User prefers functional programming patterns")
+
+## Development Commands
+
+### Local Development (API + UI)
+```bash
+# Start both API server and control plane UI
+./scripts/dev/start.sh
+```
+
+### API Server (Python/FastAPI)
+```bash
+# Start API server only (loads .env automatically)
+./scripts/dev/start-api.sh
+
+# Run all tests (parallelized with pytest-xdist)
+cd hindsight-api-slim && uv run pytest tests/
+
+# Run specific test file
+cd hindsight-api-slim && uv run pytest tests/test_http_api_integration.py -v
+
+# Run single test function
+cd hindsight-api-slim && uv run pytest tests/test_retain.py::test_retain_simple -v
+
+# Lint and format
+cd hindsight-api-slim && uv run ruff check .
+cd hindsight-api-slim && uv run ruff format .
+
+# Type checking (uses ty - extremely fast type checker from Astral)
+cd hindsight-api-slim && uv run ty check hindsight_api/
+```
+
+### Control Plane (Next.js)
+```bash
+./scripts/dev/start-control-plane.sh
+# Or manually:
+cd hindsight-control-plane && npm run dev
+```
+
+### Documentation Site (Docusaurus)
+```bash
+./scripts/dev/start-docs.sh
+```
+
+
+### Generating Clients/OpenAPI
+```bash
+# Regenerate OpenAPI spec after API changes (REQUIRED after changing endpoints)
+./scripts/generate-openapi.sh
+
+# Regenerate all client SDKs (Python, TypeScript, Rust)
+./scripts/generate-clients.sh
+```
+
+### Benchmarks
+```bash
+# Accuracy benchmarks
+./scripts/benchmarks/run-longmemeval.sh
+./scripts/benchmarks/run-locomo.sh
+
+# Performance benchmarks
+./scripts/benchmarks/run-consolidation.sh
+./scripts/benchmarks/run-retain-perf.sh --document <path>  # Requires API server running
+
+# Results viewer
+./scripts/benchmarks/start-visualizer.sh  # View results at localhost:8001
+```
+
+## Architecture
+
+### Monorepo Structure
+- **hindsight-api-slim/**: Core FastAPI server with memory engine (Python, uv)
+- **hindsight-control-plane/**: Admin UI (Next.js, npm)
+- **hindsight-cli/**: CLI tool (Rust, cargo, uses progenitor for API client)
+- **hindsight-clients/**: Generated SDK clients (Python, TypeScript, Rust)
+- **hindsight-docs/**: Docusaurus documentation site
+- **hindsight-integrations/**: Framework integrations (LiteLLM, CrewAI, LangGraph, Pydantic AI, AG2, Claude Code, etc.)
+- **hindsight-dev/**: Development tools and benchmarks
+
+### Core Engine (hindsight-api-slim/hindsight_api/engine/)
+- `memory_engine.py`: Main orchestrator for retain/recall/reflect operations
+- `llm_wrapper.py`: LLM abstraction supporting OpenAI, Anthropic, Gemini, VertexAI, Groq, MiniMax, Ollama, LM Studio, LiteLLM, Claude Code
+- `embeddings.py`: Embedding generation (local sentence-transformers or TEI)
+- `cross_encoder.py`: Reranking (local or TEI)
+- `entity_resolver.py`: Entity extraction and normalization
+- `query_analyzer.py`: Query intent analysis
+
+**retain/**: Memory ingestion pipeline
+- `orchestrator.py`: Coordinates the retain flow
+- `fact_extraction.py`: LLM-based fact extraction from content
+- `link_utils.py`: Entity link creation and management
+
+**search/**: Multi-strategy retrieval
+- `retrieval.py`: Main retrieval orchestrator
+- `graph_retrieval.py`: Graph retrieval abstract base class
+- `link_expansion_retrieval.py`: Link expansion graph retrieval
+- `fusion.py`: Reciprocal rank fusion for combining results
+- `reranking.py`: Cross-encoder reranking
+
+### API Layer (hindsight-api-slim/hindsight_api/api/)
+- `http.py`: FastAPI HTTP routers for all REST endpoints
+- `mcp.py`: Model Context Protocol server implementation
+
+Main operations:
+- **Retain**: Store memories, extracts facts/entities/relationships
+- **Recall**: Retrieve memories via 4 parallel strategies (semantic, BM25, graph, temporal) + reranking
+- **Reflect**: Disposition-aware reasoning using memories and mental models.
+
+### Database
+PostgreSQL with pgvector. Schema managed via Alembic migrations in `hindsight-api-slim/hindsight_api/alembic/`. Migrations run automatically on API startup.
+
+Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
+
+### Adding Database Migrations
+
+1. **Create a new migration file** in `hindsight-api-slim/hindsight_api/alembic/versions/`:
+   - File name format: `<revision_id>_<description>.py` (e.g., `f1a2b3c4d5e6_add_new_index.py`)
+   - Use a unique hex revision ID (12 chars)
+   - Set `down_revision` to the previous migration's revision ID
+
+2. **Migration template**:
+   ```python
+   """Description of the migration
+
+   Revision ID: f1a2b3c4d5e6
+   Revises: <previous_revision_id>
+   Create Date: YYYY-MM-DD
+   """
+   from collections.abc import Sequence
+   from alembic import context, op
+
+   revision: str = "f1a2b3c4d5e6"
+   down_revision: str | Sequence[str] | None = "<previous_revision_id>"
+   branch_labels: str | Sequence[str] | None = None
+   depends_on: str | Sequence[str] | None = None
+
+   def _get_schema_prefix() -> str:
+       """Get schema prefix for table names (required for multi-tenant support)."""
+       schema = context.config.get_main_option("target_schema")
+       return f'"{schema}".' if schema else ""
+
+   def upgrade() -> None:
+       schema = _get_schema_prefix()
+       op.execute(f"CREATE INDEX ... ON {schema}table_name(...)")
+
+   def downgrade() -> None:
+       schema = _get_schema_prefix()
+       op.execute(f"DROP INDEX IF EXISTS {schema}index_name")
+   ```
+
+3. **Run migrations locally**:
+   ```bash
+   # Set database URL and run migrations for the base schema plus all tenants
+   uv run hindsight-admin run-db-migration
+
+   # Run on a specific tenant schema
+   uv run hindsight-admin run-db-migration --schema tenant_xyz
+   ```
+
+## Key Conventions
+
+### Code Quality
+
+**Before writing code, read `.claude/skills/code-review/SKILL.md`** for the full coding standards (Python style, type safety, TypeScript style, general principles).
+
+**Always run the lint script after making Python or TypeScript/Node changes:**
+```bash
+./scripts/hooks/lint.sh
+```
+
+**After completing any implementation work, run `/code-review`** to verify your changes against project standards (missing tests, dead code, type safety, etc.). Fix any "must fix" issues before considering the task done.
+
+**MANDATORY: Run `/code-review` before pushing code or creating a pull request.** Do not push or create a PR until all "must fix" issues are resolved.
+
+### Memory Banks
+- Each bank is an isolated memory store (like a "brain" for one user/agent)
+- Banks have dispositions (skepticism, literalism, empathy traits 1-5) affecting reflect
+- Banks can have background context
+- Bank isolation is strict - no cross-bank data leakage
+
+### API Design
+- All endpoints operate on a single bank per request
+- Multi-bank queries are client responsibility to orchestrate
+- Disposition traits only affect reflect, not recall
+
+### Control Plane API Routes
+
+When adding or modifying parameters in the dataplane API (hindsight-api), you must also update the control plane routes that proxy to it:
+
+1. **API Routes** (`hindsight-control-plane/src/app/api/`):
+   - `recall/route.ts` - proxies to `/v1/default/banks/{bank_id}/memories/recall`
+   - `reflect/route.ts` - proxies to `/v1/default/banks/{bank_id}/reflect`
+   - `memories/retain/route.ts` - proxies to `/v1/default/banks/{bank_id}/memories/retain`
+   - Other routes follow the same pattern
+
+2. **Client types** (`hindsight-control-plane/src/lib/api.ts`):
+   - Update the TypeScript type definitions for `recall()`, `reflect()`, `retain()` etc.
+
+3. **Checklist when adding new API parameters**:
+   - Add parameter extraction in the route handler (destructure from `body`)
+   - Pass the parameter to the SDK call
+   - Update the client type definition in `lib/api.ts`
+   - Update any UI components that need to use the new parameter
+
+### Adding New Integrations
+
+Every new integration in `hindsight-integrations/` must satisfy all of the following before it can be merged:
+
+1. **Tests are required** — tests must simulate or exercise the external system (mock the framework's interfaces and verify the integration actually calls Hindsight correctly). Pure unit tests of helper functions are not sufficient.
+2. **CI job** — add a test job in `.github/workflows/test.yml` following the existing pattern (e.g., `test-crewai-integration`). The job must build, install deps, and run `uv run pytest tests -v`. Also add the integration to `detect-changes` outputs so it only runs when its files change.
+3. **Release process** — add the integration name to the `VALID_INTEGRATIONS` array in `scripts/release-integration.sh` so it can be released via the standard release workflow.
+4. **Follow project code standards** — Python style, type safety, no raw dicts for structured data, no multi-item tuple returns (see `.claude/skills/code-review/SKILL.md`).
+
+If any of these are missing, the integration is incomplete and must not be pushed or merged.
+
+### Adding New API Configuration Flags
+
+Configuration follows a hierarchical system: **Global (env vars) → Tenant (via extension) → Bank (database)**.
+
+Fields must be categorized as either **hierarchical** (can be overridden per-tenant/bank) or **static** (server-level only).
+
+#### Adding a New Configuration Field
+
+1. **config.py** (`hindsight-api-slim/hindsight_api/config.py`):
+   - Add `ENV_*` constant for the environment variable name (e.g., `ENV_MY_SETTING = "HINDSIGHT_API_MY_SETTING"`)
+   - Add `DEFAULT_*` constant for the default value
+   - Add field to `HindsightConfig` dataclass with type annotation
+   - **Mark as configurable** by adding to `_CONFIGURABLE_FIELDS` set if the field should be overridable per-tenant/bank via API
+   - Add initialization in `from_env()` method
+
+   ```python
+   # Configurable field (can be overridden per-tenant/bank via API)
+   _CONFIGURABLE_FIELDS = {
+       ...,
+       "my_setting",  # Add here for configurable
+   }
+
+   # Static field - just don't add to _CONFIGURABLE_FIELDS
+   ```
+
+2. **main.py** (`hindsight-api-slim/hindsight_api/main.py`):
+   - Add field to the manual `HindsightConfig()` constructor call (search for "CLI override")
+
+3. **Use hierarchical config in MemoryEngine**:
+   ```python
+   # Config is resolved automatically per bank via ConfigResolver
+   config_dict = await self._config_resolver.get_bank_config(bank_id, context)
+   value = config_dict["my_setting"]
+   ```
+
+4. **Use static config** (non-hierarchical):
+   ```python
+   from ...config import get_config
+   config = get_config()
+   value = config.my_static_field
+   ```
+
+5. **Documentation** (`hindsight-docs/docs/developer/configuration.md`):
+   - Add to appropriate section table with Variable, Description, Default
+   - Mark if it's hierarchical (can be overridden per-bank)
+
+#### Hierarchical vs Static Guidelines
+
+**Hierarchical** (per-bank overridable):
+- LLM settings (provider, model, API key, base URL)
+- Operation-specific settings (retain mode, chunk size, etc.)
+- Feature flags that vary by customer/bank
+
+**Static** (server-level only):
+- Infrastructure settings (database URL, port, host)
+- Global limits (max concurrent operations)
+- System-wide feature flags
+
+## Environment Setup
+
+```bash
+cp .env.example .env
+# Edit .env with LLM API key
+
+# Python deps
+uv sync --directory hindsight-api-slim/
+
+# Node deps (uses npm workspaces)
+npm install
+```
+
+Required env vars:
+- `HINDSIGHT_API_LLM_PROVIDER`: openai, anthropic, gemini, groq, minimax, ollama, lmstudio
+- `HINDSIGHT_API_LLM_API_KEY`: Your API key
+- `HINDSIGHT_API_LLM_MODEL`: Model name (e.g., gpt-4o-mini, claude-sonnet-4-20250514)
+
+Optional (uses local models by default):
+- `HINDSIGHT_API_EMBEDDINGS_PROVIDER`: local (default) or tei
+- `HINDSIGHT_API_RERANKER_PROVIDER`: local (default) or tei
+- `HINDSIGHT_API_DATABASE_URL`: External PostgreSQL (uses embedded pg0 by default)
+- `HINDSIGHT_API_ENABLE_BANK_CONFIG_API`: Enable per-bank config API (default: true)

CODE_OF_CONDUCT.md

@@ -0,0 +1,127 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

CONTRIBUTING.md

@@ -5,13 +5,23 @@ Thanks for your interest in contributing to Hindsight!
 ## Getting Started
 
 1. Fork and clone the repository
-2. Install dependencies:
    ```bash
-   cd hindsight-api && uv sync
+   git clone [email protected]:vectorize-io/hindsight.git
+   cd hindsight
    ```
-3. Set up your environment:
+2. Set up your environment:
    ```bash
-   export OPENAI_API_KEY=your-key
+   cp .env.example .env
+   ```
+   Edit the .env to add LLM API key and config as required
+
+3. Install dependencies:
+   ```bash
+   # Python dependencies
+   uv sync --directory hindsight-api/
+
+   # Node dependencies (uses npm workspaces)
+   npm install
    ```
 
 ## Development
@@ -41,7 +51,36 @@ cd hindsight-api
 uv run pytest tests/
 ```
 
-### Code style
+### Code Style
+
+We use [Ruff](https://docs.astral.sh/ruff/) for Python linting and formatting, and ESLint/Prettier for TypeScript.
+
+#### Setting up git hooks (recommended)
+
+Set up git hooks to automatically lint and format code before each commit:
+
+```bash
+./scripts/setup-hooks.sh
+```
+
+This configures git to use the hooks in `.githooks/`, which run all scripts in `scripts/hooks/` on commit. The lint hook runs in parallel:
+- **Python**: `ruff check --fix`, `ruff format`, `ty check`
+- **TypeScript**: `eslint --fix`, `prettier`
+
+#### Manual linting and formatting
+
+```bash
+# Run all lints (same as pre-commit)
+./scripts/hooks/lint.sh
+
+# Or run individually for Python:
+cd hindsight-api
+uv run ruff check --fix .   # Lint and auto-fix
+uv run ruff format .        # Format code
+uv run ty check hindsight_api  # Type check
+```
+
+#### Style guidelines
 
 - Use Python type hints
 - Follow existing code patterns
@@ -54,6 +93,34 @@ uv run pytest tests/
 3. Run tests to ensure nothing breaks
 4. Submit a PR with a clear description of changes
 
+## Release Process
+
+The project uses `scripts/release.sh` for creating releases. This script automates the entire release workflow:
+
+1. Bumps version in all components (API, clients, CLI, control plane, Helm)
+2. **Regenerates OpenAPI spec and client SDKs** (Python, TypeScript, Rust)
+3. Updates documentation versioning
+4. Creates a commit and git tag
+5. Pushes to GitHub (triggers CI/CD to publish packages)
+
+### Usage
+
+```bash
+./scripts/release.sh <version>
+```
+
+**Example:**
+```bash
+./scripts/release.sh 0.5.0
+```
+
+### Important for Developers
+
+- During development, version bumps in `__init__.py` do NOT require client regeneration
+- Clients are only regenerated during releases
+- Do not manually run `./scripts/generate-clients.sh` unless testing generation changes
+- Client version comments will reflect the API version from the latest release
+
 ## Reporting Issues
 
 Open an issue on GitHub with:

HINDSIGHT_PAPER.md

@@ -1,933 +0,0 @@
-# Hindsight: A Unified Memory System for AI Agents with Temporal Retrieval and Personality-Driven Reasoning
-
-## Abstract
-
-We present **Hindsight**, a comprehensive memory architecture for conversational AI agents that combines multi-strategy retrieval with personality-driven reasoning to enable both high-recall factual search and consistent, trait-based opinion formation. The system consists of two integrated components: **TEMPR (Temporal Entity Memory Priming Retrieval)** for memory recall, and **CARA (Coherent Adaptive Reasoning Agents)** for personality-aware reflection. TEMPR achieves strong retrieval performance through four parallel search strategies—semantic vector search, BM25 keyword matching, graph-based spreading activation incorporating multiple link types (entity, semantic, temporal, causal), and temporal-aware graph traversal—achieving 73.50% on LoComo and 80.60% on LongMemEval benchmarks, with particularly strong performance on multi-hop reasoning (+15.8% over baseline). CARA builds on TEMPR's four-network architecture (world facts, bank experiences, opinions, and observations) to enable personality-driven reasoning using the Big Five model, allowing agents to form and evolve opinions influenced by configurable traits while maintaining epistemic clarity between objective information and subjective beliefs. A novel observation paradigm automatically synthesizes entity-level summaries from multiple facts, creating structured mental models of people, organizations, and concepts without personality influence. The combination enables AI agents with long-term memory that can both retrieve information accurately and reason consistently with stable character traits.
-
----
-
-# Part I: Recall - TEMPR (Temporal Entity Memory Priming Retrieval)
-
-## 1. Introduction to Recall
-
-Conversational AI agents face a fundamental challenge: maintaining coherent, context-aware memories across extended interactions. Traditional search systems are optimized for human users with top-k ranking and relevance feedback, but AI agents have fundamentally different requirements: they need to retrieve variable amounts of information based on reasoning complexity while respecting LLM context windows. Existing approaches rely either on vector similarity search, which captures semantic relationships but misses entity-level connections, or on keyword matching, which provides precision but lacks conceptual understanding. Neither approach adequately handles the temporal aspects of memory or entity-based reasoning that enable multi-hop information discovery.
-
-We propose TEMPR, a memory retrieval architecture designed specifically for AI agents that combines established information retrieval techniques—semantic vector search, BM25 keyword matching, spreading activation graph traversal (Anderson 1983), and neural reranking—into a unified system optimized for agent workflows. The key architectural choices are:
-
-1. **Agent-Optimized Interface**: budget and max_tokens parameters instead of traditional top-k ranking
-2. **Comprehensive Narrative Fact Extraction with Temporal Ranges**: LLM-powered extraction that creates self-contained narrative facts preserving full conversational context, extracting temporal ranges (occurred_start/end) to distinguish point events from periods
-3. **Entity-Aware Graph Structure with Multiple Link Types**: LLM-based entity resolution and linking that connects memories through shared identities, along with temporal, semantic, and causal link types
-4. **Four-Way Parallel Retrieval**: Semantic, keyword, graph-based (spreading activation), and temporal range retrieval strategies executed in parallel and fused using RRF (Cormack et al. 2009)
-5. **Neural Cross-Encoder Reranking**: Learned query-document relevance with temporal awareness and token budget filtering
-
-This combination of techniques enables agents to discover indirectly related information through graph traversal while maintaining temporal awareness, achieving strong performance on multi-hop reasoning tasks.
-
-### 1.1 Contributions
-
-Our key contributions for the recall system are:
-
-1. **Agent-Optimized Retrieval Interface**: Unlike traditional top-k search optimized for human users, we introduce budget and max_tokens parameters that allow AI agents to dynamically trade off latency for recall based on reasoning complexity and context window constraints
-
-2. **Four-Way Parallel Retrieval**: We combine semantic vector search, BM25 keyword matching, graph-based spreading activation (Anderson 1983), and temporal-aware graph traversal into a unified parallel retrieval pipeline using Reciprocal Rank Fusion (Cormack et al. 2009) and neural cross-encoder reranking. The graph traversal incorporates multiple link types (entity, semantic, temporal, causal) with configurable weighting during activation spreading.
-
-3. **LLM-Based Knowledge Graph Construction with Temporal Ranges**: We leverage open-source LLMs for comprehensive narrative fact extraction, entity recognition, and entity disambiguation. The system extracts temporal ranges (occurred_start, occurred_end) to represent both point events and extended periods, distinguishing when facts occurred from when they were mentioned.
-
-4. **Strong Performance on Multi-Hop Reasoning**: 73.50% on LoComo and 80.60% on LongMemEval, with particularly strong performance on multi-hop queries (+15.8% over Mem0), demonstrating the effectiveness of combining these techniques for discovering indirectly related information in conversational contexts
-
-## 2. Memory Organization
-
-### 2.1 Four Memory Networks
-
-TEMPR organizes memories into four distinct networks for epistemic clarity:
-
-**World Network** (fact_type='world'): Objective information about the world
-- Example: "Alice works at Google in Mountain View on the AI team"
-- Stores facts received from external sources
-- No confidence scores (facts are information received, not beliefs)
-
-**Bank Network** (fact_type='bank'): Biographical information about the agent itself
-- Example: "I recommended Yosemite National Park to Alice for hiking"
-- Stores the agent's own actions and experiences
-- Uses first-person perspective ("I recommended..." not "The agent recommended...")
-
-**Opinion Network** (fact_type='opinion'): Subjective beliefs formed by the agent
-- Example: "Python is better for data science because of libraries like pandas (confidence: 0.85)"
-- Stores judgments and opinions with confidence scores
-- Evolved through opinion reinforcement when new evidence arrives
-- Influenced by personality traits (see Part II: Reflect)
-
-**Observation Network** (fact_type='observation'): Synthesized entity summaries
-- Example: "Alice is a software engineer at Google specializing in machine learning"
-- Objective syntheses from multiple facts about an entity
-- Generated WITHOUT personality influence (unlike opinions)
-- Automatically created and updated in background processes
-- Provides structured "mental models" of entities
-
-This separation provides:
-- **Epistemic Clarity**: Facts represent information encountered; opinions represent personality-driven judgments; observations represent objective syntheses
-- **Traceability**: Opinion reinforcement traces facts; observations trace entity-related facts
-- **Debugging**: Developers can separately inspect factual knowledge, formed beliefs, and entity models
-- **Confidence Semantics**: Facts and observations lack confidence scores; opinions have confidence scores representing conviction strength
-- **Personality Independence**: Observations remain objective while opinions reflect personality
-
-### 2.2 Memory Unit Structure
-
-Each memory is represented as a self-contained node with:
-
-- id: Unique UUID
-- bank_id: Identifier for the memory bank this memory belongs to
-- text: Self-contained comprehensive narrative fact
-- embedding: 384-dimensional vector (BAAI/bge-small-en-v1.5)
-- event_date: Timestamp when the fact became true (maintained for backward compatibility)
-- occurred_start: Timestamp when the fact/event started (temporal range support)
-- occurred_end: Timestamp when the fact/event ended (temporal range support)
-- mentioned_at: Timestamp when the fact was mentioned/learned
-- context: Optional contextual metadata
-- fact_type: One of 'world', 'bank', 'opinion'
-- confidence_score: For opinions only, strength of conviction (0.0-1.0)
-- access_count: Frequency-based importance signal
-- search_vector: Full-text search tsvector for BM25 ranking
-
-### 2.3 LLM-Powered Comprehensive Narrative Fact Extraction
-
-TEMPR employs **LLM-powered comprehensive narrative fact extraction** using open-source models. This approach provides more context-aware extraction compared to traditional rule-based NLP pipelines, though at higher computational cost.
-
-#### 2.3.1 Extraction Principles
-
-**Chunking Strategy**: TEMPR uses a coarse-grained chunking approach, extracting 2-5 comprehensive facts per conversation rather than dozens of atomic fragments. This is a deliberate tradeoff: larger chunks preserve more context and narrative flow, at the cost of reduced precision when only a small portion of the chunk is relevant.
-
-Each fact should:
-1. **Capture entire conversations or exchanges** - Include the full back-and-forth discussion
-2. **Be narrative and comprehensive** - Tell the complete story with all context
-3. **Be self-contained** - Readable without the original text
-4. **Include all participants** - WHO said/did WHAT, with their reasoning
-5. **Preserve the flow** - Keep related exchanges together in one fact
-
-**Example Comparison**:
-
-❌ **Fragmented Approach** (traditional):
-- "Bob suggested Summer Vibes"
-- "Alice wanted something unique"
-- "They considered Sunset Sessions"
-- "Alice likes Beach Beats"
-- "They chose Beach Beats"
-
-✅ **Comprehensive Approach** (TEMPR):
-- "Alice and Bob discussed naming their summer party playlist. Bob suggested 'Summer Vibes' because it's catchy and seasonal, but Alice wanted something more unique. Bob then proposed 'Sunset Sessions' and 'Beach Beats', with Alice favoring 'Beach Beats' for its playful and fun tone. They ultimately decided on 'Beach Beats' as the final name."
-
-#### 2.3.2 Open-Source LLM Extraction Pipeline
-
-The extraction process leverages open-source LLMs with structured output (Pydantic schemas). This follows the established practice of using LLMs for information extraction, which has been shown to improve context understanding compared to rule-based NLP pipelines, particularly for:
-- Coreference resolution in conversational text
-- Domain-specific entity recognition
-- Maintaining narrative coherence across multi-turn exchanges
-
-**LLM Extraction Steps**:
-1. **Pronoun Resolution**: "She loves hiking" → "Alice loves hiking"
-2. **Temporal Normalization**: "last year" → "in 2023" (absolute dates)
-3. **Temporal Range Extraction**: Identify when facts occurred vs. when mentioned
-   - Point events: "on July 14" → occurred_start = occurred_end = 2023-07-14
-   - Period events: "in February 2023" → occurred_start = 2023-02-01, occurred_end = 2023-02-28
-   - Vague periods: "lately" → estimated range based on context
-   - mentioned_at = conversation date (when fact was learned)
-4. **Participant Attribution**: Preserve WHO said/did WHAT
-5. **Reasoning Preservation**: Include WHY decisions were made
-6. **Fact Type Classification**: Determine fact categories (world, bank, opinion)
-7. **Entity Extraction**: Identify all entities (PERSON, ORG, LOCATION, PRODUCT, CONCEPT)
-
-**Temporal Augmentation**: Before embedding, facts are augmented with readable temporal information:
-- Original: "Alice started working at Google"
-- Augmented for embedding: "Alice started working at Google (happened in November 2023)"
-
-This augmentation helps semantic search understand temporal relevance without modifying the stored fact text.
-
-### 2.4 Entity Resolution and Linking
-
-Entity resolution creates strong connections between memories that share common entities, solving the problem where semantically dissimilar facts are related through shared identities.
-
-#### 2.4.1 LLM-Based Entity Recognition
-
-TEMPR uses the same open-source LLM that performs fact extraction to also identify and extract entities during the narrative fact creation process. This unified approach eliminates the brittleness of traditional NER pipelines that struggle with domain-specific entities, novel names, and context-dependent disambiguation.
-
-**Entity Types**:
-- PERSON: "Alice", "Bob Chen"
-- ORGANIZATION: "Google", "Stanford University"
-- LOCATION: "Yosemite National Park", "California"
-- PRODUCT: "Python", "pandas library"
-- CONCEPT: "machine learning", "remote work"
-- OTHER: Miscellaneous proper nouns
-
-#### 2.4.2 LLM-Based Entity Disambiguation
-
-Multiple mentions of entities (e.g., "Alice", "Alice Chen", "Alice C.") must be resolved to a single canonical entity. TEMPR uses the LLM to perform entity disambiguation, analyzing the surrounding context to determine if two entity mentions refer to the same entity. This handles complex cases like:
-- Nicknames and formal names ("Bob" vs. "Robert Chen")
-- Partial mentions ("Alice" vs. "Alice Chen")
-- Context-dependent disambiguation ("Apple the company" vs. "apple the fruit")
-
-The LLM considers multiple signals:
-- **Name Similarity**: String similarity using Levenshtein distance
-- **Co-occurrence Patterns**: Entities mentioned together frequently are likely distinct
-- **Temporal Proximity**: Recent mentions are more likely to refer to the same entity
-
-#### 2.4.3 Entity Link Structure
-
-Each entity creates a link_type='entity' edge between all memories mentioning it:
-
-**Properties**:
-- weight=1.0 (constant, no temporal decay)
-- entity_id: Reference to resolved canonical entity
-- Bidirectional connections between all mentioning memories
-
-**Impact on Retrieval**: Entity links enable graph traversal to discover indirectly related facts:
-
-**Example Query**: "What does Alice do?"
-1. **Semantic Match**: "Alice works at Google in Mountain View..." (direct match)
-2. **Entity Traversal**: Follow entity links for "Alice" →
-   - "Alice loves hiking in Yosemite..." (different semantic space)
-   - "I recommended technical books to Alice" (Bank Network, via "Alice")
-3. **Chained Traversal**: Follow "Google" entity →
-   - "Google's office in Mountain View has excellent amenities"
-
-### 2.5 Link Types and Graph Structure
-
-The memory graph contains four types of edges connecting memory units:
-
-#### 2.5.1 Temporal Links
-
-Temporal links connect memories close in time, enabling temporal reasoning:
-
-**Creation Logic**:
-
-**Properties**:
-- Decays linearly with time distance
-- Minimum weight 0.3 to maintain some connectivity
-- Enables "What happened around the same time?" queries
-
-#### 2.5.2 Semantic Links
-
-Semantic links connect memories with similar meanings:
-
-**Creation Logic**:
-
-**Properties**:
-- Uses pgvector HNSW index for efficient nearest-neighbor search
-- Higher threshold (0.7) than retrieval (0.3) to avoid over-connection
-- Weight equals cosine similarity score
-
-#### 2.5.3 Entity Links
-
-Entity links (described in Section 2.4.3) create the strongest connections:
-
-**Properties**:
-- weight=1.0 (constant, never decays)
-- Connects all memories mentioning the same resolved entity
-- Most reliable traversal path during graph search
-
-#### 2.5.4 Causal Links
-
-Causal links represent identified cause-effect relationships between facts. During fact extraction, the LLM attempts to identify causal relationships between facts extracted from the same conversation. These links are incorporated as one component of the graph retrieval system.
-
-**Causal Relationship Types**:
-- causes: This fact directly causes the target fact
-- caused_by: This fact was caused by the target fact (inverse of causes)
-- enables: This fact enables or allows the target fact to happen
-- prevents: This fact prevents or blocks the target fact
-
-**Properties**:
-- weight: Strength of causal relationship ∈ [0.0, 1.0] (default 1.0)
-- Directional edges (from cause to effect)
-- Prioritized during graph traversal with 2x activation boost
-
-**Role in Retrieval**: Causal links provide an additional signal during graph-based retrieval. When present, they allow the system to traverse explanatory relationships in addition to semantic, temporal, and entity-based connections.
-
-**Example**: For a query "Why does Alice spend time in the garden?", the system may find both direct semantic matches ("Alice spends time in the garden to find comfort") and traverse causal links to related facts ("Alice lost her friend Karlie in February 2023").
-
-**Graph Density**: Each memory unit typically has:
-- 5-10 temporal links (to nearby memories)
-- 3-5 semantic links (to similar content)
-- Variable entity links (depending on entity mention frequency)
-- 0-3 causal links (when causal relationships are identified)
-
-### 2.6 The Observation Paradigm
-
-A critical challenge in long-term memory systems is maintaining structured, high-level understanding of entities (people, organizations, places, concepts) without re-reading all individual facts each time. Traditional approaches either retrieve all entity-related facts (expensive, noisy) or maintain no entity-level state (losing structured understanding). Hindsight introduces **observations**—automatically synthesized entity summaries that provide structured "mental models" without personality influence.
-
-#### 2.6.1 Motivation and Design
-
-**The Problem**: When a system accumulates dozens of facts about an entity like "Alice," queries about Alice must either:
-1. Retrieve all 50+ individual facts (expensive, overwhelming)
-2. Rely only on top-k semantic matches (may miss key attributes)
-3. Manually maintain entity profiles (doesn't scale, requires human curation)
-
-**The Solution**: Observations provide a fourth fact type that synthesizes multiple facts into coherent, objective entity summaries, automatically maintained as new information arrives.
-
-**Key Properties**:
-- **Objective Synthesis**: Generated WITHOUT personality influence (unlike opinions)
-- **Entity-Scoped**: Each observation is about a single entity
-- **Automatic Maintenance**: Generated in background after fact ingestion
-- **Multi-Fact Fusion**: Combines information scattered across multiple facts
-- **Response Augmentation**: NOT used for retrieval/search, but returned alongside results when include_entities=True to provide entity context
-
-#### 2.6.2 Observation Generation
-
-Observations are generated through an LLM-powered synthesis process:
-
-**Trigger**: When new facts mentioning an entity are ingested via retain(), a background task is queued to regenerate observations for that entity.
-
-**Process**:
-
-**LLM Prompt Structure**:
-
-**Example Transformation**:
-
-**Input Facts**:
-- "Alice works at Google"
-- "Alice is a software engineer"
-- "Alice specializes in ML and deep learning"
-- "Alice joined Google in 2023"
-- "Alice is detail-oriented and methodical"
-
-**Generated Observations**:
-- "Alice is a software engineer at Google specializing in machine learning and deep learning"
-- "Alice joined Google in 2023"
-- "Alice is detail-oriented and methodical in her approach"
-
-#### 2.6.3 Storage and Retrieval
-
-**Storage**: Observations are stored as regular memory_units with fact_type='observation':
-
-
-**Entity Links**: Observations are linked to their entity via the entity_links table, enabling efficient lookup of all observations for an entity.
-
-**Important**: Observations are NOT used during the retrieval/search process itself. They do not participate in the 4-way parallel search (semantic, keyword, graph, temporal). Instead, they are **response augmentations**—additional context returned alongside search results.
-
-**Response Augmentation**: When calling recall() with include_entities=True:
-
-
-**Response Structure**:
-
-#### 2.6.4 Observations vs. Opinions
-
-A critical distinction separates observations from opinions:
-
-| Dimension | Observations | Opinions |
-|-----------|-------------|----------|
-| **Influence** | No personality influence | Influenced by Big Five traits |
-| **Purpose** | Objective entity summaries | Subjective beliefs and judgments |
-| **Confidence** | No confidence score | Confidence score (0.0-1.0) |
-| **Generation** | Background synthesis from facts | Formed during reflect() reasoning |
-| **Update Mechanism** | Regenerated when entity facts change | Updated via opinion reinforcement |
-| **Example** | "Alice is a software engineer at Google" | "Alice is an excellent engineer" |
-
-**Why Both?**: Observations provide factual entity understanding for retrieval contexts, while opinions represent the memory bank's personality-driven beliefs for reasoning contexts. A memory bank can have objective observations about Alice (she works at Google, specializes in ML) AND personality-influenced opinions about Alice (she's a talented engineer, she'd be great for project X).
-
-#### 2.6.5 Background Processing
-
-Observation generation is asynchronous to avoid blocking retain() operations:
-
-**Flow**:
-
-This design ensures low-latency writes while maintaining fresh entity summaries.
-
-#### 2.6.6 Benefits and Use Cases
-
-**Benefits**:
-
-1. **Contextual Entity Summaries**: After retrieving facts that mention entities, observations provide synthesized context about those entities without requiring separate queries
-2. **Structured Entity Understanding**: Provides coherent mental models of entities as response augmentation
-3. **Token Efficiency**: 3-5 observations provide more structured context than retrieving all entity-related facts
-4. **Objective Grounding**: When reflecting with personality, observations provide objective entity context
-5. **Scalability**: Automatically maintained as facts accumulate, always fresh when needed
-6. **Separation of Concerns**: Search focuses on relevant facts through semantic similarity, keyword matching, and graph traversal; observations provide entity context post-retrieval
-
-**Note on Observation Stability**: While observations are regenerated when entity facts change, the core retrieval mechanism remains grounded in the original facts. The four-way parallel search (semantic, keyword, graph, temporal) retrieves facts based on query relevance, semantic co-occurrence, and entity relationships—not based on observations. This ensures that the most relevant factual information is surfaced regardless of how observations may evolve over time.
-
-**Use Cases**:
-
-**Multi-Agent Conversations**: When retrieving facts that mention people, observations provide shared, objective entity context:
-
-**Entity-Centric Queries**: "Tell me about Alice" retrieves facts about Alice, and observations provide synthesized entity summary in the response.
-
-**Contextual Reasoning**: When forming opinions during reflect(), observations provide factual entity grounding alongside retrieved facts.
-
-**Knowledge Graph Interfaces**: Observations can be exposed as structured entity profiles in UIs or APIs via dedicated entity endpoints.
-
-## 3. Retrieval Architecture
-
-Our retrieval pipeline addresses the fundamental challenge of long-term memory: achieving both **high recall** (finding all relevant information) and **high precision** (ranking the most relevant items first).
-
-### 3.1 Four-Way Parallel Retrieval
-
-We execute four complementary retrieval strategies in parallel, each capturing different aspects of relevance:
-
-#### 3.1.1 Semantic Retrieval (Vector Similarity)
-
-**Method**: Cosine similarity between query embedding and memory embeddings
-**Index**: pgvector HNSW (Hierarchical Navigable Small World)
-**Threshold**: ≥ 0.3 similarity
-
-**Implementation**:
-
-**Advantages**:
-- Captures conceptual similarity
-- Handles synonyms and paraphrasing
-- Language-model understanding of meaning
-
-**Limitations**:
-- Misses exact proper nouns if not in training data
-- Cannot reason about temporal relationships
-- Weak at entity disambiguation
-
-#### 3.1.2 Keyword Retrieval (BM25 Full-Text Search)
-
-**Method**: PostgreSQL full-text search with BM25 ranking (ts_rank_cd)
-**Index**: GIN index on to_tsvector('english', text)
-
-**Advantages**:
-- High precision for proper nouns and technical terms
-- Exact phrase matching
-- Fast execution with GIN index
-
-**Limitations**:
-- No semantic understanding
-- Requires exact or stemmed matches
-
-**Complementarity**: Semantic + Keyword achieves >90% recall: vector search catches concepts, BM25 catches exact names.
-
-#### 3.1.3 Graph Retrieval (Spreading Activation)
-
-**Method**: Activation spreading from semantic entry points through the memory graph, following the spreading activation model of memory (Anderson 1983).
-
-**Algorithm**:
-
-**Decay Mechanism**: Activation decays by 0.8 per hop, limiting spread to ~4-5 hops.
-
-**Link Weighting with Causal Boosting**:
-- **Causal links**: Base weight × 2.0 boost (causes/caused_by) or × 1.5 boost (enables/prevents)
-- **Entity links**: weight 1.0 (no boost, already strong signal)
-- **Semantic links**: weight ∈ [0.7, 1.0] (cosine similarity, no boost)
-- **Temporal links**: weight ∈ [0.3, 1.0] (time-based decay, no boost)
-
-**Advantages**:
-- Discovers indirectly related facts through graph connectivity
-- Leverages entity links to traverse knowledge graph
-- Finds context-adjacent memories via temporal links
-- Prioritizes explanatory relationships through causal boosting
-
-#### 3.1.4 Temporal Graph Retrieval (Time-Constrained + Spreading)
-
-**Activation Condition**: Only triggered when temporal constraint detected in query
-
-**Temporal Parsing**: Uses google/flan-t5-small (80M parameters) to extract temporal constraints from natural language queries:
-- "last spring" → 2024-03-01 to 2024-05-31
-- "in June" → 2024-06-01 to 2024-06-30
-- "last year" → 2024-01-01 to 2024-12-31
-- "between March and May" → 2025-03-01 to 2025-05-31
-
-**Temporal Range Matching**: Facts are matched against time constraints using their temporal range (occurred_start, occurred_end):
-
-
-**Algorithm**:
-
-### 3.2 Reciprocal Rank Fusion (RRF)
-
-After parallel retrieval, we merge 3-4 ranked lists using Reciprocal Rank Fusion (Cormack et al. 2009):
-
-**Algorithm**:
-
-**Advantages over Score-Based Fusion**:
-- **Rank-based**: Position matters more than absolute scores
-- **Robust to missing items**: Missing from a list contributes 0, not a penalty
-- **Multi-evidence weighting**: Items appearing in multiple lists rank higher
-
-### 3.3 Neural Cross-Encoder Reranking
-
-After RRF fusion, TEMPR applies neural cross-encoder reranking to refine precision:
-
-**Model**: cross-encoder/ms-marco-MiniLM-L-6-v2 (pretrained on MS MARCO passage ranking)
-
-**Algorithm**:
-
-**Advantages**:
-- Learns query-document relevance patterns from supervised data
-- Considers full query-document interaction
-- Temporal awareness through formatted date context
-
-### 3.4 Token Budget Filtering
-
-Final stage applies token budget filtering to limit context window usage:
-
-**Algorithm**:
-
-**Purpose**: Ensures retrieved facts fit within LLM context windows while maximizing information density.
-
-### 3.5 Complete Retrieval Pipeline
-
-**End-to-End Flow**:
-
-## 4. Evaluation
-
-We evaluate TEMPR on two established long-term memory benchmarks: LoComo (Long-term Conversation Memory) and LongMemEval.
-
-### 4.1 LoComo Benchmark
-
-LoComo evaluates conversational memory systems across four dimensions: single-hop queries, multi-hop queries, open-domain queries, and temporal queries.
-
-**Results**:
-
-| Method | Single Hop J ↑ | Multi-Hop J ↑ | Open Domain J ↑ | Temporal J ↑ | Overall |
-|--------|---------------|---------------|-----------------|--------------|---------|
-| A-Mem* | 39.79 | 18.85 | 54.05 | 31.08 | 48.38 |
-| LangMem | 62.23 | 47.92 | 71.12 | 23.43 | 58.10 |
-| Zep (Mem0 paper) | 61.70 | 41.35 | 76.60 | 49.31 | 65.99 |
-| OpenAI | 63.79 | 42.92 | 62.29 | 21.71 | 52.90 |
-| Mem0 | 67.13 | 51.15 | 72.93 | 55.51 | 66.88 |
-| Mem0 w/ Graph | 65.71 | 47.19 | 75.71 | 58.13 | 68.44 |
-| **TEMPR** | **73.20** | **66.90** | **78.60** | **56.30** | **73.50** |
-
-**Analysis**: TEMPR achieves strong performance across all query types:
-- **Single-Hop (+6.1% vs Mem0)**: Superior performance due to comprehensive narrative facts and BM25 keyword matching
-- **Multi-Hop (+15.8% vs Mem0)**: Largest improvement, demonstrating effectiveness of graph-based spreading activation
-- **Open Domain (+2.9% vs Mem0)**: Strong performance through multi-strategy parallel retrieval
-- **Temporal (-1.8% vs Mem0 w/ Graph)**: Competitive temporal reasoning
-
-### 4.2 LongMemEval Benchmark
-
-LongMemEval assesses memory systems across six dimensions:
-
-**Results**:
-
-| Method | Single-Session Preference | Single-Session Assistant | Temporal Reasoning | Multi-Session | Knowledge Update | Single-Session User | Overall |
-|--------|--------------------------|-------------------------|-------------------|---------------|-----------------|-------------------|---------|
-| Zep gpt-4o-mini | 53.30% | 75.00% | 54.10% | 47.40% | 74.40% | 92.90% | 63.80% |
-| Zep gpt-4o | 56.70% | 80.40% | 62.40% | 57.90% | 83.30% | 92.90% | 71.00% |
-| **TEMPR** | **83.30%** | **80.40%** | **75.90%** | **75.20%** | **85.90%** | **92.90%** | **80.60%** |
-| Mastra gpt-4o | 46.70% | 100.00% | 75.20% | 76.70% | 84.60% | 97.10% | 80.05% |
-
-**Analysis**: TEMPR achieves competitive performance:
-- **Single-Session Preference (+26.6% vs Zep gpt-4o)**: Dramatic improvement enabled by comprehensive narrative facts
-- **Temporal Reasoning (+13.5% vs Zep gpt-4o)**: Strong performance through dedicated temporal graph retrieval
-- **Multi-Session (+17.3% vs Zep gpt-4o)**: Entity-aware graph linking maintains consistency
-
-The 80.60% overall score represents a 9.6 percentage point improvement over Zep gpt-4o (71.00%).
-
----
-
-# Part II: Reflect - CARA (Coherent Adaptive Reasoning Agents)
-
-## 5. Introduction to Reflect
-
-Conversational AI agents increasingly need to maintain consistent perspectives and form judgments that reflect stable character traits. Current systems either provide purely objective information retrieval without perspective, or generate responses that lack consistency across interactions. Human conversation partners expect agents to have stable viewpoints, preferences, and reasoning styles—characteristics that emerge from personality.
-
-We propose CARA (Coherent Adaptive Reasoning Agents), a personality framework that addresses these limitations through:
-
-1. **Big Five Personality Integration**: Configurable traits (OCEAN model) that influence how agents interpret facts and form opinions
-2. **TEMPR Memory Integration**: Leverages TEMPR's three-network architecture (world facts, bank experiences, opinions) for sophisticated memory access
-3. **Opinion Reinforcement**: Dynamic belief updating when new evidence reinforces, weakens, or contradicts existing opinions
-4. **Personality Bias Control**: Adjustable influence strength allowing agents to range from objective to strongly personality-driven
-5. **Background Merging**: LLM-powered integration of biographical information with intelligent conflict resolution
-
-This architecture enables agents to maintain consistent identities while allowing beliefs to evolve naturally with new information.
-
-### 5.1 Motivation
-
-Consider an agent discussing remote work. With high openness (0.9) and low conscientiousness (0.2), the agent might form the opinion: "Remote work enables creative flexibility and spontaneous innovation." The same facts presented to an agent with low openness (0.2) and high conscientiousness (0.9) might yield: "Remote work lacks the structure and accountability needed for consistent performance."
-
-Both agents access identical factual information, but personality traits bias how they weight different aspects (flexibility vs. structure) and what conclusions they draw. This mirrors human reasoning—our personalities influence what we attend to and how we integrate information into our worldview.
-
-### 5.2 Contributions
-
-Our key contributions for the reflect system are:
-
-1. **Personality-Aware Reasoning**: A prompt engineering framework that injects Big Five traits into LLM reasoning, demonstrating how personality consistently biases opinion formation
-
-2. **TEMPR-Based Three-Network Architecture**: Integration with TEMPR to manage three distinct networks (world facts, bank experiences, opinions), enabling architectural separation between objective information and subjective beliefs with epistemic clarity and traceability
-
-3. **Opinion Reinforcement Mechanism**: An automatic belief update system that adjusts confidence scores when new evidence arrives, creating dynamic belief systems that evolve with information
-
-4. **Background Merging with Conflict Resolution**: An LLM-powered method for maintaining coherent agent identities when new biographical information contradicts existing background
-
-5. **Bias Strength Control**: A meta-parameter that allows tuning personality influence from objective (0.0) to strongly subjective (1.0), enabling task-appropriate personality expression
-
-## 6. Personality Model
-
-### 6.1 Big Five Framework
-
-We adopt the **Big Five** personality model (OCEAN), which is empirically validated across cultures and provides continuous trait dimensions:
-
-**Trait Dimensions** (each 0.0-1.0):
-
-1. **Openness (O)**: Receptiveness to new ideas, creativity, abstract thinking
-   - High: "I embrace novel approaches", "innovation over tradition"
-   - Low: "I prefer proven methods", "tradition over experimentation"
-
-2. **Conscientiousness (C)**: Organization, goal-directed behavior, dependability
-   - High: "I plan systematically", "evidence-based decisions"
-   - Low: "I work flexibly", "intuition-based decisions"
-
-3. **Extraversion (E)**: Sociability, assertiveness, energy from interaction
-   - High: "I seek collaboration", "enthusiastic communication"
-   - Low: "I prefer solitude", "measured communication"
-
-4. **Agreeableness (A)**: Cooperation, empathy, conflict avoidance
-   - High: "I seek consensus", "consider social harmony"
-   - Low: "I express dissent", "prioritize accuracy over harmony"
-
-5. **Neuroticism (N)**: Emotional sensitivity, anxiety, stress response
-   - High: "I consider risks carefully", "emotionally engaged"
-   - Low: "I remain calm under uncertainty", "emotionally detached"
-
-**Bias Strength** (0.0-1.0): Meta-parameter controlling how much personality influences opinions
-- 0.0: Neutral, fact-based reasoning (no personality bias)
-- 0.5: Moderate personality influence, balanced with objective analysis
-- 1.0: Strong personality influence, facts filtered through trait lens
-
-### 6.2 Psychological Basis
-
-The Big Five model has several advantages for AI agents:
-
-1. **Empirical Validation**: Decades of psychological research demonstrate cross-cultural stability and predictive validity
-2. **Continuous Dimensions**: Unlike categorical types, continuous scales allow fine-grained personality tuning
-3. **Behavioral Prediction**: Traits predict information processing styles, decision-making approaches, and communication preferences
-4. **Interpretability**: Well-understood trait meanings enable users to anticipate agent behavior
-
-**Trait Influence on Reasoning**:
-- **High Openness**: Favors novel solutions, abstract thinking, considers unconventional perspectives
-- **High Conscientiousness**: Emphasizes systematic analysis, evidence quality, long-term consequences
-- **High Extraversion**: Considers social aspects, collaborative solutions, enthusiastic expression
-- **High Agreeableness**: Weights harmony, considers multiple viewpoints, seeks consensus
-- **High Neuroticism**: Attends to risks, emotional implications, uncertainty
-
-## 7. Bank Profile Structure
-
-### 7.1 Profile Schema
-
-Each memory bank has an associated profile containing identity information:
-
-
-**Name Field**: Memory bank's name used in prompts and self-reference ("Your name: Marcus")
-
-**Personality Field**: JSONB containing six continuous values (five traits + bias strength)
-
-**Background Field**: First-person narrative describing the agent's biographical context:
-- "I am a software engineer with 10 years of startup experience"
-- "I was born in Texas and value innovation over tradition"
-- "I am a creative artist interested in digital media"
-
-### 7.2 Trait Description Generation
-
-Personality traits are translated into natural language descriptions for LLM prompts:
-
-
-**Example Output** (openness=0.9, conscientiousness=0.2, extraversion=0.7, agreeableness=0.3, neuroticism=0.5):
-
-This verbalization makes traits interpretable to the LLM, enabling personality-biased reasoning.
-
-## 8. Opinion Network and Opinion Formation
-
-### 8.1 Opinion Structure
-
-Opinions are stored as memory units in the dedicated opinion network (fact_type='opinion'):
-
-**Core Attributes**:
-- text: The opinion statement with explicit reasoning
-- confidence_score: Opinion strength and resistance to change (0.0-1.0)
-- event_date: When the opinion was formed
-- bank_id: Which memory bank holds this opinion
-- entities: Mentioned entities (for reinforcement triggering)
-
-**Example Opinion**:
-
-**Fact vs. Opinion Separation**:
-
-A critical architectural distinction separates **facts** (objective information stored in world/bank networks) from **opinions** (subjective beliefs stored in the opinion network). This separation provides:
-
-1. **Epistemic Clarity**: Facts represent information encountered; opinions represent judgments formed
-2. **Traceability**: Opinion reinforcement can trace which facts influenced belief updates
-3. **Debugging**: Developers can separately inspect factual knowledge vs. formed beliefs
-4. **Confidence Semantics**: Facts lack confidence scores; opinions have confidence scores
-
-### 8.2 Opinion Formation
-
-Opinions are generated during "reflect" operations—when the agent is asked to reason about a topic and form a judgment.
-
-**Formation Process**:
-1. Retrieve relevant facts from all memory networks (world, bank, existing opinions) using TEMPR
-2. Inject bank profile (name, personality, background) into LLM prompt
-3. Generate reasoning with personality bias applied
-4. Extract new opinions from response using structured output
-5. Store opinions with confidence scores in opinion network
-
-**Prompt Structure** (bias_strength=0.8):
-
-### 8.3 System Message Adaptation
-
-The system message adjusts based on bias strength to control personality influence:
-
-**High bias (≥0.7)**:
-
-**Moderate bias (0.4-0.7)**:
-
-**Low bias (<0.4)**:
-
-### 8.4 Confidence Score Semantics
-
-Confidence scores represent opinion strength—how firmly the agent holds the belief:
-
-- **0.9-1.0**: Very strong conviction, deeply held belief
-- **0.7-0.9**: Strong conviction, firmly held opinion
-- **0.5-0.7**: Moderate conviction, open to revision
-- **0.3-0.5**: Weak conviction, easily influenced
-- **0.0-0.3**: Very weak conviction, highly malleable
-
-**LLM Generation**: Confidence scores are extracted using structured output (Pydantic schema):
-
-
-## 9. Opinion Reinforcement
-
-### 9.1 Motivation
-
-Human beliefs evolve as we encounter new information. Supporting evidence strengthens beliefs, contradictory evidence weakens them, and sufficient contradiction causes belief revision. Opinion reinforcement implements this dynamic belief updating.
-
-### 9.2 Reinforcement Mechanism
-
-When new facts are ingested (via retain), the system:
-
-1. **Identify Related Opinions**: Find existing opinions that mention entities in the new facts
-2. **Evaluate Evidence Relationship**: Use LLM to determine if new facts:
-   - **Reinforce**: Support the existing opinion (increase confidence)
-   - **Weaken**: Contradict the existing opinion (decrease confidence)
-   - **Contradict**: Strongly contradict, requiring opinion revision
-   - **Neutral**: Unrelated or no clear relationship
-3. **Update Opinions**: Adjust confidence scores or revise opinion text based on evaluation
-
-**Example Reinforcement**:
-
-**Existing Opinion** (confidence: 0.7):
-
-**New Fact**:
-
-**LLM Evaluation**: "This evidence REINFORCES the opinion with strong quantitative support."
-
-**Updated Opinion** (confidence: 0.85):
-
-### 9.3 Reinforcement Algorithm
-
-
-### 9.4 Reinforcement Guarantees
-
-**Consistency**: Opinions are only updated when new facts genuinely relate to existing beliefs
-
-**Personality Coherence**: Reinforcement evaluation incorporates bank personality, ensuring updates align with trait-driven reasoning
-
-**Transparency**: Each update records the triggering facts and reasoning, providing an audit trail
-
-**Bounded Updates**: Confidence changes are bounded (±0.1-0.15 per update) to prevent extreme swings
-
-## 10. Background Merging
-
-### 10.1 Challenge
-
-Memory bank backgrounds accumulate biographical information over time. New information may:
-- **Complement**: Add new facts without contradiction
-- **Conflict**: Contradict existing facts ("born in Texas" vs. "born in Colorado")
-- **Refine**: Provide more specific versions of existing facts
-
-Naive concatenation creates incoherent backgrounds with contradictions. We need intelligent merging.
-
-### 10.2 LLM-Powered Merging
-
-We use an LLM to merge backgrounds with conflict resolution:
-
-**Merge Rules**:
-1. **New overwrites old** when contradictory
-2. **Add non-conflicting** information
-3. **Maintain first-person** perspective ("I..." not "You...")
-4. **Keep concise** (under 500 characters)
-
-**Prompt Template**:
-
-**Example Merges**:
-
-**Conflict Resolution**:
-- Current: "I was born in Colorado"
-- New: "You were born in Texas"
-- Result: "I was born in Texas"
-
-**Addition**:
-- Current: "I was born in Texas"
-- New: "I have 10 years of startup experience"
-- Result: "I was born in Texas. I have 10 years of startup experience."
-
-### 10.3 First-Person Normalization
-
-Users may provide background in second person ("You are..."), but internal storage maintains first person for consistency in prompts.
-
-**Normalization**: LLM automatically converts:
-- "You are a creative engineer" → "I am a creative engineer"
-- "You were born in 1990" → "I was born in 1990"
-- "You value innovation" → "I value innovation"
-
-## 11. Personality-Driven Reasoning Examples
-
-### 11.1 Example: Remote Work Discussion
-
-**Scenario**: Two memory banks with opposite personalities discuss remote work given identical facts.
-
-**Facts** (both banks receive):
-- "Remote work eliminates commute time (average 1 hour/day saved)"
-- "Office work provides spontaneous collaboration and mentorship"
-- "Studies show 65% of remote workers report higher productivity"
-- "Some managers report difficulty monitoring remote employee performance"
-
-**Bank A** (High Openness=0.9, Low Conscientiousness=0.2, bias=0.8):
-
-**Bank B** (Low Openness=0.2, High Conscientiousness=0.9, bias=0.8):
-
-**Analysis**: Both banks accessed identical facts but formed opposite conclusions based on personality:
-- Bank A (high openness) weighted autonomy, flexibility, innovation
-- Bank B (high conscientiousness) weighted structure, monitoring, discipline
-
-### 11.2 Example: Opinion Evolution
-
-**Scenario**: Bank forms initial opinion, then encounters reinforcing and contradictory evidence.
-
-**Initial State** (t=0):
-
-**Reinforcement** (t=1):
-- New Fact: "Python dominates AI/ML with 75% market share; TensorFlow and PyTorch are Python-first"
-- Update: Confidence → 0.85, text adds "Python's dominance in AI/ML frameworks..."
-
-**Partial Contradiction** (t=2):
-- New Fact: "Julia offers 10x faster numerical computation; increasingly adopted in research"
-- Update: Confidence → 0.75, text revised to include nuance about specialized languages
-
-**Strong Contradiction** (t=3):
-- New Fact: "Major tech companies migrating data pipelines to Rust for performance"
-- Update: Confidence → 0.55, text revised to acknowledge Python's shifting role
-
-**Trajectory**: The opinion evolved from strong conviction (0.7 → 0.85) to weaker, more malleable belief (0.55) as evidence accumulated.
-
-# Part III: Unified Hindsight Architecture
-
-## 13. Integration: TEMPR + CARA
-
-The Hindsight system integrates TEMPR (recall) and CARA (reflect) into a unified architecture:
-
-### 13.1 Three Core Operations
-
-**1. Retain** (retain()): Store information into memory banks
-- LLM-powered fact extraction with temporal ranges
-- Entity recognition and resolution
-- Graph link construction (temporal, semantic, entity, causal)
-- Automatic opinion reinforcement for existing beliefs
-
-**2. Recall** (recall()): Retrieve memories using multi-strategy search
-- Four-way parallel retrieval (semantic, keyword, graph, temporal)
-- Reciprocal Rank Fusion
-- Neural cross-encoder reranking
-- Token budget filtering
-
-**3. Reflect** (reflect()): Generate personality-aware responses
-- Retrieves relevant memories from all networks using TEMPR
-- Loads bank personality and background
-- Generates response influenced by Big Five traits
-- Forms new opinions with confidence scores
-- Stores opinions for future retrieval
-
-### 13.2 Unified Data Flow
-
-
-### 13.3 PostgreSQL Schema
-
-The system uses PostgreSQL with pgvector for storage:
-
-
-## 14. System Properties
-
-### 14.1 Epistemic Clarity
-
-The three-network architecture provides clear separation:
-- **World**: What the bank knows about the world
-- **Bank**: What the bank has done
-- **Opinion**: What the bank believes
-
-This enables:
-- Transparent reasoning (trace opinions back to facts)
-- Debugging (identify missing facts vs. flawed reasoning)
-- Confidence calibration (opinions have confidence, facts don't)
-
-### 14.2 Temporal Awareness
-
-Multi-dimensional temporal representation:
-- occurred_start / occurred_end: When events actually happened
-- mentioned_at: When the bank learned about it
-- event_date: Backward compatibility
-
-Enables:
-- Precise historical queries ("What happened in June?")
-- Recency-aware ranking (newer mentions prioritized)
-- Period matching (events spanning weeks or months)
-
-### 14.3 Entity-Aware Reasoning
-
-LLM-based entity resolution creates knowledge graph:
-- Connects semantically distant facts through shared entities
-- Enables multi-hop discovery ("Alice's manager's team")
-- Disambiguates mentions ("Alice" vs. "Alice Chen")
-
-### 14.4 Multiple Link Types
-
-The graph incorporates multiple relationship types:
-- Entity links connect memories mentioning the same entities
-- Semantic links connect conceptually similar memories
-- Temporal links connect temporally proximate memories
-- Causal links represent identified cause-effect relationships
-- Links are weighted differently during graph traversal
-
-### 14.5 Personality Consistency
-
-Big Five traits ensure stable reasoning style:
-- Configurable bias strength (objective to subjective)
-- Trait-appropriate opinion formation
-- Consistent voice across interactions
-
-### 14.6 Dynamic Belief Systems
-
-Opinion reinforcement enables belief evolution:
-- Confidence increases with supporting evidence
-- Confidence decreases with contradictory evidence
-- Opinion text revised when strongly contradicted
-- Audit trail of belief changes
-
-## 15. Conclusion
-
-We present Hindsight, a unified memory architecture for AI agents that combines TEMPR's multi-strategy retrieval with CARA's personality-driven reasoning. The system achieves strong performance on established benchmarks (73.50% on LoComo, 80.60% on LongMemEval) while enabling personality-consistent opinion formation through the Big Five model.
-
-The integration of four parallel search strategies (semantic, keyword, graph with multiple link types, temporal) with three-network architecture (world, bank, opinion) and opinion reinforcement creates a comprehensive memory system that:
-- Retrieves information with high recall and precision
-- Maintains epistemic clarity between facts and beliefs
-- Enables personality-driven reasoning with stable traits
-- Supports dynamic belief evolution with evidence
-
-Real-world deployment in sports content generation demonstrates the system's ability to maintain consistent yet adaptive perspectives across extended interactions. Future work will explore personality evolution, multi-agent belief systems, and richer personality models incorporating values and cultural factors.
-
-By combining temporal-aware retrieval with personality-driven reasoning, Hindsight moves toward conversational agents that exhibit not just memory and intelligence, but character—stable traits and evolving beliefs that enable more natural, trustworthy human-AI interaction.
-
-## References
-
-1. Anderson, J. R. (1983). A spreading activation theory of memory. *Journal of Verbal Learning and Verbal Behavior*, 22(3), 261-295.
-
-2. Cormack, G. V., Clarke, C. L., & Buettcher, S. (2009). Reciprocal rank fusion outperforms condorcet and individual rank learning methods. In *SIGIR'09* (pp. 758-759).
-
-3. McCrae, R. R., & Costa, P. T. (1997). Personality trait structure as a human universal. *American Psychologist*, 52(5), 509.
-
-4. Goldberg, L. R. (1993). The structure of phenotypic personality traits. *American Psychologist*, 48(1), 26.
-
-5. Malkov, Y. A., & Yashunin, D. A. (2018). Efficient and robust approximate nearest neighbor search using hierarchical navigable small world graphs. *IEEE Transactions on Pattern Analysis and Machine Intelligence*, 42(4), 824-836.
-
-6. Robertson, S., & Zaragoza, H. (2009). The probabilistic relevance framework: BM25 and beyond. *Foundations and Trends in Information Retrieval*, 3(4), 333-489.
-
-7. Brown, T. B., Mann, B., Ryder, N., Subbiah, M., Kaplan, J., Dhariwal, P., ... & Amodei, D. (2020). Language models are few-shot learners. *Advances in Neural Information Processing Systems*, 33, 1877-1901.
-
-8. Petroni, F., Rocktäschel, T., Riedel, S., Lewis, P., Bakhtin, A., Wu, Y., & Miller, A. (2019). Language models as knowledge bases?. In *Proceedings of EMNLP-IJCNLP* (pp. 2463-2473).

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Vectorize AI, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,98 +1,313 @@
-# Hindsight
+<div align="center">
 
-**Long-term memory for AI agents.**
+![Hindsight Banner](./hindsight-docs/static/img/hindsight-github-banner.png)
 
-## Why Hindsight?
+[Documentation](https://hindsight.vectorize.io) • [Paper](https://arxiv.org/abs/2512.12818) • [Cookbook](https://hindsight.vectorize.io/cookbook) • [Hindsight Cloud](https://ui.hindsight.vectorize.io/signup)
 
-AI assistants forget everything between sessions. Every conversation starts from zero—no context about who you are, what you've discussed, or what the memory bank has learned. This isn't just inconvenient; it fundamentally limits what AI memory banks can do.
+[![CI](https://github.com/vectorize-io/hindsight/actions/workflows/release.yml/badge.svg)](https://github.com/vectorize-io/hindsight/actions/workflows/release.yml)
+[![Slack Community](https://img.shields.io/badge/Slack-Join%20Community-4A154B?logo=slack)](https://join.slack.com/t/hindsight-space/shared_invite/zt-3nhbm4w29-LeSJ5Ixi6j8PdiYOCPlOgg)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![gitcgr](https://gitcgr.com/badge/vectorize-io/hindsight.svg)](https://gitcgr.com/vectorize-io/hindsight)
+![PyPI - Downloads](https://img.shields.io/pypi/dm/hindsight-api?label=PyPI)
+![NPM Downloads](https://img.shields.io/npm/dm/%40vectorize-io%2Fhindsight-client?logoColor=orange&label=NPM&color=blue&link=https%3A%2F%2Fwww.npmjs.com%2Fpackage%2F%40vectorize-io%2Fhindsight-client)
+<br/>
 
-**The problem is harder than it looks:**
+<a href="https://trendshift.io/repositories/15603" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15603" alt="vectorize-io%2Fhindsight | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
 
-- **Simple vector search isn't enough** — "What did Alice do last spring?" requires temporal reasoning, not just semantic similarity
-- **Facts get disconnected** — Knowing "Alice works at Google" and "Google is in Mountain View" should let you answer "Where does Alice work?" even if you never stored that directly
-- **Memory banks need opinions** — A coding assistant that remembers "the user prefers functional programming" should weigh that when making recommendations
-- **Context matters** — The same information means different things to different memory banks with different personalities
+---
 
-Hindsight solves these problems with a memory system designed specifically for AI memory banks.
+## What is Hindsight?
+
+Hindsight™ is an agent memory system built to create smarter agents that learn over time. Most agent memory systems focus on recalling conversation history. Hindsight is focused on making agents that learn, not just remember.
+
+
+<video src="https://github.com/user-attachments/assets/923b798d-3581-4897-bb62-9cfa5a931682" controls></video>
+
+It eliminates the shortcomings of alternative techniques such as RAG and knowledge graph and delivers state-of-the-art performance on long term memory tasks.
+
+## Memory Performance & Accuracy
+
+Hindsight is the most accurate agent memory system ever tested according to benchmark performance. It has achieved state-of-the-art performance on the LongMemEval benchmark, widely used to assess memory system performance across a variety of conversational AI scenarios. The current reported performance of Hindsight and other agent memory solutions as of January 2026 is shown here:
+
+![Overview](./hindsight-docs/static/img/hindsight-bench.jpg)
+
+The benchmark performance data for Hindsight has been independently reproduced by research collaborators at the Virginia Tech [Sanghani Center for Artificial Intelligence and Data Analytics](https://sanghani.cs.vt.edu/) and The Washington Post. Other scores are self-reported by software vendors.
+
+Hindsight is being used in production at Fortune 500 enterprises and by a growing number of AI startups. 
+
+## Adding Hindsight to Your AI Agents
+
+The easiest way to use Hindsight with an existing agent is with the LLM Wrapper. You can add memory to your agent with 2 lines of code. That will swap your current LLM client out with the Hindsight wrapper. After that, memories will be stored and retrieved automatically as you make LLM calls.
+
+If you need more control over how and when your agent stores and recalls memories, there's also a simple API you can integrate with using the SDKs or directly via HTTP.
+
+![Hindsight Banner](./hindsight-docs/static/img/migration-code.png)
+
+---
+
+> 🤖 **Using a coding agent?** Install the Hindsight documentation skill for instant access to docs while you code:
+> ```bash
+> npx skills add https://github.com/vectorize-io/hindsight --skill hindsight-docs
+> ```
+> Works with Claude Code, Cursor, and other AI coding assistants.
+
+---
 
 
 ## Quick Start
 
-### Option 1: Docker (recommended)
-
-Get the full experience with the API and Control Plane UI:
+### Docker (recommended)
 
 ```bash
-export OPENAI_API_KEY=your-key
-docker run -p 8888:8888 -p 9999:9999 \
-  -e HINDSIGHT_API_LLM_PROVIDER=openai \
+export OPENAI_API_KEY=sk-xxx
+
+docker run --rm -it --pull always -p 8888:8888 -p 9999:9999 \
   -e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \
-  -e HINDSIGHT_API_LLM_MODEL=gpt-4o-mini \
-  vectorize/hindsight
+  -v $HOME/.hindsight-docker:/home/hindsight/.pg0 \
+  ghcr.io/vectorize-io/hindsight:latest
 ```
 
-- **API**: http://localhost:8888
-- **Control Plane UI**: http://localhost:9999
+>API: http://localhost:8888
+>UI: http://localhost:9999
 
-Then use the Python client:
+You can modify the LLM provider by setting `HINDSIGHT_API_LLM_PROVIDER`. Valid options are `openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`, and `minimax`. The documentation provides more details on [supported models](https://hindsight.vectorize.io/developer/models).
+
+
+
+### Docker (external PostgreSQL)
 
 ```bash
-pip install hindsight-client
+export OPENAI_API_KEY=sk-xxx
+export HINDSIGHT_DB_PASSWORD=choose-a-password
+cd docker/docker-compose
+docker compose up 
 ```
 
+
+>API: http://localhost:8888
+>UI: http://localhost:9999
+
+### Client
+
+```bash
+pip install hindsight-client -U
+# or
+npm install @vectorize-io/hindsight-client
+```
+
+#### Python
+
 ```python
-from hindsight import HindsightClient
+from hindsight_client import Hindsight
 
-client = HindsightClient(base_url="http://localhost:8888")
+client = Hindsight(base_url="http://localhost:8888")
 
-# Store memories
-client.retain(bank_id="my-agent", content="Alice works at Google as a software engineer")
-client.retain(bank_id="my-agent", content="Alice mentioned she loves hiking in the mountains")
+# Retain: Store information
+client.retain(bank_id="my-bank", content="Alice works at Google as a software engineer")
 
-# Query with temporal reasoning
-results = client.recall(bank_id="my-agent", query="What does Alice do for work?")
+# Recall: Search memories
+client.recall(bank_id="my-bank", query="What does Alice do?")
 
-# Get a synthesized perspective
-response = client.reflect(bank_id="my-agent", query="Tell me about Alice")
-print(response.text)
+# Reflect: Generate disposition-aware response
+client.reflect(bank_id="my-bank", query="Tell me about Alice")
 ```
 
-### Option 2: Embedded (no docker/server required)
-
-For quick prototyping, run everything in-process:
+#### Node.js / TypeScript
 
 ```bash
-pip install hindsight-all
-export OPENAI_API_KEY=your-key
+npm install @vectorize-io/hindsight-client
+```
+
+```javascript
+const { HindsightClient } = require('@vectorize-io/hindsight-client');
+
+const main = async () => {
+  const client = new HindsightClient({ baseUrl: 'http://localhost:8888' });
+
+  await client.retain('my-bank', 'Alice loves hiking in Yosemite');
+
+  const results = await client.recall('my-bank', 'What does Alice like?');
+  console.log(results);
+}
+
+main();
+```
+
+
+### Python Embedded (no server required)
+
+```bash
+pip install hindsight-all -U
 ```
 
 ```python
 import os
 from hindsight import HindsightServer, HindsightClient
 
-with HindsightServer(llm_provider="openai", llm_model="gpt-4o-mini", llm_api_key=os.environ["OPENAI_API_KEY"]) as server:
+with HindsightServer(
+    llm_provider="openai",
+    llm_model="gpt-5-mini", 
+    llm_api_key=os.environ["OPENAI_API_KEY"]
+) as server:
     client = HindsightClient(base_url=server.url)
-
-    client.retain(bank_id="my-user", content="User prefers functional programming")
-    response = client.reflect(bank_id="my-user", query="What coding style should I use?")
-    print(response.text)
+    client.retain(bank_id="my-bank", content="Alice works at Google")
+    results = client.recall(bank_id="my-bank", query="Where does Alice work?")
 ```
 
 
+---
 
-## Documentation
+## Use Cases
 
-Full documentation: [vectorize-io.github.io/hindsight](https://vectorize-io.github.io/hindsight)
 
-- [Architecture](https://vectorize-io.github.io/hindsight/developer/architecture) — How ingestion, storage, and retrieval work
-- [Python Client](https://vectorize-io.github.io/hindsight/sdks/python) — Full API reference
-- [API Reference](https://vectorize-io.github.io/hindsight/api-reference) — REST API endpoints
-- [Personality](https://vectorize-io.github.io/hindsight/developer/personality) — Big Five traits and opinion formation
+Hindsight is built to support conversational AI agents as well as agents that are intended to perform tasks autonomously. The ideal use case for Hindsight are agents that require a blend of these features such as AI employees that need to handle open-ended tasks, change behavior based on user feedback, and learn to perform complex tasks to automate work at a level that approximates a human work. Hindsight can be used with simple AI workflows like those built with n8n and other similar tools, but may be overkill for such applications.
+
+### Per-User Memories and Chat History
+
+One of the simpler use cases you can use Hindsight for is to personalize AI chatbots and other conversational agents by storing and recalling memories associated with individual users.
+
+The requirements for this use case usually look something like this:
+
+![Per-User Memories](./hindsight-docs/static/img/per-user-memory-requirements.png)
+
+<video src="https://github.com/user-attachments/assets/4805e8e1-e7d1-47c6-a4f8-2344a5ec8906" controls></video>
+
+Satisfying these requirements in Hindsight is straightforward. When new user inputs and tool calls are ingested into Hindsight using the retain operation, custom metadata can be used to enrich the new memories. Metadata provides a convenient way to isolate memories that need to be restricted to a given user. Once these are fed into the retain operation, any raw memories and mental models that get created can be filtered when retrieving relevant memories. 
+
+![Per-User Memories](./hindsight-docs/static/img/per-user-memory-howto.png)
+
+---
+
+## Architecture & Operations
+
+![Overview](./hindsight-docs/static/img/hindsight-overview.webp)
+
+Most agent memory implementations rely on basic vector search or sometimes use a knowledge graph. Hindsight uses biomimetic data structures to organize agent memories in a way that is more like how human memory works:
+
+- **World:** Facts about the world ("The stove gets hot")
+- **Experiences:** Agent's own experiences ("I touched the stove and it really hurt")
+- **Mental Models:** Learned understanding of the agent's world formed by reflecting on raw memories and experiences.
+
+Memories in Hindsight are stored in banks (i.e. memory banks). When memories are added to Hindsight, they are pushed into either the world facts or experiences memory pathway. They are then represented as a combination of entities, relationships, and time series with sparse/dense vector representations to aid in later recall.
+
+Hindsight provides three simple methods to interact with the system:
+
+- **Retain:** Provide information to Hindsight that you want it to remember
+- **Recall:** Retrieve memories from Hindsight
+- **Reflect:** Reflect on memories and experiences to generate new observations and insights from existing memories.
+
+### Retain
+
+The `retain` operation is used to push new memories into Hindsight. It tells Hindsight to _retain_ the information you pass in as an input.
+
+```python
+from hindsight_client import Hindsight
+
+client = Hindsight(base_url="http://localhost:8888")
+
+# Simple
+client.retain(
+    bank_id="my-bank",
+    content="Alice works at Google as a software engineer"
+)
+
+# With context and timestamp
+client.retain(
+    bank_id="my-bank",
+    content="Alice got promoted to senior engineer",
+    context="career update",
+    timestamp="2025-06-15T10:00:00Z"
+)
+```
+
+Behind the scenes, the retain operation uses an LLM to extract key facts, temporal data, entities, and relationships. It passes these through a normalization process to transform extracted data into canonical entities, time series, and search indexes along with metadata. These representations create the pathways for accurate memory retrieval in the recall and reflect operations. 
+
+![Retain Operation](hindsight-docs/static/img/retain-operation.webp)
+
+### Recall
+
+The recall operation is used to retrieve memories. These memories can come from any of the memory types (world, experiences, etc.)
+
+```python
+from hindsight_client import Hindsight
+
+client = Hindsight(base_url="http://localhost:8888")
+
+# Simple
+client.recall(bank_id="my-bank", query="What does Alice do?")
+
+# Temporal
+client.recall(bank_id="my-bank", query="What happened in June?")
+```
+
+Recall performs 4 retrieval strategies in parallel:
+- Semantic: Vector similarity
+- Keyword: BM25 exact matching
+- Graph: Entity/temporal/causal links
+- Temporal: Time range filtering
+
+![Retain Operation](hindsight-docs/static/img/recall-operation.webp)
+
+The individual results from the retrievals are merged, then ordered by relevance using reciprocal rank fusion and a cross-encoder reranking model.
+
+The final output is trimmed as needed to fit within the token limit.
+
+### Reflect
+
+The reflect operation is used to perform a more thorough analysis of existing memories. This allows the agent to form new connections between memories and build a more thorough understanding of its world.
+
+For example, the `reflect` operation can be used to support use cases such as:
+
+- An **AI Project Manager** reflecting on what risks need to be mitigated on a project.
+- A **Sales Agent** reflecting on why certain outreach messages have gotten responses while others haven't.
+- A **Support Agent** reflecting on opportunities where customers have questions not answered by current product documentation.
+
+The `reflect` operation can also be used to handle on-demand question answering or analysis which require more deep thinking.
+
+```python
+from hindsight_client import Hindsight
+
+client = Hindsight(base_url="http://localhost:8888")
+
+client.reflect(bank_id="my-bank", query="What should I know about Alice?")
+```
+
+![Retain Operation](hindsight-docs/static/img/reflect-operation.webp)
+
+---
+
+## Resources
+
+**Documentation:** 
+- [https://hindsight.vectorize.io](https://hindsight.vectorize.io)
+
+**Clients:**
+- [Python](http://hindsight.vectorize.io/sdks/python)
+- [Node.js](http://hindsight.vectorize.io/sdks/nodejs)
+- [REST API](https://hindsight.vectorize.io/api-reference)
+- [CLI](https://hindsight.vectorize.io/sdks/cli)
+
+**Community:**
+- [Slack](https://join.slack.com/t/hindsight-space/shared_invite/zt-3nhbm4w29-LeSJ5Ixi6j8PdiYOCPlOgg)
+- [GitHub Issues](https://github.com/vectorize-io/hindsight/issues)
+
+---
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=vectorize-io/hindsight&type=date&legend=top-left)](https://www.star-history.com/#vectorize-io/hindsight&type=date&legend=top-left)
+---
 
 ## Contributing
 
-We welcome contributions! See [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines.
+See [CONTRIBUTING.md](./CONTRIBUTING.md).
 
 ## License
 
-MIT
+MIT — see [LICENSE](./LICENSE)
+
+---
+
+Built by [Vectorize.io](https://vectorize.io)
+
+<img src="https://umami-pixel.chris-latimer.workers.dev/?id=a8b043e6-6964-454d-80df-69b69d3f0d50&host=github.com&url=/vectorize-io/hindsight" width="1" height="1" alt="" />

SECURITY.md

@@ -0,0 +1,39 @@
+# Security Policy
+
+## Supported Versions
+
+We release patches for security vulnerabilities. Which versions are eligible for
+receiving such patches depends on the CVSS v3.0 Rating:
+
+| Version | Supported          |
+| ------- | ------------------ |
+| latest  | :white_check_mark: |
+
+## Reporting a Vulnerability
+
+Please report (suspected) security vulnerabilities to the maintainers privately.
+You can do this by opening a [GitHub Security Advisory](https://github.com/vectorize-io/hindsight/security/advisories/new).
+
+You will receive a response from us within 48 hours. If the issue is confirmed,
+we will release a patch as soon as possible depending on complexity but
+typically within a few days.
+
+Please include the following information in your report:
+
+- Type of issue (e.g., buffer overflow, SQL injection, cross-site scripting, etc.)
+- Full paths of source file(s) related to the manifestation of the issue
+- The location of the affected source code (tag/branch/commit or direct URL)
+- Any special configuration required to reproduce the issue
+- Step-by-step instructions to reproduce the issue
+- Proof-of-concept or exploit code (if possible)
+- Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+We follow the principle of [Coordinated Vulnerability Disclosure](https://www.cisa.gov/resources-tools/programs/coordinated-vulnerability-disclosure-program).

deno.lock

@@ -0,0 +1,139 @@
+{
+  "version": "5",
+  "specifiers": {
+    "jsr:@std/assert@^1.0.17": "1.0.19",
+    "jsr:@std/assert@^1.0.19": "1.0.19",
+    "jsr:@std/expect@*": "1.0.18",
+    "jsr:@std/internal@^1.0.12": "1.0.12",
+    "jsr:@std/path@^1.1.4": "1.1.4",
+    "jsr:@std/testing@*": "1.0.17"
+  },
+  "jsr": {
+    "@std/[email protected]": {
+      "integrity": "eaada96ee120cb980bc47e040f82814d786fe8162ecc53c91d8df60b8755991e",
+      "dependencies": [
+        "jsr:@std/internal"
+      ]
+    },
+    "@std/[email protected]": {
+      "integrity": "8566eab35200466f8609eb7e7aed062ed0db314e9a258d5d201b1b8997ce801a",
+      "dependencies": [
+        "jsr:@std/assert@^1.0.19",
+        "jsr:@std/internal",
+        "jsr:@std/path"
+      ]
+    },
+    "@std/[email protected]": {
+      "integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027"
+    },
+    "@std/[email protected]": {
+      "integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5",
+      "dependencies": [
+        "jsr:@std/internal"
+      ]
+    },
+    "@std/[email protected]": {
+      "integrity": "87bdc2700fa98249d48a17cd72413352d3d3680dcfbdb64947fd0982d6bbf681",
+      "dependencies": [
+        "jsr:@std/assert@^1.0.17",
+        "jsr:@std/internal"
+      ]
+    }
+  },
+  "workspace": {
+    "members": {
+      "hindsight-clients/typescript": {
+        "packageJson": {
+          "dependencies": [
+            "npm:@hey-api/[email protected]",
+            "npm:@types/jest@29",
+            "npm:@types/node@20",
+            "npm:jest@29",
+            "npm:ts-jest@29",
+            "npm:tsup@^8.5.1",
+            "npm:typescript@5"
+          ]
+        }
+      },
+      "hindsight-control-plane": {
+        "packageJson": {
+          "dependencies": [
+            "npm:@eslint/eslintrc@^3.3.3",
+            "npm:@eslint/js@^9.39.2",
+            "npm:@radix-ui/react-alert-dialog@^1.1.15",
+            "npm:@radix-ui/react-checkbox@^1.3.3",
+            "npm:@radix-ui/react-dialog@^1.1.15",
+            "npm:@radix-ui/react-dropdown-menu@^2.1.16",
+            "npm:@radix-ui/react-label@^2.1.8",
+            "npm:@radix-ui/react-popover@^1.1.15",
+            "npm:@radix-ui/react-radio-group@^1.3.8",
+            "npm:@radix-ui/react-select@^2.2.6",
+            "npm:@radix-ui/react-slider@^1.3.6",
+            "npm:@radix-ui/react-slot@^1.2.4",
+            "npm:@radix-ui/react-switch@^1.2.6",
+            "npm:@radix-ui/react-tabs@^1.1.13",
+            "npm:@radix-ui/react-tooltip@^1.2.8",
+            "npm:@tailwindcss/postcss@^4.1.17",
+            "npm:@tailwindcss/typography@~0.5.19",
+            "npm:@types/cytoscape@^3.21.9",
+            "npm:@types/node@^24.10.0",
+            "npm:@types/react-dom@^19.2.2",
+            "npm:@types/react@^19.2.2",
+            "npm:autoprefixer@^10.4.21",
+            "npm:class-variance-authority@~0.7.1",
+            "npm:clsx@^2.1.1",
+            "npm:cmdk@^1.1.1",
+            "npm:cytoscape-fcose@^2.2.0",
+            "npm:cytoscape@^3.33.1",
+            "npm:eslint-config-next@^16.0.1",
+            "npm:eslint-plugin-react-hooks@^7.0.1",
+            "npm:eslint-plugin-react@^7.37.5",
+            "npm:eslint@^9.39.1",
+            "npm:[email protected]",
+            "npm:next-themes@~0.4.6",
+            "npm:next@^16.1.6",
+            "npm:postcss@^8.5.6",
+            "npm:prettier@^3.7.4",
+            "npm:react-chrono@^2.9.1",
+            "npm:react-dom@^19.2.0",
+            "npm:react-markdown@^10.1.0",
+            "npm:react18-json-view@~0.2.9",
+            "npm:react@^19.2.0",
+            "npm:recharts@^3.5.1",
+            "npm:remark-gfm@^4.0.1",
+            "npm:sonner@^2.0.7",
+            "npm:tailwind-merge@^3.4.0",
+            "npm:tailwindcss-animate@^1.0.7",
+            "npm:tailwindcss@^4.1.17",
+            "npm:[email protected]",
+            "npm:typescript-eslint@^8.50.0",
+            "npm:typescript@^5.9.3"
+          ]
+        }
+      },
+      "hindsight-docs": {
+        "packageJson": {
+          "dependencies": [
+            "npm:@docusaurus/[email protected]",
+            "npm:@docusaurus/[email protected]",
+            "npm:@docusaurus/[email protected]",
+            "npm:@docusaurus/theme-common@^3.9.2",
+            "npm:@docusaurus/theme-mermaid@^3.9.2",
+            "npm:@docusaurus/[email protected]",
+            "npm:@docusaurus/[email protected]",
+            "npm:@easyops-cn/docusaurus-search-local@~0.52.2",
+            "npm:@mdx-js/react@3",
+            "npm:clsx@2",
+            "npm:prism-react-renderer@^2.3.0",
+            "npm:raw-loader@^4.0.2",
+            "npm:react-dom@19",
+            "npm:react-icons@^5.6.0",
+            "npm:react@19",
+            "npm:redocusaurus@^2.5.0",
+            "npm:typescript@~5.6.2"
+          ]
+        }
+      }
+    }
+  }
+}

docker/README.md

@@ -1,155 +0,0 @@
-# Hindsight Docker
-
-Run Hindsight with Docker in standalone or distributed mode.
-
-## Quick Start (Standalone)
-
-```bash
-cd docker
-./start.sh
-```
-
-**Force rebuild after code changes:**
-```bash
-./start.sh --build        # Quick: rebuild and start
-# or
-./rebuild.sh              # Complete: rebuild from scratch (no cache)
-```
-
-Access:
-- **Control Plane**: http://localhost:3000
-- **API**: http://localhost:8888
-
-Press `Ctrl+C` to stop.
-
-## What You Get
-
-**Standalone** (default, simple):
-- One container with API + Control Plane + embedded database
-- Perfect for local development and simple deployments
-
-**Distributed** (advanced):
-- Separate containers for API and Control Plane
-- Better for production, scaling, or custom configurations
-
-## Deployment Modes
-
-### 1. Standalone (Recommended)
-
-All-in-one container with embedded pg0 database.
-
-```bash
-./start.sh
-# or
-cd standalone
-docker-compose up
-```
-
-**Data storage:** `/app/data` volume
-
-### 2. Distributed (Advanced)
-
-Separate API and Control Plane containers.
-
-```bash
-cd services
-docker-compose up
-```
-
-**Data storage:** `api_data` volume
-
-See `services/README.md` for details.
-
-## Data Management
-
-**Reset data:**
-```bash
-# Standalone
-cd standalone && docker-compose down -v
-
-# Distributed
-cd services && docker-compose down -v
-```
-
-## Building Images
-
-```bash
-# Standalone
-cd standalone
-docker build -f Dockerfile -t hindsight:latest ../..
-
-# Services
-cd services
-./build-all.sh
-```
-
-## Using External Database
-
-Both modes use embedded pg0 by default. To use external PostgreSQL:
-
-```bash
-export HINDSIGHT_API_DATABASE_URL=postgresql://user:pass@host:5432/db
-```
-
-## Directory Structure
-
-```
-docker/
-├── start.sh              # Quick start (standalone)
-├── README.md             # This file
-├── standalone/           # All-in-one deployment
-│   ├── Dockerfile
-│   ├── docker-compose.yml
-│   └── start-all.sh
-└── services/             # Distributed deployment
-    ├── docker-compose.yml
-    ├── api.Dockerfile
-    ├── control-plane.Dockerfile
-    ├── build-all.sh
-    └── README.md
-```
-
-## Advanced Usage
-
-**Background mode:**
-```bash
-cd standalone
-docker-compose up -d
-docker-compose logs -f
-docker-compose down
-```
-
-**Custom configuration:**
-Edit `standalone/docker-compose.yml` or `services/docker-compose.yml`
-
-## Environment Variables
-
-Hindsight requires configuration through environment variables (all prefixed with `HINDSIGHT_`).
-
-### Required:
-- `HINDSIGHT_API_LLM_API_KEY` - Your LLM API key (OpenAI, Anthropic, etc.)
-
-### Optional:
-- `HINDSIGHT_API_LLM_MODEL` - Model name (default: gpt-4o-mini)
-- `HINDSIGHT_API_LLM_BASE_URL` - API base URL (default: https://api.openai.com/v1)
-- `HINDSIGHT_API_LOG_LEVEL` - Logging level: debug, info, warning, error
-- `HINDSIGHT_API_DATABASE_URL` - External PostgreSQL connection (uses embedded pg0 by default)
-
-### Setup Options:
-
-**Option 1: .env file (recommended)**
-```bash
-# Copy example file
-cp .env.example .env
-
-# Edit .env and add your API key
-HINDSIGHT_API_LLM_API_KEY=sk-...
-```
-
-**Option 2: Export in shell**
-```bash
-export HINDSIGHT_API_LLM_API_KEY=sk-...
-export HINDSIGHT_API_LLM_MODEL=gpt-4o-mini
-```
-
-The `start.sh` script automatically loads `.env` if it exists and validates the API key is set.

docker/docker-compose/external-pg/docker-compose.yaml

@@ -0,0 +1,54 @@
+# Docker Compose file for Hindsight with PostgreSQL and pgvector
+#
+# Make sure to set the required environment variables before running:
+# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
+# - Configure LLM provider variables as needed (see below in the hindsight service)
+#
+# Usage:
+# docker compose up -d
+#
+# Optional environment variables with defaults:
+# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
+# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
+# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
+# - HINDSIGHT_DB_VERSION: PostgreSQL version (default: 18)
+
+services:
+  db:
+    # Use a PostgreSQL-Image with pgvector extension pre-installed
+    # see https://hub.docker.com/r/pgvector/pgvector 
+    image: pgvector/pgvector:pg${HINDSIGHT_DB_VERSION:-18}
+    container_name: hindsight-db
+    restart: always
+    # Expose PostgreSQL port
+    # ports:
+      # - "5432:5432"
+    environment:
+      POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
+      POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}
+      POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
+    volumes:
+      - pg_data:/var/lib/postgresql/${HINDSIGHT_DB_VERSION:-18}/docker
+    networks:
+      - hindsight-net
+
+  hindsight:
+    image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
+    container_name: hindsight-app
+    ports:
+      - "8888:8888"
+      - "9999:9999"
+    environment:
+      - HINDSIGHT_API_LLM_API_KEY=${OPENAI_API_KEY?Please set the OPENAI_API_KEY env variable}
+      - HINDSIGHT_API_DATABASE_URL=postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
+    depends_on:
+      - db
+    networks:
+      - hindsight-net
+
+networks:
+  hindsight-net:
+    driver: bridge
+
+volumes:
+  pg_data:

docker/docker-compose/nginx/README.md

@@ -0,0 +1,96 @@
+# Nginx Reverse Proxy with Custom Base Path
+
+Deploy Hindsight API under `/hindsight` (or any custom path) using Nginx reverse proxy.
+
+## Quick Start (Published Image - API Only)
+
+```bash
+docker-compose up
+```
+
+- **API:** http://localhost:8080/hindsight/docs
+- **Control Plane:** http://localhost:9999 (direct access, not proxied)
+
+## Full Stack with Custom Base Path (Requires Build)
+
+**Important:** You cannot rebuild from the published image with build args. You must build from source.
+
+### Build from Source with Custom Base Path
+
+1. **Clone the repository** (if you haven't):
+```bash
+git clone https://github.com/vectorize-io/hindsight.git
+cd hindsight
+```
+
+2. **Build with base path**:
+```bash
+docker build \
+  --build-arg NEXT_PUBLIC_BASE_PATH=/hindsight \
+  -f docker/standalone/Dockerfile \
+  -t hindsight:custom \
+  .
+```
+
+3. **Update docker-compose.yml** to use your built image:
+```yaml
+services:
+  hindsight:
+    image: hindsight:custom  # ← Change this
+    environment:
+      HINDSIGHT_API_BASE_PATH: /hindsight
+      NEXT_PUBLIC_BASE_PATH: /hindsight
+```
+
+4. **Update nginx.conf** to handle Control Plane routes (see below)
+
+5. **Run**:
+```bash
+docker-compose up
+```
+
+### Required nginx.conf for Full Stack
+
+Replace the current `nginx.conf` with this to proxy both API and Control Plane:
+
+```nginx
+events { worker_connections 1024; }
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    upstream hindsight_api { server hindsight:8888; }
+    upstream hindsight_cp { server hindsight:9999; }
+
+    server {
+        listen 80;
+
+        # API
+        location ~ ^/hindsight/(docs|openapi\.json|health|metrics|v1|mcp) {
+            proxy_pass http://hindsight_api;
+            proxy_set_header Host $http_host;
+        }
+
+        # Control Plane static files
+        location ~ ^/hindsight/_next/ {
+            proxy_pass http://hindsight_cp;
+            proxy_set_header Host $http_host;
+        }
+
+        # Control Plane UI
+        location /hindsight {
+            proxy_pass http://hindsight_cp;
+            proxy_set_header Host $http_host;
+        }
+
+        location = / { return 301 /hindsight; }
+    }
+}
+```
+
+### Why Build is Required
+
+Next.js requires `basePath` at **build time**. The published image was built without a custom base path, so you must rebuild from source with the `NEXT_PUBLIC_BASE_PATH` build arg to deploy the Control Plane under a subpath.
+
+The API works without rebuild because `HINDSIGHT_API_BASE_PATH` is a runtime environment variable.

docker/docker-compose/nginx/docker-compose.yml

@@ -0,0 +1,88 @@
+# Hindsight API deployment with Nginx reverse proxy (API-only)
+#
+# This example deploys Hindsight API under the path /hindsight with:
+# - Hindsight standalone image (API + Control Plane + embedded pg0)
+# - Nginx reverse proxy (API only)
+#
+# Quick Start:
+#   docker-compose -f docker/docker-compose/nginx/docker-compose.yml up
+#
+# Access:
+#   API (via nginx):         http://localhost:8080/hindsight/docs
+#   Control Plane (direct):  http://localhost:9999
+#
+# For full stack deployment (API + Control Plane both under /hindsight):
+#   See README.md in this directory for instructions on building with basePath.
+#
+# Note: This configuration uses the published image (no build required).
+#       Control Plane is served directly because Next.js basePath requires
+#       build-time configuration. See README.md for the full stack option.
+
+services:
+  # Hindsight (API + Control Plane + embedded pg0)
+  hindsight:
+    image: ghcr.io/vectorize-io/hindsight:latest
+    ports:
+      - "9999:9999"  # Control Plane (direct access, not proxied)
+    environment:
+      # API base path for reverse proxy
+      HINDSIGHT_API_BASE_PATH: /hindsight
+
+      # LLM configuration
+      # Using mock provider for testing (no API key needed)
+      # For production, set OPENAI_API_KEY or ANTHROPIC_API_KEY and use a real provider
+      HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-mock}
+      HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-not-needed-for-mock}
+      HINDSIGHT_API_LLM_MODEL: ${HINDSIGHT_API_LLM_MODEL:-mock-model}
+
+      # Production examples (uncomment and set appropriate API key):
+      # HINDSIGHT_API_LLM_PROVIDER: openai
+      # HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY}
+      # HINDSIGHT_API_LLM_MODEL: gpt-4o-mini
+
+      # HINDSIGHT_API_LLM_PROVIDER: anthropic
+      # HINDSIGHT_API_LLM_API_KEY: ${ANTHROPIC_API_KEY}
+      # HINDSIGHT_API_LLM_MODEL: claude-sonnet-4-20250514
+
+      # Server config
+      HINDSIGHT_API_HOST: 0.0.0.0
+      HINDSIGHT_API_PORT: 8888
+      HINDSIGHT_API_LOG_LEVEL: info
+
+      # Control Plane config
+      HINDSIGHT_CP_DATAPLANE_API_URL: http://localhost:8888
+    volumes:
+      # Persist embedded pg0 database
+      - hindsight_data:/app/data
+    # Note: Ports not exposed - access via Nginx at localhost:8080/hindsight/
+    # To debug directly, uncomment these ports:
+    # ports:
+    #   - "8888:8888"  # API
+    #   - "9999:9999"  # Control Plane
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8888/hindsight/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    networks:
+      - hindsight
+
+  # Nginx reverse proxy
+  nginx:
+    image: nginx:alpine
+    ports:
+      - "8080:80"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+    depends_on:
+      hindsight:
+        condition: service_healthy
+    networks:
+      - hindsight
+
+volumes:
+  hindsight_data:
+
+networks:
+  hindsight:

docker/docker-compose/nginx/nginx.conf

@@ -0,0 +1,40 @@
+# Nginx configuration for API-only reverse proxy
+# Control Plane accessed directly (not through nginx)
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    # Logging
+    access_log /var/log/nginx/access.log;
+    error_log /var/log/nginx/error.log;
+
+    # Upstream - Hindsight API
+    upstream hindsight_api {
+        server hindsight:8888;
+    }
+
+    server {
+        listen 80;
+        server_name _;
+
+        # API endpoints - forward with /hindsight prefix
+        location /hindsight/ {
+            proxy_pass http://hindsight_api;
+
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+
+        # Redirect root to API docs
+        location = / {
+            return 301 /hindsight/docs;
+        }
+    }
+}

docker/docker-compose/pg_textsearch/Dockerfile

@@ -0,0 +1,32 @@
+# PostgreSQL with pgvector and pg_textsearch extensions
+# Note: pg_textsearch requires PostgreSQL 17+
+FROM postgres:17
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    git \
+    postgresql-server-dev-17 \
+    libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install pgvector
+RUN cd /tmp && \
+    git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git && \
+    cd pgvector && \
+    make && \
+    make install
+
+# Install pg_textsearch
+RUN cd /tmp && \
+    git clone https://github.com/timescale/pg_textsearch.git && \
+    cd pg_textsearch && \
+    make && \
+    make install
+
+# Clean up source files and build dependencies
+RUN rm -rf /tmp/pgvector /tmp/pg_textsearch && \
+    apt-get purge -y --auto-remove build-essential git postgresql-server-dev-17
+
+# Ensure extensions are preloaded
+RUN echo "shared_preload_libraries = 'pg_textsearch'" >> /usr/share/postgresql/postgresql.conf.sample

docker/docker-compose/pg_textsearch/docker-compose.yaml

@@ -0,0 +1,91 @@
+name: hindsight
+# Docker Compose file for Hindsight with PostgreSQL and Timescale pg_textsearch
+# docker compose -f docker/docker-compose/pg_textsearch/docker-compose.yaml down && sleep 2 && docker compose -f docker/docker-compose/pg_textsearch/docker-compose.yaml up -d
+# Make sure to set the required environment variables before running:
+# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
+# - Configure LLM provider variables as needed (see below in the hindsight service)
+#
+# Usage:
+# docker compose up -d
+#
+# Optional environment variables with defaults:
+# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
+# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
+# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
+
+services:
+  db:
+    # Use custom PostgreSQL image with pgvector and pg_textsearch extensions
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: hindsight-db
+    restart: always
+    # Expose PostgreSQL port
+    ports:
+      - "5437:5432"
+    environment:
+      POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
+      POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:-hindsight_password}
+      POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    networks:
+      - hindsight-net
+
+  pg-textsearch-init:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      - db
+    environment:
+      - PGPASSWORD=${HINDSIGHT_DB_PASSWORD:-hindsight_password}
+    command: >
+      bash -c "
+        echo 'Waiting for PostgreSQL to be ready...';
+        until pg_isready -h hindsight-db -p 5432 -U hindsight_user; do
+          echo 'PostgreSQL is unavailable - sleeping';
+          sleep 2;
+        done;
+        echo 'PostgreSQL is ready - creating hindsight_db database';
+        psql -h hindsight-db -p 5432 -U hindsight_user -c 'CREATE DATABASE hindsight_db;' 2>/dev/null || echo 'Database already exists';
+        echo 'Creating extensions in hindsight_db database';
+        psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vector CASCADE;';
+        psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS pg_textsearch CASCADE;';
+        echo 'Database and extensions created successfully';
+      "
+    restart: "no"
+    networks:
+      - hindsight-net
+
+  hindsight:
+    image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
+    container_name: hindsight-app
+    ports:
+      - "8888:8888"
+      - "9999:9999"
+    environment:
+      # LLM Configuration
+      HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai}
+      HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key}
+
+      # Database Configuration
+      HINDSIGHT_API_DATABASE_URL: postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:-hindsight_password}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
+
+      # Vector and Text Search Extensions
+      HINDSIGHT_API_VECTOR_EXTENSION: pgvector
+      HINDSIGHT_API_TEXT_SEARCH_EXTENSION: pg_textsearch
+
+    depends_on:
+      - db
+    networks:
+      - hindsight-net
+
+
+networks:
+  hindsight-net:
+    driver: bridge
+
+volumes:
+  pg_data:

docker/docker-compose/s3-file-storage/docker-compose.yaml

@@ -0,0 +1,83 @@
+# Docker Compose file for Hindsight with S3 file storage (SeaweedFS)
+#
+# SeaweedFS (Apache 2.0) provides an S3-compatible object storage backend
+# for storing uploaded files instead of PostgreSQL BYTEA storage.
+#
+# Make sure to set the required environment variables before running:
+# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
+# - Configure LLM provider variables as needed (see below in the hindsight service)
+#
+# Usage:
+# docker compose up -d
+#
+# Optional environment variables with defaults:
+# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
+# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
+# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
+# - HINDSIGHT_DB_VERSION: PostgreSQL version (default: 18)
+# - SEAWEEDFS_S3_ACCESS_KEY: S3 access key (default: hindsight_s3_key)
+# - SEAWEEDFS_S3_SECRET_KEY: S3 secret key (default: hindsight_s3_secret)
+
+services:
+  db:
+    image: pgvector/pgvector:pg${HINDSIGHT_DB_VERSION:-18}
+    container_name: hindsight-db
+    restart: always
+    environment:
+      POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
+      POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}
+      POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
+    volumes:
+      - pg_data:/var/lib/postgresql/${HINDSIGHT_DB_VERSION:-18}/docker
+    networks:
+      - hindsight-net
+
+  seaweedfs:
+    image: chrislusf/seaweedfs:latest
+    container_name: hindsight-seaweedfs
+    restart: always
+    # Single-node mode: master + volume + filer + S3 gateway all in one process
+    command: >
+      server
+      -s3
+      -s3.port=8333
+      -s3.config=/etc/seaweedfs/s3.json
+      -ip.bind=0.0.0.0
+    volumes:
+      - seaweedfs_data:/data
+      - ./s3.json:/etc/seaweedfs/s3.json:ro
+    # Expose S3 API port (uncomment to access from host)
+    # ports:
+    #   - "8333:8333"
+    networks:
+      - hindsight-net
+
+  hindsight:
+    image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
+    container_name: hindsight-app
+    ports:
+      - "8888:8888"
+      - "9999:9999"
+    environment:
+      - HINDSIGHT_API_LLM_API_KEY=${OPENAI_API_KEY?Please set the OPENAI_API_KEY env variable}
+      - HINDSIGHT_API_DATABASE_URL=postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
+      # S3 file storage configuration (SeaweedFS)
+      - HINDSIGHT_API_FILE_STORAGE_TYPE=s3
+      - HINDSIGHT_API_FILE_STORAGE_S3_BUCKET=hindsight
+      - HINDSIGHT_API_FILE_STORAGE_S3_ENDPOINT=http://seaweedfs:8333
+      - HINDSIGHT_API_FILE_STORAGE_S3_REGION=us-east-1
+      - HINDSIGHT_API_FILE_STORAGE_S3_ACCESS_KEY_ID=${SEAWEEDFS_S3_ACCESS_KEY:-hindsight_s3_key}
+      - HINDSIGHT_API_FILE_STORAGE_S3_SECRET_ACCESS_KEY=${SEAWEEDFS_S3_SECRET_KEY:-hindsight_s3_secret}
+    depends_on:
+      - db
+      - seaweedfs
+    networks:
+      - hindsight-net
+
+networks:
+  hindsight-net:
+    driver: bridge
+
+volumes:
+  pg_data:
+  seaweedfs_data:

docker/docker-compose/s3-file-storage/s3.json

@@ -0,0 +1,19 @@
+{
+  "identities": [
+    {
+      "name": "hindsight",
+      "credentials": [
+        {
+          "accessKey": "hindsight_s3_key",
+          "secretKey": "hindsight_s3_secret"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "Write",
+        "List"
+      ]
+    }
+  ]
+}

docker/docker-compose/timescale/.dockerignore

@@ -0,0 +1,16 @@
+# Git
+.git
+.gitignore
+.gitattributes
+
+# Docker
+docker-compose.yaml
+.dockerignore
+
+# Documentation
+README.md
+*.md
+
+# Environment
+.env
+.env.example

docker/docker-compose/timescale/.env.example

@@ -0,0 +1,25 @@
+# PostgreSQL Configuration
+HINDSIGHT_DB_USER=hindsight_user
+HINDSIGHT_DB_PASSWORD=change-me-to-secure-password
+HINDSIGHT_DB_NAME=hindsight_db
+
+# Hindsight Version
+HINDSIGHT_VERSION=latest
+
+# LLM Configuration
+HINDSIGHT_API_LLM_PROVIDER=openai
+OPENAI_API_KEY=your-openai-api-key-here
+
+# Alternative LLM providers (uncomment and configure as needed):
+# HINDSIGHT_API_LLM_PROVIDER=anthropic
+# ANTHROPIC_API_KEY=your-anthropic-api-key
+
+# HINDSIGHT_API_LLM_PROVIDER=gemini
+# GEMINI_API_KEY=your-gemini-api-key
+
+# HINDSIGHT_API_LLM_PROVIDER=groq
+# GROQ_API_KEY=your-groq-api-key
+
+# Vector and Text Search (already configured in docker-compose.yaml)
+# HINDSIGHT_API_VECTOR_EXTENSION=pgvectorscale
+# HINDSIGHT_API_TEXT_SEARCH_EXTENSION=pg_textsearch

docker/docker-compose/timescale/Dockerfile

@@ -0,0 +1,55 @@
+# PostgreSQL with pgvector, pgvectorscale, and pg_textsearch extensions
+# All three extensions from Timescale/pgvector for high-performance vector and text search
+# Note: Requires PostgreSQL 16+
+FROM postgres:17
+
+# Install build dependencies and Rust toolchain
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    git \
+    postgresql-server-dev-17 \
+    libpq-dev \
+    cmake \
+    curl \
+    pkg-config \
+    libssl-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Rust toolchain (required for pgvectorscale)
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install pgvector (required by pgvectorscale)
+RUN cd /tmp && \
+    git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git && \
+    cd pgvector && \
+    make && \
+    make install && \
+    rm -rf /tmp/pgvector
+
+# Install cargo-pgrx (PostgreSQL extension framework for Rust)
+RUN cargo install cargo-pgrx --version 0.12.5 --locked && \
+    cargo pgrx init --pg17 /usr/bin/pg_config
+
+# Install pgvectorscale (DiskANN index support)
+RUN cd /tmp && \
+    git clone --branch 0.5.1 https://github.com/timescale/pgvectorscale.git && \
+    cd pgvectorscale/pgvectorscale && \
+    cargo pgrx install --release && \
+    rm -rf /tmp/pgvectorscale
+
+# Install pg_textsearch (BM25 text search)
+RUN cd /tmp && \
+    git clone https://github.com/timescale/pg_textsearch.git && \
+    cd pg_textsearch && \
+    make && \
+    make install && \
+    rm -rf /tmp/pg_textsearch
+
+# Clean up build dependencies (keep runtime dependencies)
+RUN apt-get purge -y --auto-remove git cmake curl && \
+    rm -rf /root/.cargo/registry /root/.cargo/git
+
+# Ensure extensions are preloaded (pg_textsearch requires preloading)
+RUN echo "shared_preload_libraries = 'pg_textsearch'" >> /usr/share/postgresql/postgresql.conf.sample
+

docker/docker-compose/timescale/README.md

@@ -0,0 +1,101 @@
+# Hindsight with Timescale Extensions
+
+This Docker Compose setup provides a complete Hindsight deployment with **Timescale extensions**:
+- **pgvectorscale** - DiskANN algorithm for disk-based scalable vector search  
+- **pg_textsearch** - High-performance BM25 text search
+
+Both extensions are from [Timescale](https://github.com/timescale) and provide production-grade performance.
+
+## Prerequisites
+
+- Docker and Docker Compose installed
+- OpenAI API key (or another LLM provider)
+
+## Quick Start
+
+```bash
+# Set environment variables
+export HINDSIGHT_DB_PASSWORD="your-secure-password"
+export OPENAI_API_KEY="your-openai-api-key"
+
+# Build and start
+docker compose -f docker/docker-compose/timescale/docker-compose.yaml up -d --build
+
+# Check logs
+
+docker compose -f docker/docker-compose/timescale/docker-compose.yaml logs -f
+```
+
+**Access:**
+- API: http://localhost:8888
+- Control Plane: http://localhost:9999
+
+## Stop and Clean Up
+
+```bash
+# Stop services
+docker compose -f docker/docker-compose/timescale/docker-compose.yaml down
+
+# Remove volumes (deletes all data)
+docker compose -f docker/docker-compose/timescale/docker-compose.yaml down -v
+```
+
+## Configuration
+
+### Environment Variables
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `HINDSIGHT_DB_PASSWORD` | PostgreSQL password | `hindsight_password` |
+| `HINDSIGHT_DB_USER` | PostgreSQL username | `hindsight_user` |
+| `HINDSIGHT_DB_NAME` | Database name | `hindsight_db` |
+| `HINDSIGHT_VERSION` | Hindsight Docker image version | `latest` |
+| `OPENAI_API_KEY` | OpenAI API key | (required) |
+| `HINDSIGHT_API_LLM_PROVIDER` | LLM provider | `openai` |
+
+### Why Timescale Extensions?
+
+**pgvectorscale (DiskANN):**
+- 28x lower p95 latency vs dedicated vector databases
+- 16x higher query throughput at 99% recall
+- 60-75% cost reduction (disk is cheaper than RAM)
+- Best for large datasets (10M+ vectors)
+
+**pg_textsearch (BM25):**
+- High-performance keyword retrieval
+- Native BM25 ranking algorithm
+- Optimized for full-text search
+
+## Troubleshooting
+
+### Extensions not installed
+
+Check if extensions are available:
+
+```bash
+docker exec -it hindsight-db-timescale psql -U hindsight_user -d hindsight_db -c "\dx"
+```
+
+You should see:
+- `vector` (pgvector)
+- `vectorscale` (pgvectorscale/DiskANN)
+- `pg_textsearch` (BM25 search)
+
+### Build fails
+
+If the Docker build fails during pgvectorscale compilation:
+
+1. Ensure you have sufficient memory (recommended: 4GB+)
+2. Check Docker build logs for Rust compilation errors
+3. Try building with more resources: `docker compose build --no-cache --memory 4g`
+
+### Port conflicts
+
+If port 5438 is already in use, modify the `ports` section in docker-compose.yaml.
+
+## Learn More
+
+- [pgvectorscale GitHub](https://github.com/timescale/pgvectorscale)
+- [pg_textsearch GitHub](https://github.com/timescale/pg_textsearch)
+- [HNSW vs DiskANN](https://www.tigerdata.com/learn/hnsw-vs-diskann)
+- [Hindsight Documentation](https://hindsight.dev)

docker/docker-compose/timescale/docker-compose.yaml

@@ -0,0 +1,108 @@
+name: hindsight
+# Docker Compose file for Hindsight with Timescale extensions
+# - pgvectorscale: DiskANN vector search (disk-based, scalable)
+# - pg_textsearch: BM25 text search (high-performance keyword retrieval)
+#
+# Quick start:
+# docker compose -f docker/docker-compose/timescale/docker-compose.yaml up -d --build
+#
+# Required environment variables:
+# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
+# - OPENAI_API_KEY (or configure another LLM provider)
+#
+# Optional environment variables with defaults:
+# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
+# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
+# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
+
+services:
+  db:
+    # Custom PostgreSQL image with Timescale extensions (pgvectorscale + pg_textsearch)
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: hindsight-db-timescale
+    restart: always
+    # Expose PostgreSQL port (using 5438 to avoid conflicts with other setups)
+    ports:
+      - "5438:5432"
+    environment:
+      POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
+      POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:-hindsight_password}
+      POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    networks:
+      - hindsight-net
+    # Health check to ensure database is ready
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U hindsight_user"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+  timescale-init:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      db:
+        condition: service_healthy
+    environment:
+      - PGPASSWORD=${HINDSIGHT_DB_PASSWORD:-hindsight_password}
+    command: >
+      bash -c "
+        echo 'PostgreSQL is ready - creating hindsight_db database';
+        psql -h hindsight-db-timescale -p 5432 -U hindsight_user -c 'CREATE DATABASE hindsight_db;' 2>/dev/null || echo 'Database already exists';
+        echo 'Installing Timescale extensions...';
+        echo '1/3: Installing pgvector (required by pgvectorscale)...';
+        psql -h hindsight-db-timescale -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vector CASCADE;';
+        echo '2/3: Installing pgvectorscale (DiskANN vector search)...';
+        psql -h hindsight-db-timescale -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vectorscale CASCADE;';
+        echo '3/3: Installing pg_textsearch (BM25 text search)...';
+        psql -h hindsight-db-timescale -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS pg_textsearch CASCADE;';
+        echo '';
+        echo '✅ Timescale extensions installed successfully';
+        echo '';
+        echo 'Installed extensions:';
+        psql -h hindsight-db-timescale -p 5432 -U hindsight_user -d hindsight_db -c \"\\dx\" | grep -E '(vector|vectorscale|pg_textsearch)';
+      "
+    restart: "no"
+    networks:
+      - hindsight-net
+
+  hindsight:
+    image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
+    container_name: hindsight-app-timescale
+    ports:
+      - "8888:8888"
+      - "9999:9999"
+    environment:
+      # LLM Configuration
+      HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai}
+      HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key}
+
+      # Database Configuration
+      HINDSIGHT_API_DATABASE_URL: postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:-hindsight_password}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
+
+      # Timescale Extensions
+      # pgvectorscale: DiskANN algorithm for disk-based scalable vector search
+      HINDSIGHT_API_VECTOR_EXTENSION: pgvectorscale
+      # pg_textsearch: High-performance BM25 text search
+      HINDSIGHT_API_TEXT_SEARCH_EXTENSION: pg_textsearch
+
+    depends_on:
+      db:
+        condition: service_healthy
+      timescale-init:
+        condition: service_completed_successfully
+    networks:
+      - hindsight-net
+
+
+networks:
+  hindsight-net:
+    driver: bridge
+
+volumes:
+  pg_data:

docker/docker-compose/vchord/docker-compose.yaml

@@ -0,0 +1,93 @@
+name: hindsight
+# Docker Compose file for Hindsight with PostgreSQL and vectorchord
+# docker compose -f docker/docker-compose/docker-compose.yaml down && sleep 2 && docker compose -f docker/docker-compose/docker-compose.yaml up -d
+# Make sure to set the required environment variables before running:
+# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
+# - Configure LLM provider variables as needed (see below in the hindsight service)
+#
+# Usage:
+# docker compose up -d
+#
+# Optional environment variables with defaults:
+# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
+# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
+# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
+# - HINDSIGHT_DB_VERSION: PostgreSQL version (default: 18)
+
+services:
+  db:
+    # Use a PostgreSQL-Image with vectorchord extension pre-installed
+    image: tensorchord/vchord-suite:pg${HINDSIGHT_DB_VERSION:-18-latest}
+    container_name: hindsight-db
+    restart: always
+    # Expose PostgreSQL port
+    ports:
+      - "5436:5432"
+    environment:
+      POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
+      POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:-hindsight_password}
+      POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
+    volumes:
+      - pg_data:/var/lib/postgresql/${HINDSIGHT_DB_VERSION:-18}/docker
+    networks:
+      - hindsight-net
+
+  vectorchord-init:
+    image: tensorchord/vchord-suite:pg18-latest
+    #container_name: vectorchord-init
+    depends_on:
+      - db
+    environment:
+      - PGPASSWORD=${HINDSIGHT_DB_PASSWORD:-hindsight_password}
+    command: >
+      bash -c "
+        echo 'Waiting for PostgreSQL to be ready...';
+        until pg_isready -h hindsight-db -p 5432 -U hindsight_user; do
+          echo 'PostgreSQL is unavailable - sleeping';
+          sleep 2;
+        done;
+        echo 'PostgreSQL is ready - creating hindsight_db database';
+        psql -h hindsight-db -p 5432 -U hindsight_user -c 'CREATE DATABASE hindsight_db;' 2>/dev/null || echo 'Database already exists';
+        echo 'Creating extensions in hindsight_db database';
+        psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vchord CASCADE;';
+        psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS pg_tokenizer CASCADE;';
+        psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vchord_bm25 CASCADE;';
+        echo 'Creating llmlingua2 tokenizer';
+        psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c \"SELECT create_tokenizer('llmlingua2', \\$\\$ model = \\\"llmlingua2\\\" \\$\\$);\" 2>/dev/null || echo 'Tokenizer already exists or creation skipped';
+        echo 'Database and extensions created successfully';
+      "
+    restart: "no"
+    networks:
+      - hindsight-net
+
+  hindsight:
+    image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
+    container_name: hindsight-app
+    ports:
+      - "8888:8888"
+      - "9999:9999"
+    environment:
+      # LLM Configuration (uses OpenAI for testing vchord)
+      # LLM configuration
+      HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai}
+      HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key}
+
+      # Database Configuration
+      HINDSIGHT_API_DATABASE_URL: postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:-hindsight_password}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
+
+      # Vector and Text Search Extensions
+      HINDSIGHT_API_VECTOR_EXTENSION: vchord
+      HINDSIGHT_API_TEXT_SEARCH_EXTENSION: vchord
+
+    depends_on:
+      - db
+    networks:
+      - hindsight-net
+
+
+networks:
+  hindsight-net:
+    driver: bridge
+
+volumes:
+  pg_data:

docker/standalone/Dockerfile

@@ -2,16 +2,25 @@
 # Supports building API-only, Control Plane-only, or both
 #
 # Build args:
-#   INCLUDE_API=true/false  - Include API (default: true)
-#   INCLUDE_CP=true/false   - Include Control Plane (default: true)
+#   INCLUDE_API=true/false         - Include API (default: true)
+#   INCLUDE_CP=true/false          - Include Control Plane (default: true)
+#   INCLUDE_LOCAL_MODELS=true/false - Include local ML models for embeddings/reranking (default: true)
+#                                     Set to false when using external providers (TEI, OpenAI, Cohere)
+#   PRELOAD_ML_MODELS=true/false   - Pre-download ML models during build (default: true)
+#                                     Only effective when INCLUDE_LOCAL_MODELS=true
+#                                     NOTE: tiktoken encodings are ALWAYS preloaded (required for air-gapped deployments)
 #
 # Examples:
-#   docker build -t hindsight .                                    # Both (standalone)
-#   docker build -t hindsight-api --build-arg INCLUDE_CP=false .   # API only
-#   docker build -t hindsight-cp --build-arg INCLUDE_API=false .   # Control Plane only
+#   docker build -t hindsight .                                          # Both (standalone)
+#   docker build -t hindsight-api --build-arg INCLUDE_CP=false .         # API only
+#   docker build -t hindsight-cp --build-arg INCLUDE_API=false .         # Control Plane only
+#   docker build -t hindsight --build-arg PRELOAD_ML_MODELS=false .      # Skip ML model preload
+#   docker build -t hindsight --build-arg INCLUDE_LOCAL_MODELS=false .   # Skip local ML deps (for external providers)
 
 ARG INCLUDE_API=true
 ARG INCLUDE_CP=true
+ARG PRELOAD_ML_MODELS=true
+ARG INCLUDE_LOCAL_MODELS=true
 
 # =============================================================================
 # Stage: API Builder
@@ -19,6 +28,7 @@ ARG INCLUDE_CP=true
 FROM python:3.11-slim AS api-builder
 
 ARG INCLUDE_API
+ARG INCLUDE_LOCAL_MODELS
 RUN if [ "$INCLUDE_API" != "true" ]; then echo "Skipping API build" && exit 0; fi
 
 WORKDIR /app
@@ -32,17 +42,25 @@ RUN apt-get update && apt-get install -y \
     && pip install --no-cache-dir uv
 
 # Copy dependency files and README (required by pyproject.toml)
-COPY hindsight-api/pyproject.toml ./api/
-COPY hindsight-api/README.md ./api/
+COPY hindsight-api-slim/pyproject.toml ./api/
+COPY hindsight-api-slim/README.md ./api/
 
 WORKDIR /app/api
 
-# Sync dependencies (will create lock file if needed)
-RUN uv sync
+# Sync dependencies using appropriate extras based on INCLUDE_LOCAL_MODELS
+# local-ml: torch, sentence-transformers, transformers, einops, flashrank, mlx (optional)
+# embedded-db: pg0-embedded (always included for embedded PostgreSQL support)
+RUN if [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
+        uv sync --extra local-ml --extra embedded-db; \
+    else \
+        uv sync --extra embedded-db; \
+    fi
 
-# Copy source code and alembic migrations
-COPY hindsight-api/hindsight_api ./hindsight_api
-COPY hindsight-api/alembic ./alembic
+# Copy source code (alembic migrations are inside hindsight_api/)
+COPY hindsight-api-slim/hindsight_api ./hindsight_api
+
+# Install the local package (uv sync only installed dependencies, not the package itself)
+RUN uv pip install -e .
 
 # =============================================================================
 # Stage: SDK Builder (needed for Control Plane)
@@ -52,13 +70,15 @@ FROM node:20-slim AS sdk-builder
 ARG INCLUDE_CP
 RUN if [ "$INCLUDE_CP" != "true" ]; then echo "Skipping SDK build" && exit 0; fi
 
-WORKDIR /app/sdk
+WORKDIR /app
 
-COPY hindsight-clients/typescript/package*.json ./
-RUN npm ci
+# Copy root package files for npm workspaces
+COPY package.json package-lock.json ./
+COPY hindsight-clients/typescript/ ./hindsight-clients/typescript/
 
-COPY hindsight-clients/typescript/ ./
-RUN npm run build
+# Install and build SDK using workspace (--ignore-scripts skips git hooks setup)
+RUN npm ci --ignore-scripts -w @vectorize-io/hindsight-client
+RUN npm run build -w @vectorize-io/hindsight-client
 
 # =============================================================================
 # Stage: Control Plane Builder
@@ -68,30 +88,52 @@ FROM node:20-slim AS cp-builder
 ARG INCLUDE_CP
 RUN if [ "$INCLUDE_CP" != "true" ]; then echo "Skipping CP build" && exit 0; fi
 
-WORKDIR /app
-
-# Copy built SDK
-COPY --from=sdk-builder /app/sdk /app/sdk
+# Create directory structure matching the monorepo layout
+# This is required because build:standalone script expects .next/standalone/memory-poc/hindsight-control-plane
+WORKDIR /app/memory-poc/hindsight-control-plane
 
 # Install Control Plane dependencies
 # Only copy package.json (not package-lock.json) to ensure npm installs
 # correct platform-specific native bindings for lightningcss/tailwindcss
 COPY hindsight-control-plane/package.json ./
+# Remove the file: dependency on SDK (we'll copy it directly later)
+RUN sed -i '/"@vectorize-io\/hindsight-client":/d' package.json
 RUN npm install
 
 # Copy Control Plane source (excluding node_modules via .dockerignore)
 COPY hindsight-control-plane/ ./
 # Remove package-lock.json to avoid conflicts with installed native bindings
-RUN rm -f package-lock.json
+# Also remove the file: dependency from package.json (restored by COPY above)
+RUN rm -f package-lock.json && sed -i '/"@vectorize-io\/hindsight-client":/d' package.json
 
-# Link SDK (temporary for build)
-RUN cd /app/sdk && npm link && cd /app && npm link @vectorize-io/hindsight-client
+# Copy built SDK directly into node_modules (more reliable than npm link in Docker)
+COPY --from=sdk-builder /app/hindsight-clients/typescript ./node_modules/@vectorize-io/hindsight-client
 
-# Build Control Plane
-RUN npm run build
+# Accept base path as build argument for reverse proxy deployments
+# Usage: docker build --build-arg NEXT_PUBLIC_BASE_PATH=/hindsight ...
+ARG NEXT_PUBLIC_BASE_PATH=""
 
-# Create public directory if it doesn't exist
-RUN mkdir -p public
+# Build Control Plane - run next build first, then custom standalone copy
+# (The build:standalone script expects a specific path structure that differs in Docker)
+RUN npm exec -- next build
+
+# Create standalone directory structure manually
+# Note: Must exclude node_modules from find to avoid wrong server.js from next/dist/experimental/testmode/
+# Note: Must explicitly copy .next since glob * doesn't match hidden directories
+RUN STANDALONE_ROOT=$(find .next/standalone -path '*/node_modules' -prune -o -name 'server.js' -print | head -1 | xargs dirname) && \
+    mkdir -p standalone && \
+    cp -r "$STANDALONE_ROOT"/* standalone/ && \
+    cp -r "$STANDALONE_ROOT"/.next standalone/.next && \
+    # Copy node_modules if separate from app dir (monorepo structure)
+    if [ -d ".next/standalone/node_modules" ] && [ "$STANDALONE_ROOT" != ".next/standalone" ]; then \
+      cp -r .next/standalone/node_modules standalone/node_modules; \
+    fi && \
+    cp -r .next/static standalone/.next/static && \
+    mkdir -p standalone/public && \
+    cp -r public/* standalone/public/ 2>/dev/null || true && \
+    # Verify required files exist
+    test -f standalone/server.js || (echo "ERROR: server.js missing!" && exit 1) && \
+    test -f standalone/.next/BUILD_ID || (echo "ERROR: BUILD_ID missing!" && exit 1)
 
 # =============================================================================
 # Stage: Final Image - API Only
@@ -100,18 +142,18 @@ FROM python:3.11-slim AS api-only
 
 WORKDIR /app
 
-# Install pg0 dependencies
+# Note: libicu version varies by Debian version - try common versions in order
 RUN apt-get update && apt-get install -y \
     curl \
+    procps \
     libxml2 \
     libssl3 \
     libgssapi-krb5-2 \
     libossp-uuid16 \
-    && apt-get install -y libicu72 || apt-get install -y libicu74 || apt-get install -y libicu* \
+    && (apt-get install -y libicu72 2>/dev/null || apt-get install -y libicu74 2>/dev/null || apt-get install -y libicu76 2>/dev/null || true) \
     && rm -rf /var/lib/apt/lists/* \
     && pip install --no-cache-dir uv
 
-# Create non-root user (PostgreSQL cannot run as root)
 RUN useradd -m -s /bin/bash hindsight
 
 # Copy API with virtual environment from builder
@@ -121,43 +163,70 @@ COPY --from=api-builder /app/api /app/api
 COPY docker/standalone/start-all.sh /app/start-all.sh
 RUN chmod +x /app/start-all.sh
 
-# Create data directory for pg0 and set ownership
-RUN mkdir -p /app/data && chown -R hindsight:hindsight /app
+RUN chown -R hindsight:hindsight /app
 
-# Switch to non-root user
 USER hindsight
 
-# Set PATH for hindsight user
-ENV PATH="/home/hindsight/.hindsight/bin:/app/api/.venv/bin:${PATH}"
+# Create pg0 data directory as hindsight user so that Docker seeds new named
+# volumes with correct ownership (UID 1000) on first use, avoiding the
+# "Permission denied" error when mounting a fresh root-owned volume.
+RUN mkdir -p /home/hindsight/.pg0
 
-# Install pg0 binary
-RUN mkdir -p /home/hindsight/.hindsight/bin && \
-    ARCH=$(uname -m) && \
-    if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \
-        PG0_BINARY="pg0-linux-aarch64-gnu"; \
-    elif [ "$ARCH" = "x86_64" ]; then \
-        PG0_BINARY="pg0-linux-x86_64-gnu"; \
-    else \
-        echo "Unsupported architecture: $ARCH" && exit 1; \
-    fi && \
-    echo "Installing pg0 binary: $PG0_BINARY" && \
-    for i in 1 2 3 4 5; do \
-        curl -fsSL -o /home/hindsight/.hindsight/bin/pg0 \
-            "https://github.com/vectorize-io/pg0/releases/latest/download/$PG0_BINARY" && \
-        chmod +x /home/hindsight/.hindsight/bin/pg0 && \
-        break || (echo "Retry $i failed, waiting..." && sleep 10); \
-    done && \
-    /home/hindsight/.hindsight/bin/pg0 --version
+ENV PATH="/app/api/.venv/bin:${PATH}"
 
-# Pre-download PostgreSQL binaries
-ENV PG0_HOME=/home/hindsight/.pg0-cache
-RUN pg0 start --help && \
-    (pg0 start --name hindsight --port 5555 --username hindsight --password hindsight --database hindsight && \
-    sleep 2 && \
-    pg0 stop --name hindsight && \
-    echo "PostgreSQL pre-cached to $PG0_HOME") || echo "Pre-download skipped"
+# Pre-download tiktoken encoding (ALWAYS - required for token counting even in air-gapped envs)
+# Tiktoken is a core runtime dependency, not an optional ML model
+RUN MAX_RETRIES=3; \
+    RETRY_DELAY=5; \
+    for i in $(seq 1 $MAX_RETRIES); do \
+      echo "Attempt $i/$MAX_RETRIES: Downloading tiktoken encoding..."; \
+      /app/api/.venv/bin/python -c "\
+import tiktoken; \
+print('Downloading cl100k_base encoding...'); \
+tiktoken.get_encoding('cl100k_base'); \
+print('Tiktoken encoding cached successfully')" && break; \
+      if [ $i -lt $MAX_RETRIES ]; then \
+        echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
+        sleep $RETRY_DELAY; \
+        RETRY_DELAY=$((RETRY_DELAY * 2)); \
+      fi; \
+    done; \
+    if [ $i -eq $MAX_RETRIES ]; then \
+      echo "ERROR: Failed to download tiktoken encoding after $MAX_RETRIES attempts"; \
+      exit 1; \
+    fi
 
-ENV PG0_HOME=/home/hindsight/.pg0
+# Pre-download ML models to avoid runtime download (conditional)
+# Only runs if both PRELOAD_ML_MODELS=true AND INCLUDE_LOCAL_MODELS=true
+# Includes retry logic with exponential backoff for transient network failures
+ARG PRELOAD_ML_MODELS
+ARG INCLUDE_LOCAL_MODELS
+ENV HF_HUB_DOWNLOAD_TIMEOUT=600
+RUN if [ "$PRELOAD_ML_MODELS" = "true" ] && [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
+    MAX_RETRIES=3; \
+    RETRY_DELAY=10; \
+    for i in $(seq 1 $MAX_RETRIES); do \
+      echo "Attempt $i/$MAX_RETRIES: Downloading ML models..."; \
+      /app/api/.venv/bin/python -c "\
+import os; os.environ['HF_HUB_DOWNLOAD_TIMEOUT'] = '600'; \
+from sentence_transformers import SentenceTransformer, CrossEncoder; \
+print('Downloading embedding model...'); \
+SentenceTransformer('BAAI/bge-small-en-v1.5'); \
+print('Downloading cross-encoder model...'); \
+CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'); \
+print('Models cached successfully')" && break; \
+      if [ $i -lt $MAX_RETRIES ]; then \
+        echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
+        sleep $RETRY_DELAY; \
+        RETRY_DELAY=$((RETRY_DELAY * 2)); \
+      fi; \
+    done; \
+    if [ $i -eq $MAX_RETRIES ] && ! /app/api/.venv/bin/python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('BAAI/bge-small-en-v1.5')" 2>/dev/null; then \
+      echo "ERROR: Failed to download models after $MAX_RETRIES attempts"; \
+      exit 1; \
+    fi; \
+    elif [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then echo "Skipping ML model preload (local-models not included)"; \
+    else echo "Skipping ML model preload"; fi
 
 EXPOSE 8888
 
@@ -166,6 +235,11 @@ ENV HINDSIGHT_API_PORT=8888
 ENV HINDSIGHT_API_LOG_LEVEL=info
 ENV HINDSIGHT_ENABLE_API=true
 ENV HINDSIGHT_ENABLE_CP=false
+ENV PYTHONUNBUFFERED=1
+# Suppress verbose transformers/HuggingFace model loading warnings
+ENV TRANSFORMERS_VERBOSITY=error
+ENV HF_HUB_VERBOSITY=error
+ENV TOKENIZERS_PARALLELISM=false
 
 CMD ["/app/start-all.sh"]
 
@@ -177,13 +251,13 @@ FROM node:20-alpine AS cp-only
 WORKDIR /app
 
 # Copy built SDK
-COPY --from=sdk-builder /app/sdk /app/sdk
+COPY --from=sdk-builder /app/hindsight-clients/typescript /app/sdk
 
 # Copy Control Plane standalone build
 WORKDIR /app/control-plane
-COPY --from=cp-builder /app/.next/standalone ./
-COPY --from=cp-builder /app/.next/static ./.next/static
-COPY --from=cp-builder /app/public ./public
+COPY --from=cp-builder /app/memory-poc/hindsight-control-plane/standalone ./
+COPY --from=cp-builder /app/memory-poc/hindsight-control-plane/.next/static ./.next/static
+COPY --from=cp-builder /app/memory-poc/hindsight-control-plane/public ./public
 
 WORKDIR /app
 
@@ -210,33 +284,34 @@ FROM python:3.11-slim AS standalone
 
 WORKDIR /app
 
-# Install Node.js, curl, uv, and pg0 dependencies
+# Install Node.js, curl, uv, and system dependencies
+# Note: libicu version varies by Debian version - try common versions in order
 RUN apt-get update && apt-get install -y \
     curl \
+    procps \
     libxml2 \
     libssl3 \
     libgssapi-krb5-2 \
     libossp-uuid16 \
-    && apt-get install -y libicu72 || apt-get install -y libicu74 || apt-get install -y libicu* \
+    && (apt-get install -y libicu72 2>/dev/null || apt-get install -y libicu74 2>/dev/null || apt-get install -y libicu76 2>/dev/null || true) \
     && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
     && apt-get install -y nodejs \
     && rm -rf /var/lib/apt/lists/* \
     && pip install --no-cache-dir uv
 
-# Create non-root user (PostgreSQL cannot run as root)
 RUN useradd -m -s /bin/bash hindsight
 
 # Copy API with virtual environment from builder
 COPY --from=api-builder /app/api /app/api
 
 # Copy built SDK
-COPY --from=sdk-builder /app/sdk /app/sdk
+COPY --from=sdk-builder /app/hindsight-clients/typescript /app/sdk
 
 # Copy Control Plane standalone build
 WORKDIR /app/control-plane
-COPY --from=cp-builder /app/.next/standalone ./
-COPY --from=cp-builder /app/.next/static ./.next/static
-COPY --from=cp-builder /app/public ./public
+COPY --from=cp-builder /app/memory-poc/hindsight-control-plane/standalone ./
+COPY --from=cp-builder /app/memory-poc/hindsight-control-plane/.next/static ./.next/static
+COPY --from=cp-builder /app/memory-poc/hindsight-control-plane/public ./public
 
 WORKDIR /app
 
@@ -244,43 +319,70 @@ WORKDIR /app
 COPY docker/standalone/start-all.sh /app/start-all.sh
 RUN chmod +x /app/start-all.sh
 
-# Create data directory for pg0 and set ownership
-RUN mkdir -p /app/data && chown -R hindsight:hindsight /app
+RUN chown -R hindsight:hindsight /app
 
-# Switch to non-root user
 USER hindsight
 
-# Set PATH for hindsight user
-ENV PATH="/home/hindsight/.hindsight/bin:/app/api/.venv/bin:${PATH}"
+# Create pg0 data directory as hindsight user so that Docker seeds new named
+# volumes with correct ownership (UID 1000) on first use, avoiding the
+# "Permission denied" error when mounting a fresh root-owned volume.
+RUN mkdir -p /home/hindsight/.pg0
 
-# Install pg0 binary
-RUN mkdir -p /home/hindsight/.hindsight/bin && \
-    ARCH=$(uname -m) && \
-    if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \
-        PG0_BINARY="pg0-linux-aarch64-gnu"; \
-    elif [ "$ARCH" = "x86_64" ]; then \
-        PG0_BINARY="pg0-linux-x86_64-gnu"; \
-    else \
-        echo "Unsupported architecture: $ARCH" && exit 1; \
-    fi && \
-    echo "Installing pg0 binary: $PG0_BINARY" && \
-    for i in 1 2 3 4 5; do \
-        curl -fsSL -o /home/hindsight/.hindsight/bin/pg0 \
-            "https://github.com/vectorize-io/pg0/releases/latest/download/$PG0_BINARY" && \
-        chmod +x /home/hindsight/.hindsight/bin/pg0 && \
-        break || (echo "Retry $i failed, waiting..." && sleep 10); \
-    done && \
-    /home/hindsight/.hindsight/bin/pg0 --version
+ENV PATH="/app/api/.venv/bin:${PATH}"
 
-# Pre-download PostgreSQL binaries
-ENV PG0_HOME=/home/hindsight/.pg0-cache
-RUN pg0 start --help && \
-    (pg0 start --name hindsight --port 5555 --username hindsight --password hindsight --database hindsight && \
-    sleep 2 && \
-    pg0 stop --name hindsight && \
-    echo "PostgreSQL pre-cached to $PG0_HOME") || echo "Pre-download skipped"
+# Pre-download tiktoken encoding (ALWAYS - required for token counting even in air-gapped envs)
+# Tiktoken is a core runtime dependency, not an optional ML model
+RUN MAX_RETRIES=3; \
+    RETRY_DELAY=5; \
+    for i in $(seq 1 $MAX_RETRIES); do \
+      echo "Attempt $i/$MAX_RETRIES: Downloading tiktoken encoding..."; \
+      /app/api/.venv/bin/python -c "\
+import tiktoken; \
+print('Downloading cl100k_base encoding...'); \
+tiktoken.get_encoding('cl100k_base'); \
+print('Tiktoken encoding cached successfully')" && break; \
+      if [ $i -lt $MAX_RETRIES ]; then \
+        echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
+        sleep $RETRY_DELAY; \
+        RETRY_DELAY=$((RETRY_DELAY * 2)); \
+      fi; \
+    done; \
+    if [ $i -eq $MAX_RETRIES ]; then \
+      echo "ERROR: Failed to download tiktoken encoding after $MAX_RETRIES attempts"; \
+      exit 1; \
+    fi
 
-ENV PG0_HOME=/home/hindsight/.pg0
+# Pre-download ML models to avoid runtime download (conditional)
+# Only runs if both PRELOAD_ML_MODELS=true AND INCLUDE_LOCAL_MODELS=true
+# Includes retry logic with exponential backoff for transient network failures
+ARG PRELOAD_ML_MODELS
+ARG INCLUDE_LOCAL_MODELS
+ENV HF_HUB_DOWNLOAD_TIMEOUT=600
+RUN if [ "$PRELOAD_ML_MODELS" = "true" ] && [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
+    MAX_RETRIES=3; \
+    RETRY_DELAY=10; \
+    for i in $(seq 1 $MAX_RETRIES); do \
+      echo "Attempt $i/$MAX_RETRIES: Downloading ML models..."; \
+      /app/api/.venv/bin/python -c "\
+import os; os.environ['HF_HUB_DOWNLOAD_TIMEOUT'] = '600'; \
+from sentence_transformers import SentenceTransformer, CrossEncoder; \
+print('Downloading embedding model...'); \
+SentenceTransformer('BAAI/bge-small-en-v1.5'); \
+print('Downloading cross-encoder model...'); \
+CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'); \
+print('Models cached successfully')" && break; \
+      if [ $i -lt $MAX_RETRIES ]; then \
+        echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
+        sleep $RETRY_DELAY; \
+        RETRY_DELAY=$((RETRY_DELAY * 2)); \
+      fi; \
+    done; \
+    if [ $i -eq $MAX_RETRIES ] && ! /app/api/.venv/bin/python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('BAAI/bge-small-en-v1.5')" 2>/dev/null; then \
+      echo "ERROR: Failed to download models after $MAX_RETRIES attempts"; \
+      exit 1; \
+    fi; \
+    elif [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then echo "Skipping ML model preload (local-models not included)"; \
+    else echo "Skipping ML model preload"; fi
 
 EXPOSE 8888 9999
 
@@ -291,6 +393,11 @@ ENV NODE_ENV=production
 ENV HINDSIGHT_CP_DATAPLANE_API_URL=http://localhost:8888
 ENV HINDSIGHT_ENABLE_API=true
 ENV HINDSIGHT_ENABLE_CP=true
+ENV PYTHONUNBUFFERED=1
+# Suppress verbose transformers/HuggingFace model loading warnings
+ENV TRANSFORMERS_VERBOSITY=error
+ENV HF_HUB_VERBOSITY=error
+ENV TOKENIZERS_PARALLELISM=false
 
 CMD ["/app/start-all.sh"]
 

docker/standalone/docker-compose.yml

@@ -1,25 +0,0 @@
-services:
-  hindsight:
-    image: hindsight
-    build:
-      context: ../..
-      dockerfile: docker/standalone/Dockerfile
-    env_file:
-      - ../../.env
-    ports:
-      - "9999:9999"
-      - "8888:8888"
-    environment:
-      # These override env_file values only when set in host shell
-      # Default values are applied only when not set in env_file or host
-      HINDSIGHT_API_HOST: ${HINDSIGHT_API_HOST:-0.0.0.0}
-      HINDSIGHT_API_PORT: ${HINDSIGHT_API_PORT:-8888}
-      HINDSIGHT_API_LOG_LEVEL: ${HINDSIGHT_API_LOG_LEVEL:-info}
-      # HINDSIGHT_API_DATABASE_URL can be set if you want to use an external database
-      # If not set, embedded pg0 will be used automatically
-    volumes:
-      - hindsight_data:/home/hindsight/.pg0
-    restart: unless-stopped
-
-volumes:
-  hindsight_data:

docker/standalone/start-all.sh

@@ -1,57 +1,201 @@
 #!/bin/bash
 set -e
 
-echo "🚀 Starting Hindsight..."
-echo ""
+# =============================================================================
+# Embedded pg0 data integrity check (#675)
+#
+# When using embedded pg0, check if the data directory has existing PostgreSQL
+# data before starting. If the directory exists but appears empty/corrupt
+# (e.g., missing PG_VERSION file), log a warning. This helps diagnose data
+# loss scenarios where a container restart caused the data directory to be
+# wiped despite a volume mount being present.
+# =============================================================================
+PG0_DATA_DIR="${HOME}/.pg0"
+if [ -d "$PG0_DATA_DIR" ]; then
+    # Look for actual PostgreSQL data directories (pg0 creates subdirs per instance)
+    if compgen -G "$PG0_DATA_DIR"/*/PG_VERSION > /dev/null 2>&1; then
+        echo "✅ Existing pg0 data directory detected at $PG0_DATA_DIR"
+    elif [ "$(ls -A "$PG0_DATA_DIR" 2>/dev/null)" ]; then
+        echo "⚠️  WARNING: pg0 data directory exists at $PG0_DATA_DIR but no PG_VERSION found."
+        echo "   This may indicate data corruption or an incomplete previous shutdown."
+        echo "   If you see all migrations running from scratch after this, your data may have been lost."
+        echo "   See: https://github.com/vectorize-io/hindsight/issues/675"
+    fi
+fi
 
 # Service flags (default to true if not set)
 ENABLE_API="${HINDSIGHT_ENABLE_API:-true}"
 ENABLE_CP="${HINDSIGHT_ENABLE_CP:-true}"
 
-# Copy pre-cached PostgreSQL data if runtime directory is empty (first run with volume)
-if [ "$ENABLE_API" = "true" ]; then
-    PG0_CACHE="/home/hindsight/.pg0-cache"
-    PG0_HOME="/home/hindsight/.pg0"
-    if [ -d "$PG0_CACHE" ] && [ "$(ls -A $PG0_CACHE 2>/dev/null)" ]; then
-        if [ ! "$(ls -A $PG0_HOME 2>/dev/null)" ]; then
-            echo "📦 Copying pre-cached PostgreSQL data..."
-            cp -r "$PG0_CACHE"/* "$PG0_HOME"/ 2>/dev/null || true
-        fi
+# =============================================================================
+# Dependency waiting (opt-in via HINDSIGHT_WAIT_FOR_DEPS=true)
+#
+# Problem: When running with LM Studio, the LLM may take time to load models.
+# If Hindsight starts before LM Studio is ready, it fails on LLM verification.
+# This wait loop ensures dependencies are ready before starting.
+# =============================================================================
+if [ "${HINDSIGHT_WAIT_FOR_DEPS:-false}" = "true" ]; then
+    LLM_BASE_URL="${HINDSIGHT_API_LLM_BASE_URL:-http://host.docker.internal:1234/v1}"
+    MAX_RETRIES="${HINDSIGHT_RETRY_MAX:-0}"  # 0 = infinite
+    RETRY_INTERVAL="${HINDSIGHT_RETRY_INTERVAL:-10}"
+
+    # Check if external database is configured (skip check for embedded pg0)
+    SKIP_DB_CHECK=false
+    if [ -z "${HINDSIGHT_API_DATABASE_URL}" ]; then
+        SKIP_DB_CHECK=true
+    else
+        DB_CHECK_HOST=$(echo "$HINDSIGHT_API_DATABASE_URL" | sed -E 's|.*@([^:/]+):([0-9]+)/.*|\1 \2|')
     fi
+
+    check_db() {
+        if $SKIP_DB_CHECK; then
+            return 0
+        fi
+        if command -v pg_isready &> /dev/null; then
+            pg_isready -h $(echo $DB_CHECK_HOST | cut -d' ' -f1) -p $(echo $DB_CHECK_HOST | cut -d' ' -f2) &>/dev/null
+        else
+            python3 -c "import socket; s=socket.socket(); s.settimeout(5); exit(0 if s.connect_ex(('$(echo $DB_CHECK_HOST | cut -d' ' -f1)', $(echo $DB_CHECK_HOST | cut -d' ' -f2))) == 0 else 1)" 2>/dev/null
+        fi
+    }
+
+    check_llm() {
+        curl -sf "${LLM_BASE_URL}/models" --connect-timeout 5 &>/dev/null
+    }
+
+    echo "⏳ Waiting for dependencies to be ready..."
+    attempt=1
+
+    while true; do
+        db_ok=false
+        llm_ok=false
+
+        if check_db; then
+            db_ok=true
+        fi
+
+        if check_llm; then
+            llm_ok=true
+        fi
+
+        if $db_ok && $llm_ok; then
+            echo "✅ Dependencies ready!"
+            break
+        fi
+
+        if [ "$MAX_RETRIES" -ne 0 ] && [ "$attempt" -ge "$MAX_RETRIES" ]; then
+            echo "❌ Max retries ($MAX_RETRIES) reached. Dependencies not available."
+            exit 1
+        fi
+
+        echo "   Attempt $attempt: DB=$( $db_ok && echo 'ok' || echo 'waiting' ), LLM=$( $llm_ok && echo 'ok' || echo 'waiting' )"
+        sleep "$RETRY_INTERVAL"
+        ((attempt++))
+    done
 fi
 
+# =============================================================================
+# Graceful shutdown handler (#675)
+#
+# Docker sends SIGTERM on `docker stop`/`docker restart`. Without a trap, child
+# processes (hindsight-api + pg0, control-plane) are killed abruptly. For the
+# embedded pg0 database this can cause data loss when the data directory is on
+# a Docker volume that gets remounted after restart.
+#
+# The trap forwards SIGTERM to all tracked child PIDs so that:
+#   - hindsight-api receives the signal and can run its shutdown hooks
+#   - pg0 gets a clean PostgreSQL shutdown (checkpoint + WAL flush)
+#   - The control-plane Node.js process exits cleanly
+# =============================================================================
+# Guard against concurrent cleanup (e.g., child crash + SIGTERM arriving together)
+SHUTTING_DOWN=false
+
+cleanup() {
+    if $SHUTTING_DOWN; then return; fi
+    SHUTTING_DOWN=true
+
+    echo ""
+    echo "🛑 Received shutdown signal, stopping services gracefully..."
+    for pid in "${PIDS[@]}"; do
+        if kill -0 "$pid" 2>/dev/null; then
+            kill -TERM "$pid" 2>/dev/null
+        fi
+    done
+    # Give processes time to shut down cleanly (pg0 needs to flush WAL).
+    # NOTE: Docker's default stop_grace_period is 10s. If you use the default,
+    # either set stop_grace_period: 30s in your compose file / docker stop -t 30,
+    # or Docker will SIGKILL the container before this timeout expires.
+    local timeout=30
+    for ((i=1; i<=timeout; i++)); do
+        local all_stopped=true
+        for pid in "${PIDS[@]}"; do
+            if kill -0 "$pid" 2>/dev/null; then
+                all_stopped=false
+                break
+            fi
+        done
+        if $all_stopped; then
+            echo "✅ All services stopped cleanly"
+            exit 0
+        fi
+        sleep 1
+    done
+    # Force kill if still running after timeout
+    echo "⚠️  Timeout reached, forcing shutdown..."
+    for pid in "${PIDS[@]}"; do
+        if kill -0 "$pid" 2>/dev/null; then
+            kill -9 "$pid" 2>/dev/null
+        fi
+    done
+    exit 1
+}
+trap cleanup SIGTERM SIGINT
+
 # Track PIDs for wait
 PIDS=()
 
 # Start API if enabled
 if [ "$ENABLE_API" = "true" ]; then
     cd /app/api
-    python -m hindsight_api.web.server 2>&1 | sed -u 's/^/[api] /' &
+    API_HEALTH_URL="${HINDSIGHT_API_HEALTH_URL:-http://localhost:8888/health}"
+    API_STARTUP_WAIT_SECONDS="${HINDSIGHT_API_STARTUP_WAIT_SECONDS:-300}"
+
+    # Run API directly - Python's PYTHONUNBUFFERED=1 handles output buffering
+    hindsight-api &
     API_PID=$!
     PIDS+=($API_PID)
 
     # Wait for API to be ready
-    echo "⏳ Waiting for API..."
-    for i in {1..60}; do
-        if curl -sf http://localhost:8888/health &>/dev/null; then
-            echo "✅ API is ready"
+    api_ready=false
+    for ((i=1; i<=API_STARTUP_WAIT_SECONDS; i++)); do
+        if ! kill -0 "$API_PID" 2>/dev/null; then
+            wait "$API_PID"
+            exit $?
+        fi
+        if curl -sf "$API_HEALTH_URL" &>/dev/null; then
+            api_ready=true
             break
         fi
         sleep 1
     done
+
+    if [ "$api_ready" != "true" ]; then
+        echo "❌ API did not become healthy within ${API_STARTUP_WAIT_SECONDS}s"
+        exit 1
+    fi
 else
-    echo "⏭️  API disabled (HINDSIGHT_ENABLE_API=false)"
+    echo "API disabled (HINDSIGHT_ENABLE_API=false)"
 fi
 
 # Start Control Plane if enabled
 if [ "$ENABLE_CP" = "true" ]; then
     echo "🎛️  Starting Control Plane..."
     cd /app/control-plane
-    PORT=9999 node server.js 2>&1 | grep -v -E "^[[:space:]]*(▲|✓|-|$)" | sed -u 's/^/[control-plane] /' &
+    export HOSTNAME="${HINDSIGHT_CP_HOSTNAME:-0.0.0.0}"
+    PORT="${HINDSIGHT_CP_PORT:-9999}" node server.js &
     CP_PID=$!
     PIDS+=($CP_PID)
 else
-    echo "⏭️  Control Plane disabled (HINDSIGHT_ENABLE_CP=false)"
+    echo "Control Plane disabled (HINDSIGHT_ENABLE_CP=false)"
 fi
 
 # Print status
@@ -60,7 +204,7 @@ echo "✅ Hindsight is running!"
 echo ""
 echo "📍 Access:"
 if [ "$ENABLE_CP" = "true" ]; then
-    echo "   Control Plane: http://localhost:9999"
+    echo "   Control Plane: http://localhost:${HINDSIGHT_CP_PORT:-9999}"
 fi
 if [ "$ENABLE_API" = "true" ]; then
     echo "   API:           http://localhost:8888"
@@ -73,8 +217,21 @@ if [ ${#PIDS[@]} -eq 0 ]; then
     exit 1
 fi
 
-# Wait for any process to exit
-wait -n
-
-# Exit with status of first exited process
-exit $?
+# Wait for any process to exit (use wait -n with trap-safe loop)
+while true; do
+    # wait -n returns when any child exits; it also returns on signal delivery
+    # (the trap handler will run and exit, so this loop is just for robustness).
+    # `&& true` prevents `set -e` from killing the script when wait -n returns
+    # non-zero (child exited with error or no backgrounded children remain).
+    wait -n && true
+    # Check if any tracked PID has exited
+    for pid in "${PIDS[@]}"; do
+        if ! kill -0 "$pid" 2>/dev/null; then
+            wait "$pid" 2>/dev/null
+            exit_code=$?
+            echo "⚠️  Service (PID $pid) exited with code $exit_code"
+            # Trigger cleanup for remaining services
+            cleanup
+        fi
+    done
+done

docker/start.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-# Start Hindsight (standalone all-in-one)
-
-cd "$(dirname "$0")"
-
-# Check for --build flag
-BUILD_FLAG=""
-if [[ "$1" == "--build" ]] || [[ "$1" == "-b" ]]; then
-  BUILD_FLAG="--build"
-  echo "🔨 Forcing rebuild of images..."
-  echo ""
-fi
-
-echo "🚀 Starting Hindsight..."
-echo ""
-
-# Load .env file from project root if it exists
-if [ -f ../.env ]; then
-  echo "📝 Loading environment variables from .env file..."
-  export $(grep -v '^#' ../.env | grep -v '^$' | xargs)
-fi
-
-# Check for required HINDSIGHT_API_LLM_API_KEY
-if [ -z "$HINDSIGHT_API_LLM_API_KEY" ]; then
-  echo "⚠️  Warning: HINDSIGHT_API_LLM_API_KEY is not set"
-  echo ""
-  echo "Set it by either:"
-  echo "  1. Creating a .env file in the project root with: HINDSIGHT_API_LLM_API_KEY=your-key"
-  echo "  2. Exporting: export HINDSIGHT_API_LLM_API_KEY=your-key"
-  echo ""
-  read -p "Continue anyway? (y/N) " -n 1 -r
-  echo
-  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
-    exit 1
-  fi
-fi
-
-cd standalone
-
-# Run docker-compose with optional --build flag
-docker-compose up $BUILD_FLAG

docker/test-image.sh

@@ -0,0 +1,235 @@
+#!/bin/bash
+#
+# Docker Smoke Test Script
+#
+# Tests that a Hindsight Docker image starts correctly and becomes healthy.
+# Can be run locally or in CI pipelines.
+#
+# Usage:
+#   ./docker/test-image.sh <image> [target]
+#
+# Arguments:
+#   image   - Docker image to test (e.g., hindsight-api:test, ghcr.io/vectorize-io/hindsight:latest)
+#   target  - Optional: 'cp-only' for control plane, otherwise assumes API image (default: api)
+#
+# Environment variables:
+#   HINDSIGHT_API_LLM_API_KEY                   - Required for API/standalone images (LLM verification)
+#   HINDSIGHT_API_LLM_PROVIDER                  - LLM provider (default: openai)
+#   HINDSIGHT_API_LLM_MODEL                     - LLM model (default: gpt-4o-mini)
+#   HINDSIGHT_API_EMBEDDINGS_PROVIDER           - Embeddings provider (optional, for slim images: openai, cohere, tei)
+#   HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY     - OpenAI API key for embeddings (optional)
+#   HINDSIGHT_API_RERANKER_PROVIDER             - Reranker provider (optional, for slim images: cohere, tei)
+#   HINDSIGHT_API_COHERE_API_KEY                - Cohere API key for reranking (optional)
+#   SMOKE_TEST_TIMEOUT                          - Timeout in seconds (default: 120)
+#   SMOKE_TEST_CONTAINER_NAME                   - Container name (default: hindsight-smoke-test)
+#
+# Examples:
+#   # Test a locally built full image
+#   ./docker/test-image.sh hindsight-api:test
+#
+#   # Test a released image
+#   ./docker/test-image.sh ghcr.io/vectorize-io/hindsight:latest
+#
+#   # Test control plane image
+#   ./docker/test-image.sh hindsight-control-plane:test cp-only
+#
+#   # Test slim image with external providers
+#   export HINDSIGHT_API_LLM_API_KEY=sk_xxx
+#   export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai
+#   export HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=sk-xxx
+#   export HINDSIGHT_API_RERANKER_PROVIDER=cohere
+#   export HINDSIGHT_API_COHERE_API_KEY=xxx
+#   ./docker/test-image.sh hindsight-slim:test
+#
+# Exit codes:
+#   0 - Success (container healthy)
+#   1 - Failure (container not healthy within timeout)
+#   2 - Invalid arguments
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(dirname "$SCRIPT_DIR")"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
+NC='\033[0m' # No Color
+
+# Configuration
+IMAGE="${1:-}"
+TARGET="${2:-api}"
+TIMEOUT="${SMOKE_TEST_TIMEOUT:-120}"
+CONTAINER_NAME="${SMOKE_TEST_CONTAINER_NAME:-hindsight-smoke-test}"
+LLM_PROVIDER="${HINDSIGHT_API_LLM_PROVIDER:-openai}"
+LLM_MODEL="${HINDSIGHT_API_LLM_MODEL:-gpt-4o-mini}"
+
+# Validate arguments
+if [ -z "$IMAGE" ]; then
+    echo -e "${RED}Error: Image argument is required${NC}"
+    echo ""
+    echo "Usage: $0 <image> [target]"
+    echo ""
+    echo "Examples:"
+    echo "  $0 hindsight-api:test"
+    echo "  $0 ghcr.io/vectorize-io/hindsight:latest"
+    echo "  $0 hindsight-control-plane:test cp-only"
+    exit 2
+fi
+
+# Determine health endpoint based on target
+if [ "$TARGET" = "cp-only" ]; then
+    HEALTH_PORT=9999
+    HEALTH_PATH="/api/health"
+    NEEDS_LLM=false
+else
+    HEALTH_PORT=8888
+    HEALTH_PATH="/health"
+    NEEDS_LLM=true
+fi
+
+# Check for required environment variables
+if [ "$NEEDS_LLM" = true ] && [ "$LLM_PROVIDER" != "vertexai" ] && [ -z "${HINDSIGHT_API_LLM_API_KEY:-}" ]; then
+    echo -e "${RED}Error: HINDSIGHT_API_LLM_API_KEY environment variable is required for API/standalone images${NC}"
+    echo "Set it with: export HINDSIGHT_API_LLM_API_KEY=your-api-key"
+    exit 2
+fi
+
+# Cleanup function
+cleanup() {
+    echo "Cleaning up..."
+    docker stop "$CONTAINER_NAME" 2>/dev/null || true
+    docker rm "$CONTAINER_NAME" 2>/dev/null || true
+}
+
+# Set trap to cleanup on exit
+trap cleanup EXIT
+
+echo -e "${YELLOW}Starting smoke test for: ${IMAGE}${NC}"
+echo "  Target: $TARGET"
+echo "  Health endpoint: http://localhost:${HEALTH_PORT}${HEALTH_PATH}"
+echo "  Timeout: ${TIMEOUT}s"
+echo ""
+
+# Remove any existing container with the same name
+docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
+
+# Start container based on target type
+echo "Starting container..."
+if [ "$TARGET" = "cp-only" ]; then
+    docker run -d --name "$CONTAINER_NAME" \
+        -p "${HEALTH_PORT}:${HEALTH_PORT}" \
+        "$IMAGE"
+else
+    # Build docker run command with required and optional env vars
+    DOCKER_CMD="docker run -d --name $CONTAINER_NAME"
+    DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_PROVIDER=$LLM_PROVIDER"
+    if [ -n "${HINDSIGHT_API_LLM_API_KEY:-}" ]; then
+        DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_API_KEY=${HINDSIGHT_API_LLM_API_KEY}"
+    fi
+    DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_MODEL=$LLM_MODEL"
+
+    # Add Vertex AI config if provider is vertexai
+    if [ "$LLM_PROVIDER" = "vertexai" ]; then
+        if [ -n "${HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY:-}" ]; then
+            DOCKER_CMD="$DOCKER_CMD -v ${HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY}:/tmp/gcp-credentials.json:ro"
+            DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY=/tmp/gcp-credentials.json"
+        fi
+        if [ -n "${HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID:-}" ]; then
+            DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=${HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID}"
+        fi
+        if [ -n "${HINDSIGHT_API_LLM_VERTEXAI_REGION:-}" ]; then
+            DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_VERTEXAI_REGION=${HINDSIGHT_API_LLM_VERTEXAI_REGION}"
+        fi
+    fi
+
+    # Add optional embeddings provider config
+    if [ -n "${HINDSIGHT_API_EMBEDDINGS_PROVIDER:-}" ]; then
+        DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_EMBEDDINGS_PROVIDER=${HINDSIGHT_API_EMBEDDINGS_PROVIDER}"
+    fi
+    if [ -n "${HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY:-}" ]; then
+        DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=${HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY}"
+    fi
+
+    # Add optional reranker provider config
+    if [ -n "${HINDSIGHT_API_RERANKER_PROVIDER:-}" ]; then
+        DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_RERANKER_PROVIDER=${HINDSIGHT_API_RERANKER_PROVIDER}"
+    fi
+    if [ -n "${HINDSIGHT_API_COHERE_API_KEY:-}" ]; then
+        DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_COHERE_API_KEY=${HINDSIGHT_API_COHERE_API_KEY}"
+    fi
+
+    DOCKER_CMD="$DOCKER_CMD -p ${HEALTH_PORT}:${HEALTH_PORT}"
+    DOCKER_CMD="$DOCKER_CMD $IMAGE"
+
+    eval $DOCKER_CMD
+fi
+
+# Wait for health endpoint
+echo "Waiting for health endpoint at http://localhost:${HEALTH_PORT}${HEALTH_PATH}..."
+start_time=$(date +%s)
+
+for i in $(seq 1 "$TIMEOUT"); do
+    if curl -sf "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" > /dev/null 2>&1; then
+        end_time=$(date +%s)
+        duration=$((end_time - start_time))
+        echo ""
+        echo -e "${GREEN}Container is healthy after ${duration}s${NC}"
+        echo ""
+        echo "=== Health Response ==="
+        curl -s "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" | python3 -m json.tool 2>/dev/null || curl -s "http://localhost:${HEALTH_PORT}${HEALTH_PATH}"
+        echo ""
+
+        # Run retain/recall smoke test for API targets
+        if [ "$TARGET" != "cp-only" ]; then
+            echo ""
+            echo "=== Retain/Recall Smoke Test ==="
+            if ! "$REPO_ROOT/scripts/smoke-test-slim.sh" "http://localhost:${HEALTH_PORT}"; then
+                echo ""
+                echo "=== Container Logs (last 50 lines) ==="
+                docker logs "$CONTAINER_NAME" 2>&1 | tail -50
+                echo ""
+                echo -e "${RED}Smoke test FAILED${NC}"
+                exit 1
+            fi
+        fi
+
+        echo ""
+        echo "=== Container Logs (last 50 lines) ==="
+        docker logs "$CONTAINER_NAME" 2>&1 | tail -50
+        echo ""
+        echo -e "${GREEN}Smoke test PASSED${NC}"
+        exit 0
+    fi
+
+    # Show progress every 10 seconds
+    if [ $((i % 10)) -eq 0 ]; then
+        echo "  Still waiting... (${i}s)"
+    fi
+
+    # Check if container is still running
+    if ! docker ps -q -f "name=$CONTAINER_NAME" | grep -q .; then
+        echo ""
+        echo -e "${RED}Container exited unexpectedly!${NC}"
+        echo ""
+        echo "=== Container Logs ==="
+        docker logs "$CONTAINER_NAME" 2>&1
+        echo ""
+        echo -e "${RED}Smoke test FAILED${NC}"
+        exit 1
+    fi
+
+    sleep 1
+done
+
+# Timeout reached
+echo ""
+echo -e "${RED}Container failed to become healthy after ${TIMEOUT}s${NC}"
+echo ""
+echo "=== Container Logs ==="
+docker logs "$CONTAINER_NAME" 2>&1
+echo ""
+echo -e "${RED}Smoke test FAILED${NC}"
+exit 1

docker/test-slim-local.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Local Test Script for Slim Docker Images
+#
+# This script makes it easy to test slim images locally with external providers.
+# It expects API keys to be set in environment variables.
+#
+# Usage:
+#   export OPENAI_API_KEY=sk-xxx
+#   export COHERE_API_KEY=xxx
+#   ./docker/test-slim-local.sh
+#
+# Or inline:
+#   OPENAI_API_KEY=sk_xxx COHERE_API_KEY=xxx ./docker/test-slim-local.sh
+#
+
+set -euo pipefail
+
+# Check for required API keys
+if [ -z "${OPENAI_API_KEY:-}" ]; then
+    echo "❌ Error: OPENAI_API_KEY environment variable is required"
+    echo "Set it with: export OPENAI_API_KEY=sk-xxx"
+    exit 1
+fi
+
+if [ -z "${COHERE_API_KEY:-}" ]; then
+    echo "❌ Error: COHERE_API_KEY environment variable is required"
+    echo "Set it with: export COHERE_API_KEY=xxx"
+    exit 1
+fi
+
+# Configuration
+IMAGE="${1:-hindsight-slim:test}"
+echo "Testing image: $IMAGE"
+echo ""
+
+# Set up LLM and external providers
+export HINDSIGHT_API_LLM_PROVIDER=openai
+export HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY
+export HINDSIGHT_API_LLM_MODEL=gpt-4o-mini
+export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai
+export HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=$OPENAI_API_KEY
+export HINDSIGHT_API_RERANKER_PROVIDER=cohere
+export HINDSIGHT_API_COHERE_API_KEY=$COHERE_API_KEY
+
+# Run the test
+exec "$(dirname "$0")/test-image.sh" "$IMAGE" standalone

helm/INSTALL.txt

@@ -1,135 +0,0 @@
-HINDSIGHT HELM CHART INSTALLATION GUIDE
-=====================================
-
-PREREQUISITES
--------------
-- Kubernetes cluster (1.19+)
-- kubectl configured
-- Helm 3.x installed
-- PostgreSQL database with pgvector extension (if not using bundled PostgreSQL)
-
-BASIC INSTALLATION
-------------------
-
-1. Install with default values (requires external PostgreSQL):
-
-   helm install hindsight ./hindsight \
-     --set postgresql.external.host=your-postgres-host \
-     --set postgresql.external.password=your-password \
-     --set api.secrets.MEMORY_LLM_API_KEY=your-api-key
-
-2. Install with custom values file:
-
-   helm install hindsight ./hindsight -f hindsight/values-production.yaml
-
-3. Install in a specific namespace:
-
-   kubectl create namespace hindsight
-   helm install hindsight ./hindsight -n hindsight
-
-CONFIGURATION OPTIONS
----------------------
-
-Development setup (using values-development.yaml):
-   helm install hindsight ./hindsight -f hindsight/values-development.yaml
-
-Production setup (using values-production.yaml):
-   helm install hindsight ./hindsight -f hindsight/values-production.yaml
-
-Custom LLM provider:
-   helm install hindsight ./hindsight \
-     --set api.env.MEMORY_LLM_PROVIDER=openai \
-     --set api.env.MEMORY_LLM_MODEL=gpt-4 \
-     --set api.secrets.MEMORY_LLM_API_KEY=sk-your-key
-
-Enable ingress:
-   helm install hindsight ./hindsight \
-     --set ingress.enabled=true \
-     --set ingress.hosts[0].host=hindsight.example.com
-
-Enable autoscaling:
-   helm install hindsight ./hindsight \
-     --set autoscaling.enabled=true \
-     --set autoscaling.minReplicas=2 \
-     --set autoscaling.maxReplicas=10
-
-UPGRADE
--------
-
-Upgrade existing installation:
-   helm upgrade hindsight ./hindsight
-
-Upgrade with new values:
-   helm upgrade hindsight ./hindsight -f hindsight/values-production.yaml
-
-UNINSTALL
----------
-
-Remove the Helm release:
-   helm uninstall hindsight
-
-Remove with namespace:
-   helm uninstall hindsight -n hindsight
-
-TESTING
--------
-
-Test the installation with dry-run:
-   helm install hindsight ./hindsight --dry-run --debug
-
-Validate templates:
-   helm template hindsight ./hindsight
-
-Lint the chart:
-   helm lint ./hindsight
-
-ACCESSING THE SERVICES
-----------------------
-
-Port-forward control plane:
-   kubectl port-forward svc/hindsight-control-plane 3000:3000
-
-Port-forward API:
-   kubectl port-forward svc/hindsight-api 8888:8888
-
-Get service URLs:
-   helm status hindsight
-
-DATABASE INITIALIZATION
------------------------
-
-NOTE: Database migrations now run automatically when the API service starts.
-      You typically don't need to run migrations manually.
-
-If you want to pre-initialize the database before deploying (optional):
-   kubectl run hindsight-init --rm -it --restart=Never \
-     --image=hindsight/api:latest \
-     --env="DATABASE_URL=postgresql://user:pass@host:5432/hindsight" \
-     -- python -c "from hindsight.migrations import run_migrations; run_migrations()"
-
-TROUBLESHOOTING
----------------
-
-Check pod status:
-   kubectl get pods -l app.kubernetes.io/name=hindsight
-
-View logs for API:
-   kubectl logs -l app.kubernetes.io/component=api
-
-View logs for control plane:
-   kubectl logs -l app.kubernetes.io/component=control-plane
-
-Describe a pod:
-   kubectl describe pod <pod-name>
-
-Check configuration:
-   kubectl get configmap hindsight-config -o yaml
-   kubectl get secret hindsight-secret -o yaml
-
-NOTES
------
-- Make sure PostgreSQL has pgvector extension enabled
-- Run database migrations before first use
-- Configure proper resource limits for production
-- Use external secrets management for production
-- Enable TLS/SSL for production deployments

helm/hindsight/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: postgresql
+  repository: https://charts.bitnami.com/bitnami
+  version: 15.5.38
+digest: sha256:f67c7612736803ece8a669f8ca6b0555f3b78557bc0ecb732aa2e43f0df7750d
+generated: "2025-12-10T17:20:57.058794+01:00"

helm/hindsight/Chart.yaml

@@ -1,9 +1,9 @@
 apiVersion: v2
 name: hindsight
-description: A Helm chart for Hindsight - temporal-semantic-entity memory system for AI agents
+description: Hindsight helm chart
 type: application
-version: 0.0.15
-appVersion: "0.0.15"
+version: 0.4.22
+appVersion: "0.4.22"
 keywords:
   - ai
   - memory

helm/hindsight/README.md

@@ -0,0 +1,182 @@
+# Hindsight Helm Chart
+
+Helm chart for deploying Hindsight - a temporal-semantic-entity memory system for AI agents.
+
+## Prerequisites
+
+- Kubernetes 1.19+
+- Helm 3.0+
+- PostgreSQL database (external or bundled)
+
+## Quick Start
+
+```bash
+# Update dependencies first
+helm dependency update ./helm/hindsight
+
+# Install (PostgreSQL included by default)
+export OPENAI_API_KEY="sk-your-openai-key"
+helm upgrade hindsight --install ./helm/hindsight -n hindsight --create-namespace \
+  --set api.secrets.HINDSIGHT_API_LLM_API_KEY="$OPENAI_API_KEY"
+```
+
+To use an external database instead:
+
+```bash
+helm install hindsight ./helm/hindsight -n hindsight --create-namespace \
+  --set api.secrets.HINDSIGHT_API_LLM_API_KEY="sk-your-openai-key" \
+  --set postgresql.enabled=false \
+  --set postgresql.external.host=my-postgres.example.com \
+  --set postgresql.external.password=mypassword
+```
+
+## Installation
+
+### Add the repository (if published)
+
+```bash
+helm repo add hindsight https://your-helm-repo.com
+helm repo update
+```
+
+### Install with custom values file
+
+Create a `values-override.yaml`:
+
+```yaml
+api:
+  secrets:
+    HINDSIGHT_API_LLM_API_KEY: "sk-your-openai-key"
+
+postgresql:
+  external:
+    host: "my-postgres.example.com"
+    password: "mypassword"
+```
+
+Then install:
+
+```bash
+helm install hindsight ./helm/hindsight -n hindsight --create-namespace -f values-override.yaml
+```
+
+## Configuration
+
+### Key Values
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `version` | Default image tag for all components | `0.1.0` |
+| `api.enabled` | Enable the API component | `true` |
+| `api.image.repository` | API image repository | `hindsight/api` |
+| `api.image.tag` | API image tag (defaults to `version`) | - |
+| `api.service.port` | API service port | `8888` |
+| `controlPlane.enabled` | Enable the control plane | `true` |
+| `controlPlane.image.repository` | Control plane image repository | `hindsight/control-plane` |
+| `controlPlane.image.tag` | Control plane image tag (defaults to `version`) | - |
+| `controlPlane.service.port` | Control plane service port | `3000` |
+| `postgresql.enabled` | Deploy PostgreSQL as subchart | `true` |
+| `postgresql.external.host` | External PostgreSQL host | `postgresql` |
+| `postgresql.external.port` | External PostgreSQL port | `5432` |
+| `postgresql.external.database` | Database name | `hindsight` |
+| `postgresql.external.username` | Database username | `hindsight` |
+| `ingress.enabled` | Enable ingress | `false` |
+| `autoscaling.enabled` | Enable HPA | `false` |
+
+### Environment Variables
+
+All environment variables in `api.env` and `controlPlane.env` are automatically added to the respective pods. Sensitive values should go in `api.secrets` or `controlPlane.secrets`.
+
+```yaml
+api:
+  env:
+    HINDSIGHT_API_LLM_PROVIDER: "openai"
+    HINDSIGHT_API_LLM_MODEL: "gpt-4"
+  secrets:
+    HINDSIGHT_API_LLM_API_KEY: "your-api-key"
+    HINDSIGHT_API_LLM_BASE_URL: "https://api.openai.com/v1"
+
+controlPlane:
+  env:
+    NODE_ENV: "production"
+  secrets: {}
+```
+
+### External Database
+
+To connect to an external PostgreSQL database:
+
+```yaml
+postgresql:
+  enabled: false
+  external:
+    host: "my-postgres.example.com"
+    port: 5432
+    database: "hindsight"
+    username: "hindsight"
+    password: "your-password"
+```
+
+### Ingress
+
+To expose the services via ingress:
+
+```yaml
+ingress:
+  enabled: true
+  className: "nginx"
+  annotations:
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+  hosts:
+    - host: hindsight.example.com
+      paths:
+        - path: /
+          pathType: Prefix
+          service: controlPlane
+        - path: /api
+          pathType: Prefix
+          service: api
+  tls:
+    - secretName: hindsight-tls
+      hosts:
+        - hindsight.example.com
+```
+
+## Upgrading
+
+```bash
+helm upgrade hindsight ./helm/hindsight -n hindsight
+```
+
+## Uninstalling
+
+```bash
+helm uninstall hindsight -n hindsight
+```
+
+## Components
+
+The chart deploys:
+
+- **API**: The main Hindsight API server for memory operations
+- **Control Plane**: Web UI for managing agents and viewing memories
+
+## Development
+
+### Lint the chart
+
+```bash
+helm lint ./helm/hindsight
+```
+
+### Template locally
+
+```bash
+helm template hindsight ./helm/hindsight --debug
+```
+
+### Dry run installation
+
+```bash
+helm install hindsight ./helm/hindsight --dry-run --debug
+```

helm/hindsight/templates/NOTES.txt

@@ -1,71 +1,2 @@
-Thank you for installing {{ .Chart.Name }}!
-
-Your release is named {{ .Release.Name }}.
-
-To learn more about the release, try:
-
-  $ helm status {{ .Release.Name }}
-  $ helm get all {{ .Release.Name }}
-
-{{- if .Values.ingress.enabled }}
-
-The application is accessible via the following URL(s):
-{{- range .Values.ingress.hosts }}
-  - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .host }}
-{{- end }}
-
-{{- else }}
-
-1. Get the Control Plane URL by running these commands:
-{{- if contains "NodePort" .Values.controlPlane.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "hindsight.fullname" . }}-control-plane)
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo "Control Plane URL: http://$NODE_IP:$NODE_PORT"
-{{- else if contains "LoadBalancer" .Values.controlPlane.service.type }}
-  NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-  You can watch the status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "hindsight.fullname" . }}-control-plane'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "hindsight.fullname" . }}-control-plane --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo "Control Plane URL: http://$SERVICE_IP:{{ .Values.controlPlane.service.port }}"
-{{- else if contains "ClusterIP" .Values.controlPlane.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/component=control-plane,app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
-  echo "Control Plane URL: http://127.0.0.1:3000"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 3000:$CONTAINER_PORT
-{{- end }}
-
-2. Get the API URL by running these commands:
-{{- if contains "NodePort" .Values.api.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "hindsight.fullname" . }}-api)
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo "API URL: http://$NODE_IP:$NODE_PORT"
-{{- else if contains "LoadBalancer" .Values.api.service.type }}
-  NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-  You can watch the status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "hindsight.fullname" . }}-api'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "hindsight.fullname" . }}-api --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo "API URL: http://$SERVICE_IP:{{ .Values.api.service.port }}"
-{{- else if contains "ClusterIP" .Values.api.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/component=api,app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
-  echo "API URL: http://127.0.0.1:8888"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8888:$CONTAINER_PORT
-{{- end }}
-
-{{- end }}
-
-{{- if not .Values.postgresql.enabled }}
-
-NOTE: You are using an external PostgreSQL database.
-Please ensure that:
-  1. The database is accessible from the cluster
-  2. The pgvector extension is enabled
-
-Database migrations run automatically when the API service starts.
-
-If you want to pre-initialize the database before deploying (optional):
-  kubectl run --namespace {{ .Release.Namespace }} hindsight-init --rm -it --restart=Never \
-    --image={{ .Values.api.image.repository }}:{{ .Values.api.image.tag }} \
-    --env="DATABASE_URL={{ include "hindsight.databaseUrl" . }}" \
-    -- python -c "from hindsight.migrations import run_migrations; run_migrations()"
-{{- end }}
-
-For more information, visit: https://github.com/yourusername/hindsight
+Hindsight installed. Access the control plane:
+  kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "hindsight.fullname" . }}-control-plane 3000:3000

helm/hindsight/templates/_helpers.tpl

@@ -80,6 +80,22 @@ Control plane selector labels
 app.kubernetes.io/component: control-plane
 {{- end }}
 
+{{/*
+Worker labels
+*/}}
+{{- define "hindsight.worker.labels" -}}
+{{ include "hindsight.labels" . }}
+app.kubernetes.io/component: worker
+{{- end }}
+
+{{/*
+Worker selector labels
+*/}}
+{{- define "hindsight.worker.selectorLabels" -}}
+{{ include "hindsight.selectorLabels" . }}
+app.kubernetes.io/component: worker
+{{- end }}
+
 {{/*
 Create the name of the service account to use
 */}}
@@ -98,7 +114,7 @@ Generate database URL
 {{- if .Values.databaseUrl }}
 {{- .Values.databaseUrl }}
 {{- else if .Values.postgresql.enabled }}
-{{- printf "postgresql://%s:%s@%s-postgresql:%d/%s" .Values.postgresql.auth.username .Values.postgresql.auth.password (include "hindsight.fullname" .) (.Values.postgresql.primary.service.port | int) .Values.postgresql.auth.database }}
+{{- printf "postgresql://%s:%s@%s-postgresql:%d/%s" .Values.postgresql.auth.username .Values.postgresql.auth.password (include "hindsight.fullname" .) (.Values.postgresql.service.port | int) .Values.postgresql.auth.database }}
 {{- else }}
 {{- printf "postgresql://%s:$(POSTGRES_PASSWORD)@%s:%d/%s" .Values.postgresql.external.username .Values.postgresql.external.host (.Values.postgresql.external.port | int) .Values.postgresql.external.database }}
 {{- end }}
@@ -110,3 +126,46 @@ API URL for control plane
 {{- define "hindsight.apiUrl" -}}
 {{- printf "http://%s-api:%d" (include "hindsight.fullname" .) (.Values.api.service.port | int) }}
 {{- end }}
+
+{{/*
+TEI reranker labels
+*/}}
+{{- define "hindsight.tei.reranker.labels" -}}
+{{ include "hindsight.labels" . }}
+app.kubernetes.io/component: tei-reranker
+{{- end }}
+
+{{/*
+TEI reranker selector labels
+*/}}
+{{- define "hindsight.tei.reranker.selectorLabels" -}}
+{{ include "hindsight.selectorLabels" . }}
+app.kubernetes.io/component: tei-reranker
+{{- end }}
+
+{{/*
+TEI embedding labels
+*/}}
+{{- define "hindsight.tei.embedding.labels" -}}
+{{ include "hindsight.labels" . }}
+app.kubernetes.io/component: tei-embedding
+{{- end }}
+
+{{/*
+TEI embedding selector labels
+*/}}
+{{- define "hindsight.tei.embedding.selectorLabels" -}}
+{{ include "hindsight.selectorLabels" . }}
+app.kubernetes.io/component: tei-embedding
+{{- end }}
+
+{{/*
+Get the name of the secret to use
+*/}}
+{{- define "hindsight.secretName" -}}
+{{- if .Values.existingSecret }}
+{{- .Values.existingSecret }}
+{{- else }}
+{{- printf "%s-secret" (include "hindsight.fullname" .) }}
+{{- end }}
+{{- end }}

helm/hindsight/templates/api-deployment.yaml

@@ -15,8 +15,9 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- if not .Values.existingSecret }}
         checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+        {{- end }}
         {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -32,45 +33,61 @@ spec:
       - name: api
         securityContext:
           {{- toYaml .Values.securityContext | nindent 10 }}
-        image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag }}"
+        image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag | default .Values.version | default .Chart.AppVersion }}"
         imagePullPolicy: {{ .Values.api.image.pullPolicy }}
         ports:
         - name: http
           containerPort: {{ .Values.api.service.targetPort }}
           protocol: TCP
+        {{- if .Values.existingSecret }}
+        envFrom:
+        - secretRef:
+            name: {{ .Values.existingSecret }}
+        {{- end }}
         env:
-        - name: HINDSIGHT_API_DATABASE_URL
-          value: {{ include "hindsight.databaseUrl" . | quote }}
+        {{- /* POSTGRES_PASSWORD must be defined before DATABASE_URL for $(VAR) interpolation */}}
         {{- if not .Values.postgresql.enabled }}
         - name: POSTGRES_PASSWORD
           valueFrom:
             secretKeyRef:
-              name: {{ include "hindsight.fullname" . }}-secret
+              name: {{ include "hindsight.secretName" . }}
               key: postgres-password
         {{- end }}
-        - name: HINDSIGHT_API_LLM_PROVIDER
-          valueFrom:
-            configMapKeyRef:
-              name: {{ include "hindsight.fullname" . }}-config
-              key: llm-provider
-        - name: HINDSIGHT_API_LLM_MODEL
-          valueFrom:
-            configMapKeyRef:
-              name: {{ include "hindsight.fullname" . }}-config
-              key: llm-model
-        {{- if and .Values.api.secrets (hasKey .Values.api.secrets "HINDSIGHT_API_LLM_API_KEY") }}
-        - name: HINDSIGHT_API_LLM_API_KEY
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "hindsight.fullname" . }}-secret
-              key: llm-api-key
+        - name: HINDSIGHT_API_DATABASE_URL
+          value: {{ include "hindsight.databaseUrl" . | quote }}
+        {{- /* Disable internal worker when dedicated workers are enabled */}}
+        {{- if .Values.worker.enabled }}
+        - name: HINDSIGHT_API_WORKER_ENABLED
+          value: "false"
         {{- end }}
-        {{- if and .Values.api.secrets (hasKey .Values.api.secrets "HINDSIGHT_API_LLM_BASE_URL") }}
-        - name: HINDSIGHT_API_LLM_BASE_URL
+        {{- /* Explicitly set port to override K8s service discovery env var (HINDSIGHT_API_PORT) */}}
+        - name: HINDSIGHT_API_PORT
+          value: {{ .Values.api.service.targetPort | quote }}
+        {{- range $key, $value := .Values.api.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- if .Values.tei.reranker.enabled }}
+        - name: HINDSIGHT_API_RERANKER_PROVIDER
+          value: "tei"
+        - name: HINDSIGHT_API_RERANKER_TEI_URL
+          value: "http://{{ include "hindsight.fullname" . }}-tei-reranker:{{ .Values.tei.reranker.port }}"
+        {{- end }}
+        {{- if .Values.tei.embedding.enabled }}
+        - name: HINDSIGHT_API_EMBEDDINGS_PROVIDER
+          value: "tei"
+        - name: HINDSIGHT_API_EMBEDDINGS_TEI_URL
+          value: "http://{{ include "hindsight.fullname" . }}-tei-embedding:{{ .Values.tei.embedding.port }}"
+        {{- end }}
+        {{- /* Only use api.secrets when not using existingSecret (for chart-managed secrets) */}}
+        {{- if not .Values.existingSecret }}
+        {{- range $key, $value := .Values.api.secrets }}
+        - name: {{ $key }}
           valueFrom:
             secretKeyRef:
-              name: {{ include "hindsight.fullname" . }}-secret
-              key: llm-base-url
+              name: {{ include "hindsight.secretName" $ }}
+              key: {{ $key }}
+        {{- end }}
         {{- end }}
         livenessProbe:
           {{- toYaml .Values.api.livenessProbe | nindent 10 }}
@@ -78,11 +95,32 @@ spec:
           {{- toYaml .Values.api.readinessProbe | nindent 10 }}
         resources:
           {{- toYaml .Values.api.resources | nindent 10 }}
+        {{- if or .Values.api.persistence.modelCache.enabled .Values.api.extraVolumeMounts }}
+        volumeMounts:
+        {{- if .Values.api.persistence.modelCache.enabled }}
+        - name: model-cache
+          mountPath: /home/hindsight/.cache
+        {{- end }}
+        {{- with .Values.api.extraVolumeMounts }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- end }}
+      {{- if or .Values.api.persistence.modelCache.enabled .Values.api.extraVolumes }}
+      volumes:
+      {{- if .Values.api.persistence.modelCache.enabled }}
+      - name: model-cache
+        persistentVolumeClaim:
+          claimName: {{ include "hindsight.fullname" . }}-api-model-cache
+      {{- end }}
+      {{- with .Values.api.extraVolumes }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
+      {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.affinity }}
+      {{- with (.Values.api.affinity | default .Values.affinity) }}
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}

helm/hindsight/templates/api-model-cache-pvc.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.api.enabled .Values.api.persistence.modelCache.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "hindsight.fullname" . }}-api-model-cache
+  labels:
+    {{- include "hindsight.api.labels" . | nindent 4 }}
+  {{- with .Values.api.persistence.modelCache.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  accessModes:
+    {{- toYaml .Values.api.persistence.modelCache.accessModes | nindent 4 }}
+  {{- if .Values.api.persistence.modelCache.storageClass }}
+  storageClassName: {{ .Values.api.persistence.modelCache.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.api.persistence.modelCache.size }}
+{{- end }}

helm/hindsight/templates/configmap.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "hindsight.fullname" . }}-config
-  labels:
-    {{- include "hindsight.labels" . | nindent 4 }}
-data:
-  # API configuration
-  llm-provider: {{ .Values.api.env.HINDSIGHT_API_LLM_PROVIDER | quote }}
-  llm-model: {{ .Values.api.env.HINDSIGHT_API_LLM_MODEL | quote }}
-
-  # Control plane configuration
-  node-env: {{ .Values.controlPlane.env.NODE_ENV | quote }}
-  hostname: {{ .Values.controlPlane.env.HINDSIGHT_CP_HOSTNAME | quote }}
-  control-plane-port: {{ .Values.controlPlane.env.HINDSIGHT_CP_PORT | quote }}

helm/hindsight/templates/controlplane-deployment.yaml

@@ -15,7 +15,9 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- if not .Values.existingSecret }}
+        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+        {{- end }}
         {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -31,30 +33,34 @@ spec:
       - name: control-plane
         securityContext:
           {{- toYaml .Values.securityContext | nindent 10 }}
-        image: "{{ .Values.controlPlane.image.repository }}:{{ .Values.controlPlane.image.tag }}"
+        image: "{{ .Values.controlPlane.image.repository }}:{{ .Values.controlPlane.image.tag | default .Values.version | default .Chart.AppVersion }}"
         imagePullPolicy: {{ .Values.controlPlane.image.pullPolicy }}
         ports:
         - name: http
           containerPort: {{ .Values.controlPlane.service.targetPort }}
           protocol: TCP
+        {{- if .Values.existingSecret }}
+        envFrom:
+        - secretRef:
+            name: {{ .Values.existingSecret }}
+        {{- end }}
         env:
-        - name: NODE_ENV
-          valueFrom:
-            configMapKeyRef:
-              name: {{ include "hindsight.fullname" . }}-config
-              key: node-env
-        - name: HINDSIGHT_CP_HOSTNAME
-          valueFrom:
-            configMapKeyRef:
-              name: {{ include "hindsight.fullname" . }}-config
-              key: hostname
-        - name: HINDSIGHT_CP_PORT
-          valueFrom:
-            configMapKeyRef:
-              name: {{ include "hindsight.fullname" . }}-config
-              key: control-plane-port
         - name: HINDSIGHT_CP_DATAPLANE_API_URL
           value: {{ include "hindsight.apiUrl" . | quote }}
+        {{- range $key, $value := .Values.controlPlane.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- /* Only use controlPlane.secrets when not using existingSecret (for chart-managed secrets) */}}
+        {{- if not .Values.existingSecret }}
+        {{- range $key, $value := .Values.controlPlane.secrets }}
+        - name: {{ $key }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "hindsight.secretName" $ }}
+              key: {{ $key }}
+        {{- end }}
+        {{- end }}
         livenessProbe:
           {{- toYaml .Values.controlPlane.livenessProbe | nindent 10 }}
         readinessProbe:
@@ -65,7 +71,7 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.affinity }}
+      {{- with (.Values.controlPlane.affinity | default .Values.affinity) }}
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}

helm/hindsight/templates/pdb.yaml

@@ -0,0 +1,56 @@
+{{- if and .Values.api.enabled .Values.api.podDisruptionBudget.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "hindsight.fullname" . }}-api
+  labels:
+    {{- include "hindsight.api.labels" . | nindent 4 }}
+spec:
+  {{- if .Values.api.podDisruptionBudget.minAvailable }}
+  minAvailable: {{ .Values.api.podDisruptionBudget.minAvailable }}
+  {{- end }}
+  {{- if .Values.api.podDisruptionBudget.maxUnavailable }}
+  maxUnavailable: {{ .Values.api.podDisruptionBudget.maxUnavailable }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "hindsight.api.selectorLabels" . | nindent 6 }}
+{{- end }}
+---
+{{- if and .Values.controlPlane.enabled .Values.controlPlane.podDisruptionBudget.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "hindsight.fullname" . }}-control-plane
+  labels:
+    {{- include "hindsight.controlPlane.labels" . | nindent 4 }}
+spec:
+  {{- if .Values.controlPlane.podDisruptionBudget.minAvailable }}
+  minAvailable: {{ .Values.controlPlane.podDisruptionBudget.minAvailable }}
+  {{- end }}
+  {{- if .Values.controlPlane.podDisruptionBudget.maxUnavailable }}
+  maxUnavailable: {{ .Values.controlPlane.podDisruptionBudget.maxUnavailable }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "hindsight.controlPlane.selectorLabels" . | nindent 6 }}
+{{- end }}
+---
+{{- if and .Values.worker.enabled .Values.worker.podDisruptionBudget.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "hindsight.fullname" . }}-worker
+  labels:
+    {{- include "hindsight.worker.labels" . | nindent 4 }}
+spec:
+  {{- if .Values.worker.podDisruptionBudget.minAvailable }}
+  minAvailable: {{ .Values.worker.podDisruptionBudget.minAvailable }}
+  {{- end }}
+  {{- if .Values.worker.podDisruptionBudget.maxUnavailable }}
+  maxUnavailable: {{ .Values.worker.podDisruptionBudget.maxUnavailable }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "hindsight.worker.selectorLabels" . | nindent 6 }}
+{{- end }}

helm/hindsight/templates/postgresql-service.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.postgresql.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hindsight.fullname" . }}-postgresql
+  labels:
+    {{- include "hindsight.labels" . | nindent 4 }}
+    app.kubernetes.io/component: postgresql
+spec:
+  type: ClusterIP
+  ports:
+  - port: {{ .Values.postgresql.service.port }}
+    targetPort: postgresql
+    protocol: TCP
+    name: postgresql
+  selector:
+    {{- include "hindsight.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: postgresql
+{{- end }}

helm/hindsight/templates/postgresql-statefulset.yaml

@@ -0,0 +1,85 @@
+{{- if .Values.postgresql.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "hindsight.fullname" . }}-postgresql
+  labels:
+    {{- include "hindsight.labels" . | nindent 4 }}
+    app.kubernetes.io/component: postgresql
+spec:
+  serviceName: {{ include "hindsight.fullname" . }}-postgresql
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "hindsight.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: postgresql
+  template:
+    metadata:
+      labels:
+        {{- include "hindsight.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: postgresql
+    spec:
+      containers:
+      - name: postgresql
+        image: "{{ .Values.postgresql.image.repository }}:{{ .Values.postgresql.image.tag }}"
+        imagePullPolicy: {{ .Values.postgresql.image.pullPolicy }}
+        ports:
+        - name: postgresql
+          containerPort: 5432
+          protocol: TCP
+        env:
+        - name: POSTGRES_USER
+          value: {{ .Values.postgresql.auth.username | quote }}
+        - name: POSTGRES_PASSWORD
+          value: {{ .Values.postgresql.auth.password | quote }}
+        - name: POSTGRES_DB
+          value: {{ .Values.postgresql.auth.database | quote }}
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        livenessProbe:
+          exec:
+            command:
+            - pg_isready
+            - -U
+            - {{ .Values.postgresql.auth.username }}
+            - -d
+            - {{ .Values.postgresql.auth.database }}
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 3
+        readinessProbe:
+          exec:
+            command:
+            - pg_isready
+            - -U
+            - {{ .Values.postgresql.auth.username }}
+            - -d
+            - {{ .Values.postgresql.auth.database }}
+          initialDelaySeconds: 5
+          periodSeconds: 5
+          timeoutSeconds: 3
+          failureThreshold: 3
+        resources:
+          {{- toYaml .Values.postgresql.resources | nindent 10 }}
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/postgresql/data
+  {{- if .Values.postgresql.persistence.enabled }}
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      {{- if .Values.postgresql.persistence.storageClass }}
+      storageClassName: {{ .Values.postgresql.persistence.storageClass | quote }}
+      {{- end }}
+      resources:
+        requests:
+          storage: {{ .Values.postgresql.persistence.size }}
+  {{- else }}
+      volumes:
+      - name: data
+        emptyDir: {}
+  {{- end }}
+{{- end }}

helm/hindsight/templates/secret.yaml

@@ -1,19 +1,19 @@
+{{- if not .Values.existingSecret }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ include "hindsight.fullname" . }}-secret
+  name: {{ include "hindsight.secretName" . }}
   labels:
     {{- include "hindsight.labels" . | nindent 4 }}
 type: Opaque
 data:
-  {{- if and .Values.api.secrets (hasKey .Values.api.secrets "MEMORY_LLM_API_KEY") }}
-  llm-api-key: {{ .Values.api.secrets.MEMORY_LLM_API_KEY | b64enc | quote }}
+  {{- range $key, $value := .Values.api.secrets }}
+  {{ $key }}: {{ $value | b64enc | quote }}
   {{- end }}
-  {{- if and .Values.api.secrets (hasKey .Values.api.secrets "MEMORY_LLM_BASE_URL") }}
-  llm-base-url: {{ .Values.api.secrets.MEMORY_LLM_BASE_URL | b64enc | quote }}
+  {{- range $key, $value := .Values.controlPlane.secrets }}
+  {{ $key }}: {{ $value | b64enc | quote }}
   {{- end }}
-  {{- if not .Values.postgresql.enabled }}
-  {{- if .Values.postgresql.external.password }}
+  {{- if and (not .Values.postgresql.enabled) .Values.postgresql.external.password }}
   postgres-password: {{ .Values.postgresql.external.password | b64enc | quote }}
   {{- end }}
-  {{- end }}
+{{- end }}

helm/hindsight/templates/tei-embedding-deployment.yaml

@@ -0,0 +1,76 @@
+{{- if .Values.tei.embedding.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "hindsight.fullname" . }}-tei-embedding
+  labels:
+    {{- include "hindsight.tei.embedding.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.tei.embedding.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "hindsight.tei.embedding.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "hindsight.tei.embedding.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- if .Values.serviceAccount.create }}
+      serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+      - name: tei-embedding
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 10 }}
+        image: "{{ .Values.tei.embedding.image.repository }}:{{ .Values.tei.embedding.image.tag }}"
+        imagePullPolicy: {{ .Values.tei.embedding.image.pullPolicy }}
+        args:
+          - "--model-id"
+          - {{ .Values.tei.embedding.model | quote }}
+          - "--hostname"
+          - "0.0.0.0"
+          {{- range .Values.tei.embedding.args }}
+          - {{ . | quote }}
+          {{- end }}
+        ports:
+        - name: http
+          containerPort: {{ .Values.tei.embedding.port }}
+          protocol: TCP
+        env:
+        - name: PORT
+          value: {{ .Values.tei.embedding.port | quote }}
+        {{- range $key, $value := .Values.tei.embedding.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        livenessProbe:
+          {{- toYaml .Values.tei.embedding.livenessProbe | nindent 10 }}
+        readinessProbe:
+          {{- toYaml .Values.tei.embedding.readinessProbe | nindent 10 }}
+        resources:
+          {{- toYaml .Values.tei.embedding.resources | nindent 10 }}
+        volumeMounts:
+        - name: model-cache
+          mountPath: /data
+      volumes:
+      - name: model-cache
+        emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

helm/hindsight/templates/tei-embedding-service.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.tei.embedding.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hindsight.fullname" . }}-tei-embedding
+  labels:
+    {{- include "hindsight.tei.embedding.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+  - port: {{ .Values.tei.embedding.port }}
+    targetPort: http
+    protocol: TCP
+    name: http
+  selector:
+    {{- include "hindsight.tei.embedding.selectorLabels" . | nindent 4 }}
+{{- end }}

helm/hindsight/templates/tei-reranker-deployment.yaml

@@ -0,0 +1,76 @@
+{{- if .Values.tei.reranker.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "hindsight.fullname" . }}-tei-reranker
+  labels:
+    {{- include "hindsight.tei.reranker.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.tei.reranker.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "hindsight.tei.reranker.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "hindsight.tei.reranker.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- if .Values.serviceAccount.create }}
+      serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+      - name: tei-reranker
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 10 }}
+        image: "{{ .Values.tei.reranker.image.repository }}:{{ .Values.tei.reranker.image.tag }}"
+        imagePullPolicy: {{ .Values.tei.reranker.image.pullPolicy }}
+        args:
+          - "--model-id"
+          - {{ .Values.tei.reranker.model | quote }}
+          - "--hostname"
+          - "0.0.0.0"
+          {{- range .Values.tei.reranker.args }}
+          - {{ . | quote }}
+          {{- end }}
+        ports:
+        - name: http
+          containerPort: {{ .Values.tei.reranker.port }}
+          protocol: TCP
+        env:
+        - name: PORT
+          value: {{ .Values.tei.reranker.port | quote }}
+        {{- range $key, $value := .Values.tei.reranker.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        livenessProbe:
+          {{- toYaml .Values.tei.reranker.livenessProbe | nindent 10 }}
+        readinessProbe:
+          {{- toYaml .Values.tei.reranker.readinessProbe | nindent 10 }}
+        resources:
+          {{- toYaml .Values.tei.reranker.resources | nindent 10 }}
+        volumeMounts:
+        - name: model-cache
+          mountPath: /data
+      volumes:
+      - name: model-cache
+        emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

helm/hindsight/templates/tei-reranker-service.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.tei.reranker.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hindsight.fullname" . }}-tei-reranker
+  labels:
+    {{- include "hindsight.tei.reranker.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+  - port: {{ .Values.tei.reranker.port }}
+    targetPort: http
+    protocol: TCP
+    name: http
+  selector:
+    {{- include "hindsight.tei.reranker.selectorLabels" . | nindent 4 }}
+{{- end }}

helm/hindsight/templates/worker-service.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.worker.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hindsight.fullname" . }}-worker
+  labels:
+    {{- include "hindsight.worker.labels" . | nindent 4 }}
+  {{- if .Values.podAnnotations }}
+  annotations:
+    {{- /* Common Prometheus annotations for metrics scraping */}}
+    prometheus.io/scrape: "true"
+    prometheus.io/port: {{ .Values.worker.service.port | quote }}
+    prometheus.io/path: "/metrics"
+  {{- end }}
+spec:
+  # Headless service for StatefulSet (enables stable DNS names like worker-0.worker.namespace)
+  clusterIP: None
+  ports:
+  - port: {{ .Values.worker.service.port }}
+    targetPort: {{ .Values.worker.service.targetPort }}
+    protocol: TCP
+    name: http
+  selector:
+    {{- include "hindsight.worker.selectorLabels" . | nindent 4 }}
+{{- end }}

helm/hindsight/templates/worker-statefulset.yaml

@@ -0,0 +1,142 @@
+{{- if .Values.worker.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "hindsight.fullname" . }}-worker
+  labels:
+    {{- include "hindsight.worker.labels" . | nindent 4 }}
+spec:
+  serviceName: {{ include "hindsight.fullname" . }}-worker
+  replicas: {{ .Values.worker.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "hindsight.worker.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        {{- if not .Values.existingSecret }}
+        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+        {{- end }}
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "hindsight.worker.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- if .Values.serviceAccount.create }}
+      serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+      - name: worker
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 10 }}
+        image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Values.version | default .Chart.AppVersion }}"
+        imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+        command: ["hindsight-worker"]
+        ports:
+        - name: http
+          containerPort: {{ .Values.worker.service.targetPort }}
+          protocol: TCP
+        {{- if .Values.existingSecret }}
+        envFrom:
+        - secretRef:
+            name: {{ .Values.existingSecret }}
+        {{- end }}
+        env:
+        {{- /* POSTGRES_PASSWORD must be defined before DATABASE_URL for $(VAR) interpolation */}}
+        {{- if not .Values.postgresql.enabled }}
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "hindsight.secretName" . }}
+              key: postgres-password
+        {{- end }}
+        - name: HINDSIGHT_API_DATABASE_URL
+          value: {{ include "hindsight.databaseUrl" . | quote }}
+        {{- /* Worker ID uses pod name (StatefulSet provides stable names like worker-0, worker-1) */}}
+        - name: HINDSIGHT_API_WORKER_ID
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        {{- /* Inherit LLM config from api.env */}}
+        {{- range $key, $value := .Values.api.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- /* Worker-specific env vars */}}
+        {{- range $key, $value := .Values.worker.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- /* Only use secrets when not using existingSecret */}}
+        {{- if not .Values.existingSecret }}
+        {{- /* Inherit secrets from api.secrets */}}
+        {{- range $key, $value := .Values.api.secrets }}
+        - name: {{ $key }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "hindsight.secretName" $ }}
+              key: {{ $key }}
+        {{- end }}
+        {{- /* Worker-specific secrets (can override api.secrets) */}}
+        {{- range $key, $value := .Values.worker.secrets }}
+        - name: {{ $key }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "hindsight.secretName" $ }}
+              key: {{ $key }}
+        {{- end }}
+        {{- end }}
+        livenessProbe:
+          {{- toYaml .Values.worker.livenessProbe | nindent 10 }}
+        readinessProbe:
+          {{- toYaml .Values.worker.readinessProbe | nindent 10 }}
+        resources:
+          {{- toYaml .Values.worker.resources | nindent 10 }}
+        {{- if or .Values.worker.persistence.modelCache.enabled .Values.worker.extraVolumeMounts }}
+        volumeMounts:
+        {{- if .Values.worker.persistence.modelCache.enabled }}
+        - name: model-cache
+          mountPath: /home/hindsight/.cache
+        {{- end }}
+        {{- with .Values.worker.extraVolumeMounts }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with (.Values.worker.affinity | default .Values.affinity) }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.worker.extraVolumes }}
+      volumes:
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
+  {{- if .Values.worker.persistence.modelCache.enabled }}
+  volumeClaimTemplates:
+  - metadata:
+      name: model-cache
+      {{- with .Values.worker.persistence.modelCache.annotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      accessModes:
+        {{- toYaml .Values.worker.persistence.modelCache.accessModes | nindent 8 }}
+      {{- if .Values.worker.persistence.modelCache.storageClass }}
+      storageClassName: {{ .Values.worker.persistence.modelCache.storageClass }}
+      {{- end }}
+      resources:
+        requests:
+          storage: {{ .Values.worker.persistence.modelCache.size }}
+  {{- end }}
+{{- end }}

helm/hindsight/values.yaml

@@ -1,5 +1,18 @@
 # Default values for hindsight
 
+# Global version override - use this to set a consistent image tag across all components
+# If not set, defaults to Chart.appVersion from Chart.yaml
+# version: ""
+
+# Use an existing secret instead of creating one from values
+# When set, all keys from this secret are injected as environment variables via envFrom
+# Required keys:
+#   - postgres-password: PostgreSQL password (when postgresql.enabled=false)
+# Optional keys (any key becomes an env var):
+#   - HINDSIGHT_API_LLM_API_KEY: API key for LLM provider
+#   - Any other env vars you want to inject
+# existingSecret: "my-hindsight-secret"
+
 # Global settings
 replicaCount: 1
 
@@ -8,9 +21,9 @@ api:
   enabled: true
   replicaCount: 1
   image:
-    repository: hindsight/api
+    repository: ghcr.io/vectorize-io/hindsight-api
     pullPolicy: IfNotPresent
-    tag: "latest"
+    # tag defaults to .Values.version if not specified
 
   service:
     type: ClusterIP
@@ -29,7 +42,7 @@ api:
   # Liveness and readiness probes
   livenessProbe:
     httpGet:
-      path: /
+      path: /health
       port: 8888
     initialDelaySeconds: 30
     periodSeconds: 10
@@ -38,16 +51,52 @@ api:
 
   readinessProbe:
     httpGet:
-      path: /
+      path: /health
       port: 8888
     initialDelaySeconds: 10
     periodSeconds: 5
     timeoutSeconds: 3
     failureThreshold: 3
 
+  # Pod disruption budget
+  podDisruptionBudget:
+    enabled: false
+    minAvailable: 1
+    # maxUnavailable: 1
+
+  # Pod affinity/anti-affinity (overrides global affinity for this component)
+  # affinity: {}
+
+  # Persistent volume for local model cache (reranker, embeddings)
+  # Models are downloaded to /home/hindsight/.cache on first use.
+  # Without persistence, models are re-downloaded on every pod restart.
+  persistence:
+    modelCache:
+      enabled: false
+      size: 5Gi
+      storageClass: ""
+      accessModes:
+        - ReadWriteOnce
+      annotations: {}
+
+  # Extra volume mounts for the api container
+  # e.g.
+  # extraVolumeMounts:
+  #   - name: my-volume
+  #     mountPath: /mnt/my-volume
+  extraVolumeMounts: []
+
+  # Extra volumes for the api pod
+  # e.g.
+  # extraVolumes:
+  #   - name: my-volume
+  #     configMap:
+  #       name: my-configmap
+  extraVolumes: []
+
   # Environment variables
   env:
-    HINDSIGHT_API_LLM_PROVIDER: "groq"
+    #HINDSIGHT_API_LLM_PROVIDER: "groq"
     HINDSIGHT_API_LLM_MODEL: "openai/gpt-oss-120b"
 
   # Secret environment variables
@@ -55,14 +104,106 @@ api:
     # HINDSIGHT_API_LLM_API_KEY: "your-api-key"
     # HINDSIGHT_API_LLM_BASE_URL: "https://api.groq.com/openai/v1"
 
+# Worker settings (distributed task processing)
+# When enabled, dedicated worker pods process tasks and the API's internal worker is disabled
+worker:
+  enabled: false
+  replicaCount: 2
+  image:
+    repository: ghcr.io/vectorize-io/hindsight-api
+    pullPolicy: IfNotPresent
+    # tag: ""  # defaults to .Values.version, then Chart.appVersion if not specified
+
+  service:
+    # Service for metrics scraping (headless for StatefulSet)
+    port: 8889
+    targetPort: 8889
+
+  # Resource limits and requests
+  resources:
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+    requests:
+      cpu: 500m
+      memory: 1Gi
+
+  # Liveness and readiness probes
+  livenessProbe:
+    httpGet:
+      path: /health
+      port: 8889
+    initialDelaySeconds: 30
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 3
+
+  readinessProbe:
+    httpGet:
+      path: /health
+      port: 8889
+    initialDelaySeconds: 10
+    periodSeconds: 5
+    timeoutSeconds: 3
+    failureThreshold: 3
+
+  # Worker-specific environment variables
+  env:
+    # Poll interval in milliseconds (how often to check for new tasks)
+    HINDSIGHT_API_WORKER_POLL_INTERVAL_MS: "500"
+    # Number of tasks to claim per poll cycle
+    HINDSIGHT_API_WORKER_BATCH_SIZE: "10"
+    # Max retries before marking a task as failed
+    HINDSIGHT_API_WORKER_MAX_RETRIES: "3"
+    # HTTP port for metrics/health (matches service.targetPort)
+    HINDSIGHT_API_WORKER_HTTP_PORT: "8889"
+
+  # Pod disruption budget
+  podDisruptionBudget:
+    enabled: false
+    minAvailable: 1
+    # maxUnavailable: 1
+
+  # Pod affinity/anti-affinity (overrides global affinity for this component)
+  # affinity: {}
+
+  # Persistent volume for local model cache (reranker, embeddings)
+  # Uses volumeClaimTemplates since worker is a StatefulSet.
+  persistence:
+    modelCache:
+      enabled: false
+      size: 5Gi
+      storageClass: ""
+      accessModes:
+        - ReadWriteOnce
+      annotations: {}
+
+  # Extra volume mounts for the worker container
+  # e.g.
+  # extraVolumeMounts:
+  #   - name: my-volume
+  #     mountPath: /mnt/my-volume
+  extraVolumeMounts: []
+
+  # Extra volumes for the worker pod
+  # e.g.
+  # extraVolumes:
+  #   - name: my-volume
+  #     configMap:
+  #       name: my-configmap
+  extraVolumes: []
+
+  # Secret environment variables (inherited from api.secrets if not specified)
+  secrets: {}
+
 # Image settings for control plane
 controlPlane:
   enabled: true
   replicaCount: 1
   image:
-    repository: hindsight/hindsight-control-plane
+    repository: ghcr.io/vectorize-io/hindsight-control-plane
     pullPolicy: IfNotPresent
-    tag: "latest"
+    # tag defaults to .Values.version if not specified
 
   service:
     type: ClusterIP
@@ -78,10 +219,9 @@ controlPlane:
       cpu: 250m
       memory: 512Mi
 
-  # Liveness and readiness probes
+  # Liveness and readiness probes (TCP check)
   livenessProbe:
-    httpGet:
-      path: /
+    tcpSocket:
       port: 3000
     initialDelaySeconds: 30
     periodSeconds: 10
@@ -89,14 +229,22 @@ controlPlane:
     failureThreshold: 3
 
   readinessProbe:
-    httpGet:
-      path: /
+    tcpSocket:
       port: 3000
     initialDelaySeconds: 10
     periodSeconds: 5
     timeoutSeconds: 3
     failureThreshold: 3
 
+  # Pod disruption budget
+  podDisruptionBudget:
+    enabled: false
+    minAvailable: 1
+    # maxUnavailable: 1
+
+  # Pod affinity/anti-affinity (overrides global affinity for this component)
+  # affinity: {}
+
   # Environment variables
   env:
     NODE_ENV: "production"
@@ -106,21 +254,43 @@ controlPlane:
 # PostgreSQL configuration
 postgresql:
   # Set to true to deploy PostgreSQL as part of this chart
-  enabled: false
+  enabled: true
+
+  image:
+    repository: ankane/pgvector
+    tag: latest
+    pullPolicy: IfNotPresent
+
+  auth:
+    username: "hindsight"
+    password: "hindsight"
+    database: "hindsight"
+
+  service:
+    port: 5432
+
+  persistence:
+    enabled: true
+    size: 8Gi
+    # storageClass: ""
+
+  resources:
+    limits:
+      cpu: 1000m
+      memory: 1Gi
+    requests:
+      cpu: 250m
+      memory: 256Mi
 
   # External PostgreSQL connection details
-  # If postgresql.enabled is false, provide external database details
+  # Only used if postgresql.enabled is false
   external:
     host: "postgresql"
     port: 5432
     database: "hindsight"
     username: "hindsight"
-    # Password should be provided via secret
     # password: ""
 
-# Database URL (auto-generated from postgresql config if not provided)
-# databaseUrl: "postgresql://user:pass@host:5432/database"
-
 # Ingress configuration
 ingress:
   enabled: false
@@ -173,9 +343,87 @@ nodeSelector: {}
 # Tolerations
 tolerations: []
 
-# Affinity
+# Affinity (applied to all components unless overridden per-component)
 affinity: {}
 
+# TEI (Text Embeddings Inference) - optional standalone deployments
+# for reranking and/or embedding models
+tei:
+  reranker:
+    enabled: false
+    replicaCount: 1
+    image:
+      repository: ghcr.io/huggingface/text-embeddings-inference
+      tag: cpu-1.8.3
+      pullPolicy: IfNotPresent
+    model: "cross-encoder/ms-marco-MiniLM-L-6-v2"
+    port: 8090
+    args:
+      - "--auto-truncate"
+    env:
+      PAYLOAD_LIMIT: "10000000"
+      MAX_CLIENT_BATCH_SIZE: "256"
+    resources:
+      limits:
+        cpu: 2000m
+        memory: 2Gi
+      requests:
+        cpu: 500m
+        memory: 1Gi
+    livenessProbe:
+      httpGet:
+        path: /health
+        port: 8090
+      initialDelaySeconds: 30
+      periodSeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 6
+    readinessProbe:
+      httpGet:
+        path: /health
+        port: 8090
+      initialDelaySeconds: 15
+      periodSeconds: 5
+      timeoutSeconds: 3
+      failureThreshold: 3
+
+  embedding:
+    enabled: false
+    replicaCount: 1
+    image:
+      repository: ghcr.io/huggingface/text-embeddings-inference
+      tag: cpu-1.8.3
+      pullPolicy: IfNotPresent
+    model: "sentence-transformers/all-MiniLM-L6-v2"
+    port: 8091
+    args: []
+    env:
+      PAYLOAD_LIMIT: "10000000"
+      MAX_CLIENT_BATCH_SIZE: "256"
+    resources:
+      limits:
+        cpu: 2000m
+        memory: 2Gi
+      requests:
+        cpu: 500m
+        memory: 1Gi
+    livenessProbe:
+      httpGet:
+        path: /health
+        port: 8091
+      initialDelaySeconds: 30
+      periodSeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 6
+    readinessProbe:
+      httpGet:
+        path: /health
+        port: 8091
+      initialDelaySeconds: 15
+      periodSeconds: 5
+      timeoutSeconds: 3
+      failureThreshold: 3
+
 # Autoscaling
 autoscaling:
   enabled: false

hindsight-all-slim/README.md

@@ -0,0 +1,48 @@
+# hindsight-all
+
+All-in-one package for Hindsight - Agent Memory That Works Like Human Memory
+
+## Quick Start
+
+```python
+from hindsight import start_server, HindsightClient
+
+# Start server with embedded PostgreSQL
+server = start_server(
+    llm_provider="groq",
+    llm_api_key="your-api-key",
+    llm_model="openai/gpt-oss-120b"
+)
+
+# Create client
+client = HindsightClient(base_url=server.url)
+
+# Store memories
+client.put(agent_id="assistant", content="User prefers Python for data analysis")
+
+# Search memories
+results = client.search(agent_id="assistant", query="programming preferences")
+
+# Generate contextual response
+response = client.think(agent_id="assistant", query="What languages should I recommend?")
+
+# Stop server when done
+server.stop()
+```
+
+## Using Context Manager
+
+```python
+from hindsight import HindsightServer, HindsightClient
+
+with HindsightServer(llm_provider="groq", llm_api_key="...") as server:
+    client = HindsightClient(base_url=server.url)
+    # ... use client ...
+# Server automatically stops
+```
+
+## Installation
+
+```bash
+pip install hindsight-all
+```

hindsight-all-slim/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hindsight-all-slim"
+version = "0.4.22"
+description = "Hindsight: Agent Memory That Works Like Human Memory - Slim All-in-One Bundle"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "hindsight-api-slim>=0.4.17",
+    "hindsight-client>=0.0.7",
+    "hindsight-embed>=0.1.0",
+]
+
+[tool.uv.sources]
+hindsight-api-slim = { workspace = true }
+hindsight-client = { workspace = true }
+hindsight-embed = { workspace = true }
+
+[project.optional-dependencies]
+test = [
+    "pytest>=7.0.0",
+    "pytest-asyncio>=0.21.0",
+]
+
+[tool.setuptools]
+packages = []
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"

hindsight-all/README.md

@@ -0,0 +1,48 @@
+# hindsight-all
+
+All-in-one package for Hindsight - Agent Memory That Works Like Human Memory
+
+## Quick Start
+
+```python
+from hindsight import start_server, HindsightClient
+
+# Start server with embedded PostgreSQL
+server = start_server(
+    llm_provider="groq",
+    llm_api_key="your-api-key",
+    llm_model="openai/gpt-oss-120b"
+)
+
+# Create client
+client = HindsightClient(base_url=server.url)
+
+# Store memories
+client.put(agent_id="assistant", content="User prefers Python for data analysis")
+
+# Search memories
+results = client.search(agent_id="assistant", query="programming preferences")
+
+# Generate contextual response
+response = client.think(agent_id="assistant", query="What languages should I recommend?")
+
+# Stop server when done
+server.stop()
+```
+
+## Using Context Manager
+
+```python
+from hindsight import HindsightServer, HindsightClient
+
+with HindsightServer(llm_provider="groq", llm_api_key="...") as server:
+    client = HindsightClient(base_url=server.url)
+    # ... use client ...
+# Server automatically stops
+```
+
+## Installation
+
+```bash
+pip install hindsight-all
+```

hindsight-all/hindsight/__init__.py

@@ -0,0 +1,69 @@
+"""
+Hindsight - All-in-one semantic memory system for AI agents.
+
+This package provides a simple way to run Hindsight locally with embedded PostgreSQL.
+
+Easiest way - Embedded client (recommended):
+    ```python
+    from hindsight import HindsightEmbedded
+
+    # Server starts automatically on first use
+    client = HindsightEmbedded(
+        profile="myapp",
+        llm_provider="groq",
+        llm_api_key="your-api-key",
+    )
+
+    # Use immediately - no manual server management needed
+    client.retain(bank_id="alice", content="Alice loves AI")
+    results = client.recall(bank_id="alice", query="What does Alice like?")
+    ```
+
+Manual server management:
+    ```python
+    from hindsight import start_server, HindsightClient
+
+    # Start server with embedded PostgreSQL (pg0)
+    server = start_server(
+        llm_provider="groq",
+        llm_api_key="your-api-key",
+        llm_model="openai/gpt-oss-120b"
+    )
+
+    # Create client
+    client = HindsightClient(base_url=server.url)
+
+    # Store memories
+    client.retain(bank_id="assistant", content="User prefers Python for data analysis")
+
+    # Search memories
+    results = client.recall(bank_id="assistant", query="programming preferences")
+
+    # Generate contextual response
+    response = client.reflect(bank_id="assistant", query="What are my interests?")
+
+    # Stop server when done
+    server.stop()
+    ```
+
+Using context manager:
+    ```python
+    from hindsight import HindsightServer, HindsightClient
+
+    with HindsightServer(llm_provider="groq", llm_api_key="...") as server:
+        client = HindsightClient(base_url=server.url)
+        # ... use client ...
+    # Server automatically stops
+    ```
+"""
+
+from .client_wrapper import HindsightClient
+from .embedded import HindsightEmbedded
+from .server import Server as HindsightServer, start_server
+
+__all__ = [
+    "HindsightServer",
+    "start_server",
+    "HindsightClient",
+    "HindsightEmbedded",
+]

hindsight-all/hindsight/api_namespaces.py

@@ -0,0 +1,193 @@
+"""
+API namespace classes for organizing client methods.
+
+These classes provide organized access to different parts of the Hindsight API
+while ensuring the daemon is running before each call.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from .embedded import HindsightEmbedded
+
+
+class BanksAPI:
+    """Namespace for bank-related operations."""
+
+    def __init__(self, embedded: "HindsightEmbedded"):
+        self._embedded = embedded
+
+    def create(
+        self,
+        bank_id: str,
+        name: str | None = None,
+        mission: str | None = None,
+        disposition: dict[str, Any] | None = None,
+    ):
+        """Create a new bank."""
+        self._embedded._ensure_started()
+        return self._embedded._client.create_bank(
+            bank_id=bank_id,
+            name=name,
+            mission=mission,
+            disposition=disposition,
+        )
+
+    def delete(self, bank_id: str):
+        """Delete a bank."""
+        self._embedded._ensure_started()
+        return self._embedded._client.delete_bank(bank_id=bank_id)
+
+    def set_mission(self, bank_id: str, mission: str):
+        """Set or update the mission for a bank."""
+        self._embedded._ensure_started()
+        return self._embedded._client.set_mission(bank_id=bank_id, mission=mission)
+
+    def set_disposition(self, bank_id: str, disposition: dict[str, Any]):
+        """Set or update the disposition for a bank."""
+        self._embedded._ensure_started()
+        return self._embedded._client.set_disposition(bank_id=bank_id, disposition=disposition)
+
+
+class MentalModelsAPI:
+    """Namespace for mental model operations."""
+
+    def __init__(self, embedded: "HindsightEmbedded"):
+        self._embedded = embedded
+
+    def create(
+        self,
+        bank_id: str,
+        name: str,
+        content: str,
+        tags: list[str] | None = None,
+    ):
+        """Create a new mental model."""
+        self._embedded._ensure_started()
+        return self._embedded._client.create_mental_model(
+            bank_id=bank_id,
+            name=name,
+            content=content,
+            tags=tags,
+        )
+
+    def list(self, bank_id: str, tags: list[str] | None = None):
+        """List all mental models for a bank."""
+        self._embedded._ensure_started()
+        return self._embedded._client.list_mental_models(bank_id=bank_id, tags=tags)
+
+    def get(self, bank_id: str, mental_model_id: str):
+        """Get a specific mental model."""
+        self._embedded._ensure_started()
+        return self._embedded._client.get_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
+
+    def refresh(self, bank_id: str, mental_model_id: str):
+        """Refresh a mental model."""
+        self._embedded._ensure_started()
+        return self._embedded._client.refresh_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
+
+    def update(
+        self,
+        bank_id: str,
+        mental_model_id: str,
+        name: str | None = None,
+        content: str | None = None,
+        tags: list[str] | None = None,
+    ):
+        """Update a mental model."""
+        self._embedded._ensure_started()
+        return self._embedded._client.update_mental_model(
+            bank_id=bank_id,
+            mental_model_id=mental_model_id,
+            name=name,
+            content=content,
+            tags=tags,
+        )
+
+    def delete(self, bank_id: str, mental_model_id: str):
+        """Delete a mental model."""
+        self._embedded._ensure_started()
+        return self._embedded._client.delete_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
+
+
+class DirectivesAPI:
+    """Namespace for directive operations."""
+
+    def __init__(self, embedded: "HindsightEmbedded"):
+        self._embedded = embedded
+
+    def create(
+        self,
+        bank_id: str,
+        name: str,
+        content: str,
+        tags: list[str] | None = None,
+    ):
+        """Create a new directive."""
+        self._embedded._ensure_started()
+        return self._embedded._client.create_directive(
+            bank_id=bank_id,
+            name=name,
+            content=content,
+            tags=tags,
+        )
+
+    def list(self, bank_id: str, tags: list[str] | None = None):
+        """List all directives for a bank."""
+        self._embedded._ensure_started()
+        return self._embedded._client.list_directives(bank_id=bank_id, tags=tags)
+
+    def get(self, bank_id: str, directive_id: str):
+        """Get a specific directive."""
+        self._embedded._ensure_started()
+        return self._embedded._client.get_directive(bank_id=bank_id, directive_id=directive_id)
+
+    def update(
+        self,
+        bank_id: str,
+        directive_id: str,
+        name: str | None = None,
+        content: str | None = None,
+        tags: list[str] | None = None,
+    ):
+        """Update a directive."""
+        self._embedded._ensure_started()
+        return self._embedded._client.update_directive(
+            bank_id=bank_id,
+            directive_id=directive_id,
+            name=name,
+            content=content,
+            tags=tags,
+        )
+
+    def delete(self, bank_id: str, directive_id: str):
+        """Delete a directive."""
+        self._embedded._ensure_started()
+        return self._embedded._client.delete_directive(bank_id=bank_id, directive_id=directive_id)
+
+
+class MemoriesAPI:
+    """Namespace for memory operations."""
+
+    def __init__(self, embedded: "HindsightEmbedded"):
+        self._embedded = embedded
+
+    def list(
+        self,
+        bank_id: str,
+        type: str | None = None,
+        search_query: str | None = None,
+        limit: int = 100,
+        offset: int = 0,
+    ):
+        """List memories in a bank."""
+        self._embedded._ensure_started()
+        return self._embedded._client.list_memories(
+            bank_id=bank_id,
+            type=type,
+            search_query=search_query,
+            limit=limit,
+            offset=offset,
+        )

hindsight-all/hindsight/client_wrapper.py

@@ -0,0 +1,423 @@
+"""
+Wrapper for Hindsight client that adds API namespaces.
+
+Provides organized access to different parts of the Hindsight API through
+namespaces like .banks, .mental_models, etc.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from hindsight_client import Hindsight
+
+
+class BanksAPI:
+    """Namespace for bank-related operations.
+    
+    Provides methods to create, delete, and manage memory banks.
+    """
+
+    def __init__(self, client: Hindsight):
+        self._client = client
+
+    def create(
+        self,
+        bank_id: str,
+        name: str | None = None,
+        mission: str | None = None,
+        disposition: dict[str, Any] | None = None,
+    ) -> Any:
+        """Create a new bank.
+        
+        Args:
+            bank_id: Unique identifier for the bank.
+            name: Optional display name for the bank.
+            mission: Optional mission statement for the bank.
+            disposition: Optional disposition configuration dict.
+            
+        Returns:
+            Bank creation response from the API.
+        """
+        return self._client.create_bank(
+            bank_id=bank_id,
+            name=name,
+            mission=mission,
+            disposition=disposition,
+        )
+
+    def delete(self, bank_id: str) -> Any:
+        """Delete a bank.
+        
+        Args:
+            bank_id: The ID of the bank to delete.
+            
+        Returns:
+            Deletion response from the API.
+        """
+        return self._client.delete_bank(bank_id=bank_id)
+
+    def set_mission(self, bank_id: str, mission: str) -> Any:
+        """Set or update the mission for a bank.
+        
+        Args:
+            bank_id: The ID of the bank.
+            mission: The mission statement to set.
+            
+        Returns:
+            API response confirming the update.
+        """
+        return self._client.set_mission(bank_id=bank_id, mission=mission)
+
+    def set_disposition(self, bank_id: str, disposition: dict[str, Any]) -> Any:
+        """Set or update the disposition for a bank.
+        
+        Args:
+            bank_id: The ID of the bank.
+            disposition: The disposition configuration dict.
+            
+        Returns:
+            API response confirming the update.
+        """
+        return self._client.set_disposition(bank_id=bank_id, disposition=disposition)
+
+    def list(self) -> Any:
+        """List all banks.
+        
+        Returns:
+            List of banks from the API.
+        """
+        from hindsight_client.hindsight_client import _run_async
+
+        return _run_async(self._client._banks_api.list_banks())
+
+
+class MentalModelsAPI:
+    """Namespace for mental model operations.
+    
+    Mental models are reusable knowledge structures that guide agent behavior.
+    """
+
+    def __init__(self, client: Hindsight):
+        self._client = client
+
+    def create(
+        self,
+        bank_id: str,
+        name: str,
+        content: str,
+        tags: list[str] | None = None,
+    ) -> Any:
+        """Create a new mental model.
+        
+        Args:
+            bank_id: The ID of the bank to add the model to.
+            name: Name for the mental model.
+            content: The content/instructions for the mental model.
+            tags: Optional list of tags for categorization.
+            
+        Returns:
+            Creation response from the API.
+        """
+        return self._client.create_mental_model(
+            bank_id=bank_id,
+            name=name,
+            content=content,
+            tags=tags,
+        )
+
+    def list(self, bank_id: str, tags: list[str] | None = None) -> Any:
+        """List all mental models for a bank.
+        
+        Args:
+            bank_id: The ID of the bank.
+            tags: Optional filter by tags.
+            
+        Returns:
+            List of mental models.
+        """
+        return self._client.list_mental_models(bank_id=bank_id, tags=tags)
+
+    def get(self, bank_id: str, mental_model_id: str) -> Any:
+        """Get a specific mental model.
+        
+        Args:
+            bank_id: The ID of the bank.
+            mental_model_id: The ID of the mental model.
+            
+        Returns:
+            The mental model details.
+        """
+        return self._client.get_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
+
+    def refresh(self, bank_id: str, mental_model_id: str) -> Any:
+        """Refresh a mental model.
+        
+        Args:
+            bank_id: The ID of the bank.
+            mental_model_id: The ID of the mental model to refresh.
+            
+        Returns:
+            Refresh response from the API.
+        """
+        return self._client.refresh_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
+
+    def update(
+        self,
+        bank_id: str,
+        mental_model_id: str,
+        name: str | None = None,
+        content: str | None = None,
+        tags: list[str] | None = None,
+    ) -> Any:
+        """Update a mental model.
+        
+        Args:
+            bank_id: The ID of the bank.
+            mental_model_id: The ID of the mental model to update.
+            name: Optional new name.
+            content: Optional new content.
+            tags: Optional new tags list.
+            
+        Returns:
+            Update response from the API.
+        """
+        return self._client.update_mental_model(
+            bank_id=bank_id,
+            mental_model_id=mental_model_id,
+            name=name,
+            content=content,
+            tags=tags,
+        )
+
+    def delete(self, bank_id: str, mental_model_id: str) -> Any:
+        """Delete a mental model.
+        
+        Args:
+            bank_id: The ID of the bank.
+            mental_model_id: The ID of the mental model to delete.
+            
+        Returns:
+            Deletion response from the API.
+        """
+        return self._client.delete_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
+
+
+class DirectivesAPI:
+    """Namespace for directive operations.
+    
+    Directives are explicit instructions that guide agent behavior.
+    """
+
+    def __init__(self, client: Hindsight):
+        self._client = client
+
+    def create(
+        self,
+        bank_id: str,
+        name: str,
+        content: str,
+        tags: list[str] | None = None,
+    ) -> Any:
+        """Create a new directive.
+        
+        Args:
+            bank_id: The ID of the bank to add the directive to.
+            name: Name for the directive.
+            content: The directive content/instructions.
+            tags: Optional list of tags for categorization.
+            
+        Returns:
+            Creation response from the API.
+        """
+        return self._client.create_directive(
+            bank_id=bank_id,
+            name=name,
+            content=content,
+            tags=tags,
+        )
+
+    def list(self, bank_id: str, tags: list[str] | None = None) -> Any:
+        """List all directives for a bank.
+        
+        Args:
+            bank_id: The ID of the bank.
+            tags: Optional filter by tags.
+            
+        Returns:
+            List of directives.
+        """
+        return self._client.list_directives(bank_id=bank_id, tags=tags)
+
+    def get(self, bank_id: str, directive_id: str) -> Any:
+        """Get a specific directive.
+        
+        Args:
+            bank_id: The ID of the bank.
+            directive_id: The ID of the directive.
+            
+        Returns:
+            The directive details.
+        """
+        return self._client.get_directive(bank_id=bank_id, directive_id=directive_id)
+
+    def update(
+        self,
+        bank_id: str,
+        directive_id: str,
+        name: str | None = None,
+        content: str | None = None,
+        tags: list[str] | None = None,
+    ) -> Any:
+        """Update a directive.
+        
+        Args:
+            bank_id: The ID of the bank.
+            directive_id: The ID of the directive to update.
+            name: Optional new name.
+            content: Optional new content.
+            tags: Optional new tags list.
+            
+        Returns:
+            Update response from the API.
+        """
+        return self._client.update_directive(
+            bank_id=bank_id,
+            directive_id=directive_id,
+            name=name,
+            content=content,
+            tags=tags,
+        )
+
+    def delete(self, bank_id: str, directive_id: str) -> Any:
+        """Delete a directive.
+        
+        Args:
+            bank_id: The ID of the bank.
+            directive_id: The ID of the directive to delete.
+            
+        Returns:
+            Deletion response from the API.
+        """
+        return self._client.delete_directive(bank_id=bank_id, directive_id=directive_id)
+
+
+class MemoriesAPI:
+    """Namespace for memory operations.
+    
+    Provides methods to query and retrieve stored memories.
+    """
+
+    def __init__(self, client: Hindsight):
+        self._client = client
+
+    def list(
+        self,
+        bank_id: str,
+        type: str | None = None,
+        search_query: str | None = None,
+        limit: int = 100,
+        offset: int = 0,
+    ) -> Any:
+        """List memories in a bank.
+        
+        Args:
+            bank_id: The ID of the bank to query.
+            type: Optional filter by memory type.
+            search_query: Optional search query for filtering.
+            limit: Maximum number of results to return (default: 100).
+            offset: Number of results to skip for pagination (default: 0).
+            
+        Returns:
+            List of memories matching the criteria.
+        """
+        return self._client.list_memories(
+            bank_id=bank_id,
+            type=type,
+            search_query=search_query,
+            limit=limit,
+            offset=offset,
+        )
+
+
+class HindsightClient(Hindsight):
+    """
+    Enhanced Hindsight client with organized API namespaces.
+
+    This wrapper extends the auto-generated Hindsight client with organized
+    access to different parts of the API through namespaces.
+
+    Example:
+        ```python
+        from hindsight import HindsightClient
+
+        client = HindsightClient(base_url="http://localhost:8888")
+
+        # Core operations (inherited from Hindsight)
+        client.retain(bank_id="test", content="Hello")
+        results = client.recall(bank_id="test", query="Hello")
+
+        # Organized API access through namespaces
+        client.banks.create(bank_id="test", name="Test Bank")
+        models = client.mental_models.list(bank_id="test")
+        directives = client.directives.list(bank_id="test")
+        memories = client.memories.list(bank_id="test")
+        ```
+        
+    Attributes:
+        banks: Namespace for bank management operations.
+        mental_models: Namespace for mental model operations.
+        directives: Namespace for directive operations.
+        memories: Namespace for memory listing operations.
+    """
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self._banks_namespace: BanksAPI | None = None
+        self._mental_models_namespace: MentalModelsAPI | None = None
+        self._directives_namespace: DirectivesAPI | None = None
+        self._memories_namespace: MemoriesAPI | None = None
+
+    @property
+    def banks(self) -> BanksAPI:
+        """Access bank management operations.
+        
+        Returns:
+            BanksAPI instance for bank operations.
+        """
+        if self._banks_namespace is None:
+            self._banks_namespace = BanksAPI(self)
+        return self._banks_namespace
+
+    @property
+    def mental_models(self) -> MentalModelsAPI:
+        """Access mental model operations.
+        
+        Returns:
+            MentalModelsAPI instance for mental model operations.
+        """
+        if self._mental_models_namespace is None:
+            self._mental_models_namespace = MentalModelsAPI(self)
+        return self._mental_models_namespace
+
+    @property
+    def directives(self) -> DirectivesAPI:
+        """Access directive operations.
+        
+        Returns:
+            DirectivesAPI instance for directive operations.
+        """
+        if self._directives_namespace is None:
+            self._directives_namespace = DirectivesAPI(self)
+        return self._directives_namespace
+
+    @property
+    def memories(self) -> MemoriesAPI:
+        """Access memory listing operations.
+        
+        Returns:
+            MemoriesAPI instance for memory operations.
+        """
+        if self._memories_namespace is None:
+            self._memories_namespace = MemoriesAPI(self)
+        return self._memories_namespace

hindsight-all/hindsight/embedded.py

@@ -0,0 +1,412 @@
+"""
+Embedded Hindsight client with automatic daemon lifecycle management.
+
+This module provides HindsightEmbedded, a client that uses the same daemon
+management interface as hindsight-embed CLI, ensuring full compatibility.
+
+Example:
+    ```python
+    from hindsight import HindsightEmbedded
+
+    # Daemon starts automatically on first use
+    client = HindsightEmbedded(
+        profile="myapp",
+        llm_provider="groq",
+        llm_api_key="your-api-key",
+    )
+
+    # Use just like HindsightClient
+    client.retain(bank_id="alice", content="Alice loves AI")
+    results = client.recall(bank_id="alice", query="What does Alice like?")
+
+    # Optional cleanup
+    client.close()
+    ```
+
+Using context manager:
+    ```python
+    from hindsight import HindsightEmbedded
+
+    with HindsightEmbedded(profile="myapp") as client:
+        client.retain(bank_id="alice", content="Alice loves AI")
+        # Daemon managed automatically
+    ```
+"""
+
+import logging
+import threading
+from typing import Optional
+
+from hindsight_client import Hindsight
+from hindsight_embed import get_embed_manager
+
+from .api_namespaces import BanksAPI, DirectivesAPI, MemoriesAPI, MentalModelsAPI
+
+logger = logging.getLogger(__name__)
+
+
+class HindsightEmbedded:
+    """
+    Hindsight client with automatic daemon lifecycle management.
+
+    This client uses the same daemon management interface as hindsight-embed CLI,
+    ensuring full compatibility and shared profiles. The daemon is started automatically
+    on first use and manages profile-specific databases.
+
+    Profile data is stored in: ~/.pg0/instances/hindsight-embed-{profile}/
+
+    All methods from HindsightClient are available:
+    - retain(), retain_batch()
+    - recall()
+    - reflect()
+    - create_bank(), set_mission(), delete_bank()
+    - create_mental_model(), list_mental_models(), etc.
+    - create_directive(), list_directives(), etc.
+    - And all async variants (aretain, arecall, areflect, etc.)
+
+    Args:
+        profile: Profile name for data isolation (default: "default")
+        llm_provider: LLM provider ("groq", "openai", "ollama", "gemini", "anthropic", "lmstudio")
+        llm_api_key: API key for the LLM provider
+        llm_model: Model name to use
+        llm_base_url: Optional custom base URL for LLM API
+        database_url: Optional database URL override (default: profile-specific pg0)
+        idle_timeout: Seconds before daemon auto-exits when idle (default: 300)
+        log_level: Daemon log level (default: "info")
+        ui: Whether to start the control plane web UI alongside the daemon (default: False)
+        ui_port: Port for the UI. Defaults to daemon_port + 10000.
+        ui_hostname: Hostname to bind the UI to. Defaults to "0.0.0.0".
+    """
+
+    def __init__(
+        self,
+        profile: str = "default",
+        llm_provider: str = "groq",
+        llm_api_key: str = "",
+        llm_model: str = "openai/gpt-oss-120b",
+        llm_base_url: Optional[str] = None,
+        database_url: Optional[str] = None,
+        idle_timeout: int = 300,
+        log_level: str = "info",
+        ui: bool = False,
+        ui_port: Optional[int] = None,
+        ui_hostname: str = "0.0.0.0",
+    ):
+        """
+        Initialize the embedded client (daemon starts on first use).
+
+        Args:
+            profile: Profile name for data isolation
+            llm_provider: LLM provider
+            llm_api_key: API key for the LLM provider
+            llm_model: Model name to use
+            llm_base_url: Optional custom base URL for LLM API
+            database_url: Optional database URL override
+            idle_timeout: Seconds before daemon auto-exits when idle
+            log_level: Daemon log level
+            ui: Whether to start the control plane web UI alongside the daemon
+            ui_port: Port for the UI (defaults to daemon_port + 10000)
+            ui_hostname: Hostname to bind the UI to (defaults to "0.0.0.0")
+        """
+        self.profile = profile
+
+        # Build config dict for daemon (matches CLI format)
+        self.config = {
+            "HINDSIGHT_API_LLM_PROVIDER": llm_provider,
+            "HINDSIGHT_API_LLM_API_KEY": llm_api_key,
+            "HINDSIGHT_API_LLM_MODEL": llm_model,
+            "HINDSIGHT_API_LOG_LEVEL": log_level,
+            "HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT": str(idle_timeout),
+        }
+
+        if llm_base_url:
+            self.config["HINDSIGHT_API_LLM_BASE_URL"] = llm_base_url
+
+        if database_url:
+            self.config["HINDSIGHT_EMBED_API_DATABASE_URL"] = database_url
+
+        self._ui = ui
+        self._ui_port = ui_port
+        self._ui_hostname = ui_hostname
+
+        self._client: Optional[Hindsight] = None
+        self._lock = threading.Lock()
+        self._started = False
+        self._closed = False
+        self._manager = get_embed_manager()
+
+        # API namespaces (initialized once, lazily)
+        self._banks_api: Optional[BanksAPI] = None
+        self._mental_models_api: Optional[MentalModelsAPI] = None
+        self._directives_api: Optional[DirectivesAPI] = None
+        self._memories_api: Optional[MemoriesAPI] = None
+
+    def _ensure_started(self):
+        """Ensure daemon is running (thread-safe)."""
+        if self._started and self._client is not None:
+            return
+
+        with self._lock:
+            # Double-check after acquiring lock
+            if self._started and self._client is not None:
+                return
+
+            if self._closed:
+                raise RuntimeError(
+                    "Cannot use HindsightEmbedded after it has been closed"
+                )
+
+            # Use embed manager interface for daemon management
+            logger.info(f"Ensuring daemon is running for profile '{self.profile}'...")
+            success = self._manager.ensure_running(self.config, self.profile)
+            if not success:
+                raise RuntimeError(
+                    f"Failed to start daemon for profile '{self.profile}'"
+                )
+
+            # Get daemon URL and create client
+            daemon_url = self._manager.get_url(self.profile)
+            self._client = Hindsight(base_url=daemon_url)
+            self._started = True
+            logger.info(f"Connected to daemon at {daemon_url}")
+
+            # Start UI if requested
+            if self._ui:
+                logger.info(f"Starting UI for profile '{self.profile}'...")
+                ui_started = self._manager.start_ui(
+                    self.profile, self._ui_port, self._ui_hostname
+                )
+                if not ui_started:
+                    logger.warning(f"Failed to start UI for profile '{self.profile}'")
+
+    def _cleanup(self, stop_daemon_on_close: bool = False):
+        """
+        Cleanup client resources (idempotent).
+
+        Args:
+            stop_daemon_on_close: If True, stops the daemon. Otherwise, daemon continues
+                running (it will auto-stop after idle timeout).
+        """
+        if self._closed:
+            return
+
+        with self._lock:
+            if self._closed:
+                return
+
+            if self._client is not None:
+                self._client.close()
+                self._client = None
+
+            # Stop UI if it was started
+            if self._ui and self._started:
+                logger.info(f"Stopping UI for profile '{self.profile}'...")
+                self._manager.stop_ui(self.profile, self._ui_port)
+
+            # Optionally stop daemon (daemon has idle timeout, so not required)
+            if stop_daemon_on_close and self._started:
+                logger.info(f"Stopping daemon for profile '{self.profile}'...")
+                self._manager.stop(self.profile)
+
+            self._closed = True
+
+    def close(self, stop_daemon: bool = False):
+        """
+        Explicitly close the client.
+
+        Args:
+            stop_daemon: If True, stops the daemon. Otherwise, daemon continues running
+                and will auto-stop after idle timeout (default: False).
+
+        Note:
+            The daemon may be shared with other clients or the CLI, so stopping it
+            might affect other users. By default, we rely on the daemon's idle timeout.
+        """
+        self._cleanup(stop_daemon_on_close=stop_daemon)
+
+    def __getattr__(self, name: str):
+        """
+        Proxy all method calls to the underlying Hindsight client.
+
+        This allows HindsightEmbedded to expose all HindsightClient methods
+        without manually wrapping each one.
+        """
+        # Ensure server is started before proxying
+        self._ensure_started()
+
+        # Get the attribute from the underlying client
+        attr = getattr(self._client, name)
+
+        # If it's a callable, wrap it to ensure server is started
+        # (shouldn't be needed since _ensure_started already called, but defensive)
+        if callable(attr):
+
+            def wrapper(*args, **kwargs):
+                self._ensure_started()
+                return attr(*args, **kwargs)
+
+            return wrapper
+
+        return attr
+
+    def __enter__(self):
+        """Context manager entry - ensures server is started."""
+        self._ensure_started()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit - stops the server."""
+        self.close()
+
+    def __del__(self):
+        """Cleanup on garbage collection."""
+        self._cleanup()
+
+    @property
+    def banks(self) -> BanksAPI:
+        """
+        Access bank management operations.
+
+        Each method call ensures the daemon is running before executing.
+
+        Example:
+            ```python
+            from hindsight import HindsightEmbedded
+
+            embedded = HindsightEmbedded(profile="myapp", ...)
+
+            # Create a bank
+            embedded.banks.create(bank_id="test", name="Test Bank")
+
+            # Set mission
+            embedded.banks.set_mission(bank_id="test", mission="Help users")
+            ```
+        """
+        if self._banks_api is None:
+            self._banks_api = BanksAPI(self)
+        return self._banks_api
+
+    @property
+    def mental_models(self) -> MentalModelsAPI:
+        """
+        Access mental model operations.
+
+        Each method call ensures the daemon is running before executing.
+
+        Example:
+            ```python
+            from hindsight import HindsightEmbedded
+
+            embedded = HindsightEmbedded(profile="myapp", ...)
+
+            # Create a mental model
+            embedded.mental_models.create(
+                bank_id="test",
+                name="User Preferences",
+                content="User prefers dark mode"
+            )
+
+            # List mental models
+            models = embedded.mental_models.list(bank_id="test")
+            ```
+        """
+        if self._mental_models_api is None:
+            self._mental_models_api = MentalModelsAPI(self)
+        return self._mental_models_api
+
+    @property
+    def directives(self) -> DirectivesAPI:
+        """
+        Access directive operations.
+
+        Each method call ensures the daemon is running before executing.
+
+        Example:
+            ```python
+            from hindsight import HindsightEmbedded
+
+            embedded = HindsightEmbedded(profile="myapp", ...)
+
+            # Create a directive
+            embedded.directives.create(
+                bank_id="test",
+                name="Response Style",
+                content="Always be concise and friendly"
+            )
+
+            # List directives
+            directives = embedded.directives.list(bank_id="test")
+            ```
+        """
+        if self._directives_api is None:
+            self._directives_api = DirectivesAPI(self)
+        return self._directives_api
+
+    @property
+    def memories(self) -> MemoriesAPI:
+        """
+        Access memory listing operations.
+
+        Each method call ensures the daemon is running before executing.
+
+        Example:
+            ```python
+            from hindsight import HindsightEmbedded
+
+            embedded = HindsightEmbedded(profile="myapp", ...)
+
+            # List memories
+            memories = embedded.memories.list(
+                bank_id="test",
+                type="world",
+                limit=50
+            )
+            ```
+        """
+        if self._memories_api is None:
+            self._memories_api = MemoriesAPI(self)
+        return self._memories_api
+
+    @property
+    def client(self) -> Hindsight:
+        """
+        Get the underlying Hindsight client for direct access.
+
+        WARNING: Using this property directly means daemon restarts won't be
+        handled automatically. Prefer using the API namespaces (banks, mental_models,
+        directives, memories) or direct method calls on HindsightEmbedded instead.
+
+        Ensures daemon is started before returning the client.
+
+        Returns:
+            Hindsight: The underlying client instance
+
+        Example:
+            ```python
+            from hindsight import HindsightEmbedded
+
+            embedded = HindsightEmbedded(profile="myapp", ...)
+
+            # Direct access (not recommended - daemon crashes won't be handled)
+            client = embedded.client
+            banks = client.list_banks()  # If daemon crashes, this will fail
+            ```
+        """
+        self._ensure_started()
+        return self._client
+
+    @property
+    def url(self) -> str:
+        """Get the daemon URL (starts daemon if needed)."""
+        self._ensure_started()
+        return self._manager.get_url(self.profile)
+
+    @property
+    def is_running(self) -> bool:
+        """Check if the client is initialized."""
+        return self._started and not self._closed and self._client is not None
+
+    @property
+    def ui_url(self) -> str:
+        """Get the UI URL for this profile."""
+        return self._manager.get_ui_url(self.profile)

hindsight-all/hindsight/server.py

@@ -0,0 +1,280 @@
+"""
+Server module for running Hindsight in a background thread.
+
+Provides a simple way to start and stop the Hindsight HTTP API server
+without blocking the main thread.
+"""
+import asyncio
+import logging
+import socket
+import threading
+import time
+from typing import Optional
+
+import uvicorn
+from uvicorn import Config
+
+from hindsight_api import MemoryEngine
+from hindsight_api.api import create_app
+
+logger = logging.getLogger(__name__)
+
+
+def _find_free_port() -> int:
+    """Find a free port on localhost."""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("", 0))
+        s.listen(1)
+        port = s.getsockname()[1]
+    return port
+
+
+class Server:
+    """
+    Hindsight server that runs in a background thread.
+
+    Example:
+        ```python
+        from hindsight import Server
+
+        server = Server(
+            db_url="pg0",
+            llm_provider="groq",
+            llm_api_key="your-api-key",
+            llm_model="openai/gpt-oss-120b"
+        )
+        server.start()
+
+        print(f"Server running at {server.url}")
+
+        # Use the server...
+
+        server.stop()
+        ```
+    """
+
+    def __init__(
+        self,
+        db_url: str = "pg0",
+        llm_provider: str = "groq",
+        llm_api_key: str = "",
+        llm_model: str = "openai/gpt-oss-120b",
+        llm_base_url: Optional[str] = None,
+        host: str = "127.0.0.1",
+        port: Optional[int] = None,
+        mcp_enabled: bool = False,
+        log_level: str = "info",
+    ):
+        """
+        Initialize the Hindsight server.
+
+        Args:
+            db_url: Database URL. Use "pg0" for embedded PostgreSQL.
+            llm_provider: LLM provider ("groq", "openai", "ollama", "gemini", "anthropic", "lmstudio")
+            llm_api_key: API key for the LLM provider
+            llm_model: Model name to use
+            llm_base_url: Optional custom base URL for LLM API
+            host: Host to bind to (default: 127.0.0.1)
+            port: Port to bind to (default: auto-select free port)
+            mcp_enabled: Whether to enable MCP server
+            log_level: Uvicorn log level (default: warning)
+        """
+        self.db_url = db_url
+        self.llm_provider = llm_provider
+        self.llm_api_key = llm_api_key
+        self.llm_model = llm_model
+        self.llm_base_url = llm_base_url
+        self.host = host
+        self.port = port or _find_free_port()
+        self.mcp_enabled = mcp_enabled
+        self.log_level = log_level
+
+        self._memory: Optional[MemoryEngine] = None
+        self._server: Optional[uvicorn.Server] = None
+        self._thread: Optional[threading.Thread] = None
+        self._started = threading.Event()
+        self._stopped = threading.Event()
+
+    @property
+    def url(self) -> str:
+        """Get the server URL."""
+        return f"http://{self.host}:{self.port}"
+
+    def _run_server(self):
+        """Run the server in a background thread."""
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        try:
+            # Create MemoryEngine
+            self._memory = MemoryEngine(
+                db_url=self.db_url,
+                memory_llm_provider=self.llm_provider,
+                memory_llm_api_key=self.llm_api_key,
+                memory_llm_model=self.llm_model,
+                memory_llm_base_url=self.llm_base_url,
+            )
+
+            # Create FastAPI app
+            app = create_app(
+                memory=self._memory,
+                mcp_api_enabled=self.mcp_enabled,
+                initialize_memory=True,
+            )
+
+            # Create uvicorn config and server
+            config = Config(
+                app=app,
+                host=self.host,
+                port=self.port,
+                log_level=self.log_level,
+                loop="asyncio",
+            )
+            self._server = uvicorn.Server(config)
+
+            # Signal that we're starting
+            self._started.set()
+
+            # Run the server
+            loop.run_until_complete(self._server.serve())
+
+        except Exception as e:
+            logger.error(f"Server error: {e}")
+            raise
+        finally:
+            # Cleanup
+            if self._memory:
+                loop.run_until_complete(self._memory.close())
+            loop.close()
+            self._stopped.set()
+
+    def start(self, timeout: float = 30.0) -> "Server":
+        """
+        Start the server in a background thread.
+
+        Args:
+            timeout: Maximum time to wait for server to start (seconds)
+
+        Returns:
+            self (for chaining)
+
+        Raises:
+            RuntimeError: If server fails to start within timeout
+        """
+        if self._thread is not None and self._thread.is_alive():
+            raise RuntimeError("Server is already running")
+
+        self._started.clear()
+        self._stopped.clear()
+
+        self._thread = threading.Thread(target=self._run_server, daemon=True)
+        self._thread.start()
+
+        # Wait for server to start
+        self._started.wait(timeout=timeout)
+
+        # Give uvicorn a moment to actually bind to the port
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            try:
+                with socket.create_connection((self.host, self.port), timeout=1):
+                    logger.info(f"Hindsight server started at {self.url}")
+                    return self
+            except (ConnectionRefusedError, socket.timeout, OSError):
+                time.sleep(0.1)
+
+        raise RuntimeError(f"Server failed to start within {timeout} seconds")
+
+    def stop(self, timeout: float = 10.0) -> None:
+        """
+        Stop the server.
+
+        Args:
+            timeout: Maximum time to wait for server to stop (seconds)
+        """
+        if self._server is None:
+            return
+
+        # Signal uvicorn to shutdown
+        self._server.should_exit = True
+
+        # Wait for thread to finish
+        if self._thread is not None:
+            self._thread.join(timeout=timeout)
+            if self._thread.is_alive():
+                logger.warning("Server thread did not stop cleanly")
+
+        self._server = None
+        self._thread = None
+        logger.info("Hindsight server stopped")
+
+    def __enter__(self) -> "Server":
+        """Context manager entry."""
+        return self.start()
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        """Context manager exit."""
+        self.stop()
+
+
+def start_server(
+    db_url: str = "pg0",
+    llm_provider: str = "groq",
+    llm_api_key: str = "",
+    llm_model: str = "openai/gpt-oss-120b",
+    llm_base_url: Optional[str] = None,
+    host: str = "127.0.0.1",
+    port: Optional[int] = None,
+    mcp_enabled: bool = False,
+    log_level: str = "warning",
+    timeout: float = 30.0,
+) -> Server:
+    """
+    Start a Hindsight server in a background thread.
+
+    This is a convenience function that creates and starts a Server instance.
+
+    Args:
+        db_url: Database URL. Use "pg0" for embedded PostgreSQL.
+        llm_provider: LLM provider ("groq", "openai", "ollama", "gemini", "anthropic", "lmstudio")
+        llm_api_key: API key for the LLM provider
+        llm_model: Model name to use
+        llm_base_url: Optional custom base URL for LLM API
+        host: Host to bind to (default: 127.0.0.1)
+        port: Port to bind to (default: auto-select free port)
+        mcp_enabled: Whether to enable MCP server
+        log_level: Uvicorn log level (default: warning)
+        timeout: Maximum time to wait for server to start (seconds)
+
+    Returns:
+        Running Server instance
+
+    Example:
+        ```python
+        from hindsight import start_server, Client
+
+        server = start_server(
+            db_url="pg0",
+            llm_provider="groq",
+            llm_api_key="your-api-key",
+            llm_model="openai/gpt-oss-120b"
+        )
+
+        client = Client(base_url=server.url)
+        client.put(agent_id="assistant", content="User likes Python")
+
+        server.stop()
+        ```
+    """
+    server = Server(
+        db_url=db_url,
+        llm_provider=llm_provider,
+        llm_api_key=llm_api_key,
+        llm_model=llm_model,
+        llm_base_url=llm_base_url,
+        host=host,
+        port=port,
+        mcp_enabled=mcp_enabled,
+        log_level=log_level,
+    )
+    return server.start(timeout=timeout)

hindsight-all/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "hindsight-all"
+version = "0.4.22"
+description = "Hindsight: Agent Memory That Works Like Human Memory - All-in-One Bundle"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "hindsight-api-slim[all]>=0.4.17",
+    "hindsight-client>=0.0.7",
+    "hindsight-embed>=0.1.0",
+]
+
+[tool.uv.sources]
+hindsight-api-slim = { workspace = true }
+hindsight-client = { workspace = true }
+hindsight-embed = { workspace = true }
+
+[project.optional-dependencies]
+test = [
+    "pytest>=7.0.0",
+    "pytest-asyncio>=0.21.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["hindsight"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"

hindsight-all/tests/test_embedded.py

@@ -0,0 +1,403 @@
+"""
+Integration tests for HindsightEmbedded client.
+
+Tests the embedded client with automatic server lifecycle management:
+1. Lazy server startup on first use
+2. Server reuse across multiple operations
+3. Context manager support
+4. Method proxying to underlying HindsightClient
+5. Proper cleanup
+
+Note: Each test uses random bank_ids to avoid conflicts and allow safe parallel execution.
+"""
+
+import os
+import uuid
+
+import pytest
+import urllib.request
+import json
+
+from hindsight import HindsightEmbedded
+
+
+@pytest.fixture(scope="session")
+def llm_config():
+    """Get LLM configuration from environment (session-scoped)."""
+    # Try both naming conventions
+    provider = os.getenv("HINDSIGHT_API_LLM_PROVIDER") or os.getenv(
+        "HINDSIGHT_LLM_PROVIDER", "groq"
+    )
+    api_key = os.getenv("HINDSIGHT_API_LLM_API_KEY") or os.getenv(
+        "HINDSIGHT_LLM_API_KEY", ""
+    )
+    model = os.getenv("HINDSIGHT_API_LLM_MODEL") or os.getenv(
+        "HINDSIGHT_LLM_MODEL", "openai/gpt-oss-120b"
+    )
+
+    if not api_key:
+        pytest.skip(
+            "LLM API key not configured. Set HINDSIGHT_API_LLM_API_KEY or HINDSIGHT_LLM_API_KEY."
+        )
+
+    return {
+        "llm_provider": provider,
+        "llm_api_key": api_key,
+        "llm_model": model,
+    }
+
+
+def test_embedded_lazy_start(llm_config):
+    """
+    Test that HindsightEmbedded starts server lazily on first use.
+    """
+    profile = f"test_lazy_{uuid.uuid4().hex[:8]}"
+    bank_id = f"bank_{uuid.uuid4().hex[:8]}"
+
+    # Create client - should NOT start server yet
+    client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
+    assert not client.is_running, "Server should not be running after initialization"
+
+    # First call should start server
+    result = client.retain(bank_id=bank_id, content="Test content for lazy start")
+
+    # Verify server is now running
+    assert client.is_running, "Server should be running after first call"
+    assert result.success, "Retain should succeed"
+    assert result.items_count >= 1, "Should have stored at least 1 item"
+
+    # Cleanup
+    client.close()
+    assert not client.is_running, "Server should stop after close()"
+
+
+def test_embedded_context_manager(llm_config):
+    """
+    Test HindsightEmbedded with context manager.
+    """
+    profile = f"test_ctx_{uuid.uuid4().hex[:8]}"
+    bank_id = f"bank_{uuid.uuid4().hex[:8]}"
+
+    # Use context manager
+    with HindsightEmbedded(profile=profile, log_level="info", **llm_config) as client:
+        assert client.is_running, "Server should be running inside context"
+
+        # Store memory
+        result = client.retain(bank_id=bank_id, content="Testing context manager")
+        assert result.success, "Retain should succeed"
+
+        # Recall memory
+        recall_results = client.recall(bank_id=bank_id, query="context")
+        assert isinstance(recall_results.results, list), (
+            "Recall should return results list"
+        )
+
+    # Server should be stopped after context exit
+    # Note: We can't check client.is_running here as client is out of scope
+
+
+def test_embedded_complete_workflow(llm_config):
+    """
+    Test complete workflow with HindsightEmbedded.
+
+    This test:
+    1. Creates a client with lazy start
+    2. Creates a memory bank
+    3. Stores multiple memories
+    4. Recalls memories
+    5. Reflects on memories
+    6. Tests cleanup
+    """
+    profile = f"test_workflow_{uuid.uuid4().hex[:8]}"
+    bank_id = f"assistant_{uuid.uuid4().hex[:8]}"
+
+    client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
+
+    try:
+        # Step 1: Create a memory bank
+        print(f"\n1. Creating memory bank: {bank_id}")
+        bank_response = client.create_bank(
+            bank_id=bank_id,
+            name="Test Assistant",
+            mission="Help with programming tasks",
+        )
+        assert bank_response.bank_id == bank_id
+
+        # Step 2: Store memories (single)
+        print("\n2. Storing single memory...")
+        retain_response = client.retain(
+            bank_id=bank_id,
+            content="User prefers Python for data analysis.",
+            context="Programming preferences",
+        )
+        assert retain_response.success
+        assert retain_response.items_count >= 1
+
+        # Step 3: Store batch memories
+        print("\n3. Storing batch memories...")
+        batch_response = client.retain_batch(
+            bank_id=bank_id,
+            items=[
+                {"content": "User works with pandas and numpy."},
+                {"content": "User likes matplotlib for visualization."},
+                {
+                    "content": "User is interested in machine learning with scikit-learn."
+                },
+            ],
+        )
+        assert batch_response.success
+        assert batch_response.items_count >= 3
+
+        # Step 4: Recall memories
+        print("\n4. Recalling memories...")
+        recall_response = client.recall(
+            bank_id=bank_id, query="What tools does the user prefer?", max_tokens=2000
+        )
+        assert isinstance(recall_response.results, list)
+        assert len(recall_response.results) > 0
+        print(f"   Found {len(recall_response.results)} relevant memories")
+
+        # Step 5: Reflect on memories
+        print("\n5. Reflecting on memories...")
+        reflect_response = client.reflect(
+            bank_id=bank_id,
+            query="What programming tools should I recommend?",
+            budget="low",
+        )
+        assert reflect_response.text
+        assert len(reflect_response.text) > 0
+        print(f"   Answer: {reflect_response.text[:150]}...")
+
+        # Verify answer mentions relevant tools
+        answer_lower = reflect_response.text.lower()
+        assert any(
+            term in answer_lower for term in ["python", "pandas", "numpy", "data"]
+        )
+
+        # Step 6: List memories
+        print("\n6. Listing memories...")
+        list_response = client.list_memories(bank_id=bank_id, limit=10)
+        assert len(list_response.items) > 0
+        print(f"   Listed {len(list_response.items)} memories")
+
+    finally:
+        # Cleanup
+        client.close()
+
+
+def test_embedded_server_reuse(llm_config):
+    """
+    Test that the same server is reused across multiple calls.
+    """
+    profile = f"test_reuse_{uuid.uuid4().hex[:8]}"
+    bank_id = f"bank_{uuid.uuid4().hex[:8]}"
+
+    client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
+
+    try:
+        # First call starts server
+        result1 = client.retain(bank_id=bank_id, content="First message")
+        url1 = client.url
+        assert client.is_running
+
+        # Second call should reuse the same server
+        result2 = client.retain(bank_id=bank_id, content="Second message")
+        url2 = client.url
+
+        # URLs should be identical (same server)
+        assert url1 == url2, "Server URL should remain the same across calls"
+        assert result1.success and result2.success
+
+        # Third call should also reuse
+        recall_result = client.recall(bank_id=bank_id, query="message")
+        url3 = client.url
+        assert url3 == url1, "Server URL should remain the same for recall"
+        assert isinstance(recall_result.results, list)
+
+    finally:
+        client.close()
+
+
+def test_embedded_method_proxying(llm_config):
+    """
+    Test that all HindsightClient methods are properly proxied.
+
+    This ensures __getattr__ proxying works for various method types.
+    """
+    profile = f"test_proxy_{uuid.uuid4().hex[:8]}"
+    bank_id = f"bank_{uuid.uuid4().hex[:8]}"
+
+    client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
+
+    try:
+        # Test bank operations
+        bank = client.create_bank(bank_id=bank_id, name="Proxy Test")
+        assert bank.bank_id == bank_id
+
+        # Test mission setting
+        mission_response = client.set_mission(
+            bank_id=bank_id, mission="Test mission for proxying"
+        )
+        assert mission_response.bank_id == bank_id
+
+        # Test retain
+        retain_result = client.retain(bank_id=bank_id, content="Test content")
+        assert retain_result.success
+
+        # Test retain_batch
+        batch_result = client.retain_batch(
+            bank_id=bank_id, items=[{"content": "Item 1"}, {"content": "Item 2"}]
+        )
+        assert batch_result.success
+        assert batch_result.items_count >= 2
+
+        # Test recall
+        recall_result = client.recall(bank_id=bank_id, query="test")
+        assert hasattr(recall_result, "results")
+
+        # Test reflect
+        reflect_result = client.reflect(bank_id=bank_id, query="What is stored?")
+        assert hasattr(reflect_result, "text")
+
+        # Test list_memories
+        list_result = client.list_memories(bank_id=bank_id, limit=5)
+        assert hasattr(list_result, "items")
+
+        print("✓ All methods successfully proxied")
+
+    finally:
+        client.close()
+
+
+def test_embedded_multiple_banks(llm_config):
+    """
+    Test that HindsightEmbedded can work with multiple banks.
+    """
+    profile = f"test_multibank_{uuid.uuid4().hex[:8]}"
+    bank1_id = f"bank1_{uuid.uuid4().hex[:8]}"
+    bank2_id = f"bank2_{uuid.uuid4().hex[:8]}"
+
+    client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
+
+    try:
+        # Create first bank and store data
+        client.create_bank(bank_id=bank1_id, name="Bank 1")
+        client.retain(bank_id=bank1_id, content="Alice prefers Python for data science")
+
+        # Create second bank and store data
+        client.create_bank(bank_id=bank2_id, name="Bank 2")
+        client.retain(
+            bank_id=bank2_id, content="Bob uses JavaScript for web development"
+        )
+
+        # Recall from both banks
+        results1 = client.recall(bank_id=bank1_id, query="programming language")
+        results2 = client.recall(bank_id=bank2_id, query="programming language")
+
+        assert len(results1.results) > 0
+        assert len(results2.results) > 0
+
+        # Verify banks are isolated (each should only see their own content)
+        # This is a basic check - content isolation is tested more thoroughly in other tests
+        assert results1.results[0].text != results2.results[0].text or len(
+            results1.results
+        ) != len(results2.results)
+
+    finally:
+        client.close()
+
+
+def test_embedded_profile_isolation(llm_config):
+    """
+    Test that different profiles create isolated data stores.
+    """
+    profile1 = f"test_iso1_{uuid.uuid4().hex[:8]}"
+    profile2 = f"test_iso2_{uuid.uuid4().hex[:8]}"
+    bank_id = "shared_bank_name"  # Same bank_id in both profiles
+
+    client1 = HindsightEmbedded(profile=profile1, log_level="info", **llm_config)
+    client2 = HindsightEmbedded(profile=profile2, log_level="info", **llm_config)
+
+    try:
+        # Store data in profile1
+        client1.retain(
+            bank_id=bank_id, content="User likes TypeScript for frontend development"
+        )
+
+        # Store different data in profile2
+        client2.retain(
+            bank_id=bank_id, content="User prefers Rust for systems programming"
+        )
+
+        # Each profile should only see its own data
+        results1 = client1.recall(bank_id=bank_id, query="programming preference")
+        results2 = client2.recall(bank_id=bank_id, query="programming preference")
+
+        # Both should have results
+        assert len(results1.results) > 0
+        assert len(results2.results) > 0
+
+        # Results should be different (basic isolation check)
+        # Note: This is a basic sanity check. Full isolation is ensured by pg0's data directory separation
+
+    finally:
+        client1.close()
+        client2.close()
+
+
+def test_embedded_error_after_close(llm_config):
+    """
+    Test that using HindsightEmbedded after close() raises an error.
+    """
+    profile = f"test_error_{uuid.uuid4().hex[:8]}"
+    bank_id = f"bank_{uuid.uuid4().hex[:8]}"
+
+    client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
+
+    # Use it once to start server
+    client.retain(bank_id=bank_id, content="Test")
+
+    # Close the client
+    client.close()
+    assert not client.is_running
+
+    # Trying to use it after close should raise an error
+    with pytest.raises(
+        RuntimeError, match="Cannot use HindsightEmbedded after it has been closed"
+    ):
+        client.retain(bank_id=bank_id, content="This should fail")
+
+
+def test_embedded_ui_flag(llm_config):
+    """
+    Test that ui=True starts the control plane UI alongside the daemon,
+    and that the UI's health endpoint reports a connected dataplane.
+    """
+    profile = f"test_ui_{uuid.uuid4().hex[:8]}"
+    bank_id = f"bank_{uuid.uuid4().hex[:8]}"
+
+    client = HindsightEmbedded(profile=profile, log_level="info", ui=True, **llm_config)
+
+    try:
+        # First use triggers daemon + UI startup
+        result = client.retain(bank_id=bank_id, content="UI integration test content")
+        assert result.success, "Retain should succeed"
+        assert client.is_running, "Daemon should be running"
+
+        # Verify UI is reachable and reports connected dataplane
+        ui_url = client.ui_url
+        assert ui_url, "ui_url should be set"
+
+        health_url = f"{ui_url}/api/health"
+        with urllib.request.urlopen(health_url, timeout=10) as resp:
+            health = json.loads(resp.read().decode())
+
+        assert health["status"] == "ok", (
+            f"UI health status should be 'ok', got: {health['status']}"
+        )
+        assert health["dataplane"]["status"] == "connected", (
+            f"Dataplane should be connected, got: {health['dataplane']}"
+        )
+
+    finally:
+        client.close()

hindsight-all/tests/test_embedded_namespaces.py

@@ -0,0 +1,170 @@
+"""Test that API namespaces ensure daemon is started before each call."""
+
+from unittest.mock import Mock, patch
+
+import pytest
+
+from hindsight import HindsightEmbedded
+
+
+@pytest.fixture
+def embedded_client():
+    """Create an embedded client for testing."""
+    return HindsightEmbedded(
+        profile="test",
+        llm_provider="openai",
+        llm_api_key="test-key",
+    )
+
+
+def test_banks_create_ensures_daemon_started(embedded_client):
+    """Test that banks.create() calls _ensure_started()."""
+    # Mock _ensure_started to track calls
+    with patch.object(embedded_client, "_ensure_started") as mock_ensure:
+        # Mock the underlying client to avoid actual API call
+        mock_client = Mock()
+        embedded_client._client = mock_client
+
+        # Call namespace method
+        try:
+            embedded_client.banks.create(bank_id="test", name="Test Bank")
+        except Exception:
+            pass  # We don't care if the actual call fails
+
+        # Verify _ensure_started was called
+        mock_ensure.assert_called_once()
+
+
+def test_mental_models_list_ensures_daemon_started(embedded_client):
+    """Test that mental_models.list() calls _ensure_started()."""
+    with patch.object(embedded_client, "_ensure_started") as mock_ensure:
+        mock_client = Mock()
+        embedded_client._client = mock_client
+
+        try:
+            embedded_client.mental_models.list(bank_id="test")
+        except Exception:
+            pass
+
+        mock_ensure.assert_called_once()
+
+
+def test_directives_list_ensures_daemon_started(embedded_client):
+    """Test that directives.list() calls _ensure_started()."""
+    with patch.object(embedded_client, "_ensure_started") as mock_ensure:
+        mock_client = Mock()
+        embedded_client._client = mock_client
+
+        try:
+            embedded_client.directives.list(bank_id="test")
+        except Exception:
+            pass
+
+        mock_ensure.assert_called_once()
+
+
+def test_memories_list_ensures_daemon_started(embedded_client):
+    """Test that memories.list() calls _ensure_started()."""
+    with patch.object(embedded_client, "_ensure_started") as mock_ensure:
+        mock_client = Mock()
+        embedded_client._client = mock_client
+
+        try:
+            embedded_client.memories.list(bank_id="test")
+        except Exception:
+            pass
+
+        mock_ensure.assert_called_once()
+
+
+def test_multiple_calls_ensure_daemon_each_time(embedded_client):
+    """Test that each namespace call ensures daemon is started."""
+    with patch.object(embedded_client, "_ensure_started") as mock_ensure:
+        mock_client = Mock()
+        embedded_client._client = mock_client
+
+        # Make multiple calls
+        try:
+            embedded_client.banks.create(bank_id="test", name="Test")
+        except Exception:
+            pass
+
+        try:
+            embedded_client.mental_models.list(bank_id="test")
+        except Exception:
+            pass
+
+        try:
+            embedded_client.directives.list(bank_id="test")
+        except Exception:
+            pass
+
+        # Should be called 3 times (once per namespace method call)
+        assert mock_ensure.call_count == 3
+
+
+def test_daemon_restart_handling(embedded_client):
+    """Test that namespace methods can recover from daemon crash."""
+    call_count = 0
+
+    def mock_ensure_started():
+        """Mock that simulates daemon restart."""
+        nonlocal call_count
+        call_count += 1
+        # Create a new mock client each time (simulating daemon restart)
+        embedded_client._client = Mock()
+        embedded_client._started = True
+
+    with patch.object(embedded_client, "_ensure_started", side_effect=mock_ensure_started):
+        # First call - daemon starts
+        embedded_client.banks.create(bank_id="test", name="Test")
+        assert call_count == 1
+
+        # Simulate daemon crash by clearing client
+        embedded_client._client = None
+        embedded_client._started = False
+
+        # Second call - daemon restarts
+        embedded_client.banks.create(bank_id="test", name="Test")
+        assert call_count == 2
+
+
+def test_ensure_started_calls_manager(embedded_client):
+    """Test that _ensure_started actually starts the daemon via manager."""
+    # Mock the manager
+    mock_manager = Mock()
+    mock_manager.ensure_running.return_value = True
+    mock_manager.get_url.return_value = "http://localhost:54321"
+
+    embedded_client._manager = mock_manager
+
+    # Mock Hindsight client constructor
+    with patch("hindsight.embedded.Hindsight") as mock_hindsight_class:
+        mock_client = Mock()
+        mock_hindsight_class.return_value = mock_client
+
+        # Call _ensure_started
+        embedded_client._ensure_started()
+
+        # Verify manager was called
+        mock_manager.ensure_running.assert_called_once_with(
+            embedded_client.config, embedded_client.profile
+        )
+        mock_manager.get_url.assert_called_once_with(embedded_client.profile)
+
+        # Verify Hindsight client was created
+        mock_hindsight_class.assert_called_once_with(base_url="http://localhost:54321")
+
+
+def test_namespace_singleton_behavior(embedded_client):
+    """Test that namespace properties return the same instance."""
+    banks1 = embedded_client.banks
+    banks2 = embedded_client.banks
+
+    # Should be the same instance
+    assert banks1 is banks2
+
+    # Same for other namespaces
+    assert embedded_client.mental_models is embedded_client.mental_models
+    assert embedded_client.directives is embedded_client.directives
+    assert embedded_client.memories is embedded_client.memories

hindsight-all/tests/test_server_integration.py

@@ -0,0 +1,272 @@
+"""
+Integration test for Hindsight server with context manager.
+
+Tests the full workflow:
+1. Starting server using context manager
+2. Creating a memory bank
+3. Storing memories (retain)
+4. Recalling memories
+5. Reflecting on memories
+
+Note: These tests use embedded PostgreSQL (pg0) with a shared server instance
+across all tests. Each test uses random bank_ids to avoid conflicts, allowing
+safe parallel execution.
+"""
+
+import os
+import uuid
+import pytest
+from hindsight import HindsightServer, HindsightClient
+
+
+@pytest.fixture(scope="session")
+def llm_config():
+    """Get LLM configuration from environment (session-scoped)."""
+    provider = os.getenv("HINDSIGHT_LLM_PROVIDER", "groq")
+    api_key = os.getenv("HINDSIGHT_LLM_API_KEY", "")
+    model = os.getenv("HINDSIGHT_LLM_MODEL", "openai/gpt-oss-120b")
+
+    # vertexai uses GCP service account credentials (HINDSIGHT_API_LLM_VERTEXAI_*),
+    # not a traditional API key
+    providers_without_api_key = ("vertexai", "ollama")
+    if not api_key and provider not in providers_without_api_key:
+        raise Exception("LLM API key not configured. Set HINDSIGHT_LLM_API_KEY environment variable.")
+
+    return {
+        "llm_provider": provider,
+        "llm_api_key": api_key,
+        "llm_model": model,
+    }
+
+
+@pytest.fixture(scope="session")
+def shared_server(llm_config):
+    """
+    Shared server instance for all tests (session-scoped).
+
+    This allows tests to run in parallel by sharing the same pg0 instance,
+    while using different bank_ids to avoid data conflicts.
+    """
+    server = HindsightServer(db_url="pg0", **llm_config)
+    server.start()
+    yield server
+    server.stop()
+
+
+@pytest.fixture
+def client(shared_server):
+    """Create a client connected to the shared server."""
+    return HindsightClient(base_url=shared_server.url)
+
+
+def test_server_context_manager_basic_workflow(client):
+    """
+    Test complete workflow using shared server.
+
+    This test:
+    1. Uses a shared server instance
+    2. Creates a memory bank with unique ID
+    3. Stores multiple memories
+    4. Recalls memories based on a query
+    5. Reflects (generates contextual answers) based on stored memories
+    """
+    # Use random bank_id to allow parallel test execution
+    bank_id = f"test_assistant_{uuid.uuid4().hex[:8]}"
+
+    # Step 1: Create a memory bank with background information
+    print(f"\n1. Creating memory bank: {bank_id}")
+    bank_response = client.create_bank(
+        bank_id=bank_id,
+        name="Test Assistant",
+        mission="An AI assistant that helps with programming and data analysis tasks."
+    )
+    assert bank_response.bank_id == bank_id
+
+    # Step 2: Store some memories about user preferences
+    print("\n2. Storing memories...")
+
+    # Store first memory
+    retain_response1 = client.retain(
+        bank_id=bank_id,
+        content="User prefers Python over JavaScript for data analysis projects.",
+        context="User conversation about programming languages"
+    )
+    assert retain_response1.success is True
+
+    # Store second memory
+    retain_response2 = client.retain(
+        bank_id=bank_id,
+        content="User is working on a machine learning project using scikit-learn.",
+        context="Discussion about ML frameworks"
+    )
+    assert retain_response2.success is True
+
+    # Store third memory
+    retain_response3 = client.retain(
+        bank_id=bank_id,
+        content="User likes visualizing data with matplotlib and seaborn.",
+        context="Conversation about data visualization"
+    )
+    assert retain_response3.success is True
+
+    # Store batch memories
+    batch_response = client.retain_batch(
+        bank_id=bank_id,
+        items=[
+            {"content": "User is interested in neural networks and deep learning."},
+            {"content": "User asked about best practices for training models."},
+        ]
+    )
+    # Check if the batch was submitted successfully (items_count shows how many were submitted)
+    assert batch_response.items_count >= 2
+
+    # Step 3: Recall memories based on a query
+    print("\n3. Recalling memories about programming preferences...")
+    recall_results = client.recall(
+        bank_id=bank_id,
+        query="What programming languages and tools does the user prefer?",
+        max_tokens=4096
+    )
+
+    # Verify recall results
+    assert isinstance(recall_results.results, list)
+    assert len(recall_results.results) > 0
+    print(f"   Found {len(recall_results.results)} relevant memories")
+
+    # Check that results have expected structure
+    for result in recall_results.results:
+        print(f"   - {result.text[:100]}")
+
+    # Step 4: Recall memories about machine learning
+    print("\n4. Recalling memories about machine learning...")
+    ml_recall_results = client.recall(
+        bank_id=bank_id,
+        query="machine learning and neural networks",
+        max_tokens=4096
+    )
+
+    # Verify recall results
+    assert isinstance(ml_recall_results.results, list)
+    assert len(ml_recall_results.results) > 0
+    print(f"   Found {len(ml_recall_results.results)} ML-related memories")
+    for result in ml_recall_results.results[:3]:  # Show first 3
+        print(f"   - {result.text[:100]}")
+
+    # Step 5: Reflect (generate contextual answer based on memories)
+    print("\n5. Reflecting on query about recommendations...")
+    reflect_response = client.reflect(
+        bank_id=bank_id,
+        query="What tools and libraries should I recommend for this user's data analysis work?",
+        budget="mid"
+    )
+
+    # Verify reflection response
+    answer = reflect_response.text
+    assert len(answer) > 0
+    print(f"   Answer: {answer[:200]}...")
+
+    # Verify the answer mentions relevant tools/libraries
+    answer_lower = answer.lower()
+    assert any(term in answer_lower for term in ["python", "scikit-learn", "matplotlib", "seaborn", "data"])
+
+    # Step 6: Another reflection with different context
+    print("\n6. Reflecting with additional context...")
+    reflect_with_context = client.reflect(
+        bank_id=bank_id,
+        query="Should I use TensorFlow or PyTorch?",
+        budget="low",
+        context="The user is starting a new deep learning project"
+    )
+
+    context_answer = reflect_with_context.text
+    assert len(context_answer) > 0
+    print(f"   Context-aware answer: {context_answer[:150]}...")
+
+
+def test_server_manual_start_stop(client):
+    """
+    Test basic operations on shared server.
+
+    Verifies that basic bank operations work correctly.
+    """
+    # Use random bank_id to allow parallel test execution
+    bank_id = f"test_manual_{uuid.uuid4().hex[:8]}"
+
+    # Create bank
+    bank_response = client.create_bank(
+        bank_id=bank_id,
+        name="Manual Test"
+    )
+    assert bank_response.bank_id == bank_id
+
+    # Store a memory
+    retain_response = client.retain(
+        bank_id=bank_id,
+        content="Testing manual server lifecycle."
+    )
+    assert retain_response.success is True
+
+    # Recall the memory
+    recall_results = client.recall(
+        bank_id=bank_id,
+        query="server testing"
+    )
+    assert len(recall_results.results) >= 0  # May or may not find results immediately
+
+
+def test_server_with_client_context_manager(client):
+    """
+    Test client context manager with shared server.
+    """
+    # Use random bank_id to allow parallel test execution
+    bank_id = f"test_nested_context_{uuid.uuid4().hex[:8]}"
+
+    # Use client context manager (client fixture already provides this)
+    # Create bank
+    client.create_bank(bank_id=bank_id, name="Nested Context Test")
+
+    # Store memory
+    response = client.retain(
+        bank_id=bank_id,
+        content="Testing nested context managers."
+    )
+    assert response.success is True
+
+    # Verify we can recall
+    results = client.recall(bank_id=bank_id, query="context")
+    assert isinstance(results.results, list)
+
+
+def test_list_banks(client, shared_server):
+    """
+    Test listing banks to verify bank_id field mapping.
+
+    This test verifies that the list_banks endpoint correctly returns
+    bank_id (not agent_id) in the response.
+    """
+    # Create a couple of banks with random IDs to allow parallel test execution
+    test_suffix = uuid.uuid4().hex[:8]
+    bank1_id = f"test_bank_1_{test_suffix}"
+    bank2_id = f"test_bank_2_{test_suffix}"
+
+    client.create_bank(bank_id=bank1_id, name="Test Bank 1", mission="First test bank")
+    client.create_bank(bank_id=bank2_id, name="Test Bank 2", mission="Second test bank")
+
+    # List all banks using the namespace API
+    response = client.banks.list()
+
+    # Verify response structure
+    assert hasattr(response, 'banks'), "Response should have 'banks' attribute"
+    assert len(response.banks) >= 2, f"Should have at least 2 banks, got {len(response.banks)}"
+
+    # Verify each bank has bank_id (not agent_id)
+    for bank in response.banks:
+        assert hasattr(bank, 'bank_id'), f"Bank should have 'bank_id' attribute"
+        assert bank.bank_id is not None, "Bank ID should not be None"
+
+    # Find our test banks
+    bank_ids = [b.bank_id if hasattr(b, 'bank_id') else b['bank_id'] for b in response.banks]
+    assert bank1_id in bank_ids, f"Should find {bank1_id} in bank list"
+    assert bank2_id in bank_ids, f"Should find {bank2_id} in bank list"
+
+    print(f"✓ Successfully listed {len(response.banks)} banks with correct bank_id field")

hindsight-api-slim/README.md

@@ -0,0 +1,137 @@
+# Hindsight API
+
+**Memory System for AI Agents** — Temporal + Semantic + Entity Memory Architecture using PostgreSQL with pgvector.
+
+Hindsight gives AI agents persistent memory that works like human memory: it stores facts, tracks entities and relationships, handles temporal reasoning ("what happened last spring?"), and forms opinions based on configurable disposition traits.
+
+## Installation
+
+```bash
+pip install hindsight-api
+```
+
+## Quick Start
+
+### Run the Server
+
+```bash
+# Set your LLM provider
+export HINDSIGHT_API_LLM_PROVIDER=openai
+export HINDSIGHT_API_LLM_API_KEY=sk-xxxxxxxxxxxx
+
+# Start the server (uses embedded PostgreSQL by default)
+hindsight-api
+```
+
+The server starts at http://localhost:8888 with:
+- REST API for memory operations
+- MCP server at `/mcp` for tool-use integration
+
+### Use the Python API
+
+```python
+from hindsight_api import MemoryEngine
+
+# Create and initialize the memory engine
+memory = MemoryEngine()
+await memory.initialize()
+
+# Create a memory bank for your agent
+bank = await memory.create_memory_bank(
+    name="my-assistant",
+    background="A helpful coding assistant"
+)
+
+# Store a memory
+await memory.retain(
+    memory_bank_id=bank.id,
+    content="The user prefers Python for data science projects"
+)
+
+# Recall memories
+results = await memory.recall(
+    memory_bank_id=bank.id,
+    query="What programming language does the user prefer?"
+)
+
+# Reflect with reasoning
+response = await memory.reflect(
+    memory_bank_id=bank.id,
+    query="Should I recommend Python or R for this ML project?"
+)
+```
+
+## CLI Options
+
+```bash
+hindsight-api --help
+
+# Common options
+hindsight-api --port 9000          # Custom port (default: 8888)
+hindsight-api --host 127.0.0.1     # Bind to localhost only
+hindsight-api --workers 4          # Multiple worker processes
+hindsight-api --log-level debug    # Verbose logging
+```
+
+## Configuration
+
+Configure via environment variables:
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `HINDSIGHT_API_DATABASE_URL` | PostgreSQL connection string | `pg0` (embedded) |
+| `HINDSIGHT_API_LLM_PROVIDER` | `openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio` | `openai` |
+| `HINDSIGHT_API_LLM_API_KEY` | API key for LLM provider | - |
+| `HINDSIGHT_API_LLM_MODEL` | Model name | `gpt-4o-mini` |
+| `HINDSIGHT_API_HOST` | Server bind address | `0.0.0.0` |
+| `HINDSIGHT_API_PORT` | Server port | `8888` |
+
+### Example with External PostgreSQL
+
+```bash
+export HINDSIGHT_API_DATABASE_URL=postgresql://user:pass@localhost:5432/hindsight
+export HINDSIGHT_API_LLM_PROVIDER=groq
+export HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx
+
+hindsight-api
+```
+
+## Docker
+
+```bash
+docker run --rm -it -p 8888:8888 \
+  -e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \
+  -v $HOME/.hindsight-docker:/home/hindsight/.pg0 \
+  ghcr.io/vectorize-io/hindsight:latest
+```
+
+## MCP Server
+
+For local MCP integration without running the full API server:
+
+```bash
+hindsight-local-mcp
+```
+
+This runs a stdio-based MCP server that can be used directly with MCP-compatible clients.
+
+## Key Features
+
+- **Multi-Strategy Retrieval (TEMPR)** — Semantic, keyword, graph, and temporal search combined with RRF fusion
+- **Entity Graph** — Automatic entity extraction and relationship tracking
+- **Temporal Reasoning** — Native support for time-based queries
+- **Disposition Traits** — Configurable skepticism, literalism, and empathy influence opinion formation
+- **Three Memory Types** — World facts, bank actions, and formed opinions with confidence scores
+
+## Documentation
+
+Full documentation: [https://hindsight.vectorize.io](https://hindsight.vectorize.io)
+
+- [Installation Guide](https://hindsight.vectorize.io/developer/installation)
+- [Configuration Reference](https://hindsight.vectorize.io/developer/configuration)
+- [API Reference](https://hindsight.vectorize.io/api-reference)
+- [Python SDK](https://hindsight.vectorize.io/sdks/python)
+
+## License
+
+Apache 2.0

hindsight-api-slim/hindsight_api/__init__.py

@@ -0,0 +1,49 @@
+"""
+Memory System for AI Agents.
+
+Temporal + Semantic Memory Architecture using PostgreSQL with pgvector.
+"""
+
+from .config import HindsightConfig, get_config
+from .engine.cross_encoder import CrossEncoderModel, LocalSTCrossEncoder, RemoteTEICrossEncoder
+from .engine.embeddings import Embeddings, LocalSTEmbeddings, RemoteTEIEmbeddings
+from .engine.llm_wrapper import LLMConfig
+from .engine.memory_engine import MemoryEngine
+from .engine.search.trace import (
+    EntryPoint,
+    LinkInfo,
+    NodeVisit,
+    PruningDecision,
+    QueryInfo,
+    SearchPhaseMetrics,
+    SearchSummary,
+    SearchTrace,
+    WeightComponents,
+)
+from .engine.search.tracer import SearchTracer
+from .models import RequestContext
+
+__all__ = [
+    "MemoryEngine",
+    "RequestContext",
+    "HindsightConfig",
+    "get_config",
+    "SearchTrace",
+    "SearchTracer",
+    "QueryInfo",
+    "EntryPoint",
+    "NodeVisit",
+    "WeightComponents",
+    "LinkInfo",
+    "PruningDecision",
+    "SearchSummary",
+    "SearchPhaseMetrics",
+    "Embeddings",
+    "LocalSTEmbeddings",
+    "RemoteTEIEmbeddings",
+    "CrossEncoderModel",
+    "LocalSTCrossEncoder",
+    "RemoteTEICrossEncoder",
+    "LLMConfig",
+]
+__version__ = "0.4.22"

hindsight-api-slim/hindsight_api/admin/__init__.py

@@ -0,0 +1 @@
+# Admin CLI for Hindsight

hindsight-api-slim/hindsight_api/admin/cli.py

@@ -0,0 +1,383 @@
+"""
+Hindsight Admin CLI - backup and restore operations.
+"""
+
+import asyncio
+import io
+import json
+import logging
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import asyncpg
+import typer
+
+from ..config import DEFAULT_DATABASE_SCHEMA, HindsightConfig
+from ..extensions import TenantExtension, load_extension
+from ..pg0 import parse_pg0_url, resolve_database_url
+
+
+def _fq_table(table: str, schema: str) -> str:
+    """Get fully-qualified table name with schema prefix."""
+    return f"{schema}.{table}"
+
+
+# Setup logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(message)s",
+)
+logger = logging.getLogger(__name__)
+
+app = typer.Typer(name="hindsight-admin", help="Hindsight administrative commands")
+
+# Tables to backup/restore in dependency order
+# Import must happen in this order due to foreign key constraints
+BACKUP_TABLES = [
+    "banks",
+    "documents",
+    "entities",
+    "chunks",
+    "memory_units",
+    "unit_entities",
+    "entity_cooccurrences",
+    "memory_links",
+]
+
+MANIFEST_VERSION = "1"
+
+
+async def _backup(database_url: str, output_path: Path, schema: str = "public") -> dict[str, Any]:
+    """Backup all tables to a zip file using binary COPY protocol."""
+    conn = await asyncpg.connect(database_url)
+    try:
+        tables: dict[str, Any] = {}
+        manifest: dict[str, Any] = {
+            "version": MANIFEST_VERSION,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+            "schema": schema,
+            "tables": tables,
+        }
+
+        # Use a transaction with REPEATABLE READ isolation to get a consistent
+        # snapshot across all tables. This prevents race conditions where
+        # entity_cooccurrences could reference entities created after the
+        # entities table was backed up.
+        async with conn.transaction(isolation="repeatable_read"):
+            with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zf:
+                for i, table in enumerate(BACKUP_TABLES, 1):
+                    typer.echo(f"  [{i}/{len(BACKUP_TABLES)}] Backing up {table}...", nl=False)
+
+                    buffer = io.BytesIO()
+
+                    # Use binary COPY for exact type preservation
+                    # asyncpg requires schema_name as separate parameter
+                    await conn.copy_from_table(table, schema_name=schema, output=buffer, format="binary")
+
+                    data = buffer.getvalue()
+                    zf.writestr(f"{table}.bin", data)
+
+                    # Get row count for manifest
+                    qualified_table = _fq_table(table, schema)
+                    row_count = await conn.fetchval(f"SELECT COUNT(*) FROM {qualified_table}")
+                    tables[table] = {
+                        "rows": row_count,
+                        "size_bytes": len(data),
+                    }
+
+                    typer.echo(f" {row_count} rows")
+
+                zf.writestr("manifest.json", json.dumps(manifest, indent=2))
+
+        return manifest
+    finally:
+        await conn.close()
+
+
+async def _restore(database_url: str, input_path: Path, schema: str = "public") -> dict[str, Any]:
+    """Restore all tables from a zip file using binary COPY protocol."""
+    conn = await asyncpg.connect(database_url)
+    try:
+        with zipfile.ZipFile(input_path, "r") as zf:
+            # Read and validate manifest
+            manifest: dict[str, Any] = json.loads(zf.read("manifest.json"))
+            if manifest.get("version") != MANIFEST_VERSION:
+                raise ValueError(f"Unsupported backup version: {manifest.get('version')}")
+
+            # Use a transaction for atomic restore - either all tables are
+            # restored or none are, preventing partial/inconsistent state.
+            async with conn.transaction():
+                typer.echo("  Clearing existing data...")
+                # Truncate tables in reverse order (respects FK constraints)
+                for table in reversed(BACKUP_TABLES):
+                    qualified_table = _fq_table(table, schema)
+                    await conn.execute(f"TRUNCATE TABLE {qualified_table} CASCADE")
+
+                # Restore tables in forward order
+                for i, table in enumerate(BACKUP_TABLES, 1):
+                    filename = f"{table}.bin"
+                    if filename not in zf.namelist():
+                        typer.echo(f"  [{i}/{len(BACKUP_TABLES)}] {table}: skipped (not in backup)")
+                        continue
+
+                    expected_rows = manifest["tables"].get(table, {}).get("rows", "?")
+                    typer.echo(f"  [{i}/{len(BACKUP_TABLES)}] Restoring {table}... {expected_rows} rows")
+
+                    data = zf.read(filename)
+                    buffer = io.BytesIO(data)
+                    # asyncpg requires schema_name as separate parameter
+                    await conn.copy_to_table(table, schema_name=schema, source=buffer, format="binary")
+
+                # Refresh materialized view
+                typer.echo("  Refreshing materialized views...")
+                await conn.execute(f"REFRESH MATERIALIZED VIEW {_fq_table('memory_units_bm25', schema)}")
+
+        return manifest
+    finally:
+        await conn.close()
+
+
+async def _run_backup(db_url: str, output: Path, schema: str = "public") -> dict[str, Any]:
+    """Resolve database URL and run backup."""
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+    return await _backup(resolved_url, output, schema)
+
+
+async def _run_restore(db_url: str, input_file: Path, schema: str = "public") -> dict[str, Any]:
+    """Resolve database URL and run restore."""
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+    return await _restore(resolved_url, input_file, schema)
+
+
+@app.command()
+def backup(
+    output: Path = typer.Argument(..., help="Output file path (.zip)"),
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema to backup"),
+):
+    """Backup the Hindsight database to a zip file."""
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    if output.suffix != ".zip":
+        output = output.with_suffix(".zip")
+
+    typer.echo(f"Backing up database (schema: {schema}) to {output}...")
+
+    manifest = asyncio.run(_run_backup(config.database_url, output, schema))
+
+    total_rows = sum(t["rows"] for t in manifest["tables"].values())
+    typer.echo(f"Backed up {total_rows} rows across {len(BACKUP_TABLES)} tables")
+    typer.echo(f"Backup saved to {output}")
+
+
+@app.command()
+def restore(
+    input_file: Path = typer.Argument(..., help="Input backup file (.zip)"),
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema to restore to"),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"),
+):
+    """Restore the database from a backup file. WARNING: This deletes all existing data."""
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    if not input_file.exists():
+        typer.echo(f"Error: File not found: {input_file}", err=True)
+        raise typer.Exit(1)
+
+    if not yes:
+        typer.confirm(
+            "This will DELETE all existing data and replace it with the backup. Continue?",
+            abort=True,
+        )
+
+    typer.echo(f"Restoring database (schema: {schema}) from {input_file}...")
+
+    manifest = asyncio.run(_run_restore(config.database_url, input_file, schema))
+
+    total_rows = sum(t["rows"] for t in manifest["tables"].values())
+    typer.echo(f"Restored {total_rows} rows across {len(BACKUP_TABLES)} tables")
+    typer.echo("Restore complete")
+
+
+async def _run_migration(
+    db_url: str,
+    schema: str | None = None,
+    base_schema: str = DEFAULT_DATABASE_SCHEMA,
+    embedding_dimension: int | None = None,
+) -> list[str]:
+    """Resolve database URL and run migrations for one schema or all discovered schemas."""
+    from ..migrations import (
+        ensure_embedding_dimension,
+        ensure_text_search_extension,
+        ensure_vector_extension,
+        run_migrations,
+    )
+
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+
+    config = HindsightConfig.from_env()
+    if schema:
+        schemas = [schema]
+    else:
+        tenant_extension = load_extension("TENANT", TenantExtension)
+
+        schemas = [base_schema or DEFAULT_DATABASE_SCHEMA]
+        if tenant_extension:
+            tenants = await tenant_extension.list_tenants()
+            schemas.extend(tenant.schema for tenant in tenants if tenant.schema)
+
+        # Preserve order while removing duplicates.
+        schemas = list(dict.fromkeys(schemas))
+
+    for schema in schemas:
+        run_migrations(resolved_url, schema=schema, migration_database_url=config.migration_database_url)
+
+    if embedding_dimension is not None:
+        for schema in schemas:
+            ensure_embedding_dimension(
+                resolved_url,
+                embedding_dimension,
+                schema=schema,
+                vector_extension=config.vector_extension,
+            )
+
+    for schema in schemas:
+        ensure_vector_extension(
+            resolved_url,
+            vector_extension=config.vector_extension,
+            schema=schema,
+        )
+
+    for schema in schemas:
+        ensure_text_search_extension(
+            resolved_url,
+            text_search_extension=config.text_search_extension,
+            schema=schema,
+        )
+
+    return schemas
+
+
+@app.command(name="run-db-migration")
+def run_db_migration(
+    schema: str | None = typer.Option(
+        None,
+        "--schema",
+        "-s",
+        help="Database schema to run migrations on. If omitted, migrate the base schema and all discovered tenant schemas.",
+    ),
+    embedding_dimension: int | None = typer.Option(
+        None,
+        "--embedding-dimension",
+        help="Expected embedding dimension to enforce after migrations. Omit to skip dimension sync.",
+    ),
+):
+    """Run database migrations to the latest version."""
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    if schema:
+        typer.echo(f"Running database migrations for schema: {schema}...")
+    else:
+        typer.echo("Running database migrations for base schema and all discovered tenant schemas...")
+
+    schemas = asyncio.run(
+        _run_migration(
+            config.database_url,
+            schema=schema,
+            base_schema=config.database_schema,
+            embedding_dimension=embedding_dimension,
+        )
+    )
+
+    typer.echo(f"Database migrations completed successfully for {len(schemas)} schema(s)")
+
+
+async def _decommission_worker(db_url: str, worker_id: str, schema: str = "public") -> int:
+    """Release all tasks owned by a worker, setting them back to pending status."""
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+
+    conn = await asyncpg.connect(resolved_url)
+    try:
+        table = _fq_table("async_operations", schema)
+        result = await conn.fetch(
+            f"""
+            UPDATE {table}
+            SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
+            WHERE worker_id = $1 AND status = 'processing'
+            RETURNING operation_id
+            """,
+            worker_id,
+        )
+        return len(result)
+    finally:
+        await conn.close()
+
+
+@app.command(name="decommission-worker")
+def decommission_worker(
+    worker_id: str = typer.Argument(..., help="Worker ID to decommission"),
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema"),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"),
+):
+    """Release all tasks owned by a worker (sets status back to pending).
+
+    Use this command when a worker has crashed or been removed without graceful shutdown.
+    All tasks that were being processed by the worker will be released back to the queue
+    so other workers can pick them up.
+    """
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    if not yes:
+        typer.confirm(
+            f"This will release all tasks owned by worker '{worker_id}' back to pending. Continue?",
+            abort=True,
+        )
+
+    typer.echo(f"Decommissioning worker '{worker_id}' (schema: {schema})...")
+
+    count = asyncio.run(_decommission_worker(config.database_url, worker_id, schema))
+
+    if count > 0:
+        typer.echo(f"Released {count} task(s) from worker '{worker_id}'")
+    else:
+        typer.echo(f"No tasks found for worker '{worker_id}'")
+
+
+def main():
+    app()
+
+
+if __name__ == "__main__":
+    main()

hindsight-api-slim/hindsight_api/alembic/env.py

@@ -0,0 +1,166 @@
+"""
+Alembic environment configuration for SQLAlchemy with pgvector.
+Uses synchronous psycopg2 driver for migrations to avoid pgbouncer issues.
+"""
+
+import logging
+import os
+from pathlib import Path
+
+from alembic import context
+from dotenv import load_dotenv
+from sqlalchemy import engine_from_config, pool
+
+# Import your models here
+from hindsight_api.models import Base
+
+
+# Load environment variables based on HINDSIGHT_API_DATABASE_URL env var or default to local
+def load_env():
+    """Load environment variables from .env"""
+    # Check if HINDSIGHT_API_DATABASE_URL is already set (e.g., by CI/CD)
+    if os.getenv("HINDSIGHT_API_DATABASE_URL"):
+        return
+
+    # Look for .env file in the parent directory (root of the workspace)
+    root_dir = Path(__file__).parent.parent.parent
+    env_file = root_dir / ".env"
+
+    if env_file.exists():
+        load_dotenv(env_file)
+
+
+load_env()
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Note: We don't call fileConfig() here to avoid overriding the application's logging configuration.
+# Alembic will use the existing logging configuration from the application.
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def get_database_url() -> str:
+    """
+    Get and process the database URL from config or environment.
+
+    Returns the URL with the correct driver (psycopg2) for migrations.
+    """
+    # Get database URL from config (set programmatically) or environment
+    database_url = config.get_main_option("sqlalchemy.url")
+    if not database_url:
+        database_url = os.getenv("HINDSIGHT_API_DATABASE_URL")
+        if not database_url:
+            raise ValueError(
+                "Database URL not found. "
+                "Set HINDSIGHT_API_DATABASE_URL environment variable or pass database_url to run_migrations()."
+            )
+
+    # For migrations, use psycopg2 (sync driver) to avoid pgbouncer prepared statement issues
+    if database_url.startswith("postgresql+asyncpg://"):
+        database_url = database_url.replace("postgresql+asyncpg://", "postgresql://", 1)
+    elif database_url.startswith("postgres+asyncpg://"):
+        database_url = database_url.replace("postgres+asyncpg://", "postgresql://", 1)
+
+    # Update config with processed URL for engine_from_config to use
+    config.set_main_option("sqlalchemy.url", database_url)
+
+    return database_url
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    logging.info("running offline")
+    database_url = get_database_url()
+
+    context.configure(
+        url=database_url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode with synchronous engine."""
+    from sqlalchemy import event, text
+
+    get_database_url()  # Process and set the database URL in config
+
+    # Check if we're targeting a specific schema (for multi-tenant isolation)
+    target_schema = config.get_main_option("target_schema")
+
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    # Add event listener to ensure connection is in read-write mode
+    # This is needed for Supabase which may start connections in read-only mode
+    @event.listens_for(connectable, "connect")
+    def set_read_write_mode(dbapi_connection, connection_record):
+        cursor = dbapi_connection.cursor()
+        cursor.execute("SET SESSION CHARACTERISTICS AS TRANSACTION READ WRITE")
+        # If targeting a specific schema, set search_path
+        # Include public in search_path for access to shared extensions (pgvector)
+        if target_schema:
+            cursor.execute(f'CREATE SCHEMA IF NOT EXISTS "{target_schema}"')
+            cursor.execute(f'SET search_path TO "{target_schema}", public')
+        cursor.close()
+
+    with connectable.connect() as connection:
+        # Also explicitly set read-write mode on this connection
+        connection.execute(text("SET SESSION CHARACTERISTICS AS TRANSACTION READ WRITE"))
+
+        # If targeting a specific schema, set search_path
+        # Include public in search_path for access to shared extensions (pgvector)
+        if target_schema:
+            connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{target_schema}"'))
+            connection.execute(text(f'SET search_path TO "{target_schema}", public'))
+
+        connection.commit()  # Commit the SET command
+
+        # Configure context with version_table_schema if using a specific schema
+        context_opts = {
+            "connection": connection,
+            "target_metadata": target_metadata,
+        }
+        if target_schema:
+            context_opts["version_table_schema"] = target_schema
+
+        context.configure(**context_opts)
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+        # Explicit commit to ensure changes are persisted (especially for Supabase)
+        connection.commit()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

hindsight-api-slim/hindsight_api/alembic/versions/2eee35aa3cfc_case_insensitive_entities_trgm_index.py

@@ -0,0 +1,45 @@
+"""Recreate entities trigram index on LOWER(canonical_name) for case-insensitive matching
+
+The previous GIN trigram index on canonical_name was case-sensitive, causing
+"Alice" and "alice" to have different trigram sets. This recreates it on
+LOWER(canonical_name) so the % operator matches case-insensitively.
+
+Revision ID: d6e7f8a9b0c1
+Revises: c5d6e7f8a9b0
+Create Date: 2026-03-31
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "d6e7f8a9b0c1"
+down_revision: str | Sequence[str] | None = "c5d6e7f8a9b0"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    # Drop the old case-sensitive trigram index
+    op.execute("DROP INDEX IF EXISTS entities_canonical_name_trgm_idx")
+    # Create case-insensitive trigram index on LOWER(canonical_name)
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS entities_canonical_name_lower_trgm_idx "
+        f"ON {schema}entities USING GIN (LOWER(canonical_name) gin_trgm_ops)"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS entities_canonical_name_lower_trgm_idx")
+    schema = _get_schema_prefix()
+    # Restore original case-sensitive index
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS entities_canonical_name_trgm_idx "
+        f"ON {schema}entities USING GIN (canonical_name gin_trgm_ops)"
+    )

hindsight-api-slim/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py

@@ -0,0 +1,525 @@
+"""initial_schema
+
+Revision ID: 5a366d414dce
+Revises:
+Create Date: 2025-11-27 11:54:19.228030
+
+"""
+
+import os
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from pgvector.sqlalchemy import Vector
+from sqlalchemy import text
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "5a366d414dce"
+down_revision: str | Sequence[str] | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _detect_vector_extension() -> str:
+    """
+    Detect or validate vector extension: 'pgvector', 'vchord', or 'pgvectorscale'.
+    Respects HINDSIGHT_API_VECTOR_EXTENSION env var if set.
+    """
+    conn = op.get_bind()
+    vector_extension = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
+
+    # Validate configured extension is installed
+    if vector_extension == "pgvectorscale":
+        # pgvectorscale/DiskANN requires pgvector
+        pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
+        if not pgvector_check:
+            raise RuntimeError(
+                "DiskANN requires pgvector. Install with: CREATE EXTENSION vector; then vectorscale or pg_diskann CASCADE;"
+            )
+        # Check for either vectorscale (open source) or pg_diskann (Azure)
+        vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar()
+        pg_diskann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_diskann'")).scalar()
+
+        if vectorscale_check:
+            return "pgvectorscale"
+        elif pg_diskann_check:
+            return "pg_diskann"
+        else:
+            raise RuntimeError(
+                "Configured vector extension 'pgvectorscale' not found. Install either:\n"
+                "  - pgvectorscale: CREATE EXTENSION vectorscale CASCADE;\n"
+                "  - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
+            )
+    elif vector_extension == "vchord":
+        vchord_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord'")).scalar()
+        if not vchord_check:
+            raise RuntimeError(
+                "Configured vector extension 'vchord' not found. Install it with: CREATE EXTENSION vchord CASCADE;"
+            )
+        return "vchord"
+    elif vector_extension == "pgvector":
+        pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
+        if not pgvector_check:
+            raise RuntimeError(
+                "Configured vector extension 'pgvector' not found. Install it with: CREATE EXTENSION vector;"
+            )
+        return "pgvector"
+    else:
+        raise ValueError(
+            f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {vector_extension}. Must be 'pgvector', 'vchord', or 'pgvectorscale'"
+        )
+
+
+def _detect_text_search_extension() -> str:
+    """
+    Detect or validate text search extension: 'native', 'vchord', or 'pg_textsearch'.
+    Respects HINDSIGHT_API_TEXT_SEARCH_EXTENSION env var.
+    Creates the extension if needed.
+    """
+    text_search_extension = os.getenv("HINDSIGHT_API_TEXT_SEARCH_EXTENSION", "native").lower()
+
+    if text_search_extension == "vchord":
+        # Create vchord_bm25 extension if not exists
+        try:
+            op.execute("CREATE EXTENSION IF NOT EXISTS vchord_bm25 CASCADE")
+        except Exception:
+            # Extension might already exist or user lacks permissions - verify it exists
+            conn = op.get_bind()
+            result = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord_bm25'")).fetchone()
+            if not result:
+                # Extension truly doesn't exist - re-raise the error
+                raise
+        return "vchord"
+    elif text_search_extension == "pg_textsearch":
+        # Create pg_textsearch extension if not exists
+        try:
+            op.execute("CREATE EXTENSION IF NOT EXISTS pg_textsearch CASCADE")
+        except Exception:
+            # Extension might already exist or user lacks permissions - verify it exists
+            conn = op.get_bind()
+            result = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_textsearch'")).fetchone()
+            if not result:
+                # Extension truly doesn't exist - re-raise the error
+                raise
+        return "pg_textsearch"
+    elif text_search_extension == "native":
+        return "native"
+    else:
+        raise ValueError(
+            f"Invalid HINDSIGHT_API_TEXT_SEARCH_EXTENSION: {text_search_extension}. Must be 'native', 'vchord', or 'pg_textsearch'"
+        )
+
+
+def upgrade() -> None:
+    """Upgrade schema - create all tables from scratch."""
+
+    # Note: pgvector extension is installed globally BEFORE migrations run
+    # See migrations.py:run_migrations() - this ensures the extension is available
+    # to all schemas, not just the one being migrated
+
+    # We keep this here as a fallback for backwards compatibility
+    # This may fail if user lacks permissions, which is fine if extension already exists
+    try:
+        op.execute("CREATE EXTENSION IF NOT EXISTS vector")
+    except Exception:
+        # Extension might already exist or user lacks permissions - verify it exists
+        conn = op.get_bind()
+        result = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).fetchone()
+        if not result:
+            # Extension truly doesn't exist - re-raise the error
+            raise
+
+    # Create banks table
+    op.create_table(
+        "banks",
+        sa.Column("bank_id", sa.Text(), nullable=False),
+        sa.Column("name", sa.Text(), nullable=True),
+        sa.Column(
+            "personality",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default=sa.text("'{}'::jsonb"),
+            nullable=False,
+        ),
+        sa.Column("background", sa.Text(), nullable=True),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.PrimaryKeyConstraint("bank_id", name=op.f("pk_banks")),
+    )
+
+    # Create documents table
+    op.create_table(
+        "documents",
+        sa.Column("id", sa.Text(), nullable=False),
+        sa.Column("bank_id", sa.Text(), nullable=False),
+        sa.Column("original_text", sa.Text(), nullable=True),
+        sa.Column("content_hash", sa.Text(), nullable=True),
+        sa.Column(
+            "metadata", postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False
+        ),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.PrimaryKeyConstraint("id", "bank_id", name=op.f("pk_documents")),
+    )
+    op.create_index("idx_documents_bank_id", "documents", ["bank_id"])
+    op.create_index("idx_documents_content_hash", "documents", ["content_hash"])
+
+    # Create async_operations table
+    op.create_table(
+        "async_operations",
+        sa.Column(
+            "operation_id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False
+        ),
+        sa.Column("bank_id", sa.Text(), nullable=False),
+        sa.Column("operation_type", sa.Text(), nullable=False),
+        sa.Column("status", sa.Text(), server_default="pending", nullable=False),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("completed_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+        sa.Column(
+            "result_metadata",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default=sa.text("'{}'::jsonb"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("operation_id", name=op.f("pk_async_operations")),
+        sa.CheckConstraint(
+            "status IN ('pending', 'processing', 'completed', 'failed')", name="async_operations_status_check"
+        ),
+    )
+    op.create_index("idx_async_operations_bank_id", "async_operations", ["bank_id"])
+    op.create_index("idx_async_operations_status", "async_operations", ["status"])
+    op.create_index("idx_async_operations_bank_status", "async_operations", ["bank_id", "status"])
+
+    # Create entities table
+    op.create_table(
+        "entities",
+        sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False),
+        sa.Column("canonical_name", sa.Text(), nullable=False),
+        sa.Column("bank_id", sa.Text(), nullable=False),
+        sa.Column(
+            "metadata", postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False
+        ),
+        sa.Column("first_seen", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("last_seen", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("mention_count", sa.Integer(), server_default="1", nullable=False),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_entities")),
+    )
+    op.create_index("idx_entities_bank_id", "entities", ["bank_id"])
+    op.create_index("idx_entities_canonical_name", "entities", ["canonical_name"])
+    op.create_index("idx_entities_bank_name", "entities", ["bank_id", "canonical_name"])
+    # Create unique index on (bank_id, LOWER(canonical_name)) for entity resolution
+    op.execute("CREATE UNIQUE INDEX idx_entities_bank_lower_name ON entities (bank_id, LOWER(canonical_name))")
+
+    # Create memory_units table
+    op.create_table(
+        "memory_units",
+        sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False),
+        sa.Column("bank_id", sa.Text(), nullable=False),
+        sa.Column("document_id", sa.Text(), nullable=True),
+        sa.Column("text", sa.Text(), nullable=False),
+        sa.Column("embedding", Vector(384), nullable=True),
+        sa.Column("context", sa.Text(), nullable=True),
+        sa.Column("event_date", postgresql.TIMESTAMP(timezone=True), nullable=False),
+        sa.Column("occurred_start", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("occurred_end", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("mentioned_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("fact_type", sa.Text(), server_default="world", nullable=False),
+        sa.Column("confidence_score", sa.Float(), nullable=True),
+        sa.Column("access_count", sa.Integer(), server_default="0", nullable=False),
+        sa.Column(
+            "metadata", postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False
+        ),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id", "bank_id"],
+            ["documents.id", "documents.bank_id"],
+            name="memory_units_document_fkey",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_memory_units")),
+        sa.CheckConstraint(
+            "fact_type IN ('world', 'bank', 'opinion', 'observation')", name="memory_units_fact_type_check"
+        ),
+        sa.CheckConstraint(
+            "confidence_score IS NULL OR (confidence_score >= 0.0 AND confidence_score <= 1.0)",
+            name="memory_units_confidence_range_check",
+        ),
+        sa.CheckConstraint(
+            "(fact_type = 'opinion' AND confidence_score IS NOT NULL) OR "
+            "(fact_type = 'observation') OR "
+            "(fact_type NOT IN ('opinion', 'observation') AND confidence_score IS NULL)",
+            name="confidence_score_fact_type_check",
+        ),
+    )
+
+    # Add search_vector column for full-text search
+    # Type depends on configured text search backend
+    text_search_ext = _detect_text_search_extension()
+
+    if text_search_ext == "vchord":
+        # VectorChord BM25: bm25vector type (no GENERATED - tokenization happens on INSERT)
+        # Note: vchord_bm25 extension creates types in bm25_catalog schema
+        op.execute("""
+            ALTER TABLE memory_units
+            ADD COLUMN search_vector bm25_catalog.bm25vector
+        """)
+    elif text_search_ext == "pg_textsearch":
+        # Timescale pg_textsearch: dummy TEXT column for consistency (indexes operate on base columns directly)
+        op.execute("""
+            ALTER TABLE memory_units
+            ADD COLUMN search_vector TEXT
+        """)
+    else:  # native
+        # Native PostgreSQL: tsvector with automatic generation
+        op.execute("""
+            ALTER TABLE memory_units
+            ADD COLUMN search_vector tsvector
+            GENERATED ALWAYS AS (to_tsvector('english', COALESCE(text, '') || ' ' || COALESCE(context, ''))) STORED
+        """)
+
+    op.create_index("idx_memory_units_bank_id", "memory_units", ["bank_id"])
+    op.create_index("idx_memory_units_document_id", "memory_units", ["document_id"])
+    op.create_index("idx_memory_units_event_date", "memory_units", [sa.text("event_date DESC")])
+    op.create_index("idx_memory_units_bank_date", "memory_units", ["bank_id", sa.text("event_date DESC")])
+    op.create_index("idx_memory_units_access_count", "memory_units", [sa.text("access_count DESC")])
+    op.create_index("idx_memory_units_fact_type", "memory_units", ["fact_type"])
+    op.create_index("idx_memory_units_bank_fact_type", "memory_units", ["bank_id", "fact_type"])
+    op.create_index(
+        "idx_memory_units_bank_type_date", "memory_units", ["bank_id", "fact_type", sa.text("event_date DESC")]
+    )
+    op.create_index(
+        "idx_memory_units_opinion_confidence",
+        "memory_units",
+        ["bank_id", sa.text("confidence_score DESC")],
+        postgresql_where=sa.text("fact_type = 'opinion'"),
+    )
+    op.create_index(
+        "idx_memory_units_opinion_date",
+        "memory_units",
+        ["bank_id", sa.text("event_date DESC")],
+        postgresql_where=sa.text("fact_type = 'opinion'"),
+    )
+    op.create_index(
+        "idx_memory_units_observation_date",
+        "memory_units",
+        ["bank_id", sa.text("event_date DESC")],
+        postgresql_where=sa.text("fact_type = 'observation'"),
+    )
+    # Create vector index - conditional based on available extension
+    vector_ext = _detect_vector_extension()
+
+    if vector_ext == "pgvectorscale":
+        # Use DiskANN index for pgvectorscale (disk-based, scalable)
+        op.execute("""
+            CREATE INDEX idx_memory_units_embedding ON memory_units
+            USING diskann (embedding vector_cosine_ops)
+            WITH (num_neighbors = 50)
+        """)
+    elif vector_ext == "pg_diskann":
+        # Use DiskANN index for pg_diskann (Azure)
+        op.execute("""
+            CREATE INDEX idx_memory_units_embedding ON memory_units
+            USING diskann (embedding vector_cosine_ops)
+            WITH (max_neighbors = 50)
+        """)
+    elif vector_ext == "vchord":
+        # Use vchordrq index for vchord (supports high-dimensional embeddings)
+        op.execute("""
+            CREATE INDEX idx_memory_units_embedding ON memory_units
+            USING vchordrq (embedding vector_l2_ops)
+        """)
+    else:  # pgvector
+        # Use HNSW index for pgvector
+        op.create_index(
+            "idx_memory_units_embedding",
+            "memory_units",
+            ["embedding"],
+            postgresql_using="hnsw",
+            postgresql_ops={"embedding": "vector_cosine_ops"},
+        )
+
+    # Create full-text search index on search_vector
+    # Index type depends on text search backend
+    if text_search_ext == "vchord":
+        # VectorChord BM25 index
+        op.execute("""
+            CREATE INDEX idx_memory_units_text_search ON memory_units
+            USING bm25 (search_vector bm25_catalog.bm25_ops)
+        """)
+    elif text_search_ext == "pg_textsearch":
+        # Timescale pg_textsearch BM25 index on text column
+        # Note: pg_textsearch doesn't support expressions, so we index the main text column
+        op.execute("""
+            CREATE INDEX idx_memory_units_text_search ON memory_units
+            USING bm25(text)
+            WITH (text_config='english')
+        """)
+    else:  # native
+        # Native PostgreSQL GIN index
+        op.execute("""
+            CREATE INDEX idx_memory_units_text_search ON memory_units
+            USING gin(search_vector)
+        """)
+
+    op.execute("""
+        CREATE MATERIALIZED VIEW memory_units_bm25 AS
+        SELECT
+            id,
+            bank_id,
+            text,
+            to_tsvector('english', text) AS text_vector,
+            log(1.0 + length(text)::float / (SELECT avg(length(text)) FROM memory_units)) AS doc_length_factor
+        FROM memory_units
+    """)
+
+    op.create_index("idx_memory_units_bm25_bank", "memory_units_bm25", ["bank_id"])
+    op.create_index("idx_memory_units_bm25_text_vector", "memory_units_bm25", ["text_vector"], postgresql_using="gin")
+
+    # Create entity_cooccurrences table
+    op.create_table(
+        "entity_cooccurrences",
+        sa.Column("entity_id_1", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("entity_id_2", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("cooccurrence_count", sa.Integer(), server_default="1", nullable=False),
+        sa.Column(
+            "last_cooccurred", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["entity_id_1"],
+            ["entities.id"],
+            name=op.f("fk_entity_cooccurrences_entity_id_1_entities"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["entity_id_2"],
+            ["entities.id"],
+            name=op.f("fk_entity_cooccurrences_entity_id_2_entities"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("entity_id_1", "entity_id_2", name=op.f("pk_entity_cooccurrences")),
+        sa.CheckConstraint("entity_id_1 < entity_id_2", name="entity_cooccurrence_order_check"),
+    )
+    op.create_index("idx_entity_cooccurrences_entity1", "entity_cooccurrences", ["entity_id_1"])
+    op.create_index("idx_entity_cooccurrences_entity2", "entity_cooccurrences", ["entity_id_2"])
+    op.create_index("idx_entity_cooccurrences_count", "entity_cooccurrences", [sa.text("cooccurrence_count DESC")])
+
+    # Create memory_links table
+    op.create_table(
+        "memory_links",
+        sa.Column("from_unit_id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("to_unit_id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("link_type", sa.Text(), nullable=False),
+        sa.Column("entity_id", postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column("weight", sa.Float(), server_default="1.0", nullable=False),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["entity_id"], ["entities.id"], name=op.f("fk_memory_links_entity_id_entities"), ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["from_unit_id"],
+            ["memory_units.id"],
+            name=op.f("fk_memory_links_from_unit_id_memory_units"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["to_unit_id"],
+            ["memory_units.id"],
+            name=op.f("fk_memory_links_to_unit_id_memory_units"),
+            ondelete="CASCADE",
+        ),
+        sa.CheckConstraint(
+            "link_type IN ('temporal', 'semantic', 'entity', 'causes', 'caused_by', 'enables', 'prevents')",
+            name="memory_links_link_type_check",
+        ),
+        sa.CheckConstraint("weight >= 0.0 AND weight <= 1.0", name="memory_links_weight_check"),
+    )
+    # Create unique constraint using COALESCE for nullable entity_id
+    op.execute(
+        "CREATE UNIQUE INDEX idx_memory_links_unique ON memory_links (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid))"
+    )
+    op.create_index("idx_memory_links_from_unit", "memory_links", ["from_unit_id"])
+    op.create_index("idx_memory_links_to_unit", "memory_links", ["to_unit_id"])
+    op.create_index("idx_memory_links_entity", "memory_links", ["entity_id"])
+    op.create_index("idx_memory_links_link_type", "memory_links", ["link_type"])
+
+    # Create unit_entities table
+    op.create_table(
+        "unit_entities",
+        sa.Column("unit_id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("entity_id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["entity_id"], ["entities.id"], name=op.f("fk_unit_entities_entity_id_entities"), ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["unit_id"], ["memory_units.id"], name=op.f("fk_unit_entities_unit_id_memory_units"), ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("unit_id", "entity_id", name=op.f("pk_unit_entities")),
+    )
+    op.create_index("idx_unit_entities_unit", "unit_entities", ["unit_id"])
+    op.create_index("idx_unit_entities_entity", "unit_entities", ["entity_id"])
+
+
+def downgrade() -> None:
+    """Downgrade schema - drop all tables."""
+
+    # Drop tables in reverse dependency order
+    op.drop_index("idx_unit_entities_entity", table_name="unit_entities")
+    op.drop_index("idx_unit_entities_unit", table_name="unit_entities")
+    op.drop_table("unit_entities")
+
+    op.drop_index("idx_memory_links_link_type", table_name="memory_links")
+    op.drop_index("idx_memory_links_entity", table_name="memory_links")
+    op.drop_index("idx_memory_links_to_unit", table_name="memory_links")
+    op.drop_index("idx_memory_links_from_unit", table_name="memory_links")
+    op.execute("DROP INDEX IF EXISTS idx_memory_links_unique")
+    op.drop_table("memory_links")
+
+    op.drop_index("idx_entity_cooccurrences_count", table_name="entity_cooccurrences")
+    op.drop_index("idx_entity_cooccurrences_entity2", table_name="entity_cooccurrences")
+    op.drop_index("idx_entity_cooccurrences_entity1", table_name="entity_cooccurrences")
+    op.drop_table("entity_cooccurrences")
+
+    # Drop BM25 materialized view and index
+    op.drop_index("idx_memory_units_bm25_text_vector", table_name="memory_units_bm25")
+    op.drop_index("idx_memory_units_bm25_bank", table_name="memory_units_bm25")
+    op.execute("DROP MATERIALIZED VIEW IF EXISTS memory_units_bm25")
+
+    op.drop_index("idx_memory_units_embedding", table_name="memory_units")
+    op.drop_index("idx_memory_units_observation_date", table_name="memory_units")
+    op.drop_index("idx_memory_units_opinion_date", table_name="memory_units")
+    op.drop_index("idx_memory_units_opinion_confidence", table_name="memory_units")
+    op.drop_index("idx_memory_units_bank_type_date", table_name="memory_units")
+    op.drop_index("idx_memory_units_bank_fact_type", table_name="memory_units")
+    op.drop_index("idx_memory_units_fact_type", table_name="memory_units")
+    op.drop_index("idx_memory_units_access_count", table_name="memory_units")
+    op.drop_index("idx_memory_units_bank_date", table_name="memory_units")
+    op.drop_index("idx_memory_units_event_date", table_name="memory_units")
+    op.drop_index("idx_memory_units_document_id", table_name="memory_units")
+    op.drop_index("idx_memory_units_bank_id", table_name="memory_units")
+    op.execute("DROP INDEX IF EXISTS idx_memory_units_text_search")
+    op.drop_table("memory_units")
+
+    op.execute("DROP INDEX IF EXISTS idx_entities_bank_lower_name")
+    op.drop_index("idx_entities_bank_name", table_name="entities")
+    op.drop_index("idx_entities_canonical_name", table_name="entities")
+    op.drop_index("idx_entities_bank_id", table_name="entities")
+    op.drop_table("entities")
+
+    op.drop_index("idx_async_operations_bank_status", table_name="async_operations")
+    op.drop_index("idx_async_operations_status", table_name="async_operations")
+    op.drop_index("idx_async_operations_bank_id", table_name="async_operations")
+    op.drop_table("async_operations")
+
+    op.drop_index("idx_documents_content_hash", table_name="documents")
+    op.drop_index("idx_documents_bank_id", table_name="documents")
+    op.drop_table("documents")
+
+    op.drop_table("banks")
+
+    # Drop extensions (optional - comment out if you want to keep them)
+    # op.execute('DROP EXTENSION IF EXISTS vector')
+    # op.execute('DROP EXTENSION IF EXISTS "uuid-ossp"')

hindsight-api-slim/hindsight_api/alembic/versions/a1b2c3d4e5f6_add_file_storage_table.py

@@ -0,0 +1,70 @@
+"""Add file_storage table for BYTEA-based file storage
+
+Revision ID: a1b2c3d4e5f6
+Revises: y0t1u2v3w4x5
+Create Date: 2026-02-16
+
+Creates a dedicated table for storing uploaded files using BYTEA.
+This provides zero-config file storage that "just works" for development
+and small deployments. For production/scale, use S3-compatible storage.
+
+Files are stored in a separate table to avoid bloating the documents table.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "a1b2c3d4e5f6"
+down_revision: str | Sequence[str] | None = "y0t1u2v3w4x5"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Create file_storage table for BYTEA storage."""
+    schema = _get_schema_prefix()
+
+    # Create file_storage table (minimal: just key + data)
+    op.execute(
+        f"""
+        CREATE TABLE IF NOT EXISTS {schema}file_storage (
+            storage_key TEXT PRIMARY KEY,
+            data BYTEA NOT NULL
+        )
+    """
+    )
+
+    # Add file tracking columns to documents table
+    op.execute(
+        f"""
+        ALTER TABLE {schema}documents
+        ADD COLUMN IF NOT EXISTS file_storage_key TEXT,
+        ADD COLUMN IF NOT EXISTS file_original_name TEXT,
+        ADD COLUMN IF NOT EXISTS file_content_type TEXT
+    """
+    )
+
+
+def downgrade() -> None:
+    """Remove file_storage table and related columns."""
+    schema = _get_schema_prefix()
+
+    # Drop columns from documents table
+    op.execute(
+        f"""
+        ALTER TABLE {schema}documents
+        DROP COLUMN IF EXISTS file_storage_key,
+        DROP COLUMN IF EXISTS file_original_name,
+        DROP COLUMN IF EXISTS file_content_type
+    """
+    )
+
+    # Drop file_storage table
+    op.execute(f"DROP TABLE IF EXISTS {schema}file_storage")

hindsight-api-slim/hindsight_api/alembic/versions/a2b3c4d5e6f7_add_text_signals_column.py

@@ -0,0 +1,88 @@
+"""Add text_signals column to memory_units for enriched BM25 indexing.
+
+text_signals stores a denormalized space-separated string of entity names
+(and future signals) to improve full-text search recall without polluting
+the stored fact text.
+
+- vchord: text_signals included in tokenize() at insert time
+- native: search_vector GENERATED column regenerated to include text_signals
+- pg_textsearch: no change (index only supports a single base column)
+
+Revision ID: a2b3c4d5e6f7
+Revises: z1u2v3w4x5y6
+Create Date: 2026-02-28
+"""
+
+import os
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "a2b3c4d5e6f7"
+down_revision: str | Sequence[str] | None = "aa2b3c4d5e6f"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def _detect_text_search_extension() -> str:
+    return os.getenv("HINDSIGHT_API_TEXT_SEARCH_EXTENSION", "native").lower()
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    table = f"{schema}memory_units"
+    text_search_ext = _detect_text_search_extension()
+
+    # Add text_signals column (nullable TEXT, populated at retain time)
+    op.execute(f"ALTER TABLE {table} ADD COLUMN IF NOT EXISTS text_signals TEXT")
+
+    if text_search_ext == "native":
+        # Native PostgreSQL: drop and recreate the GENERATED tsvector column to include text_signals
+        op.execute(f"ALTER TABLE {table} DROP COLUMN IF EXISTS search_vector")
+        op.execute(f"""
+            ALTER TABLE {table}
+            ADD COLUMN search_vector tsvector
+            GENERATED ALWAYS AS (
+                to_tsvector('english',
+                    COALESCE(text, '') || ' ' ||
+                    COALESCE(context, '') || ' ' ||
+                    COALESCE(text_signals, '')
+                )
+            ) STORED
+        """)
+        # Recreate GIN index (was dropped with the column)
+        op.execute(f"""
+            CREATE INDEX IF NOT EXISTS idx_memory_units_text_search
+            ON {table} USING gin(search_vector)
+        """)
+
+    # vchord: tokenize() call in fact_storage.py is updated to include text_signals at insert time
+    # pg_textsearch: no change — index operates on the base `text` column only
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    table = f"{schema}memory_units"
+    text_search_ext = _detect_text_search_extension()
+
+    if text_search_ext == "native":
+        op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_text_search")
+        op.execute(f"ALTER TABLE {table} DROP COLUMN IF EXISTS search_vector")
+        op.execute(f"""
+            ALTER TABLE {table}
+            ADD COLUMN search_vector tsvector
+            GENERATED ALWAYS AS (
+                to_tsvector('english', COALESCE(text, '') || ' ' || COALESCE(context, ''))
+            ) STORED
+        """)
+        op.execute(f"""
+            CREATE INDEX idx_memory_units_text_search
+            ON {table} USING gin(search_vector)
+        """)
+
+    op.execute(f"ALTER TABLE {table} DROP COLUMN IF EXISTS text_signals")

hindsight-api-slim/hindsight_api/alembic/versions/a2b3c4d5e6f8_add_gin_index_source_memory_ids.py

@@ -0,0 +1,54 @@
+"""Add GIN index on source_memory_ids for observation lookup performance
+
+Without this index, queries using the array overlap operator (&&) or array
+containment (@>) on source_memory_ids require a full sequential scan over all
+observation memory_units. At ~77k observations this was measured at 45ms per
+query, becoming a bottleneck during consolidation recall (57-64s timeouts) and
+user recall (18-27s average).
+
+The GIN index reduces these queries to index scans: 45ms → 0.049ms (927x
+speedup). Recall dropped from 18-27s to ~6s, and consolidation recall
+stabilised from timeout to ~15s.
+
+Created with CONCURRENTLY so the migration does not block reads or writes.
+CONCURRENTLY requires running outside a transaction block, so the migration
+emits an explicit COMMIT before the statement and uses IF NOT EXISTS for
+idempotency.
+
+Revision ID: a2b3c4d5e6f8
+Revises: f7g8h9i0j1k2
+Create Date: 2026-03-04
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "a2b3c4d5e6f8"
+down_revision: str | Sequence[str] | None = "f7g8h9i0j1k2"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+
+    # CREATE INDEX CONCURRENTLY cannot run inside a transaction block.
+    # Commit the current Alembic transaction first.
+    op.execute("COMMIT")
+    op.execute(
+        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
+        f"ON {schema}memory_units USING GIN (source_memory_ids) "
+        f"WHERE source_memory_ids IS NOT NULL"
+    )
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute("COMMIT")
+    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")

hindsight-api-slim/hindsight_api/alembic/versions/a3b4c5d6e7f8_add_consolidation_failed_at_to_memory_units.py

@@ -0,0 +1,52 @@
+"""Add consolidation_failed_at column to memory_units for tracking persistent LLM failures.
+
+When all LLM retries are exhausted on a single-memory batch, the memory is marked
+with consolidation_failed_at instead of consolidated_at, so it is not silently lost
+and can be retried later via the API.
+
+Revision ID: a3b4c5d6e7f8
+Revises: g7h8i9j0k1l2
+Create Date: 2026-03-17
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "a3b4c5d6e7f8"
+down_revision: str | Sequence[str] | None = "g7h8i9j0k1l2"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+
+    op.execute(
+        f"""
+        ALTER TABLE {schema}memory_units
+        ADD COLUMN IF NOT EXISTS consolidation_failed_at TIMESTAMPTZ DEFAULT NULL
+        """
+    )
+
+    # Index to efficiently query memories that failed consolidation for a given bank
+    op.execute(
+        f"""
+        CREATE INDEX IF NOT EXISTS idx_memory_units_consolidation_failed
+        ON {schema}memory_units (bank_id, consolidation_failed_at)
+        WHERE consolidation_failed_at IS NOT NULL AND fact_type IN ('experience', 'world')
+        """
+    )
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_consolidation_failed")
+    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS consolidation_failed_at")

hindsight-api-slim/hindsight_api/alembic/versions/a4b5c6d7e8f9_fix_per_bank_vector_index_type.py

@@ -0,0 +1,142 @@
+"""Fix per-bank vector indexes to match configured extension
+
+Revision ID: a4b5c6d7e8f9
+Revises: d6e7f8a9b0c1
+Create Date: 2026-04-01
+
+Migration d5e6f7a8b9c0 hardcoded HNSW when creating per-bank partial vector
+indexes, ignoring HINDSIGHT_API_VECTOR_EXTENSION. Banks that existed when that
+migration ran got HNSW indexes even when pgvectorscale (DiskANN) or vchord
+was configured.
+
+This migration detects the mismatch and recreates the affected indexes with
+the correct type. Skipped entirely when the configured extension is pgvector
+(the default), since those indexes are already correct.
+"""
+
+import os
+from collections.abc import Sequence
+
+from alembic import context, op
+from sqlalchemy import text
+
+revision: str = "a4b5c6d7e8f9"
+down_revision: str | Sequence[str] | None = "d6e7f8a9b0c1"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_FACT_TYPES: dict[str, str] = {
+    "world": "worl",
+    "experience": "expr",
+    "observation": "obsv",
+}
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def _target_index_type() -> str | None:
+    """Return the target index type, or None if pgvector (no fix needed)."""
+    ext = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
+    if ext == "pgvectorscale":
+        return "diskann"
+    elif ext == "vchord":
+        return "vchordrq"
+    return None
+
+
+def _vector_index_using_clause() -> str:
+    """Return the USING clause based on the configured vector extension."""
+    ext = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
+    if ext == "pgvectorscale":
+        return "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)"
+    elif ext == "vchord":
+        return "USING vchordrq (embedding vector_l2_ops)"
+    else:
+        return "USING hnsw (embedding vector_cosine_ops)"
+
+
+def upgrade() -> None:
+    target = _target_index_type()
+    if target is None:
+        # pgvector — indexes are already HNSW, nothing to fix
+        return
+
+    bind = op.get_bind()
+    schema_name = context.config.get_main_option("target_schema")
+    schema = _get_schema_prefix()
+    table_ref = f'"{schema_name}".memory_units' if schema_name else "memory_units"
+    banks_ref = f'"{schema_name}".banks' if schema_name else "banks"
+    using_clause = _vector_index_using_clause()
+    pg_schema = schema_name or "public"
+
+    rows = bind.execute(text(f"SELECT bank_id, internal_id FROM {banks_ref}")).fetchall()  # noqa: S608
+    for row in rows:
+        bank_id = row[0]
+        internal_id = str(row[1]).replace("-", "")[:16]
+        escaped_bank_id = bank_id.replace("'", "''")
+        for ft, ft_short in _FACT_TYPES.items():
+            idx_name = f"idx_mu_emb_{ft_short}_{internal_id}"
+
+            # Check if this index exists and what type it is
+            idx_info = bind.execute(
+                text("SELECT indexdef FROM pg_indexes WHERE schemaname = :schema AND indexname = :idx"),
+                {"schema": pg_schema, "idx": idx_name},
+            ).fetchone()
+
+            if idx_info is None:
+                # Index doesn't exist — create it with the correct type
+                bind.execute(
+                    text(
+                        f"CREATE INDEX IF NOT EXISTS {idx_name} "
+                        f"ON {table_ref} {using_clause} "
+                        f"WHERE fact_type = '{ft}' AND bank_id = '{escaped_bank_id}'"
+                    )
+                )
+                continue
+
+            indexdef = idx_info[0].lower()
+            if target in indexdef:
+                # Already the correct type
+                continue
+
+            # Wrong type — drop and recreate
+            bind.execute(text(f"DROP INDEX IF EXISTS {schema}{idx_name}"))
+            bind.execute(
+                text(
+                    f"CREATE INDEX IF NOT EXISTS {idx_name} "
+                    f"ON {table_ref} {using_clause} "
+                    f"WHERE fact_type = '{ft}' AND bank_id = '{escaped_bank_id}'"
+                )
+            )
+
+
+def downgrade() -> None:
+    # Downgrade recreates indexes as HNSW (the original hardcoded behavior)
+    target = _target_index_type()
+    if target is None:
+        return
+
+    bind = op.get_bind()
+    schema_name = context.config.get_main_option("target_schema")
+    schema = _get_schema_prefix()
+    table_ref = f'"{schema_name}".memory_units' if schema_name else "memory_units"
+    banks_ref = f'"{schema_name}".banks' if schema_name else "banks"
+
+    rows = bind.execute(text(f"SELECT bank_id, internal_id FROM {banks_ref}")).fetchall()  # noqa: S608
+    for row in rows:
+        bank_id = row[0]
+        internal_id = str(row[1]).replace("-", "")[:16]
+        escaped_bank_id = bank_id.replace("'", "''")
+        for ft, ft_short in _FACT_TYPES.items():
+            idx_name = f"idx_mu_emb_{ft_short}_{internal_id}"
+            bind.execute(text(f"DROP INDEX IF EXISTS {schema}{idx_name}"))
+            bind.execute(
+                text(
+                    f"CREATE INDEX IF NOT EXISTS {idx_name} "
+                    f"ON {table_ref} USING hnsw (embedding vector_cosine_ops) "
+                    f"WHERE fact_type = '{ft}' AND bank_id = '{escaped_bank_id}'"
+                )
+            )

hindsight-api-slim/hindsight_api/alembic/versions/aa2b3c4d5e6f_nullable_event_date.py

@@ -0,0 +1,36 @@
+"""Make event_date nullable in memory_units to support timestamp-free content
+
+Revision ID: aa2b3c4d5e6f
+Revises: z1u2v3w4x5y6
+Create Date: 2026-03-02
+
+When callers retain content without a timestamp (e.g. fictional documents, static text),
+the event_date column should be allowed to be NULL rather than defaulting to utcnow().
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "aa2b3c4d5e6f"
+down_revision: str | Sequence[str] | None = "z1u2v3w4x5y6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}memory_units ALTER COLUMN event_date DROP NOT NULL")
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    # Backfill NULLs with now() before restoring the NOT NULL constraint
+    op.execute(f"UPDATE {schema}memory_units SET event_date = now() WHERE event_date IS NULL")
+    op.execute(f"ALTER TABLE {schema}memory_units ALTER COLUMN event_date SET NOT NULL")

hindsight-api-slim/hindsight_api/alembic/versions/b3c4d5e6f7a8_add_content_hash_to_chunks.py

@@ -0,0 +1,32 @@
+"""add content_hash to chunks table for delta retain
+
+Revision ID: b3c4d5e6f7a8
+Revises: a3b4c5d6e7f8
+Create Date: 2026-03-25
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "b3c4d5e6f7a8"
+down_revision: str | Sequence[str] | None = "a3b4c5d6e7f8"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    # Add content_hash column to chunks table for delta comparison
+    op.execute(f"ALTER TABLE {schema}chunks ADD COLUMN IF NOT EXISTS content_hash TEXT")
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}chunks DROP COLUMN IF EXISTS content_hash")

hindsight-api-slim/hindsight_api/alembic/versions/b3c4d5e6f7g8_add_temporal_date_indexes.py

@@ -0,0 +1,68 @@
+"""Add partial indexes on memory_units temporal date fields for fast temporal retrieval
+
+Revision ID: b3c4d5e6f7g8
+Revises: c1a2b3d4e5f6
+Create Date: 2026-03-02
+
+The temporal retrieval entry-point query filters memory_units by occurred_start,
+occurred_end, and mentioned_at using OR conditions. Without dedicated indexes the
+planner falls back to a sequential scan of all bank rows after applying the
+(bank_id, fact_type) index, then re-checks each date field.
+
+These three partial indexes give the planner bitmap-index scan options for the
+three most common date predicates, dramatically reducing the row set before any
+embedding computation is required.
+
+All indexes are created CONCURRENTLY so the migration does not block writes on
+memory_units during production deployments. CONCURRENTLY requires running outside
+a transaction block; see migrations.py for how this is handled safely.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "b3c4d5e6f7g8"
+down_revision: str | Sequence[str] | None = "c1a2b3d4e5f6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    # Partial index on occurred_start (covers "occurred_start BETWEEN $4 AND $5")
+    op.execute("COMMIT")
+    op.execute(
+        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_occurred_start "
+        f"ON {schema}memory_units(bank_id, fact_type, occurred_start) "
+        f"WHERE occurred_start IS NOT NULL"
+    )
+    # Partial index on occurred_end (covers "occurred_end BETWEEN $4 AND $5")
+    op.execute("COMMIT")
+    op.execute(
+        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_occurred_end "
+        f"ON {schema}memory_units(bank_id, fact_type, occurred_end) "
+        f"WHERE occurred_end IS NOT NULL"
+    )
+    # Partial index on mentioned_at (covers "mentioned_at BETWEEN $4 AND $5")
+    op.execute("COMMIT")
+    op.execute(
+        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_mentioned_at "
+        f"ON {schema}memory_units(bank_id, fact_type, mentioned_at) "
+        f"WHERE mentioned_at IS NOT NULL"
+    )
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute("COMMIT")
+    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_mentioned_at")
+    op.execute("COMMIT")
+    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_occurred_end")
+    op.execute("COMMIT")
+    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_occurred_start")

hindsight-api-slim/hindsight_api/alembic/versions/b4c5d6e7f8a9_backfill_observation_scopes.py

@@ -0,0 +1,34 @@
+"""Backfill observation_scopes column if missing.
+
+This migration ensures observation_scopes exists even on databases that had
+revision z1u2v3w4x5y6 applied when it referred to the old text_signals migration
+(before it was renamed to a2b3c4d5e6f7). The ADD COLUMN IF NOT EXISTS makes this
+a no-op on databases that already have the column.
+
+Revision ID: b4c5d6e7f8a9
+Revises: a2b3c4d5e6f7
+Create Date: 2026-03-02
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "b4c5d6e7f8a9"
+down_revision: str | Sequence[str] | None = "a2b3c4d5e6f7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS observation_scopes JSONB")
+
+
+def downgrade() -> None:
+    pass  # intentionally no-op — safe to leave the column in place

hindsight-api-slim/hindsight_api/alembic/versions/b7c4d8e9f1a2_add_chunks_table.py

@@ -0,0 +1,70 @@
+"""add_chunks_table
+
+Revision ID: b7c4d8e9f1a2
+Revises: 5a366d414dce
+Create Date: 2025-11-28 00:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "b7c4d8e9f1a2"
+down_revision: str | Sequence[str] | None = "5a366d414dce"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Add chunks table and link memory_units to chunks."""
+
+    # Create chunks table with single text PK (bank_id_document_id_chunk_index)
+    op.create_table(
+        "chunks",
+        sa.Column("chunk_id", sa.Text(), nullable=False),
+        sa.Column("document_id", sa.Text(), nullable=False),
+        sa.Column("bank_id", sa.Text(), nullable=False),
+        sa.Column("chunk_index", sa.Integer(), nullable=False),
+        sa.Column("chunk_text", sa.Text(), nullable=False),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id", "bank_id"],
+            ["documents.id", "documents.bank_id"],
+            name="chunks_document_fkey",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("chunk_id", name=op.f("pk_chunks")),
+    )
+
+    # Add indexes for efficient queries
+    op.create_index("idx_chunks_document_id", "chunks", ["document_id"])
+    op.create_index("idx_chunks_bank_id", "chunks", ["bank_id"])
+
+    # Add chunk_id column to memory_units (nullable, as existing records won't have chunks)
+    op.add_column("memory_units", sa.Column("chunk_id", sa.Text(), nullable=True))
+
+    # Add foreign key constraint to chunks table
+    op.create_foreign_key(
+        "memory_units_chunk_fkey", "memory_units", "chunks", ["chunk_id"], ["chunk_id"], ondelete="SET NULL"
+    )
+
+    # Add index on chunk_id for efficient lookups
+    op.create_index("idx_memory_units_chunk_id", "memory_units", ["chunk_id"])
+
+
+def downgrade() -> None:
+    """Remove chunks table and chunk_id from memory_units."""
+
+    # Drop index and foreign key from memory_units
+    op.drop_index("idx_memory_units_chunk_id", table_name="memory_units")
+    op.drop_constraint("memory_units_chunk_fkey", "memory_units", type_="foreignkey")
+    op.drop_column("memory_units", "chunk_id")
+
+    # Drop chunks table indexes and table
+    op.drop_index("idx_chunks_bank_id", table_name="chunks")
+    op.drop_index("idx_chunks_document_id", table_name="chunks")
+    op.drop_table("chunks")

hindsight-api-slim/hindsight_api/alembic/versions/c1a2b3d4e5f6_enable_pg_trgm_and_entities_trgm_index.py

@@ -0,0 +1,59 @@
+"""Enable pg_trgm extension and add GIN trigram index on entities.canonical_name
+
+Revision ID: c1a2b3d4e5f6
+Revises: b4c5d6e7f8a9
+Create Date: 2026-03-02
+
+Index is created CONCURRENTLY so the migration does not block writes on entities
+during production deployments. CONCURRENTLY requires running outside a transaction
+block; see migrations.py for how this is handled safely.
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import context, op
+
+revision: str = "c1a2b3d4e5f6"
+down_revision: str | Sequence[str] | None = "b4c5d6e7f8a9"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    # pg_trgm ships with most PostgreSQL installations as a contrib module.
+    # It enables fast similarity lookups via GIN indexes, used for entity name matching.
+    # On managed services (e.g. Azure Flexible Server), the extension may not be
+    # available or may require manual enablement.  We gracefully skip the index
+    # creation if the extension cannot be loaded — the entity resolver will
+    # auto-detect and fall back to the "full" lookup strategy at runtime.  See #626.
+    conn = op.get_bind()
+    try:
+        conn.execute(sa.text("CREATE EXTENSION IF NOT EXISTS pg_trgm"))
+    except Exception:
+        # Extension not available (managed Postgres, insufficient privileges, etc.)
+        # Roll back the failed statement and skip index creation.
+        conn.execute(sa.text("ROLLBACK"))
+        conn.execute(sa.text("BEGIN"))
+        return
+
+    schema = _get_schema_prefix()
+    # GIN index on canonical_name enables sub-millisecond trigram similarity queries
+    # (% operator, similarity()) instead of full-table scans across all bank entities.
+    op.execute("COMMIT")
+    op.execute(
+        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS entities_canonical_name_trgm_idx "
+        f"ON {schema}entities USING GIN (canonical_name gin_trgm_ops)"
+    )
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute("COMMIT")
+    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}entities_canonical_name_trgm_idx")
+    # Note: not dropping pg_trgm extension as other indexes may depend on it

hindsight-api-slim/hindsight_api/alembic/versions/c2d3e4f5g6h7_add_audit_log_table.py

@@ -0,0 +1,61 @@
+"""Add audit_log table for feature usage tracking.
+
+Merge migration that combines the two existing heads (a3b4c5d6e7f8 + c8e5f2a3b4d1).
+
+Stores raw request/response as JSONB for expandability without future migrations.
+The metadata JSONB column allows adding arbitrary fields in the future.
+
+Revision ID: c2d3e4f5g6h7
+Revises: a3b4c5d6e7f8, c8e5f2a3b4d1
+Create Date: 2026-03-26
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "c2d3e4f5g6h7"
+down_revision: str | Sequence[str] | None = ("a3b4c5d6e7f8", "c8e5f2a3b4d1")
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+
+    op.execute(
+        f"""
+        CREATE TABLE IF NOT EXISTS {schema}audit_log (
+            id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            action TEXT NOT NULL,
+            transport TEXT NOT NULL,
+            bank_id TEXT,
+            started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+            ended_at TIMESTAMPTZ,
+            request JSONB,
+            response JSONB,
+            metadata JSONB DEFAULT '{{}}'::jsonb
+        )
+        """
+    )
+
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS idx_audit_log_action_started ON {schema}audit_log (action, started_at DESC)"
+    )
+    op.execute(f"CREATE INDEX IF NOT EXISTS idx_audit_log_bank_started ON {schema}audit_log (bank_id, started_at DESC)")
+    op.execute(f"CREATE INDEX IF NOT EXISTS idx_audit_log_started ON {schema}audit_log (started_at DESC)")
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_audit_log_started")
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_audit_log_bank_started")
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_audit_log_action_started")
+    op.execute(f"DROP TABLE IF EXISTS {schema}audit_log")

hindsight-api-slim/hindsight_api/alembic/versions/c3d4e5f6g7h8_add_history_to_mental_models.py

@@ -0,0 +1,30 @@
+"""Add history column to mental_models
+
+Revision ID: c3d4e5f6g7h8
+Revises: a2b3c4d5e6f7, a2b3c4d5e6f8
+Create Date: 2026-03-06
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "c3d4e5f6g7h8"
+down_revision: str | Sequence[str] | None = ("a2b3c4d5e6f7", "a2b3c4d5e6f8")
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}mental_models ADD COLUMN IF NOT EXISTS history JSONB DEFAULT '[]'::jsonb")
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS history")