other fix

* feat: add custom extraction prompt

* feat: add custom extraction prompt

* test

.github/workflows/test.yml

@@ -153,8 +153,15 @@ jobs:
     - name: Build docs
       run: npm run build --workspace=hindsight-docs
 
-  build-rust-cli:
+  test-rust-cli:
     runs-on: ubuntu-latest
+    env:
+      HINDSIGHT_API_LLM_PROVIDER: groq
+      HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
+      HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
+      HINDSIGHT_API_URL: http://localhost:8888
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v4
@@ -171,6 +178,10 @@ jobs:
           hindsight-cli/target
         key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
+    - name: Run unit tests
+      working-directory: hindsight-cli
+      run: cargo test
+
     - name: Build CLI
       working-directory: hindsight-cli
       run: cargo build --release
@@ -182,29 +193,6 @@ jobs:
         path: hindsight-cli/target/release/hindsight
         retention-days: 1
 
-  test-rust-cli:
-    runs-on: ubuntu-latest
-    needs: build-rust-cli
-    env:
-      HINDSIGHT_API_LLM_PROVIDER: groq
-      HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
-      HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
-      HINDSIGHT_API_URL: http://localhost:8888
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Download CLI artifact
-      uses: actions/download-artifact@v4
-      with:
-        name: hindsight-cli
-        path: /tmp/cli
-
-    - name: Make CLI executable
-      run: chmod +x /tmp/cli/hindsight
-
     - name: Install uv
       uses: astral-sh/setup-uv@v5
       with:
@@ -222,7 +210,7 @@ jobs:
 
     - name: Install API dependencies
       working-directory: ./hindsight-api
-      run: uv sync --no-install-project --index-strategy unsafe-best-match
+      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
 
     - name: Create .env file
       run: |
@@ -251,7 +239,7 @@ jobs:
 
     - name: Run CLI smoke test
       run: |
-        HINDSIGHT_CLI=/tmp/cli/hindsight ./hindsight-cli/smoke-test.sh
+        HINDSIGHT_CLI=hindsight-cli/target/release/hindsight ./hindsight-cli/smoke-test.sh
 
     - name: Show API server logs
       if: always()
@@ -325,6 +313,7 @@ jobs:
       GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
       GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
       HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -351,7 +340,7 @@ jobs:
 
     - name: Install dependencies
       working-directory: ./hindsight-api
-      run: uv sync --extra test --no-install-project --index-strategy unsafe-best-match
+      run: uv sync --frozen --extra test --no-install-project --index-strategy unsafe-best-match
 
     - name: Cache HuggingFace models
       uses: actions/cache@v4
@@ -412,11 +401,11 @@ jobs:
 
     - name: Install client test dependencies
       working-directory: ./hindsight-clients/python
-      run: uv sync --extra test --index-strategy unsafe-best-match
+      run: uv sync --frozen --extra test --index-strategy unsafe-best-match
 
     - name: Install API dependencies
       working-directory: ./hindsight-api
-      run: uv sync --no-install-project --index-strategy unsafe-best-match
+      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
 
     - name: Create .env file
       run: |
@@ -489,7 +478,7 @@ jobs:
 
     - name: Install API dependencies
       working-directory: ./hindsight-api
-      run: uv sync --no-install-project --index-strategy unsafe-best-match
+      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
 
     - name: Install TypeScript client dependencies
       working-directory: ./hindsight-clients/typescript
@@ -577,7 +566,7 @@ jobs:
 
     - name: Install API dependencies
       working-directory: ./hindsight-api
-      run: uv sync --no-install-project --index-strategy unsafe-best-match
+      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
 
     - name: Create .env file
       run: |
@@ -644,11 +633,11 @@ jobs:
 
     - name: Install API dependencies
       working-directory: ./hindsight-api
-      run: uv sync --no-install-project --index-strategy unsafe-best-match
+      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
 
     - name: Install integration test dependencies
       working-directory: ./hindsight-integration-tests
-      run: uv sync
+      run: uv sync --frozen
 
     - name: Cache HuggingFace models
       uses: actions/cache@v4
@@ -728,7 +717,7 @@ jobs:
 
     - name: Install dependencies
       working-directory: ./hindsight-integrations/litellm
-      run: uv sync --extra dev
+      run: uv sync --frozen --extra dev
 
     - name: Run tests
       working-directory: ./hindsight-integrations/litellm
@@ -759,7 +748,7 @@ jobs:
 
     - name: Install dependencies
       working-directory: ./hindsight-embed
-      run: uv sync --index-strategy unsafe-best-match
+      run: uv sync --frozen --index-strategy unsafe-best-match
 
     - name: Cache HuggingFace models
       uses: actions/cache@v4
@@ -776,7 +765,7 @@ jobs:
 
   test-doc-examples:
     runs-on: ubuntu-latest
-    needs: build-rust-cli
+    needs: test-rust-cli
     env:
       HINDSIGHT_API_LLM_PROVIDER: groq
       HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
@@ -819,11 +808,11 @@ jobs:
       working-directory: ./hindsight-api
       run: |
         uv build
-        uv sync --no-install-project --index-strategy unsafe-best-match
+        uv sync --frozen --no-install-project --index-strategy unsafe-best-match
 
     - name: Install Python client dependencies
       working-directory: ./hindsight-clients/python
-      run: uv sync --extra test --index-strategy unsafe-best-match
+      run: uv sync --frozen --extra test --index-strategy unsafe-best-match
 
     - name: Install TypeScript client
       run: |
@@ -886,6 +875,66 @@ jobs:
         echo "=== API Server Logs ==="
         cat /tmp/api-server.log || echo "No API server log found"
 
+  test-upgrade:
+    runs-on: ubuntu-latest
+    env:
+      HINDSIGHT_API_LLM_PROVIDER: groq
+      HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
+      HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # Full history needed for git clone of tags
+
+    - name: Fetch tags
+      run: git fetch --tags
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version-file: ".python-version"
+
+    - name: Cache HuggingFace models
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/huggingface
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-huggingface-
+
+    - name: Install hindsight-dev dependencies
+      working-directory: ./hindsight-dev
+      run: uv sync --frozen --extra test --index-strategy unsafe-best-match
+
+    - name: Install current hindsight-api
+      working-directory: ./hindsight-api
+      run: uv sync --frozen --index-strategy unsafe-best-match
+
+    - name: Pre-download models
+      working-directory: ./hindsight-api
+      run: |
+        uv run python -c "
+        from sentence_transformers import SentenceTransformer, CrossEncoder
+        print('Downloading embedding model...')
+        SentenceTransformer('BAAI/bge-small-en-v1.5')
+        print('Downloading cross-encoder model...')
+        CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
+        print('Models downloaded successfully')
+        "
+
+    - name: Run upgrade tests
+      working-directory: ./hindsight-dev
+      run: uv run pytest upgrade_tests/ -v --tb=short
+
   verify-generated-files:
     runs-on: ubuntu-latest
     env:
@@ -927,9 +976,9 @@ jobs:
 
     - name: Install Python dependencies
       run: |
-        cd hindsight-dev && uv sync --index-strategy unsafe-best-match
-        cd ../hindsight-api && uv sync --index-strategy unsafe-best-match
-        cd ../hindsight-embed && uv sync --index-strategy unsafe-best-match
+        cd hindsight-dev && uv sync --frozen --index-strategy unsafe-best-match
+        cd ../hindsight-api && uv sync --frozen --index-strategy unsafe-best-match
+        cd ../hindsight-embed && uv sync --frozen --index-strategy unsafe-best-match
 
     - name: Run generate-openapi
       run: ./scripts/generate-openapi.sh

.gitignore

@@ -5,7 +5,8 @@ build/
 dist/
 wheels/
 *.egg-info
-
+.mcp.json
+.osgrep
 # Virtual environments
 .venv
 
@@ -26,6 +27,10 @@ docker-compose.override.yml
 # NLTK data (will be downloaded automatically)
 nltk_data/
 
+# Monitoring stack (Prometheus/Grafana binaries and data)
+.monitoring/
+.pgbouncer/
+
 # Large benchmark datasets (will be downloaded automatically)
 **/longmemeval_s_cleaned.json
 
@@ -41,4 +46,8 @@ hindsight-docs/static/llms-full.txt
 hindsight-dev/benchmarks/locomo/results/
 hindsight-dev/benchmarks/longmemeval/results/
 hindsight-cli/target
-hindsight-clients/rust/target
+hindsight-clients/rust/target
+.claude
+whats-next.md
+TASK.md
+CHANGELOG.md

AGENTS.md

@@ -1,153 +1,3 @@
 # AGENTS.md
 
-This document captures architectural decisions and coding conventions for the Hindsight project.
-
-## Documentation
-
-- **Main documentation**: [hindsight-docs/docs/developer/](./hindsight-docs/docs/developer/)
-- **Use case patterns**: [hindsight-docs/docs/cookbook/](./hindsight-docs/docs/cookbook/)
-- **API reference**: Auto-generated from OpenAPI spec
-
-## Project Structure
-
-```
-hindsight/              # Python package for embedded usage
-hindsight-api/          # FastAPI server (core memory engine)
-hindsight-cli/          # Rust CLI client
-hindsight-embed/        # Embedded CLI (no server needed)
-hindsight-control-plane/ # Next.js admin UI
-hindsight-docs/         # Docusaurus documentation site
-hindsight-dev/          # Development tools and benchmarks
-hindsight-integrations/ # Framework integrations (LangChain, etc.)
-hindsight-clients/      # Generated API clients (Python, TypeScript, Rust)
-```
-
-## Core Concepts
-
-### Memory Banks
-- Each bank is an isolated memory store (like a "brain" for one user/agent)
-- Banks contain: memory units (facts), entities, documents, entity links
-- Banks have a **disposition** (personality traits) and **background** (context)
-- Bank isolation is strict - no cross-bank data leakage
-
-### Memory Types
-- **World facts**: General knowledge ("The sky is blue")
-- **Experience facts**: Personal experiences ("I visited Paris in 2023")
-- **Opinion facts**: Beliefs with confidence scores ("Paris is beautiful" - 0.9 confidence)
-
-### Operations
-- **Retain**: Store new memories (extracts facts, entities, relationships)
-- **Recall**: Retrieve memories (semantic, BM25, graph, temporal search)
-- **Reflect**: Deep analysis to form new insights/opinions
-
-## API Design Decisions
-
-### Single Bank Per Request
-- All API endpoints (`recall`, `reflect`, `retain`) operate on a single bank
-- Multi-bank queries are the **client/agent's responsibility** to orchestrate
-- This keeps the API simple and the isolation model clear
-
-### Disposition Traits (3-trait system)
-- **Skepticism** (1-5): How skeptical vs trusting when forming opinions
-- **Literalism** (1-5): How literally to interpret information
-- **Empathy** (1-5): How much to consider emotional context
-- These influence the `reflect` operation, not `recall`
-- Background info also only affects `reflect` (opinion formation)
-
-## Multi-Bank Architecture Patterns
-
-See [hindsight-docs/docs/cookbook/](./hindsight-docs/docs/cookbook/) for detailed guides:
-
-- **Per-User Memory**: One bank per user, simplest pattern
-- **Support Agent + Shared Knowledge**: User bank + shared docs bank, client orchestrates
-
-## Developer Guide
-
-### Running the API Server
-
-```bash
-# From project root
-./scripts/dev/start-api.sh
-
-# With options
-./scripts/dev/start-api.sh --reload --port 8888 --log-level debug
-```
-
-### Running Tests
-
-```bash
-# API tests
-cd hindsight-api
-uv run pytest tests/
-
-# Specific test
-uv run pytest tests/test_http_api_integration.py -v
-```
-
-### Generating OpenAPI Spec
-
-After changing API endpoints, regenerate the OpenAPI spec and docs:
-
-```bash
-./scripts/generate-openapi.sh
-```
-
-This will:
-1. Generate `openapi.json` at project root
-2. Copy to `hindsight-docs/openapi.json`
-3. Regenerate API reference documentation
-
-### Generating API Clients
-
-After updating the OpenAPI spec, regenerate all clients:
-
-```bash
-./scripts/generate-clients.sh
-```
-
-This generates:
-- **Rust client**: `hindsight-clients/rust/` (via progenitor in build.rs)
-- **Python client**: `hindsight-clients/python/` (via openapi-generator Docker)
-- **TypeScript client**: `hindsight-clients/typescript/` (via @hey-api/openapi-ts)
-
-Note: The maintained wrapper `hindsight_client.py` and `README.md` are preserved during regeneration.
-
-### Running the Documentation Site
-
-```bash
-./scripts/dev/start-docs.sh
-```
-
-### Running the Control Plane
-
-```bash
-./scripts/dev/start-control-plane.sh
-```
-
-## Code Style
-
-### Python (hindsight-api)
-- Use `uv` for package management
-- Async throughout (asyncpg, async FastAPI endpoints)
-- Pydantic models for request/response validation
-- No py files at project root - maintain clean directory structure
-
-### TypeScript (control-plane, clients)
-- Next.js with App Router for control plane
-- Tailwind CSS with shadcn/ui components
-
-### Rust (CLI)
-- Async with tokio
-- reqwest for HTTP client
-- progenitor for API client generation
-
-## Database
-
-- PostgreSQL with pgvector extension
-- Schema managed via Alembic migrations in `hindsight-api/alembic/`, db migrations happen during api startup, no manual commands
-- Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
-
-# Branding
-## Colors
-- Primary: gradient from #0074d9 to #009296
-
+See [CLAUDE.md](./CLAUDE.md) for project documentation and coding conventions.

CLAUDE.md

@@ -4,7 +4,10 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 
 ## Project Overview
 
-Hindsight is an agent memory system that provides long-term memory for AI agents using biomimetic data structures. It stores memories as World facts, Experiences, Opinions, and Observations across memory banks.
+Hindsight is an agent memory system that provides long-term memory for AI agents using biomimetic data structures. Memories are organized as:
+- **World facts**: General knowledge ("The sky is blue")
+- **Experience facts**: Personal experiences ("I visited Paris in 2023")
+- **Mental models**: Consolidated knowledge synthesized from facts ("User prefers functional programming patterns")
 
 ## Development Commands
 
@@ -13,14 +16,21 @@ Hindsight is an agent memory system that provides long-term memory for AI agents
 # Start API server (loads .env automatically)
 ./scripts/dev/start-api.sh
 
-# Run tests
+# Run all tests (parallelized with pytest-xdist)
 cd hindsight-api && uv run pytest tests/
 
 # Run specific test file
 cd hindsight-api && uv run pytest tests/test_http_api_integration.py -v
 
-# Lint
+# Run single test function
+cd hindsight-api && uv run pytest tests/test_retain.py::test_retain_simple -v
+
+# Lint and format
 cd hindsight-api && uv run ruff check .
+cd hindsight-api && uv run ruff format .
+
+# Type checking (uses ty - extremely fast type checker from Astral)
+cd hindsight-api && uv run ty check hindsight_api/
 ```
 
 ### Control Plane (Next.js)
@@ -37,7 +47,7 @@ cd hindsight-control-plane && npm run dev
 
 ### Generating Clients/OpenAPI
 ```bash
-# Regenerate OpenAPI spec after API changes
+# Regenerate OpenAPI spec after API changes (REQUIRED after changing endpoints)
 ./scripts/generate-openapi.sh
 
 # Regenerate all client SDKs (Python, TypeScript, Rust)
@@ -57,33 +67,92 @@ cd hindsight-control-plane && npm run dev
 - **hindsight-api/**: Core FastAPI server with memory engine (Python, uv)
 - **hindsight/**: Embedded Python bundle (hindsight-all package)
 - **hindsight-control-plane/**: Admin UI (Next.js, npm)
-- **hindsight-cli/**: CLI tool (Rust, cargo)
+- **hindsight-cli/**: CLI tool (Rust, cargo, uses progenitor for API client)
 - **hindsight-clients/**: Generated SDK clients (Python, TypeScript, Rust)
 - **hindsight-docs/**: Docusaurus documentation site
 - **hindsight-integrations/**: Framework integrations (LiteLLM, OpenAI)
 - **hindsight-dev/**: Development tools and benchmarks
 
 ### Core Engine (hindsight-api/hindsight_api/engine/)
-- `memory_engine.py`: Main orchestrator for retain/recall/reflect operations
+- `memory_engine.py`: Main orchestrator (~170KB) for retain/recall/reflect operations
 - `llm_wrapper.py`: LLM abstraction supporting OpenAI, Anthropic, Gemini, Groq, Ollama, LM Studio
-- `embeddings.py`: Embedding generation (local or TEI)
+- `embeddings.py`: Embedding generation (local sentence-transformers or TEI)
 - `cross_encoder.py`: Reranking (local or TEI)
 - `entity_resolver.py`: Entity extraction and normalization
 - `query_analyzer.py`: Query intent analysis
-- `retain/`: Memory ingestion pipeline
-- `search/`: Multi-strategy retrieval (semantic, BM25, graph, temporal)
+
+**retain/**: Memory ingestion pipeline
+- `orchestrator.py`: Coordinates the retain flow
+- `fact_extraction.py`: LLM-based fact extraction from content
+- `link_utils.py`: Entity link creation and management
+
+**search/**: Multi-strategy retrieval
+- `retrieval.py`: Main retrieval orchestrator
+- `graph_retrieval.py`: Entity/relationship graph traversal
+- `mpfp_retrieval.py`: Multi-Path Fact Propagation retrieval
+- `fusion.py`: Reciprocal rank fusion for combining results
+- `reranking.py`: Cross-encoder reranking
 
 ### API Layer (hindsight-api/hindsight_api/api/)
-FastAPI routers for all endpoints. Main operations:
+- `http.py`: FastAPI HTTP routers (~80KB) for all REST endpoints
+- `mcp.py`: Model Context Protocol server implementation
+
+Main operations:
 - **Retain**: Store memories, extracts facts/entities/relationships
-- **Recall**: Retrieve memories via parallel search strategies + reranking
-- **Reflect**: Deep analysis forming new opinions/observations
+- **Recall**: Retrieve memories via 4 parallel strategies (semantic, BM25, graph, temporal) + reranking
+- **Reflect**: Disposition-aware reasoning using memories and mental models.
 
 ### Database
 PostgreSQL with pgvector. Schema managed via Alembic migrations in `hindsight-api/hindsight_api/alembic/`. Migrations run automatically on API startup.
 
 Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
 
+### Adding Database Migrations
+
+1. **Create a new migration file** in `hindsight-api/hindsight_api/alembic/versions/`:
+   - File name format: `<revision_id>_<description>.py` (e.g., `f1a2b3c4d5e6_add_new_index.py`)
+   - Use a unique hex revision ID (12 chars)
+   - Set `down_revision` to the previous migration's revision ID
+
+2. **Migration template**:
+   ```python
+   """Description of the migration
+
+   Revision ID: f1a2b3c4d5e6
+   Revises: <previous_revision_id>
+   Create Date: YYYY-MM-DD
+   """
+   from collections.abc import Sequence
+   from alembic import context, op
+
+   revision: str = "f1a2b3c4d5e6"
+   down_revision: str | Sequence[str] | None = "<previous_revision_id>"
+   branch_labels: str | Sequence[str] | None = None
+   depends_on: str | Sequence[str] | None = None
+
+   def _get_schema_prefix() -> str:
+       """Get schema prefix for table names (required for multi-tenant support)."""
+       schema = context.config.get_main_option("target_schema")
+       return f'"{schema}".' if schema else ""
+
+   def upgrade() -> None:
+       schema = _get_schema_prefix()
+       op.execute(f"CREATE INDEX ... ON {schema}table_name(...)")
+
+   def downgrade() -> None:
+       schema = _get_schema_prefix()
+       op.execute(f"DROP INDEX IF EXISTS {schema}index_name")
+   ```
+
+3. **Run migrations locally**:
+   ```bash
+   # Set database URL and run migrations
+   uv run hindsight-admin run-db-migration
+
+   # Run on a specific tenant schema
+   uv run hindsight-admin run-db-migration --schema tenant_xyz
+   ```
+
 ## Key Conventions
 
 ### Code Quality
@@ -94,20 +163,73 @@ Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
 This runs the same checks as the pre-commit hook (Ruff for Python, ESLint/Prettier for TypeScript).
 
 ### Memory Banks
-- Each bank is isolated (no cross-bank data access)
+- Each bank is an isolated memory store (like a "brain" for one user/agent)
 - Banks have dispositions (skepticism, literalism, empathy traits 1-5) affecting reflect
 - Banks can have background context
+- Bank isolation is strict - no cross-bank data leakage
 
 ### API Design
 - All endpoints operate on a single bank per request
-- Multi-bank queries are client responsibility
+- Multi-bank queries are client responsibility to orchestrate
 - Disposition traits only affect reflect, not recall
 
+### Control Plane API Routes
+
+When adding or modifying parameters in the dataplane API (hindsight-api), you must also update the control plane routes that proxy to it:
+
+1. **API Routes** (`hindsight-control-plane/src/app/api/`):
+   - `recall/route.ts` - proxies to `/v1/default/banks/{bank_id}/memories/recall`
+   - `reflect/route.ts` - proxies to `/v1/default/banks/{bank_id}/reflect`
+   - `memories/retain/route.ts` - proxies to `/v1/default/banks/{bank_id}/memories/retain`
+   - Other routes follow the same pattern
+
+2. **Client types** (`hindsight-control-plane/src/lib/api.ts`):
+   - Update the TypeScript type definitions for `recall()`, `reflect()`, `retain()` etc.
+
+3. **Checklist when adding new API parameters**:
+   - Add parameter extraction in the route handler (destructure from `body`)
+   - Pass the parameter to the SDK call
+   - Update the client type definition in `lib/api.ts`
+   - Update any UI components that need to use the new parameter
+
 ### Python Style
 - Python 3.11+, type hints required
 - Async throughout (asyncpg, async FastAPI)
 - Pydantic models for request/response
 - Ruff for linting (line-length 120)
+- No Python files at project root - maintain clean directory structure
+- **Never use multi-item tuple return values** - prefer dataclass or Pydantic model for structured returns
+
+### Type Safety with Pydantic Models
+**NEVER use raw `dict` types for structured data.** Always use Pydantic models:
+- Use Pydantic `BaseModel` for all data structures passed between functions
+- Add `@field_validator` for type coercion (e.g., ensuring datetimes are timezone-aware)
+- Avoid `dict.get()` patterns - use typed model attributes instead
+- Parse external data (JSON, API responses) into Pydantic models at the boundary
+- This catches type errors at parse time, not deep in business logic
+
+```python
+# BAD - error-prone dict access
+def process(data: dict) -> str:
+    return data.get("name", "")  # No validation, silent failures
+
+# GOOD - typed and validated
+class UserData(BaseModel):
+    name: str
+    created_at: datetime
+
+    @field_validator("created_at", mode="before")
+    @classmethod
+    def ensure_tz_aware(cls, v):
+        if isinstance(v, str):
+            v = datetime.fromisoformat(v.replace("Z", "+00:00"))
+        if v.tzinfo is None:
+            return v.replace(tzinfo=timezone.utc)
+        return v
+
+def process(data: UserData) -> str:
+    return data.name  # Type-safe, validated at construction
+```
 
 ### TypeScript Style
 - Next.js App Router for control plane
@@ -145,7 +267,7 @@ cp .env.example .env
 # Python deps
 uv sync --directory hindsight-api/
 
-# Node deps (workspace)
+# Node deps (uses npm workspaces)
 npm install
 ```
 
@@ -153,3 +275,8 @@ Required env vars:
 - `HINDSIGHT_API_LLM_PROVIDER`: openai, anthropic, gemini, groq, ollama, lmstudio
 - `HINDSIGHT_API_LLM_API_KEY`: Your API key
 - `HINDSIGHT_API_LLM_MODEL`: Model name (e.g., o3-mini, claude-sonnet-4-20250514)
+
+Optional (uses local models by default):
+- `HINDSIGHT_API_EMBEDDINGS_PROVIDER`: local (default) or tei
+- `HINDSIGHT_API_RERANKER_PROVIDER`: local (default) or tei
+- `HINDSIGHT_API_DATABASE_URL`: External PostgreSQL (uses embedded pg0 by default)

README.md

@@ -5,7 +5,7 @@
 [Documentation](https://hindsight.vectorize.io) • [Paper](https://arxiv.org/abs/2512.12818) • [Cookbook](https://hindsight.vectorize.io/cookbook) • [Hindsight Cloud](https://vectorize.io/hindsight/cloud)
 
 [![CI](https://github.com/vectorize-io/hindsight/actions/workflows/release.yml/badge.svg)](https://github.com/vectorize-io/hindsight/actions/workflows/release.yml)
-[![Slack Community](https://img.shields.io/badge/Slack-Join%20Community-4A154B?logo=slack)](https://join.slack.com/t/hindsight-space/shared_invite/zt-3klo21kua-VUCC_zHP5rIcXFB1_5yw6A)
+[![Slack Community](https://img.shields.io/badge/Slack-Join%20Community-4A154B?logo=slack)](https://join.slack.com/t/hindsight-space/shared_invite/zt-3nhbm4w29-LeSJ5Ixi6j8PdiYOCPlOgg)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 ![PyPI - Downloads](https://img.shields.io/pypi/dm/hindsight-api?label=PyPI)
 ![NPM Downloads](https://img.shields.io/npm/dm/%40vectorize-io%2Fhindsight-client?logoColor=orange&label=NPM&color=blue&link=https%3A%2F%2Fwww.npmjs.com%2Fpackage%2F%40vectorize-io%2Fhindsight-client)
@@ -242,7 +242,7 @@ client.reflect(bank_id="my-bank", query="What should I know about Alice?")
 - [CLI](https://hindsight.vectorize.io/sdks/cli)
 
 **Community:**
-- [Slack](https://join.slack.com/t/hindsight-space/shared_invite/zt-3klo21kua-VUCC_zHP5rIcXFB1_5yw6A)
+- [Slack](https://join.slack.com/t/hindsight-space/shared_invite/zt-3nhbm4w29-LeSJ5Ixi6j8PdiYOCPlOgg)
 - [GitHub Issues](https://github.com/vectorize-io/hindsight/issues)
 
 ---

docker/standalone/Dockerfile

@@ -2,19 +2,24 @@
 # Supports building API-only, Control Plane-only, or both
 #
 # Build args:
-#   INCLUDE_API=true/false       - Include API (default: true)
-#   INCLUDE_CP=true/false        - Include Control Plane (default: true)
-#   PRELOAD_ML_MODELS=true/false - Pre-download ML models during build (default: true)
+#   INCLUDE_API=true/false         - Include API (default: true)
+#   INCLUDE_CP=true/false          - Include Control Plane (default: true)
+#   INCLUDE_LOCAL_MODELS=true/false - Include local ML models for embeddings/reranking (default: true)
+#                                     Set to false when using external providers (TEI, OpenAI, Cohere)
+#   PRELOAD_ML_MODELS=true/false   - Pre-download ML models during build (default: true)
+#                                     Only effective when INCLUDE_LOCAL_MODELS=true
 #
 # Examples:
 #   docker build -t hindsight .                                          # Both (standalone)
 #   docker build -t hindsight-api --build-arg INCLUDE_CP=false .         # API only
 #   docker build -t hindsight-cp --build-arg INCLUDE_API=false .         # Control Plane only
 #   docker build -t hindsight --build-arg PRELOAD_ML_MODELS=false .      # Skip ML model preload
+#   docker build -t hindsight --build-arg INCLUDE_LOCAL_MODELS=false .   # Skip local ML deps (for external providers)
 
 ARG INCLUDE_API=true
 ARG INCLUDE_CP=true
 ARG PRELOAD_ML_MODELS=true
+ARG INCLUDE_LOCAL_MODELS=true
 
 # =============================================================================
 # Stage: API Builder
@@ -22,6 +27,7 @@ ARG PRELOAD_ML_MODELS=true
 FROM python:3.11-slim AS api-builder
 
 ARG INCLUDE_API
+ARG INCLUDE_LOCAL_MODELS
 RUN if [ "$INCLUDE_API" != "true" ]; then echo "Skipping API build" && exit 0; fi
 
 WORKDIR /app
@@ -40,6 +46,15 @@ COPY hindsight-api/README.md ./api/
 
 WORKDIR /app/api
 
+# Remove local ML model dependencies if INCLUDE_LOCAL_MODELS=false
+# This creates a smaller image when using external providers (TEI, OpenAI, Cohere)
+RUN if [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then \
+        echo "Removing local-models dependencies (sentence-transformers, torch, transformers)..." && \
+        sed -i '/"sentence-transformers/d' pyproject.toml && \
+        sed -i '/"transformers/d' pyproject.toml && \
+        sed -i '/"torch/d' pyproject.toml; \
+    fi
+
 # Sync dependencies (will create lock file if needed)
 RUN uv sync
 
@@ -153,8 +168,10 @@ USER hindsight
 ENV PATH="/app/api/.venv/bin:${PATH}"
 
 # Pre-download ML models to avoid runtime download (conditional)
+# Only runs if both PRELOAD_ML_MODELS=true AND INCLUDE_LOCAL_MODELS=true
 ARG PRELOAD_ML_MODELS
-RUN if [ "$PRELOAD_ML_MODELS" = "true" ]; then \
+ARG INCLUDE_LOCAL_MODELS
+RUN if [ "$PRELOAD_ML_MODELS" = "true" ] && [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
     /app/api/.venv/bin/python -c "\
 from sentence_transformers import SentenceTransformer, CrossEncoder; \
 print('Downloading embedding model...'); \
@@ -162,6 +179,7 @@ SentenceTransformer('BAAI/bge-small-en-v1.5'); \
 print('Downloading cross-encoder model...'); \
 CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'); \
 print('Models cached successfully')"; \
+    elif [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then echo "Skipping ML model preload (local-models not included)"; \
     else echo "Skipping ML model preload"; fi
 
 EXPOSE 8888
@@ -258,8 +276,10 @@ USER hindsight
 ENV PATH="/app/api/.venv/bin:${PATH}"
 
 # Pre-download ML models to avoid runtime download (conditional)
+# Only runs if both PRELOAD_ML_MODELS=true AND INCLUDE_LOCAL_MODELS=true
 ARG PRELOAD_ML_MODELS
-RUN if [ "$PRELOAD_ML_MODELS" = "true" ]; then \
+ARG INCLUDE_LOCAL_MODELS
+RUN if [ "$PRELOAD_ML_MODELS" = "true" ] && [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
     /app/api/.venv/bin/python -c "\
 from sentence_transformers import SentenceTransformer, CrossEncoder; \
 print('Downloading embedding model...'); \
@@ -267,6 +287,7 @@ SentenceTransformer('BAAI/bge-small-en-v1.5'); \
 print('Downloading cross-encoder model...'); \
 CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'); \
 print('Models cached successfully')"; \
+    elif [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then echo "Skipping ML model preload (local-models not included)"; \
     else echo "Skipping ML model preload"; fi
 
 EXPOSE 8888 9999

helm/hindsight/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: hindsight
 description: Hindsight helm chart
 type: application
-version: 0.2.1
-appVersion: "0.2.1"
+version: 0.3.0
+appVersion: "0.3.0"
 keywords:
   - ai
   - memory

helm/hindsight/templates/_helpers.tpl

@@ -80,6 +80,22 @@ Control plane selector labels
 app.kubernetes.io/component: control-plane
 {{- end }}
 
+{{/*
+Worker labels
+*/}}
+{{- define "hindsight.worker.labels" -}}
+{{ include "hindsight.labels" . }}
+app.kubernetes.io/component: worker
+{{- end }}
+
+{{/*
+Worker selector labels
+*/}}
+{{- define "hindsight.worker.selectorLabels" -}}
+{{ include "hindsight.selectorLabels" . }}
+app.kubernetes.io/component: worker
+{{- end }}
+
 {{/*
 Create the name of the service account to use
 */}}
@@ -110,3 +126,14 @@ API URL for control plane
 {{- define "hindsight.apiUrl" -}}
 {{- printf "http://%s-api:%d" (include "hindsight.fullname" .) (.Values.api.service.port | int) }}
 {{- end }}
+
+{{/*
+Get the name of the secret to use
+*/}}
+{{- define "hindsight.secretName" -}}
+{{- if .Values.existingSecret }}
+{{- .Values.existingSecret }}
+{{- else }}
+{{- printf "%s-secret" (include "hindsight.fullname" .) }}
+{{- end }}
+{{- end }}

helm/hindsight/templates/api-deployment.yaml

@@ -15,7 +15,9 @@ spec:
   template:
     metadata:
       annotations:
+        {{- if not .Values.existingSecret }}
         checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+        {{- end }}
         {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -37,27 +39,41 @@ spec:
         - name: http
           containerPort: {{ .Values.api.service.targetPort }}
           protocol: TCP
+        {{- if .Values.existingSecret }}
+        envFrom:
+        - secretRef:
+            name: {{ .Values.existingSecret }}
+        {{- end }}
         env:
-        - name: HINDSIGHT_API_DATABASE_URL
-          value: {{ include "hindsight.databaseUrl" . | quote }}
+        {{- /* POSTGRES_PASSWORD must be defined before DATABASE_URL for $(VAR) interpolation */}}
         {{- if not .Values.postgresql.enabled }}
         - name: POSTGRES_PASSWORD
           valueFrom:
             secretKeyRef:
-              name: {{ include "hindsight.fullname" . }}-secret
+              name: {{ include "hindsight.secretName" . }}
               key: postgres-password
         {{- end }}
+        - name: HINDSIGHT_API_DATABASE_URL
+          value: {{ include "hindsight.databaseUrl" . | quote }}
+        {{- /* Disable internal worker when dedicated workers are enabled */}}
+        {{- if .Values.worker.enabled }}
+        - name: HINDSIGHT_API_WORKER_ENABLED
+          value: "false"
+        {{- end }}
         {{- range $key, $value := .Values.api.env }}
         - name: {{ $key }}
           value: {{ $value | quote }}
         {{- end }}
+        {{- /* Only use api.secrets when not using existingSecret (for chart-managed secrets) */}}
+        {{- if not .Values.existingSecret }}
         {{- range $key, $value := .Values.api.secrets }}
         - name: {{ $key }}
           valueFrom:
             secretKeyRef:
-              name: {{ include "hindsight.fullname" $ }}-secret
+              name: {{ include "hindsight.secretName" $ }}
               key: {{ $key }}
         {{- end }}
+        {{- end }}
         livenessProbe:
           {{- toYaml .Values.api.livenessProbe | nindent 10 }}
         readinessProbe:

helm/hindsight/templates/controlplane-deployment.yaml

@@ -15,7 +15,9 @@ spec:
   template:
     metadata:
       annotations:
+        {{- if not .Values.existingSecret }}
         checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+        {{- end }}
         {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -37,6 +39,11 @@ spec:
         - name: http
           containerPort: {{ .Values.controlPlane.service.targetPort }}
           protocol: TCP
+        {{- if .Values.existingSecret }}
+        envFrom:
+        - secretRef:
+            name: {{ .Values.existingSecret }}
+        {{- end }}
         env:
         - name: HINDSIGHT_CP_DATAPLANE_API_URL
           value: {{ include "hindsight.apiUrl" . | quote }}
@@ -44,13 +51,16 @@ spec:
         - name: {{ $key }}
           value: {{ $value | quote }}
         {{- end }}
+        {{- /* Only use controlPlane.secrets when not using existingSecret (for chart-managed secrets) */}}
+        {{- if not .Values.existingSecret }}
         {{- range $key, $value := .Values.controlPlane.secrets }}
         - name: {{ $key }}
           valueFrom:
             secretKeyRef:
-              name: {{ include "hindsight.fullname" $ }}-secret
+              name: {{ include "hindsight.secretName" $ }}
               key: {{ $key }}
         {{- end }}
+        {{- end }}
         livenessProbe:
           {{- toYaml .Values.controlPlane.livenessProbe | nindent 10 }}
         readinessProbe:

helm/hindsight/templates/secret.yaml

@@ -1,7 +1,8 @@
+{{- if not .Values.existingSecret }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ include "hindsight.fullname" . }}-secret
+  name: {{ include "hindsight.secretName" . }}
   labels:
     {{- include "hindsight.labels" . | nindent 4 }}
 type: Opaque
@@ -15,3 +16,4 @@ data:
   {{- if and (not .Values.postgresql.enabled) .Values.postgresql.external.password }}
   postgres-password: {{ .Values.postgresql.external.password | b64enc | quote }}
   {{- end }}
+{{- end }}

helm/hindsight/templates/worker-service.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.worker.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hindsight.fullname" . }}-worker
+  labels:
+    {{- include "hindsight.worker.labels" . | nindent 4 }}
+  {{- if .Values.podAnnotations }}
+  annotations:
+    {{- /* Common Prometheus annotations for metrics scraping */}}
+    prometheus.io/scrape: "true"
+    prometheus.io/port: {{ .Values.worker.service.port | quote }}
+    prometheus.io/path: "/metrics"
+  {{- end }}
+spec:
+  # Headless service for StatefulSet (enables stable DNS names like worker-0.worker.namespace)
+  clusterIP: None
+  ports:
+  - port: {{ .Values.worker.service.port }}
+    targetPort: {{ .Values.worker.service.targetPort }}
+    protocol: TCP
+    name: http
+  selector:
+    {{- include "hindsight.worker.selectorLabels" . | nindent 4 }}
+{{- end }}

helm/hindsight/templates/worker-statefulset.yaml

@@ -0,0 +1,110 @@
+{{- if .Values.worker.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "hindsight.fullname" . }}-worker
+  labels:
+    {{- include "hindsight.worker.labels" . | nindent 4 }}
+spec:
+  serviceName: {{ include "hindsight.fullname" . }}-worker
+  replicas: {{ .Values.worker.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "hindsight.worker.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        {{- if not .Values.existingSecret }}
+        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+        {{- end }}
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "hindsight.worker.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- if .Values.serviceAccount.create }}
+      serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+      - name: worker
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 10 }}
+        image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Values.version }}"
+        imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+        command: ["hindsight-worker"]
+        ports:
+        - name: http
+          containerPort: {{ .Values.worker.service.targetPort }}
+          protocol: TCP
+        {{- if .Values.existingSecret }}
+        envFrom:
+        - secretRef:
+            name: {{ .Values.existingSecret }}
+        {{- end }}
+        env:
+        {{- /* POSTGRES_PASSWORD must be defined before DATABASE_URL for $(VAR) interpolation */}}
+        {{- if not .Values.postgresql.enabled }}
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "hindsight.secretName" . }}
+              key: postgres-password
+        {{- end }}
+        - name: HINDSIGHT_API_DATABASE_URL
+          value: {{ include "hindsight.databaseUrl" . | quote }}
+        {{- /* Worker ID uses pod name (StatefulSet provides stable names like worker-0, worker-1) */}}
+        - name: HINDSIGHT_API_WORKER_ID
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        {{- /* Inherit LLM config from api.env */}}
+        {{- range $key, $value := .Values.api.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- /* Worker-specific env vars */}}
+        {{- range $key, $value := .Values.worker.env }}
+        - name: {{ $key }}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- /* Only use secrets when not using existingSecret */}}
+        {{- if not .Values.existingSecret }}
+        {{- /* Inherit secrets from api.secrets */}}
+        {{- range $key, $value := .Values.api.secrets }}
+        - name: {{ $key }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "hindsight.secretName" $ }}
+              key: {{ $key }}
+        {{- end }}
+        {{- /* Worker-specific secrets (can override api.secrets) */}}
+        {{- range $key, $value := .Values.worker.secrets }}
+        - name: {{ $key }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "hindsight.secretName" $ }}
+              key: {{ $key }}
+        {{- end }}
+        {{- end }}
+        livenessProbe:
+          {{- toYaml .Values.worker.livenessProbe | nindent 10 }}
+        readinessProbe:
+          {{- toYaml .Values.worker.readinessProbe | nindent 10 }}
+        resources:
+          {{- toYaml .Values.worker.resources | nindent 10 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

helm/hindsight/values.yaml

@@ -3,6 +3,15 @@
 # Chart version - use this to set a consistent image tag across all components
 version: "0.1.1"
 
+# Use an existing secret instead of creating one from values
+# When set, all keys from this secret are injected as environment variables via envFrom
+# Required keys:
+#   - postgres-password: PostgreSQL password (when postgresql.enabled=false)
+# Optional keys (any key becomes an env var):
+#   - HINDSIGHT_API_LLM_API_KEY: API key for LLM provider
+#   - Any other env vars you want to inject
+# existingSecret: "my-hindsight-secret"
+
 # Global settings
 replicaCount: 1
 
@@ -58,6 +67,63 @@ api:
     # HINDSIGHT_API_LLM_API_KEY: "your-api-key"
     # HINDSIGHT_API_LLM_BASE_URL: "https://api.groq.com/openai/v1"
 
+# Worker settings (distributed task processing)
+# When enabled, dedicated worker pods process tasks and the API's internal worker is disabled
+worker:
+  enabled: false
+  replicaCount: 2
+  image:
+    repository: ghcr.io/vectorize-io/hindsight-api
+    pullPolicy: IfNotPresent
+    # tag defaults to .Values.version if not specified
+
+  service:
+    # Service for metrics scraping (headless for StatefulSet)
+    port: 8889
+    targetPort: 8889
+
+  # Resource limits and requests
+  resources:
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+    requests:
+      cpu: 500m
+      memory: 1Gi
+
+  # Liveness and readiness probes
+  livenessProbe:
+    httpGet:
+      path: /health
+      port: 8889
+    initialDelaySeconds: 30
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 3
+
+  readinessProbe:
+    httpGet:
+      path: /health
+      port: 8889
+    initialDelaySeconds: 10
+    periodSeconds: 5
+    timeoutSeconds: 3
+    failureThreshold: 3
+
+  # Worker-specific environment variables
+  env:
+    # Poll interval in milliseconds (how often to check for new tasks)
+    HINDSIGHT_API_WORKER_POLL_INTERVAL_MS: "500"
+    # Number of tasks to claim per poll cycle
+    HINDSIGHT_API_WORKER_BATCH_SIZE: "10"
+    # Max retries before marking a task as failed
+    HINDSIGHT_API_WORKER_MAX_RETRIES: "3"
+    # HTTP port for metrics/health (matches service.targetPort)
+    HINDSIGHT_API_WORKER_HTTP_PORT: "8889"
+
+  # Secret environment variables (inherited from api.secrets if not specified)
+  secrets: {}
+
 # Image settings for control plane
 controlPlane:
   enabled: true

hindsight-api/hindsight_api/admin/cli.py

@@ -214,6 +214,95 @@ def restore(
     typer.echo("Restore complete")
 
 
+async def _run_migration(db_url: str, schema: str = "public") -> None:
+    """Resolve database URL and run migrations."""
+    from ..migrations import run_migrations
+
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+    run_migrations(resolved_url, schema=schema)
+
+
+@app.command(name="run-db-migration")
+def run_db_migration(
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema to run migrations on"),
+):
+    """Run database migrations to the latest version."""
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    typer.echo(f"Running database migrations (schema: {schema})...")
+
+    asyncio.run(_run_migration(config.database_url, schema))
+
+    typer.echo("Database migrations completed successfully")
+
+
+async def _decommission_worker(db_url: str, worker_id: str, schema: str = "public") -> int:
+    """Release all tasks owned by a worker, setting them back to pending status."""
+    is_pg0, instance_name, _ = parse_pg0_url(db_url)
+    if is_pg0:
+        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
+    resolved_url = await resolve_database_url(db_url)
+
+    conn = await asyncpg.connect(resolved_url)
+    try:
+        table = _fq_table("async_operations", schema)
+        result = await conn.fetch(
+            f"""
+            UPDATE {table}
+            SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
+            WHERE worker_id = $1 AND status = 'processing'
+            RETURNING operation_id
+            """,
+            worker_id,
+        )
+        return len(result)
+    finally:
+        await conn.close()
+
+
+@app.command(name="decommission-worker")
+def decommission_worker(
+    worker_id: str = typer.Argument(..., help="Worker ID to decommission"),
+    schema: str = typer.Option("public", "--schema", "-s", help="Database schema"),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"),
+):
+    """Release all tasks owned by a worker (sets status back to pending).
+
+    Use this command when a worker has crashed or been removed without graceful shutdown.
+    All tasks that were being processed by the worker will be released back to the queue
+    so other workers can pick them up.
+    """
+    config = HindsightConfig.from_env()
+
+    if not config.database_url:
+        typer.echo("Error: Database URL not configured.", err=True)
+        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
+        raise typer.Exit(1)
+
+    if not yes:
+        typer.confirm(
+            f"This will release all tasks owned by worker '{worker_id}' back to pending. Continue?",
+            abort=True,
+        )
+
+    typer.echo(f"Decommissioning worker '{worker_id}' (schema: {schema})...")
+
+    count = asyncio.run(_decommission_worker(config.database_url, worker_id, schema))
+
+    if count > 0:
+        typer.echo(f"Released {count} task(s) from worker '{worker_id}'")
+    else:
+        typer.echo(f"No tasks found for worker '{worker_id}'")
+
+
 def main():
     app()
 

hindsight-api/hindsight_api/alembic/versions/f1a2b3c4d5e6_add_memory_links_composite_index.py

@@ -0,0 +1,44 @@
+"""add_memory_links_from_type_weight_index
+
+Revision ID: f1a2b3c4d5e6
+Revises: e0a1b2c3d4e5
+Create Date: 2025-01-12
+
+Add composite index on memory_links (from_unit_id, link_type, weight DESC)
+to optimize MPFP graph traversal queries that need top-k edges per type.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "f1a2b3c4d5e6"
+down_revision: str | Sequence[str] | None = "e0a1b2c3d4e5"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (e.g., 'tenant_x.' or '' for public)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add composite index for efficient MPFP edge loading."""
+    schema = _get_schema_prefix()
+    # Create composite index for efficient top-k per (from_node, link_type) queries
+    # This enables LATERAL joins to use index-only scans with early termination
+    # Note: Not using CONCURRENTLY here as it requires running outside a transaction
+    # For production with large tables, consider running this manually with CONCURRENTLY
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS idx_memory_links_from_type_weight "
+        f"ON {schema}memory_links(from_unit_id, link_type, weight DESC)"
+    )
+
+
+def downgrade() -> None:
+    """Remove the composite index."""
+    schema = _get_schema_prefix()
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_links_from_type_weight")

hindsight-api/hindsight_api/alembic/versions/g2a3b4c5d6e7_add_tags_column.py

@@ -0,0 +1,48 @@
+"""add_tags_column
+
+Revision ID: g2a3b4c5d6e7
+Revises: f1a2b3c4d5e6
+Create Date: 2025-01-13
+
+Add tags column to memory_units and documents tables for visibility scoping.
+Tags enable filtering memories by scope (e.g., user IDs, session IDs) during recall/reflect.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "g2a3b4c5d6e7"
+down_revision: str | Sequence[str] | None = "f1a2b3c4d5e6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (e.g., 'tenant_x.' or '' for public)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add tags column to memory_units and documents tables."""
+    schema = _get_schema_prefix()
+
+    # Add tags column to memory_units table
+    op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS tags VARCHAR[] NOT NULL DEFAULT '{{}}'")
+
+    # Create GIN index for efficient array containment queries (tags && ARRAY['x'])
+    op.execute(f"CREATE INDEX IF NOT EXISTS idx_memory_units_tags ON {schema}memory_units USING GIN (tags)")
+
+    # Add tags column to documents table for document-level tags
+    op.execute(f"ALTER TABLE {schema}documents ADD COLUMN IF NOT EXISTS tags VARCHAR[] NOT NULL DEFAULT '{{}}'")
+
+
+def downgrade() -> None:
+    """Remove tags columns and index."""
+    schema = _get_schema_prefix()
+
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_tags")
+    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS tags")
+    op.execute(f"ALTER TABLE {schema}documents DROP COLUMN IF EXISTS tags")

hindsight-api/hindsight_api/alembic/versions/h3c4d5e6f7g8_mental_models_v4.py

@@ -0,0 +1,112 @@
+"""mental_models_v4
+
+Revision ID: h3c4d5e6f7g8
+Revises: g2a3b4c5d6e7
+Create Date: 2026-01-08 00:00:00.000000
+
+This migration implements the v4 mental models system:
+1. Deletes existing observation memory_units (observations now in mental models)
+2. Adds mission column to banks (replacing background)
+3. Creates mental_models table with final schema
+
+Mental models can reference entities when an entity is "promoted" to a mental model.
+Summary content is stored as JSONB observations with per-observation fact attribution.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "h3c4d5e6f7g8"
+down_revision: str | Sequence[str] | None = "g2a3b4c5d6e7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Apply mental models v4 changes."""
+    schema = _get_schema_prefix()
+
+    # Step 1: Delete observation memory_units (cascades to unit_entities links)
+    # Observations are now handled through mental models, not memory_units
+    op.execute(f"DELETE FROM {schema}memory_units WHERE fact_type = 'observation'")
+
+    # Step 2: Drop observation-specific index (if it exists)
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_observation_date")
+
+    # Step 3: Add mission column to banks (replacing background)
+    op.execute(f"ALTER TABLE {schema}banks ADD COLUMN IF NOT EXISTS mission TEXT")
+
+    # Migrate: copy background to mission if background column exists
+    # Use DO block to check column existence first (idempotent for re-runs)
+    schema_name = context.config.get_main_option("target_schema") or "public"
+    op.execute(f"""
+        DO $$
+        BEGIN
+            IF EXISTS (
+                SELECT 1 FROM information_schema.columns
+                WHERE table_schema = '{schema_name}' AND table_name = 'banks' AND column_name = 'background'
+            ) THEN
+                UPDATE {schema}banks
+                SET mission = background
+                WHERE mission IS NULL;
+            END IF;
+        END $$;
+    """)
+
+    # Remove background column (replaced by mission)
+    op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS background")
+
+    # Step 4: Create mental_models table with final v4 schema (if not exists)
+    op.execute(f"""
+        CREATE TABLE IF NOT EXISTS {schema}mental_models (
+            id VARCHAR(64) NOT NULL,
+            bank_id VARCHAR(64) NOT NULL,
+            subtype VARCHAR(32) NOT NULL,
+            name VARCHAR(256) NOT NULL,
+            description TEXT NOT NULL,
+            entity_id UUID,
+            observations JSONB DEFAULT '{{"observations": []}}'::jsonb,
+            links VARCHAR[],
+            tags VARCHAR[] DEFAULT '{{}}',
+            last_updated TIMESTAMP WITH TIME ZONE,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+            PRIMARY KEY (id, bank_id),
+            FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE,
+            FOREIGN KEY (entity_id) REFERENCES {schema}entities(id) ON DELETE SET NULL,
+            CONSTRAINT ck_mental_models_subtype CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
+        )
+    """)
+
+    # Step 5: Create indexes for efficient queries (if not exist)
+    op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_bank_id ON {schema}mental_models(bank_id)")
+    op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_subtype ON {schema}mental_models(bank_id, subtype)")
+    op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_entity_id ON {schema}mental_models(entity_id)")
+    # GIN index for efficient tags array filtering
+    op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_tags ON {schema}mental_models USING GIN(tags)")
+
+
+def downgrade() -> None:
+    """Revert mental models v4 changes."""
+    schema = _get_schema_prefix()
+
+    # Drop mental_models table (cascades to indexes)
+    op.execute(f"DROP TABLE IF EXISTS {schema}mental_models CASCADE")
+
+    # Add back background column to banks
+    op.execute(f"ALTER TABLE {schema}banks ADD COLUMN IF NOT EXISTS background TEXT")
+
+    # Migrate mission back to background
+    op.execute(f"UPDATE {schema}banks SET background = mission WHERE background IS NULL")
+
+    # Remove mission column
+    op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS mission")
+
+    # Note: Cannot restore deleted observations - they are lost on downgrade

hindsight-api/hindsight_api/alembic/versions/i4d5e6f7g8h9_delete_opinions.py

@@ -0,0 +1,41 @@
+"""delete_opinions
+
+Revision ID: i4d5e6f7g8h9
+Revises: h3c4d5e6f7g8
+Create Date: 2026-01-15 00:00:00.000000
+
+This migration removes opinion facts from memory_units.
+Opinions are no longer a separate fact type - they are now represented
+through mental model observations with confidence scores.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "i4d5e6f7g8h9"
+down_revision: str | Sequence[str] | None = "h3c4d5e6f7g8"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Delete opinion memory_units."""
+    schema = _get_schema_prefix()
+
+    # Delete opinion memory_units (cascades to unit_entities links)
+    # Opinions are now handled through mental model observations
+    op.execute(f"DELETE FROM {schema}memory_units WHERE fact_type = 'opinion'")
+
+
+def downgrade() -> None:
+    """Cannot restore deleted opinions."""
+    # Note: Cannot restore deleted opinions - they are lost on downgrade
+    pass

hindsight-api/hindsight_api/alembic/versions/j5e6f7g8h9i0_mental_model_versions.py

@@ -0,0 +1,95 @@
+"""mental_model_versions
+
+Revision ID: j5e6f7g8h9i0
+Revises: i4d5e6f7g8h9
+Create Date: 2026-01-16 00:00:00.000000
+
+This migration adds versioning support for mental models:
+1. Creates mental_model_versions table to store observation snapshots
+2. Adds version column to mental_models for tracking current version
+
+This enables changelog/diff functionality for mental model observations.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "j5e6f7g8h9i0"
+down_revision: str | Sequence[str] | None = "i4d5e6f7g8h9"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Create mental_model_versions table and add version tracking."""
+    schema = _get_schema_prefix()
+
+    # Create mental_model_versions table for storing observation snapshots
+    op.execute(f"""
+        CREATE TABLE {schema}mental_model_versions (
+            id SERIAL PRIMARY KEY,
+            mental_model_id VARCHAR(64) NOT NULL,
+            bank_id VARCHAR(64) NOT NULL,
+            version INT NOT NULL,
+            observations JSONB NOT NULL DEFAULT '{{"observations": []}}'::jsonb,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+            FOREIGN KEY (mental_model_id, bank_id)
+                REFERENCES {schema}mental_models(id, bank_id) ON DELETE CASCADE,
+            UNIQUE (mental_model_id, bank_id, version)
+        )
+    """)
+
+    # Index for efficient version queries (get latest, list versions)
+    op.execute(f"""
+        CREATE INDEX idx_mental_model_versions_lookup
+        ON {schema}mental_model_versions(mental_model_id, bank_id, version DESC)
+    """)
+
+    # Add version column to mental_models to track current version
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD COLUMN IF NOT EXISTS version INT NOT NULL DEFAULT 0
+    """)
+
+    # Migrate existing mental models: create version 1 for any that have observations
+    op.execute(f"""
+        INSERT INTO {schema}mental_model_versions (mental_model_id, bank_id, version, observations, created_at)
+        SELECT id, bank_id, 1, observations, COALESCE(last_updated, created_at)
+        FROM {schema}mental_models
+        WHERE observations IS NOT NULL
+          AND observations != '{{"observations": []}}'::jsonb
+          AND (observations->'observations') IS NOT NULL
+          AND jsonb_array_length(observations->'observations') > 0
+    """)
+
+    # Update version to 1 for migrated mental models
+    op.execute(f"""
+        UPDATE {schema}mental_models
+        SET version = 1
+        WHERE observations IS NOT NULL
+          AND observations != '{{"observations": []}}'::jsonb
+          AND (observations->'observations') IS NOT NULL
+          AND jsonb_array_length(observations->'observations') > 0
+    """)
+
+
+def downgrade() -> None:
+    """Remove mental_model_versions table and version column."""
+    schema = _get_schema_prefix()
+
+    # Drop index
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_mental_model_versions_lookup")
+
+    # Drop versions table
+    op.execute(f"DROP TABLE IF EXISTS {schema}mental_model_versions")
+
+    # Remove version column from mental_models
+    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS version")

hindsight-api/hindsight_api/alembic/versions/k6f7g8h9i0j1_add_directive_subtype.py

@@ -0,0 +1,58 @@
+"""add_directive_subtype
+
+Revision ID: k6f7g8h9i0j1
+Revises: j5e6f7g8h9i0
+Create Date: 2026-01-16 00:00:00.000000
+
+This migration adds 'directive' to the mental_models subtype constraint.
+Directives are hard rules with user-provided observations that the reflect agent must follow.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "k6f7g8h9i0j1"
+down_revision: str | Sequence[str] | None = "j5e6f7g8h9i0"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add 'directive' to mental_models subtype constraint."""
+    schema = _get_schema_prefix()
+
+    # Drop existing constraint
+    op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
+
+    # Create new constraint with 'directive' added
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD CONSTRAINT ck_mental_models_subtype
+        CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned', 'directive'))
+    """)
+
+
+def downgrade() -> None:
+    """Remove 'directive' from mental_models subtype constraint."""
+    schema = _get_schema_prefix()
+
+    # First delete any directives (cannot downgrade if they exist)
+    op.execute(f"DELETE FROM {schema}mental_models WHERE subtype = 'directive'")
+
+    # Drop constraint with directive
+    op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
+
+    # Recreate original constraint without directive
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD CONSTRAINT ck_mental_models_subtype
+        CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
+    """)

hindsight-api/hindsight_api/alembic/versions/l7g8h9i0j1k2_add_worker_columns.py

@@ -0,0 +1,109 @@
+"""add_worker_columns
+
+Revision ID: l7g8h9i0j1k2
+Revises: k6f7g8h9i0j1
+Create Date: 2026-01-19 00:00:00.000000
+
+This migration adds columns to async_operations for distributed worker support:
+- worker_id: ID of the worker that claimed the task
+- claimed_at: When the task was claimed
+- retry_count: Number of retry attempts
+- task_payload: The serialized task dictionary
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import context, op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "l7g8h9i0j1k2"
+down_revision: str | Sequence[str] | None = "k6f7g8h9i0j1"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add worker columns to async_operations."""
+    schema = _get_schema_prefix()
+
+    # Add worker_id column (ID of worker that claimed the task)
+    op.add_column(
+        "async_operations",
+        sa.Column("worker_id", sa.Text(), nullable=True),
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+
+    # Add claimed_at column (when task was claimed by worker)
+    op.add_column(
+        "async_operations",
+        sa.Column("claimed_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+
+    # Add retry_count column (number of retry attempts)
+    op.add_column(
+        "async_operations",
+        sa.Column("retry_count", sa.Integer(), server_default="0", nullable=False),
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+
+    # Add task_payload column (serialized task dictionary)
+    op.add_column(
+        "async_operations",
+        sa.Column(
+            "task_payload",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+        ),
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+
+    # Add index for efficient worker polling (pending tasks ordered by creation time)
+    op.execute(
+        f"CREATE INDEX idx_async_operations_pending_claim ON {schema}async_operations (status, created_at) "
+        f"WHERE status = 'pending' AND task_payload IS NOT NULL"
+    )
+
+    # Add index for finding tasks by worker_id (for decommissioning)
+    op.execute(
+        f"CREATE INDEX idx_async_operations_worker_id ON {schema}async_operations (worker_id) WHERE worker_id IS NOT NULL"
+    )
+
+
+def downgrade() -> None:
+    """Remove worker columns from async_operations."""
+    schema = _get_schema_prefix()
+
+    # Drop indexes
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_pending_claim")
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_worker_id")
+
+    # Drop columns
+    op.drop_column(
+        "async_operations",
+        "task_payload",
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+    op.drop_column(
+        "async_operations",
+        "retry_count",
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+    op.drop_column(
+        "async_operations",
+        "claimed_at",
+        schema=context.config.get_main_option("target_schema") or None,
+    )
+    op.drop_column(
+        "async_operations",
+        "worker_id",
+        schema=context.config.get_main_option("target_schema") or None,
+    )

hindsight-api/hindsight_api/alembic/versions/m8h9i0j1k2l3_mental_model_id_to_text.py

@@ -0,0 +1,41 @@
+"""mental_model_id_to_text
+
+Revision ID: m8h9i0j1k2l3
+Revises: l7g8h9i0j1k2
+Create Date: 2026-01-19 00:00:00.000000
+
+This migration changes the mental_models.id column from VARCHAR(64) to TEXT
+to support longer model IDs (e.g., entity names that exceed 64 characters).
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "m8h9i0j1k2l3"
+down_revision: str | Sequence[str] | None = "l7g8h9i0j1k2"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Change mental_models.id from VARCHAR(64) to TEXT."""
+    schema = _get_schema_prefix()
+
+    # Alter the id column type from VARCHAR(64) to TEXT
+    op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE TEXT")
+
+
+def downgrade() -> None:
+    """Revert mental_models.id from TEXT to VARCHAR(64)."""
+    schema = _get_schema_prefix()
+
+    # Note: This may fail if any id values exceed 64 characters
+    op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE VARCHAR(64)")

hindsight-api/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py

@@ -0,0 +1,134 @@
+"""learnings_and_pinned_reflections
+
+Revision ID: n9i0j1k2l3m4
+Revises: m8h9i0j1k2l3
+Create Date: 2026-01-21 00:00:00.000000
+
+This migration:
+1. Creates the 'learnings' table for automatic bottom-up consolidation
+2. Creates the 'pinned_reflections' table for user-curated living documents
+3. Adds consolidation tracking columns to the 'banks' table
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "n9i0j1k2l3m4"
+down_revision: str | Sequence[str] | None = "m8h9i0j1k2l3"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Create learnings and pinned_reflections tables."""
+    schema = _get_schema_prefix()
+
+    # 1. Create learnings table
+    op.execute(f"""
+        CREATE TABLE {schema}learnings (
+            id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            bank_id VARCHAR(64) NOT NULL,
+            text TEXT NOT NULL,
+            proof_count INT NOT NULL DEFAULT 1,
+            history JSONB DEFAULT '[]'::jsonb,
+            mission_context VARCHAR(64),
+            pre_mission_change BOOLEAN DEFAULT FALSE,
+            embedding vector(384),
+            tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+            updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
+        )
+    """)
+
+    # Add foreign key constraint
+    op.execute(f"""
+        ALTER TABLE {schema}learnings
+        ADD CONSTRAINT fk_learnings_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+    # Indexes for learnings
+    op.execute(f"CREATE INDEX idx_learnings_bank_id ON {schema}learnings(bank_id)")
+    op.execute(f"""
+        CREATE INDEX idx_learnings_embedding ON {schema}learnings
+        USING hnsw (embedding vector_cosine_ops)
+    """)
+    op.execute(f"CREATE INDEX idx_learnings_tags ON {schema}learnings USING GIN(tags)")
+
+    # Full-text search for learnings
+    op.execute(f"""
+        ALTER TABLE {schema}learnings ADD COLUMN search_vector tsvector
+        GENERATED ALWAYS AS (to_tsvector('english', text)) STORED
+    """)
+    op.execute(f"CREATE INDEX idx_learnings_text_search ON {schema}learnings USING gin(search_vector)")
+
+    # 2. Create pinned_reflections table
+    op.execute(f"""
+        CREATE TABLE {schema}pinned_reflections (
+            id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            bank_id VARCHAR(64) NOT NULL,
+            name VARCHAR(256) NOT NULL,
+            source_query TEXT NOT NULL,
+            content TEXT NOT NULL,
+            embedding vector(384),
+            tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
+            last_refreshed_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
+        )
+    """)
+
+    # Add foreign key constraint
+    op.execute(f"""
+        ALTER TABLE {schema}pinned_reflections
+        ADD CONSTRAINT fk_pinned_reflections_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+    # Indexes for pinned_reflections
+    op.execute(f"CREATE INDEX idx_pinned_reflections_bank_id ON {schema}pinned_reflections(bank_id)")
+    op.execute(f"""
+        CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
+        USING hnsw (embedding vector_cosine_ops)
+    """)
+    op.execute(f"CREATE INDEX idx_pinned_reflections_tags ON {schema}pinned_reflections USING GIN(tags)")
+
+    # Full-text search for pinned_reflections
+    op.execute(f"""
+        ALTER TABLE {schema}pinned_reflections ADD COLUMN search_vector tsvector
+        GENERATED ALWAYS AS (to_tsvector('english', COALESCE(name, '') || ' ' || content)) STORED
+    """)
+    op.execute(f"""
+        CREATE INDEX idx_pinned_reflections_text_search ON {schema}pinned_reflections
+        USING gin(search_vector)
+    """)
+
+    # 3. Add consolidation tracking columns to banks table
+    op.execute(f"""
+        ALTER TABLE {schema}banks
+        ADD COLUMN IF NOT EXISTS last_consolidated_at TIMESTAMP WITH TIME ZONE
+    """)
+    op.execute(f"""
+        ALTER TABLE {schema}banks
+        ADD COLUMN IF NOT EXISTS mission_changed_at TIMESTAMP WITH TIME ZONE
+    """)
+
+
+def downgrade() -> None:
+    """Drop learnings and pinned_reflections tables."""
+    schema = _get_schema_prefix()
+
+    # Drop tables
+    op.execute(f"DROP TABLE IF EXISTS {schema}learnings CASCADE")
+    op.execute(f"DROP TABLE IF EXISTS {schema}pinned_reflections CASCADE")
+
+    # Remove columns from banks
+    op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS last_consolidated_at")
+    op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS mission_changed_at")

hindsight-api/hindsight_api/alembic/versions/o0j1k2l3m4n5_migrate_mental_models_data.py

@@ -0,0 +1,113 @@
+"""migrate_mental_models_data
+
+Revision ID: o0j1k2l3m4n5
+Revises: n9i0j1k2l3m4
+Create Date: 2026-01-21 00:00:00.000000
+
+This migration:
+1. Migrates existing 'pinned' mental models to the new 'pinned_reflections' table
+2. Migrates existing 'learned' mental models to the new 'learnings' table
+3. Deletes non-directive mental models (structural, emergent, pinned, learned)
+4. Drops the mental_model_versions table (no longer used)
+5. Adds a CHECK constraint that only 'directive' subtype is allowed
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "o0j1k2l3m4n5"
+down_revision: str | Sequence[str] | None = "n9i0j1k2l3m4"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Migrate data and clean up old mental models."""
+    schema = _get_schema_prefix()
+
+    # 1. Migrate 'pinned' mental models to pinned_reflections
+    # For pinned models, the first observation's content becomes the pinned reflection content
+    op.execute(f"""
+        INSERT INTO {schema}pinned_reflections (bank_id, name, source_query, content, tags, created_at)
+        SELECT
+            bank_id,
+            name,
+            description AS source_query,
+            COALESCE(
+                observations->'observations'->0->>'content',
+                description,
+                ''
+            ) AS content,
+            tags,
+            created_at
+        FROM {schema}mental_models
+        WHERE subtype = 'pinned'
+        ON CONFLICT DO NOTHING
+    """)
+
+    # 2. Migrate 'learned' mental models to learnings
+    # Each observation in a learned model becomes a separate learning
+    op.execute(f"""
+        INSERT INTO {schema}learnings (bank_id, text, proof_count, tags, created_at)
+        SELECT
+            mm.bank_id,
+            obs->>'content' AS text,
+            GREATEST(1, COALESCE(jsonb_array_length(obs->'evidence'), 1)) AS proof_count,
+            mm.tags,
+            mm.created_at
+        FROM {schema}mental_models mm,
+        LATERAL jsonb_array_elements(mm.observations->'observations') AS obs
+        WHERE mm.subtype = 'learned'
+          AND obs->>'content' IS NOT NULL
+          AND obs->>'content' != ''
+        ON CONFLICT DO NOTHING
+    """)
+
+    # 3. Delete all non-directive mental models (they've been migrated or are obsolete)
+    op.execute(f"""
+        DELETE FROM {schema}mental_models
+        WHERE subtype != 'directive'
+    """)
+
+    # 4. Drop the mental_model_versions table (no longer used)
+    op.execute(f"DROP TABLE IF EXISTS {schema}mental_model_versions CASCADE")
+
+    # 5. Drop old constraints and add new one that only allows 'directive'
+    op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD CONSTRAINT ck_mental_models_subtype CHECK (subtype = 'directive')
+    """)
+
+
+def downgrade() -> None:
+    """Reverse the migration (data migration is one-way, so this just removes constraints)."""
+    schema = _get_schema_prefix()
+
+    # Remove the directive-only constraint
+    op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
+
+    # Re-create mental_model_versions table
+    op.execute(f"""
+        CREATE TABLE IF NOT EXISTS {schema}mental_model_versions (
+            id SERIAL PRIMARY KEY,
+            bank_id VARCHAR(64) NOT NULL,
+            model_id VARCHAR(128) NOT NULL,
+            version INT NOT NULL,
+            observations JSONB NOT NULL,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
+        )
+    """)
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS idx_mm_versions_lookup ON {schema}mental_model_versions(bank_id, model_id, version DESC)"
+    )
+
+    # Note: Data migration cannot be reversed - pinned_reflections and learnings data remains

hindsight-api/hindsight_api/alembic/versions/p1k2l3m4n5o6_new_knowledge_architecture.py

@@ -0,0 +1,194 @@
+"""new_knowledge_architecture
+
+Revision ID: p1k2l3m4n5o6
+Revises: o0j1k2l3m4n5
+Create Date: 2026-01-21 00:00:00.000000
+
+This migration implements the new knowledge architecture:
+1. Drops the 'learnings' table (mental models are now in memory_units)
+2. Renames 'pinned_reflections' to 'reflections'
+3. Drops the 'mental_models' table completely
+4. Creates 'directives' table for hard rules
+5. Adds mental model support columns to 'memory_units' (proof_count, source_memory_ids, history)
+
+The new architecture:
+- Directives: Hard rules in their own table
+- Mental Models: Stored in memory_units with fact_type='mental_model'
+- Reflections: User-curated documents (renamed from pinned_reflections)
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "p1k2l3m4n5o6"
+down_revision: str | Sequence[str] | None = "o0j1k2l3m4n5"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Implement new knowledge architecture."""
+    schema = _get_schema_prefix()
+
+    # 1. Drop the learnings table (mental models will be in memory_units)
+    op.execute(f"DROP TABLE IF EXISTS {schema}learnings CASCADE")
+
+    # 2. Rename pinned_reflections to reflections
+    op.execute(f"ALTER TABLE IF EXISTS {schema}pinned_reflections RENAME TO reflections")
+
+    # Rename indexes for reflections
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_bank_id RENAME TO idx_reflections_bank_id")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_embedding RENAME TO idx_reflections_embedding")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_tags RENAME TO idx_reflections_tags")
+    op.execute(
+        f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_text_search RENAME TO idx_reflections_text_search"
+    )
+
+    # Rename foreign key constraint
+    op.execute(f"""
+        ALTER TABLE {schema}reflections
+        DROP CONSTRAINT IF EXISTS fk_pinned_reflections_bank_id
+    """)
+    op.execute(f"""
+        ALTER TABLE {schema}reflections
+        ADD CONSTRAINT fk_reflections_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+    # 3. Drop the mental_models table completely
+    op.execute(f"DROP TABLE IF EXISTS {schema}mental_models CASCADE")
+
+    # 4. Create directives table
+    op.execute(f"""
+        CREATE TABLE {schema}directives (
+            id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            bank_id VARCHAR(64) NOT NULL,
+            name VARCHAR(256) NOT NULL,
+            content TEXT NOT NULL,
+            priority INT NOT NULL DEFAULT 0,
+            is_active BOOLEAN NOT NULL DEFAULT TRUE,
+            tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+            updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
+        )
+    """)
+
+    # Add foreign key and indexes for directives
+    op.execute(f"""
+        ALTER TABLE {schema}directives
+        ADD CONSTRAINT fk_directives_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+    op.execute(f"CREATE INDEX idx_directives_bank_id ON {schema}directives(bank_id)")
+    op.execute(f"CREATE INDEX idx_directives_bank_active ON {schema}directives(bank_id, is_active)")
+    op.execute(f"CREATE INDEX idx_directives_tags ON {schema}directives USING GIN(tags)")
+
+    # 5. Add mental model support columns to memory_units
+    # proof_count: Number of memories that support this mental model
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD COLUMN IF NOT EXISTS proof_count INT DEFAULT 1
+    """)
+
+    # source_memory_ids: Array of memory IDs that consolidated into this mental model
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD COLUMN IF NOT EXISTS source_memory_ids UUID[] DEFAULT ARRAY[]::UUID[]
+    """)
+
+    # history: JSONB array tracking changes to mental models
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD COLUMN IF NOT EXISTS history JSONB DEFAULT '[]'::jsonb
+    """)
+
+    # Add index for finding mental models
+    op.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_memory_units_mental_models
+        ON {schema}memory_units(bank_id, fact_type)
+        WHERE fact_type = 'mental_model'
+    """)
+
+    # 6. Update fact_type check constraint to include 'mental_model'
+    op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD CONSTRAINT memory_units_fact_type_check
+        CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation', 'mental_model'))
+    """)
+
+
+def downgrade() -> None:
+    """Reverse the migration."""
+    schema = _get_schema_prefix()
+
+    # Restore original fact_type check constraint (without 'mental_model')
+    op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD CONSTRAINT memory_units_fact_type_check
+        CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation'))
+    """)
+
+    # Drop mental model columns from memory_units
+    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS proof_count")
+    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS source_memory_ids")
+    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS history")
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_mental_models")
+
+    # Drop directives table
+    op.execute(f"DROP TABLE IF EXISTS {schema}directives CASCADE")
+
+    # Rename reflections back to pinned_reflections
+    op.execute(f"ALTER TABLE IF EXISTS {schema}reflections RENAME TO pinned_reflections")
+
+    # Restore indexes
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_bank_id RENAME TO idx_pinned_reflections_bank_id")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_embedding RENAME TO idx_pinned_reflections_embedding")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_tags RENAME TO idx_pinned_reflections_tags")
+    op.execute(
+        f"ALTER INDEX IF EXISTS {schema}idx_reflections_text_search RENAME TO idx_pinned_reflections_text_search"
+    )
+
+    # Restore foreign key
+    op.execute(f"""
+        ALTER TABLE {schema}pinned_reflections
+        DROP CONSTRAINT IF EXISTS fk_reflections_bank_id
+    """)
+    op.execute(f"""
+        ALTER TABLE {schema}pinned_reflections
+        ADD CONSTRAINT fk_pinned_reflections_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+    # Re-create learnings table
+    op.execute(f"""
+        CREATE TABLE IF NOT EXISTS {schema}learnings (
+            id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            bank_id VARCHAR(64) NOT NULL,
+            text TEXT NOT NULL,
+            proof_count INT NOT NULL DEFAULT 1,
+            history JSONB DEFAULT '[]'::jsonb,
+            mission_context VARCHAR(64),
+            pre_mission_change BOOLEAN DEFAULT FALSE,
+            embedding vector(384),
+            tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+            updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
+        )
+    """)
+    op.execute(f"""
+        ALTER TABLE {schema}learnings
+        ADD CONSTRAINT fk_learnings_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+    # Note: mental_models table recreation is complex and would need separate handling

hindsight-api/hindsight_api/alembic/versions/q2l3m4n5o6p7_fix_mental_model_fact_type.py

@@ -0,0 +1,50 @@
+"""fix_mental_model_fact_type
+
+Revision ID: q2l3m4n5o6p7
+Revises: p1k2l3m4n5o6
+Create Date: 2026-01-21 13:30:00.000000
+
+Fix the fact_type check constraint to include 'mental_model'.
+This is a fix for p1k2l3m4n5o6 which should have included this change.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+# revision identifiers, used by Alembic.
+revision: str = "q2l3m4n5o6p7"
+down_revision: str | Sequence[str] | None = "p1k2l3m4n5o6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add 'mental_model' to the fact_type check constraint."""
+    schema = _get_schema_prefix()
+
+    # Drop the old constraint and add the new one with mental_model included
+    op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD CONSTRAINT memory_units_fact_type_check
+        CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation', 'mental_model'))
+    """)
+
+
+def downgrade() -> None:
+    """Remove 'mental_model' from the fact_type check constraint."""
+    schema = _get_schema_prefix()
+
+    op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD CONSTRAINT memory_units_fact_type_check
+        CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation'))
+    """)

hindsight-api/hindsight_api/alembic/versions/r3m4n5o6p7q8_add_reflect_response_to_reflections.py

@@ -0,0 +1,47 @@
+"""Add reflect_response JSONB column to reflections
+
+Revision ID: r3m4n5o6p7q8
+Revises: q2l3m4n5o6p7
+Create Date: 2026-01-21
+
+This migration adds a reflect_response JSONB column to store the full
+reflect API response payload, including based_on facts and trace data.
+
+Note: Table was renamed from pinned_reflections to reflections in p1k2l3m4n5o6.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "r3m4n5o6p7q8"
+down_revision: str | Sequence[str] | None = "q2l3m4n5o6p7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add reflect_response JSONB column to reflections."""
+    schema = _get_schema_prefix()
+
+    # Add reflect_response column to store the full reflect API response
+    op.execute(f"""
+        ALTER TABLE {schema}reflections
+        ADD COLUMN IF NOT EXISTS reflect_response JSONB
+    """)
+
+
+def downgrade() -> None:
+    """Remove reflect_response column from reflections."""
+    schema = _get_schema_prefix()
+
+    op.execute(f"""
+        ALTER TABLE {schema}reflections
+        DROP COLUMN IF EXISTS reflect_response
+    """)

hindsight-api/hindsight_api/alembic/versions/s4n5o6p7q8r9_add_consolidated_at_to_memory_units.py

@@ -0,0 +1,53 @@
+"""Add consolidated_at column to memory_units for incremental consolidation tracking.
+
+This allows consolidation to track progress at the memory level rather than
+using a bank-level watermark. If consolidation crashes, already-processed
+memories won't be reprocessed.
+
+Revision ID: s4n5o6p7q8r9
+Revises: r3m4n5o6p7q8
+Create Date: 2025-01-22
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "s4n5o6p7q8r9"
+down_revision: str | Sequence[str] | None = "r3m4n5o6p7q8"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+
+    # Add consolidated_at column to memory_units
+    op.execute(
+        f"""
+        ALTER TABLE {schema}memory_units
+        ADD COLUMN IF NOT EXISTS consolidated_at TIMESTAMPTZ DEFAULT NULL
+        """
+    )
+
+    # Create index for efficient querying of unconsolidated memories
+    op.execute(
+        f"""
+        CREATE INDEX IF NOT EXISTS idx_memory_units_unconsolidated
+        ON {schema}memory_units (bank_id, created_at)
+        WHERE consolidated_at IS NULL AND fact_type IN ('experience', 'world')
+        """
+    )
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_unconsolidated")
+    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS consolidated_at")

hindsight-api/hindsight_api/alembic/versions/t5o6p7q8r9s0_rename_mental_models_to_observations.py

@@ -0,0 +1,134 @@
+"""Rename mental_model fact_type to observation and reflections table to mental_models
+
+Revision ID: t5o6p7q8r9s0
+Revises: s4n5o6p7q8r9
+Create Date: 2026-01-26
+
+This migration implements the terminology rename:
+1. mental_model (fact_type in memory_units) -> observation
+2. reflections table -> mental_models table
+
+The new terminology:
+- Observations: Consolidated knowledge synthesized from facts (was mental_model)
+- Mental Models: Stored reflect responses (was reflections)
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "t5o6p7q8r9s0"
+down_revision: str | Sequence[str] | None = "s4n5o6p7q8r9"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Rename mental_model -> observation and reflections -> mental_models."""
+    schema = _get_schema_prefix()
+
+    # 1. Update fact_type values: mental_model -> observation
+    op.execute(f"""
+        UPDATE {schema}memory_units
+        SET fact_type = 'observation'
+        WHERE fact_type = 'mental_model'
+    """)
+
+    # 2. Update the CHECK constraint - remove mental_model, keep observation
+    op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD CONSTRAINT memory_units_fact_type_check
+        CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation'))
+    """)
+
+    # 3. Rename the index for observations (was for mental_models)
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_mental_models")
+    op.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_memory_units_observations
+        ON {schema}memory_units(bank_id, fact_type)
+        WHERE fact_type = 'observation'
+    """)
+
+    # 4. Update the unconsolidated index to not filter by fact_type since observations
+    # are now the consolidated type
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_unconsolidated")
+    op.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_memory_units_unconsolidated
+        ON {schema}memory_units (bank_id, created_at)
+        WHERE consolidated_at IS NULL AND fact_type IN ('experience', 'world')
+    """)
+
+    # 5. Rename reflections table to mental_models
+    op.execute(f"ALTER TABLE IF EXISTS {schema}reflections RENAME TO mental_models")
+
+    # 6. Rename indexes for mental_models (was reflections)
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_bank_id RENAME TO idx_mental_models_bank_id")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_embedding RENAME TO idx_mental_models_embedding")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_tags RENAME TO idx_mental_models_tags")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_text_search RENAME TO idx_mental_models_text_search")
+
+    # 7. Rename foreign key constraint
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        DROP CONSTRAINT IF EXISTS fk_reflections_bank_id
+    """)
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD CONSTRAINT fk_mental_models_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+
+def downgrade() -> None:
+    """Reverse: observation -> mental_model and mental_models -> reflections."""
+    schema = _get_schema_prefix()
+
+    # 1. Rename mental_models table back to reflections
+    op.execute(f"ALTER TABLE IF EXISTS {schema}mental_models RENAME TO reflections")
+
+    # 2. Rename indexes back
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_mental_models_bank_id RENAME TO idx_reflections_bank_id")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_mental_models_embedding RENAME TO idx_reflections_embedding")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_mental_models_tags RENAME TO idx_reflections_tags")
+    op.execute(f"ALTER INDEX IF EXISTS {schema}idx_mental_models_text_search RENAME TO idx_reflections_text_search")
+
+    # 3. Rename foreign key back
+    op.execute(f"""
+        ALTER TABLE {schema}reflections
+        DROP CONSTRAINT IF EXISTS fk_mental_models_bank_id
+    """)
+    op.execute(f"""
+        ALTER TABLE {schema}reflections
+        ADD CONSTRAINT fk_reflections_bank_id
+        FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
+    """)
+
+    # 4. Update fact_type values: observation -> mental_model
+    op.execute(f"""
+        UPDATE {schema}memory_units
+        SET fact_type = 'mental_model'
+        WHERE fact_type = 'observation'
+    """)
+
+    # 5. Update the CHECK constraint back
+    op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
+    op.execute(f"""
+        ALTER TABLE {schema}memory_units
+        ADD CONSTRAINT memory_units_fact_type_check
+        CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation', 'mental_model'))
+    """)
+
+    # 6. Rename index back
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_observations")
+    op.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_memory_units_mental_models
+        ON {schema}memory_units(bank_id, fact_type)
+        WHERE fact_type = 'mental_model'
+    """)

hindsight-api/hindsight_api/alembic/versions/u6p7q8r9s0t1_mental_models_text_id.py

@@ -0,0 +1,41 @@
+"""Change mental_models.id from UUID to TEXT
+
+Revision ID: u6p7q8r9s0t1
+Revises: t5o6p7q8r9s0
+Create Date: 2026-01-27
+
+This migration changes the mental_models.id column from UUID to TEXT
+to support user-defined text identifiers like 'team-communication' instead of UUIDs.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "u6p7q8r9s0t1"
+down_revision: str | Sequence[str] | None = "t5o6p7q8r9s0"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Change mental_models.id from UUID to TEXT."""
+    schema = _get_schema_prefix()
+
+    # Change the id column type from UUID to TEXT
+    # Existing UUIDs will be converted to their string representation
+    op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE TEXT USING id::TEXT")
+
+
+def downgrade() -> None:
+    """Revert mental_models.id from TEXT to UUID."""
+    schema = _get_schema_prefix()
+
+    # Note: This will fail if any id values are not valid UUIDs
+    op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE UUID USING id::UUID")

hindsight-api/hindsight_api/alembic/versions/v7q8r9s0t1u2_add_max_tokens_to_mental_models.py

@@ -0,0 +1,50 @@
+"""Add max_tokens and trigger columns to mental_models
+
+Revision ID: v7q8r9s0t1u2
+Revises: u6p7q8r9s0t1
+Create Date: 2026-01-27
+
+This migration adds:
+- max_tokens column: token limit for content generation during refresh
+- trigger column: JSONB for trigger settings (e.g., refresh_after_consolidation)
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "v7q8r9s0t1u2"
+down_revision: str | Sequence[str] | None = "u6p7q8r9s0t1"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Add max_tokens and trigger columns to mental_models."""
+    schema = _get_schema_prefix()
+
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD COLUMN IF NOT EXISTS max_tokens INT NOT NULL DEFAULT 2048
+    """)
+
+    # trigger column stores trigger settings as JSONB
+    # Default: refresh_after_consolidation = false (not "real time")
+    op.execute(f"""
+        ALTER TABLE {schema}mental_models
+        ADD COLUMN IF NOT EXISTS trigger JSONB NOT NULL DEFAULT '{{"refresh_after_consolidation": false}}'::jsonb
+    """)
+
+
+def downgrade() -> None:
+    """Remove max_tokens and trigger columns from mental_models."""
+    schema = _get_schema_prefix()
+
+    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS max_tokens")
+    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS trigger")

hindsight-api/hindsight_api/api/mcp.py

@@ -1,4 +1,4 @@
-"""Hindsight MCP Server implementation using FastMCP."""
+"""Hindsight MCP Server implementation using FastMCP (HTTP transport)."""
 
 import json
 import logging
@@ -8,9 +8,7 @@ from contextvars import ContextVar
 from fastmcp import FastMCP
 
 from hindsight_api import MemoryEngine
-from hindsight_api.api.http import BankListItem, BankListResponse, BankProfileResponse, DispositionTraits
-from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES
-from hindsight_api.models import RequestContext
+from hindsight_api.mcp_tools import MCPToolsConfig, register_mcp_tools
 
 # Configure logging from HINDSIGHT_API_LOG_LEVEL environment variable
 _log_level_str = os.environ.get("HINDSIGHT_API_LOG_LEVEL", "info").lower()
@@ -53,198 +51,15 @@ def create_mcp_server(memory: MemoryEngine) -> FastMCP:
     # Use stateless_http=True for Claude Code compatibility
     mcp = FastMCP("hindsight-mcp-server", stateless_http=True)
 
-    @mcp.tool()
-    async def retain(content: str, context: str = "general", bank_id: str | None = None) -> str:
-        """
-        Store important information to long-term memory.
+    # Configure and register tools using shared module
+    config = MCPToolsConfig(
+        bank_id_resolver=get_current_bank_id,
+        include_bank_id_param=True,  # HTTP MCP supports multi-bank via parameter
+        tools=None,  # All tools
+        retain_fire_and_forget=False,  # HTTP MCP supports sync/async modes
+    )
 
-        Use this tool PROACTIVELY whenever the user shares:
-        - Personal facts, preferences, or interests
-        - Important events or milestones
-        - User history, experiences, or background
-        - Decisions, opinions, or stated preferences
-        - Goals, plans, or future intentions
-        - Relationships or people mentioned
-        - Work context, projects, or responsibilities
-
-        Args:
-            content: The fact/memory to store (be specific and include relevant details)
-            context: Category for the memory (e.g., 'preferences', 'work', 'hobbies', 'family'). Default: 'general'
-            bank_id: Optional bank to store in (defaults to session bank). Use for cross-bank operations.
-        """
-        try:
-            target_bank = bank_id or get_current_bank_id()
-            if target_bank is None:
-                return "Error: No bank_id configured"
-            await memory.retain_batch_async(
-                bank_id=target_bank,
-                contents=[{"content": content, "context": context}],
-                request_context=RequestContext(),
-            )
-            return f"Memory stored successfully in bank '{target_bank}'"
-        except Exception as e:
-            logger.error(f"Error storing memory: {e}", exc_info=True)
-            return f"Error: {str(e)}"
-
-    @mcp.tool()
-    async def recall(query: str, max_tokens: int = 4096, bank_id: str | None = None) -> str:
-        """
-        Search memories to provide personalized, context-aware responses.
-
-        Use this tool PROACTIVELY to:
-        - Check user's preferences before making suggestions
-        - Recall user's history to provide continuity
-        - Remember user's goals and context
-        - Personalize responses based on past interactions
-
-        Args:
-            query: Natural language search query (e.g., "user's food preferences", "what projects is user working on")
-            max_tokens: Maximum tokens in the response (default: 4096)
-            bank_id: Optional bank to search in (defaults to session bank). Use for cross-bank operations.
-        """
-        try:
-            target_bank = bank_id or get_current_bank_id()
-            if target_bank is None:
-                return "Error: No bank_id configured"
-            from hindsight_api.engine.memory_engine import Budget
-
-            recall_result = await memory.recall_async(
-                bank_id=target_bank,
-                query=query,
-                fact_type=list(VALID_RECALL_FACT_TYPES),
-                budget=Budget.HIGH,
-                max_tokens=max_tokens,
-                request_context=RequestContext(),
-            )
-
-            # Use model's JSON serialization
-            return recall_result.model_dump_json(indent=2)
-        except Exception as e:
-            logger.error(f"Error searching: {e}", exc_info=True)
-            return f'{{"error": "{e}", "results": []}}'
-
-    @mcp.tool()
-    async def reflect(query: str, context: str | None = None, budget: str = "low", bank_id: str | None = None) -> str:
-        """
-        Generate thoughtful analysis by synthesizing stored memories with the bank's personality.
-
-        WHEN TO USE THIS TOOL:
-        Use reflect when you need reasoned analysis, not just fact retrieval. This tool
-        thinks through the question using everything the bank knows and its personality traits.
-
-        EXAMPLES OF GOOD QUERIES:
-        - "What patterns have emerged in how I approach debugging?"
-        - "Based on my past decisions, what architectural style do I prefer?"
-        - "What might be the best approach for this problem given what you know about me?"
-        - "How should I prioritize these tasks based on my goals?"
-
-        HOW IT DIFFERS FROM RECALL:
-        - recall: Returns raw facts matching your search (fast lookup)
-        - reflect: Reasons across memories to form a synthesized answer (deeper analysis)
-
-        Use recall for "what did I say about X?" and reflect for "what should I do about X?"
-
-        Args:
-            query: The question or topic to reflect on
-            context: Optional context about why this reflection is needed
-            budget: Search budget - 'low', 'mid', or 'high' (default: 'low')
-            bank_id: Optional bank to reflect in (defaults to session bank). Use for cross-bank operations.
-        """
-        try:
-            target_bank = bank_id or get_current_bank_id()
-            if target_bank is None:
-                return "Error: No bank_id configured"
-            from hindsight_api.engine.memory_engine import Budget
-
-            # Map string budget to enum
-            budget_map = {"low": Budget.LOW, "mid": Budget.MID, "high": Budget.HIGH}
-            budget_enum = budget_map.get(budget.lower(), Budget.LOW)
-
-            reflect_result = await memory.reflect_async(
-                bank_id=target_bank,
-                query=query,
-                budget=budget_enum,
-                context=context,
-                request_context=RequestContext(),
-            )
-
-            return reflect_result.model_dump_json(indent=2)
-        except Exception as e:
-            logger.error(f"Error reflecting: {e}", exc_info=True)
-            return f'{{"error": "{e}", "text": ""}}'
-
-    @mcp.tool()
-    async def list_banks() -> str:
-        """
-        List all available memory banks.
-
-        Use this to discover banks for orchestration or to find
-        the correct bank_id for cross-bank operations.
-
-        Returns:
-            JSON object with banks array containing bank_id, name, disposition, background, and timestamps
-        """
-        try:
-            banks = await memory.list_banks(request_context=RequestContext())
-            bank_items = [
-                BankListItem(
-                    bank_id=b.get("bank_id") or b.get("id"),
-                    name=b.get("name"),
-                    disposition=DispositionTraits(
-                        **b.get("disposition", {"skepticism": 3, "literalism": 3, "empathy": 3})
-                    ),
-                    background=b.get("background"),
-                    created_at=str(b.get("created_at")) if b.get("created_at") else None,
-                    updated_at=str(b.get("updated_at")) if b.get("updated_at") else None,
-                )
-                for b in banks
-            ]
-            return BankListResponse(banks=bank_items).model_dump_json(indent=2)
-        except Exception as e:
-            logger.error(f"Error listing banks: {e}", exc_info=True)
-            return f'{{"error": "{e}", "banks": []}}'
-
-    @mcp.tool()
-    async def create_bank(bank_id: str, name: str | None = None, background: str | None = None) -> str:
-        """
-        Create or update a memory bank.
-
-        Use this to create new banks for different agents, sessions, or purposes.
-        Banks are isolated memory stores - each bank has its own memories and personality.
-
-        Args:
-            bank_id: Unique identifier for the bank (e.g., 'orchestrator-memory', 'agent-1')
-            name: Human-readable name for the bank
-            background: Context about what this bank stores or its purpose
-        """
-        try:
-            # Get or create the bank profile (auto-creates with defaults)
-            await memory.get_bank_profile(bank_id, request_context=RequestContext())
-
-            # Update name and/or background if provided
-            if name is not None or background is not None:
-                await memory.update_bank(bank_id, name=name, background=background, request_context=RequestContext())
-
-            # Get final profile and return using BankProfileResponse model
-            profile = await memory.get_bank_profile(bank_id, request_context=RequestContext())
-            disposition = profile.get("disposition")
-            if hasattr(disposition, "model_dump"):
-                disposition_traits = DispositionTraits(**disposition.model_dump())
-            else:
-                disposition_traits = DispositionTraits(
-                    **dict(disposition or {"skepticism": 3, "literalism": 3, "empathy": 3})
-                )
-
-            response = BankProfileResponse(
-                bank_id=bank_id,
-                name=profile.get("name") or "",
-                disposition=disposition_traits,
-                background=profile.get("background") or "",
-            )
-            return response.model_dump_json(indent=2)
-        except Exception as e:
-            logger.error(f"Error creating bank: {e}", exc_info=True)
-            return json.dumps({"error": str(e)})
+    register_mcp_tools(mcp, memory, config)
 
     return mcp
 

hindsight-api/hindsight_api/config.py

@@ -4,9 +4,12 @@ Centralized configuration for Hindsight API.
 All environment variables and their defaults are defined here.
 """
 
+import json
 import logging
 import os
+import sys
 from dataclasses import dataclass
+from datetime import datetime, timezone
 
 from dotenv import find_dotenv, load_dotenv
 
@@ -25,35 +28,101 @@ ENV_LLM_MAX_CONCURRENT = "HINDSIGHT_API_LLM_MAX_CONCURRENT"
 ENV_LLM_TIMEOUT = "HINDSIGHT_API_LLM_TIMEOUT"
 ENV_LLM_GROQ_SERVICE_TIER = "HINDSIGHT_API_LLM_GROQ_SERVICE_TIER"
 
+# Per-operation LLM configuration (optional, falls back to global LLM config)
+ENV_RETAIN_LLM_PROVIDER = "HINDSIGHT_API_RETAIN_LLM_PROVIDER"
+ENV_RETAIN_LLM_API_KEY = "HINDSIGHT_API_RETAIN_LLM_API_KEY"
+ENV_RETAIN_LLM_MODEL = "HINDSIGHT_API_RETAIN_LLM_MODEL"
+ENV_RETAIN_LLM_BASE_URL = "HINDSIGHT_API_RETAIN_LLM_BASE_URL"
+
+ENV_REFLECT_LLM_PROVIDER = "HINDSIGHT_API_REFLECT_LLM_PROVIDER"
+ENV_REFLECT_LLM_API_KEY = "HINDSIGHT_API_REFLECT_LLM_API_KEY"
+ENV_REFLECT_LLM_MODEL = "HINDSIGHT_API_REFLECT_LLM_MODEL"
+ENV_REFLECT_LLM_BASE_URL = "HINDSIGHT_API_REFLECT_LLM_BASE_URL"
+
+ENV_CONSOLIDATION_LLM_PROVIDER = "HINDSIGHT_API_CONSOLIDATION_LLM_PROVIDER"
+ENV_CONSOLIDATION_LLM_API_KEY = "HINDSIGHT_API_CONSOLIDATION_LLM_API_KEY"
+ENV_CONSOLIDATION_LLM_MODEL = "HINDSIGHT_API_CONSOLIDATION_LLM_MODEL"
+ENV_CONSOLIDATION_LLM_BASE_URL = "HINDSIGHT_API_CONSOLIDATION_LLM_BASE_URL"
+
 ENV_EMBEDDINGS_PROVIDER = "HINDSIGHT_API_EMBEDDINGS_PROVIDER"
 ENV_EMBEDDINGS_LOCAL_MODEL = "HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL"
 ENV_EMBEDDINGS_TEI_URL = "HINDSIGHT_API_EMBEDDINGS_TEI_URL"
 ENV_EMBEDDINGS_OPENAI_API_KEY = "HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY"
 ENV_EMBEDDINGS_OPENAI_MODEL = "HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL"
+ENV_EMBEDDINGS_OPENAI_BASE_URL = "HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL"
+
+ENV_COHERE_API_KEY = "HINDSIGHT_API_COHERE_API_KEY"
+ENV_EMBEDDINGS_COHERE_MODEL = "HINDSIGHT_API_EMBEDDINGS_COHERE_MODEL"
+ENV_EMBEDDINGS_COHERE_BASE_URL = "HINDSIGHT_API_EMBEDDINGS_COHERE_BASE_URL"
+ENV_RERANKER_COHERE_MODEL = "HINDSIGHT_API_RERANKER_COHERE_MODEL"
+ENV_RERANKER_COHERE_BASE_URL = "HINDSIGHT_API_RERANKER_COHERE_BASE_URL"
+
+# LiteLLM gateway configuration (for embeddings and reranker via LiteLLM proxy)
+ENV_LITELLM_API_BASE = "HINDSIGHT_API_LITELLM_API_BASE"
+ENV_LITELLM_API_KEY = "HINDSIGHT_API_LITELLM_API_KEY"
+ENV_EMBEDDINGS_LITELLM_MODEL = "HINDSIGHT_API_EMBEDDINGS_LITELLM_MODEL"
+ENV_RERANKER_LITELLM_MODEL = "HINDSIGHT_API_RERANKER_LITELLM_MODEL"
 
 ENV_RERANKER_PROVIDER = "HINDSIGHT_API_RERANKER_PROVIDER"
 ENV_RERANKER_LOCAL_MODEL = "HINDSIGHT_API_RERANKER_LOCAL_MODEL"
+ENV_RERANKER_LOCAL_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_LOCAL_MAX_CONCURRENT"
 ENV_RERANKER_TEI_URL = "HINDSIGHT_API_RERANKER_TEI_URL"
+ENV_RERANKER_TEI_BATCH_SIZE = "HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE"
+ENV_RERANKER_TEI_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT"
+ENV_RERANKER_MAX_CANDIDATES = "HINDSIGHT_API_RERANKER_MAX_CANDIDATES"
+ENV_RERANKER_FLASHRANK_MODEL = "HINDSIGHT_API_RERANKER_FLASHRANK_MODEL"
+ENV_RERANKER_FLASHRANK_CACHE_DIR = "HINDSIGHT_API_RERANKER_FLASHRANK_CACHE_DIR"
 
 ENV_HOST = "HINDSIGHT_API_HOST"
 ENV_PORT = "HINDSIGHT_API_PORT"
 ENV_LOG_LEVEL = "HINDSIGHT_API_LOG_LEVEL"
+ENV_LOG_FORMAT = "HINDSIGHT_API_LOG_FORMAT"
+ENV_WORKERS = "HINDSIGHT_API_WORKERS"
 ENV_MCP_ENABLED = "HINDSIGHT_API_MCP_ENABLED"
 ENV_GRAPH_RETRIEVER = "HINDSIGHT_API_GRAPH_RETRIEVER"
+ENV_MPFP_TOP_K_NEIGHBORS = "HINDSIGHT_API_MPFP_TOP_K_NEIGHBORS"
+ENV_RECALL_MAX_CONCURRENT = "HINDSIGHT_API_RECALL_MAX_CONCURRENT"
+ENV_RECALL_CONNECTION_BUDGET = "HINDSIGHT_API_RECALL_CONNECTION_BUDGET"
 ENV_MCP_LOCAL_BANK_ID = "HINDSIGHT_API_MCP_LOCAL_BANK_ID"
 ENV_MCP_INSTRUCTIONS = "HINDSIGHT_API_MCP_INSTRUCTIONS"
-
-# Observation thresholds
-ENV_OBSERVATION_MIN_FACTS = "HINDSIGHT_API_OBSERVATION_MIN_FACTS"
-ENV_OBSERVATION_TOP_ENTITIES = "HINDSIGHT_API_OBSERVATION_TOP_ENTITIES"
+ENV_MENTAL_MODEL_REFRESH_CONCURRENCY = "HINDSIGHT_API_MENTAL_MODEL_REFRESH_CONCURRENCY"
 
 # Retain settings
 ENV_RETAIN_MAX_COMPLETION_TOKENS = "HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS"
+ENV_RETAIN_CHUNK_SIZE = "HINDSIGHT_API_RETAIN_CHUNK_SIZE"
+ENV_RETAIN_EXTRACT_CAUSAL_LINKS = "HINDSIGHT_API_RETAIN_EXTRACT_CAUSAL_LINKS"
+ENV_RETAIN_EXTRACTION_MODE = "HINDSIGHT_API_RETAIN_EXTRACTION_MODE"
+ENV_RETAIN_CUSTOM_INSTRUCTIONS = "HINDSIGHT_API_RETAIN_CUSTOM_INSTRUCTIONS"
+ENV_RETAIN_OBSERVATIONS_ASYNC = "HINDSIGHT_API_RETAIN_OBSERVATIONS_ASYNC"
+
+# Observations settings (consolidated knowledge from facts)
+ENV_ENABLE_OBSERVATIONS = "HINDSIGHT_API_ENABLE_OBSERVATIONS"
+ENV_CONSOLIDATION_BATCH_SIZE = "HINDSIGHT_API_CONSOLIDATION_BATCH_SIZE"
 
 # Optimization flags
 ENV_SKIP_LLM_VERIFICATION = "HINDSIGHT_API_SKIP_LLM_VERIFICATION"
 ENV_LAZY_RERANKER = "HINDSIGHT_API_LAZY_RERANKER"
 
+# Database migrations
+ENV_RUN_MIGRATIONS_ON_STARTUP = "HINDSIGHT_API_RUN_MIGRATIONS_ON_STARTUP"
+
+# Database connection pool
+ENV_DB_POOL_MIN_SIZE = "HINDSIGHT_API_DB_POOL_MIN_SIZE"
+ENV_DB_POOL_MAX_SIZE = "HINDSIGHT_API_DB_POOL_MAX_SIZE"
+ENV_DB_COMMAND_TIMEOUT = "HINDSIGHT_API_DB_COMMAND_TIMEOUT"
+ENV_DB_ACQUIRE_TIMEOUT = "HINDSIGHT_API_DB_ACQUIRE_TIMEOUT"
+
+# Worker configuration (distributed task processing)
+ENV_WORKER_ENABLED = "HINDSIGHT_API_WORKER_ENABLED"
+ENV_WORKER_ID = "HINDSIGHT_API_WORKER_ID"
+ENV_WORKER_POLL_INTERVAL_MS = "HINDSIGHT_API_WORKER_POLL_INTERVAL_MS"
+ENV_WORKER_MAX_RETRIES = "HINDSIGHT_API_WORKER_MAX_RETRIES"
+ENV_WORKER_BATCH_SIZE = "HINDSIGHT_API_WORKER_BATCH_SIZE"
+ENV_WORKER_HTTP_PORT = "HINDSIGHT_API_WORKER_HTTP_PORT"
+
+# Reflect agent settings
+ENV_REFLECT_MAX_ITERATIONS = "HINDSIGHT_API_REFLECT_MAX_ITERATIONS"
+
 # Default values
 DEFAULT_DATABASE_URL = "pg0"
 DEFAULT_LLM_PROVIDER = "openai"
@@ -68,20 +137,66 @@ DEFAULT_EMBEDDING_DIMENSION = 384
 
 DEFAULT_RERANKER_PROVIDER = "local"
 DEFAULT_RERANKER_LOCAL_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"
+DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT = 4  # Limit concurrent CPU-bound reranking to prevent thrashing
+DEFAULT_RERANKER_TEI_BATCH_SIZE = 128
+DEFAULT_RERANKER_TEI_MAX_CONCURRENT = 8
+DEFAULT_RERANKER_MAX_CANDIDATES = 300
+DEFAULT_RERANKER_FLASHRANK_MODEL = "ms-marco-MiniLM-L-12-v2"  # Best balance of speed and quality
+DEFAULT_RERANKER_FLASHRANK_CACHE_DIR = None  # Use default cache directory
+
+DEFAULT_EMBEDDINGS_COHERE_MODEL = "embed-english-v3.0"
+DEFAULT_RERANKER_COHERE_MODEL = "rerank-english-v3.0"
+
+# LiteLLM defaults
+DEFAULT_LITELLM_API_BASE = "http://localhost:4000"
+DEFAULT_EMBEDDINGS_LITELLM_MODEL = "text-embedding-3-small"
+DEFAULT_RERANKER_LITELLM_MODEL = "cohere/rerank-english-v3.0"
 
 DEFAULT_HOST = "0.0.0.0"
 DEFAULT_PORT = 8888
 DEFAULT_LOG_LEVEL = "info"
+DEFAULT_LOG_FORMAT = "text"  # Options: "text", "json"
+DEFAULT_WORKERS = 1
 DEFAULT_MCP_ENABLED = True
-DEFAULT_GRAPH_RETRIEVER = "bfs"  # Options: "bfs", "mpfp"
+DEFAULT_GRAPH_RETRIEVER = "link_expansion"  # Options: "link_expansion", "mpfp", "bfs"
+DEFAULT_MPFP_TOP_K_NEIGHBORS = 20  # Fan-out limit per node in MPFP graph traversal
+DEFAULT_RECALL_MAX_CONCURRENT = 32  # Max concurrent recall operations per worker
+DEFAULT_RECALL_CONNECTION_BUDGET = 4  # Max concurrent DB connections per recall operation
 DEFAULT_MCP_LOCAL_BANK_ID = "mcp"
-
-# Observation thresholds
-DEFAULT_OBSERVATION_MIN_FACTS = 5  # Min facts required to generate entity observations
-DEFAULT_OBSERVATION_TOP_ENTITIES = 5  # Max entities to process per retain batch
+DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY = 8  # Max concurrent mental model refreshes
 
 # Retain settings
 DEFAULT_RETAIN_MAX_COMPLETION_TOKENS = 64000  # Max tokens for fact extraction LLM call
+DEFAULT_RETAIN_CHUNK_SIZE = 3000  # Max chars per chunk for fact extraction
+DEFAULT_RETAIN_EXTRACT_CAUSAL_LINKS = True  # Extract causal links between facts
+DEFAULT_RETAIN_EXTRACTION_MODE = "concise"  # Extraction mode: "concise", "verbose", or "custom"
+RETAIN_EXTRACTION_MODES = ("concise", "verbose", "custom")  # Allowed extraction modes
+DEFAULT_RETAIN_CUSTOM_INSTRUCTIONS = None  # Custom extraction guidelines (only used when mode="custom")
+DEFAULT_RETAIN_OBSERVATIONS_ASYNC = False  # Run observation generation async (after retain completes)
+
+# Observations defaults (consolidated knowledge from facts)
+DEFAULT_ENABLE_OBSERVATIONS = True  # Observations enabled by default
+DEFAULT_CONSOLIDATION_BATCH_SIZE = 50  # Memories to load per batch (internal memory optimization)
+
+# Database migrations
+DEFAULT_RUN_MIGRATIONS_ON_STARTUP = True
+
+# Database connection pool
+DEFAULT_DB_POOL_MIN_SIZE = 5
+DEFAULT_DB_POOL_MAX_SIZE = 100
+DEFAULT_DB_COMMAND_TIMEOUT = 60  # seconds
+DEFAULT_DB_ACQUIRE_TIMEOUT = 30  # seconds
+
+# Worker configuration (distributed task processing)
+DEFAULT_WORKER_ENABLED = True  # API runs worker by default (standalone mode)
+DEFAULT_WORKER_ID = None  # Will use hostname if not specified
+DEFAULT_WORKER_POLL_INTERVAL_MS = 500  # Poll database every 500ms
+DEFAULT_WORKER_MAX_RETRIES = 3  # Max retries before marking task failed
+DEFAULT_WORKER_BATCH_SIZE = 10  # Tasks to claim per poll cycle
+DEFAULT_WORKER_HTTP_PORT = 8889  # HTTP port for worker metrics/health
+
+# Reflect agent settings
+DEFAULT_REFLECT_MAX_ITERATIONS = 10  # Max tool call iterations before forcing response
 
 # Default MCP tool descriptions (can be customized via env vars)
 DEFAULT_MCP_RETAIN_DESCRIPTION = """Store important information to long-term memory.
@@ -107,6 +222,48 @@ Use this tool PROACTIVELY to:
 EMBEDDING_DIMENSION = DEFAULT_EMBEDDING_DIMENSION
 
 
+class JsonFormatter(logging.Formatter):
+    """JSON formatter for structured logging.
+
+    Outputs logs in JSON format with a 'severity' field that cloud logging
+    systems (GCP, AWS CloudWatch, etc.) can parse to correctly categorize log levels.
+    """
+
+    SEVERITY_MAP = {
+        logging.DEBUG: "DEBUG",
+        logging.INFO: "INFO",
+        logging.WARNING: "WARNING",
+        logging.ERROR: "ERROR",
+        logging.CRITICAL: "CRITICAL",
+    }
+
+    def format(self, record: logging.LogRecord) -> str:
+        log_entry = {
+            "severity": self.SEVERITY_MAP.get(record.levelno, "DEFAULT"),
+            "message": record.getMessage(),
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "logger": record.name,
+        }
+
+        # Add exception info if present
+        if record.exc_info:
+            log_entry["exception"] = self.formatException(record.exc_info)
+
+        return json.dumps(log_entry)
+
+
+def _validate_extraction_mode(mode: str) -> str:
+    """Validate and normalize extraction mode."""
+    mode_lower = mode.lower()
+    if mode_lower not in RETAIN_EXTRACTION_MODES:
+        logger.warning(
+            f"Invalid extraction mode '{mode}', must be one of {RETAIN_EXTRACTION_MODES}. "
+            f"Defaulting to '{DEFAULT_RETAIN_EXTRACTION_MODE}'."
+        )
+        return DEFAULT_RETAIN_EXTRACTION_MODE
+    return mode_lower
+
+
 @dataclass
 class HindsightConfig:
     """Configuration container for Hindsight API."""
@@ -114,7 +271,7 @@ class HindsightConfig:
     # Database
     database_url: str
 
-    # LLM
+    # LLM (default, used as fallback for per-operation config)
     llm_provider: str
     llm_api_key: str | None
     llm_model: str
@@ -122,36 +279,88 @@ class HindsightConfig:
     llm_max_concurrent: int
     llm_timeout: float
 
+    # Per-operation LLM configuration (None = use default LLM config)
+    retain_llm_provider: str | None
+    retain_llm_api_key: str | None
+    retain_llm_model: str | None
+    retain_llm_base_url: str | None
+
+    reflect_llm_provider: str | None
+    reflect_llm_api_key: str | None
+    reflect_llm_model: str | None
+    reflect_llm_base_url: str | None
+
+    consolidation_llm_provider: str | None
+    consolidation_llm_api_key: str | None
+    consolidation_llm_model: str | None
+    consolidation_llm_base_url: str | None
+
     # Embeddings
     embeddings_provider: str
     embeddings_local_model: str
     embeddings_tei_url: str | None
+    embeddings_openai_base_url: str | None
+    embeddings_cohere_base_url: str | None
 
     # Reranker
     reranker_provider: str
     reranker_local_model: str
     reranker_tei_url: str | None
+    reranker_tei_batch_size: int
+    reranker_tei_max_concurrent: int
+    reranker_max_candidates: int
+    reranker_cohere_base_url: str | None
 
     # Server
     host: str
     port: int
     log_level: str
+    log_format: str
     mcp_enabled: bool
 
     # Recall
     graph_retriever: str
-
-    # Observation thresholds
-    observation_min_facts: int
-    observation_top_entities: int
+    mpfp_top_k_neighbors: int
+    recall_max_concurrent: int
+    recall_connection_budget: int
+    mental_model_refresh_concurrency: int
 
     # Retain settings
     retain_max_completion_tokens: int
+    retain_chunk_size: int
+    retain_extract_causal_links: bool
+    retain_extraction_mode: str
+    retain_custom_instructions: str | None
+    retain_observations_async: bool
+
+    # Observations settings (consolidated knowledge from facts)
+    enable_observations: bool
+    consolidation_batch_size: int
 
     # Optimization flags
     skip_llm_verification: bool
     lazy_reranker: bool
 
+    # Database migrations
+    run_migrations_on_startup: bool
+
+    # Database connection pool
+    db_pool_min_size: int
+    db_pool_max_size: int
+    db_command_timeout: int
+    db_acquire_timeout: int
+
+    # Worker configuration (distributed task processing)
+    worker_enabled: bool
+    worker_id: str | None
+    worker_poll_interval_ms: int
+    worker_max_retries: int
+    worker_batch_size: int
+    worker_http_port: int
+
+    # Reflect agent settings
+    reflect_max_iterations: int
+
     @classmethod
     def from_env(cls) -> "HindsightConfig":
         """Create configuration from environment variables."""
@@ -165,33 +374,92 @@ class HindsightConfig:
             llm_base_url=os.getenv(ENV_LLM_BASE_URL) or None,
             llm_max_concurrent=int(os.getenv(ENV_LLM_MAX_CONCURRENT, str(DEFAULT_LLM_MAX_CONCURRENT))),
             llm_timeout=float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT))),
+            # Per-operation LLM config (None = use default)
+            retain_llm_provider=os.getenv(ENV_RETAIN_LLM_PROVIDER) or None,
+            retain_llm_api_key=os.getenv(ENV_RETAIN_LLM_API_KEY) or None,
+            retain_llm_model=os.getenv(ENV_RETAIN_LLM_MODEL) or None,
+            retain_llm_base_url=os.getenv(ENV_RETAIN_LLM_BASE_URL) or None,
+            reflect_llm_provider=os.getenv(ENV_REFLECT_LLM_PROVIDER) or None,
+            reflect_llm_api_key=os.getenv(ENV_REFLECT_LLM_API_KEY) or None,
+            reflect_llm_model=os.getenv(ENV_REFLECT_LLM_MODEL) or None,
+            reflect_llm_base_url=os.getenv(ENV_REFLECT_LLM_BASE_URL) or None,
+            consolidation_llm_provider=os.getenv(ENV_CONSOLIDATION_LLM_PROVIDER) or None,
+            consolidation_llm_api_key=os.getenv(ENV_CONSOLIDATION_LLM_API_KEY) or None,
+            consolidation_llm_model=os.getenv(ENV_CONSOLIDATION_LLM_MODEL) or None,
+            consolidation_llm_base_url=os.getenv(ENV_CONSOLIDATION_LLM_BASE_URL) or None,
             # Embeddings
             embeddings_provider=os.getenv(ENV_EMBEDDINGS_PROVIDER, DEFAULT_EMBEDDINGS_PROVIDER),
             embeddings_local_model=os.getenv(ENV_EMBEDDINGS_LOCAL_MODEL, DEFAULT_EMBEDDINGS_LOCAL_MODEL),
             embeddings_tei_url=os.getenv(ENV_EMBEDDINGS_TEI_URL),
+            embeddings_openai_base_url=os.getenv(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None,
+            embeddings_cohere_base_url=os.getenv(ENV_EMBEDDINGS_COHERE_BASE_URL) or None,
             # Reranker
             reranker_provider=os.getenv(ENV_RERANKER_PROVIDER, DEFAULT_RERANKER_PROVIDER),
             reranker_local_model=os.getenv(ENV_RERANKER_LOCAL_MODEL, DEFAULT_RERANKER_LOCAL_MODEL),
             reranker_tei_url=os.getenv(ENV_RERANKER_TEI_URL),
+            reranker_tei_batch_size=int(os.getenv(ENV_RERANKER_TEI_BATCH_SIZE, str(DEFAULT_RERANKER_TEI_BATCH_SIZE))),
+            reranker_tei_max_concurrent=int(
+                os.getenv(ENV_RERANKER_TEI_MAX_CONCURRENT, str(DEFAULT_RERANKER_TEI_MAX_CONCURRENT))
+            ),
+            reranker_max_candidates=int(os.getenv(ENV_RERANKER_MAX_CANDIDATES, str(DEFAULT_RERANKER_MAX_CANDIDATES))),
+            reranker_cohere_base_url=os.getenv(ENV_RERANKER_COHERE_BASE_URL) or None,
             # Server
             host=os.getenv(ENV_HOST, DEFAULT_HOST),
             port=int(os.getenv(ENV_PORT, DEFAULT_PORT)),
             log_level=os.getenv(ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL),
+            log_format=os.getenv(ENV_LOG_FORMAT, DEFAULT_LOG_FORMAT).lower(),
             mcp_enabled=os.getenv(ENV_MCP_ENABLED, str(DEFAULT_MCP_ENABLED)).lower() == "true",
             # Recall
             graph_retriever=os.getenv(ENV_GRAPH_RETRIEVER, DEFAULT_GRAPH_RETRIEVER),
+            mpfp_top_k_neighbors=int(os.getenv(ENV_MPFP_TOP_K_NEIGHBORS, str(DEFAULT_MPFP_TOP_K_NEIGHBORS))),
+            recall_max_concurrent=int(os.getenv(ENV_RECALL_MAX_CONCURRENT, str(DEFAULT_RECALL_MAX_CONCURRENT))),
+            recall_connection_budget=int(
+                os.getenv(ENV_RECALL_CONNECTION_BUDGET, str(DEFAULT_RECALL_CONNECTION_BUDGET))
+            ),
+            mental_model_refresh_concurrency=int(
+                os.getenv(ENV_MENTAL_MODEL_REFRESH_CONCURRENCY, str(DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY))
+            ),
             # Optimization flags
             skip_llm_verification=os.getenv(ENV_SKIP_LLM_VERIFICATION, "false").lower() == "true",
             lazy_reranker=os.getenv(ENV_LAZY_RERANKER, "false").lower() == "true",
-            # Observation thresholds
-            observation_min_facts=int(os.getenv(ENV_OBSERVATION_MIN_FACTS, str(DEFAULT_OBSERVATION_MIN_FACTS))),
-            observation_top_entities=int(
-                os.getenv(ENV_OBSERVATION_TOP_ENTITIES, str(DEFAULT_OBSERVATION_TOP_ENTITIES))
-            ),
             # Retain settings
             retain_max_completion_tokens=int(
                 os.getenv(ENV_RETAIN_MAX_COMPLETION_TOKENS, str(DEFAULT_RETAIN_MAX_COMPLETION_TOKENS))
             ),
+            retain_chunk_size=int(os.getenv(ENV_RETAIN_CHUNK_SIZE, str(DEFAULT_RETAIN_CHUNK_SIZE))),
+            retain_extract_causal_links=os.getenv(
+                ENV_RETAIN_EXTRACT_CAUSAL_LINKS, str(DEFAULT_RETAIN_EXTRACT_CAUSAL_LINKS)
+            ).lower()
+            == "true",
+            retain_extraction_mode=_validate_extraction_mode(
+                os.getenv(ENV_RETAIN_EXTRACTION_MODE, DEFAULT_RETAIN_EXTRACTION_MODE)
+            ),
+            retain_custom_instructions=os.getenv(ENV_RETAIN_CUSTOM_INSTRUCTIONS) or DEFAULT_RETAIN_CUSTOM_INSTRUCTIONS,
+            retain_observations_async=os.getenv(
+                ENV_RETAIN_OBSERVATIONS_ASYNC, str(DEFAULT_RETAIN_OBSERVATIONS_ASYNC)
+            ).lower()
+            == "true",
+            # Observations settings (consolidated knowledge from facts)
+            enable_observations=os.getenv(ENV_ENABLE_OBSERVATIONS, str(DEFAULT_ENABLE_OBSERVATIONS)).lower() == "true",
+            consolidation_batch_size=int(
+                os.getenv(ENV_CONSOLIDATION_BATCH_SIZE, str(DEFAULT_CONSOLIDATION_BATCH_SIZE))
+            ),
+            # Database migrations
+            run_migrations_on_startup=os.getenv(ENV_RUN_MIGRATIONS_ON_STARTUP, "true").lower() == "true",
+            # Database connection pool
+            db_pool_min_size=int(os.getenv(ENV_DB_POOL_MIN_SIZE, str(DEFAULT_DB_POOL_MIN_SIZE))),
+            db_pool_max_size=int(os.getenv(ENV_DB_POOL_MAX_SIZE, str(DEFAULT_DB_POOL_MAX_SIZE))),
+            db_command_timeout=int(os.getenv(ENV_DB_COMMAND_TIMEOUT, str(DEFAULT_DB_COMMAND_TIMEOUT))),
+            db_acquire_timeout=int(os.getenv(ENV_DB_ACQUIRE_TIMEOUT, str(DEFAULT_DB_ACQUIRE_TIMEOUT))),
+            # Worker configuration
+            worker_enabled=os.getenv(ENV_WORKER_ENABLED, str(DEFAULT_WORKER_ENABLED)).lower() == "true",
+            worker_id=os.getenv(ENV_WORKER_ID) or DEFAULT_WORKER_ID,
+            worker_poll_interval_ms=int(os.getenv(ENV_WORKER_POLL_INTERVAL_MS, str(DEFAULT_WORKER_POLL_INTERVAL_MS))),
+            worker_max_retries=int(os.getenv(ENV_WORKER_MAX_RETRIES, str(DEFAULT_WORKER_MAX_RETRIES))),
+            worker_batch_size=int(os.getenv(ENV_WORKER_BATCH_SIZE, str(DEFAULT_WORKER_BATCH_SIZE))),
+            worker_http_port=int(os.getenv(ENV_WORKER_HTTP_PORT, str(DEFAULT_WORKER_HTTP_PORT))),
+            # Reflect agent settings
+            reflect_max_iterations=int(os.getenv(ENV_REFLECT_MAX_ITERATIONS, str(DEFAULT_REFLECT_MAX_ITERATIONS))),
         )
 
     def get_llm_base_url(self) -> str:
@@ -222,17 +490,45 @@ class HindsightConfig:
         return log_level_map.get(self.log_level.lower(), logging.INFO)
 
     def configure_logging(self) -> None:
-        """Configure Python logging based on the log level."""
-        logging.basicConfig(
-            level=self.get_python_log_level(),
-            format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
-            force=True,  # Override any existing configuration
-        )
+        """Configure Python logging based on the log level and format.
+
+        When log_format is "json", outputs structured JSON logs with a severity
+        field that GCP Cloud Logging can parse for proper log level categorization.
+        """
+        root_logger = logging.getLogger()
+        root_logger.setLevel(self.get_python_log_level())
+
+        # Remove existing handlers
+        for handler in root_logger.handlers[:]:
+            root_logger.removeHandler(handler)
+
+        # Create handler writing to stdout (GCP treats stderr as ERROR)
+        handler = logging.StreamHandler(sys.stdout)
+        handler.setLevel(self.get_python_log_level())
+
+        if self.log_format == "json":
+            handler.setFormatter(JsonFormatter())
+        else:
+            handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s"))
+
+        root_logger.addHandler(handler)
 
     def log_config(self) -> None:
         """Log the current configuration (without sensitive values)."""
         logger.info(f"Database: {self.database_url}")
         logger.info(f"LLM: provider={self.llm_provider}, model={self.llm_model}")
+        if self.retain_llm_provider or self.retain_llm_model:
+            retain_provider = self.retain_llm_provider or self.llm_provider
+            retain_model = self.retain_llm_model or self.llm_model
+            logger.info(f"LLM (retain): provider={retain_provider}, model={retain_model}")
+        if self.reflect_llm_provider or self.reflect_llm_model:
+            reflect_provider = self.reflect_llm_provider or self.llm_provider
+            reflect_model = self.reflect_llm_model or self.llm_model
+            logger.info(f"LLM (reflect): provider={reflect_provider}, model={reflect_model}")
+        if self.consolidation_llm_provider or self.consolidation_llm_model:
+            consolidation_provider = self.consolidation_llm_provider or self.llm_provider
+            consolidation_model = self.consolidation_llm_model or self.llm_model
+            logger.info(f"LLM (consolidation): provider={consolidation_provider}, model={consolidation_model}")
         logger.info(f"Embeddings: provider={self.embeddings_provider}")
         logger.info(f"Reranker: provider={self.reranker_provider}")
         logger.info(f"Graph retriever: {self.graph_retriever}")

hindsight-api/hindsight_api/engine/consolidation/__init__.py

@@ -0,0 +1,5 @@
+"""Consolidation engine for automatic learning creation from memories."""
+
+from .consolidator import run_consolidation_job
+
+__all__ = ["run_consolidation_job"]

hindsight-api/hindsight_api/engine/consolidation/consolidator.py

@@ -0,0 +1,859 @@
+"""Consolidation engine for automatic observation creation from memories.
+
+The consolidation engine runs as a background job after retain operations complete.
+It processes new memories and either:
+- Creates new observations from novel facts
+- Updates existing observations when new evidence supports/contradicts/refines them
+
+Observations are stored in memory_units with fact_type='observation' and include:
+- proof_count: Number of supporting memories
+- source_memory_ids: Array of memory UUIDs that contribute to this observation
+- history: JSONB tracking changes over time
+"""
+
+import json
+import logging
+import time
+import uuid
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any
+
+from ..memory_engine import fq_table
+from ..retain import embedding_utils
+from .prompts import (
+    CONSOLIDATION_SYSTEM_PROMPT,
+    CONSOLIDATION_USER_PROMPT,
+)
+
+if TYPE_CHECKING:
+    from asyncpg import Connection
+
+    from ...api.http import RequestContext
+    from ..memory_engine import MemoryEngine
+
+logger = logging.getLogger(__name__)
+
+
+class ConsolidationPerfLog:
+    """Performance logging for consolidation operations."""
+
+    def __init__(self, bank_id: str):
+        self.bank_id = bank_id
+        self.start_time = time.time()
+        self.lines: list[str] = []
+        self.timings: dict[str, float] = {}
+
+    def log(self, message: str) -> None:
+        """Add a log line."""
+        self.lines.append(message)
+
+    def record_timing(self, key: str, duration: float) -> None:
+        """Record a timing measurement."""
+        if key in self.timings:
+            self.timings[key] += duration
+        else:
+            self.timings[key] = duration
+
+    def flush(self) -> None:
+        """Flush all log lines to the logger."""
+        total_time = time.time() - self.start_time
+        header = f"\n{'=' * 60}\nCONSOLIDATION for bank {self.bank_id}"
+        footer = f"{'=' * 60}\nCONSOLIDATION COMPLETE: {total_time:.3f}s total\n{'=' * 60}"
+
+        log_output = header + "\n" + "\n".join(self.lines) + "\n" + footer
+        logger.info(log_output)
+
+
+async def run_consolidation_job(
+    memory_engine: "MemoryEngine",
+    bank_id: str,
+    request_context: "RequestContext",
+) -> dict[str, Any]:
+    """
+    Run consolidation job for a bank.
+
+    This is called after retain operations to consolidate new memories into mental models.
+
+    Args:
+        memory_engine: MemoryEngine instance
+        bank_id: Bank identifier
+        request_context: Request context for authentication
+
+    Returns:
+        Dict with consolidation results
+    """
+    from ...config import get_config
+
+    config = get_config()
+    perf = ConsolidationPerfLog(bank_id)
+    max_memories_per_batch = config.consolidation_batch_size
+
+    # Check if consolidation is enabled
+    if not config.enable_observations:
+        logger.debug(f"Consolidation disabled for bank {bank_id}")
+        return {"status": "disabled", "bank_id": bank_id}
+
+    pool = memory_engine._pool
+
+    # Get bank profile
+    async with pool.acquire() as conn:
+        t0 = time.time()
+        bank_row = await conn.fetchrow(
+            f"""
+            SELECT bank_id, name, mission
+            FROM {fq_table("banks")}
+            WHERE bank_id = $1
+            """,
+            bank_id,
+        )
+
+        if not bank_row:
+            logger.warning(f"Bank {bank_id} not found for consolidation")
+            return {"status": "bank_not_found", "bank_id": bank_id}
+
+        mission = bank_row["mission"] or "General memory consolidation"
+        perf.record_timing("fetch_bank", time.time() - t0)
+
+        # Count total unconsolidated memories for progress logging
+        total_count = await conn.fetchval(
+            f"""
+            SELECT COUNT(*)
+            FROM {fq_table("memory_units")}
+            WHERE bank_id = $1
+              AND consolidated_at IS NULL
+              AND fact_type IN ('experience', 'world')
+            """,
+            bank_id,
+        )
+
+    if total_count == 0:
+        logger.debug(f"No new memories to consolidate for bank {bank_id}")
+        return {"status": "no_new_memories", "bank_id": bank_id, "memories_processed": 0}
+
+    logger.info(f"[CONSOLIDATION] bank={bank_id} total_unconsolidated={total_count}")
+    perf.log(f"[1] Found {total_count} pending memories to consolidate")
+
+    # Process each memory with individual commits for crash recovery
+    stats = {
+        "memories_processed": 0,
+        "observations_created": 0,
+        "observations_updated": 0,
+        "observations_merged": 0,
+        "actions_executed": 0,
+        "skipped": 0,
+    }
+
+    batch_num = 0
+    while True:
+        batch_num += 1
+        batch_start = time.time()
+
+        # Fetch next batch of unconsolidated memories
+        async with pool.acquire() as conn:
+            t0 = time.time()
+            memories = await conn.fetch(
+                f"""
+                SELECT id, text, fact_type, occurred_start, occurred_end, event_date, tags, mentioned_at
+                FROM {fq_table("memory_units")}
+                WHERE bank_id = $1
+                  AND consolidated_at IS NULL
+                  AND fact_type IN ('experience', 'world')
+                ORDER BY created_at ASC
+                LIMIT $2
+                """,
+                bank_id,
+                max_memories_per_batch,
+            )
+            perf.record_timing("fetch_memories", time.time() - t0)
+
+        if not memories:
+            break  # No more unconsolidated memories
+
+        for memory in memories:
+            mem_start = time.time()
+
+            # Process the memory (uses its own connection internally)
+            async with pool.acquire() as conn:
+                result = await _process_memory(
+                    conn=conn,
+                    memory_engine=memory_engine,
+                    bank_id=bank_id,
+                    memory=dict(memory),
+                    mission=mission,
+                    request_context=request_context,
+                    perf=perf,
+                )
+
+                # Mark memory as consolidated (committed immediately)
+                await conn.execute(
+                    f"""
+                    UPDATE {fq_table("memory_units")}
+                    SET consolidated_at = NOW()
+                    WHERE id = $1
+                    """,
+                    memory["id"],
+                )
+
+            mem_time = time.time() - mem_start
+            perf.record_timing("process_memory_total", mem_time)
+
+            stats["memories_processed"] += 1
+
+            action = result.get("action")
+            if action == "created":
+                stats["observations_created"] += 1
+                stats["actions_executed"] += 1
+            elif action == "updated":
+                stats["observations_updated"] += 1
+                stats["actions_executed"] += 1
+            elif action == "merged":
+                stats["observations_merged"] += 1
+                stats["actions_executed"] += 1
+            elif action == "multiple":
+                stats["observations_created"] += result.get("created", 0)
+                stats["observations_updated"] += result.get("updated", 0)
+                stats["observations_merged"] += result.get("merged", 0)
+                stats["actions_executed"] += result.get("total_actions", 0)
+            elif action == "skipped":
+                stats["skipped"] += 1
+
+            # Log progress periodically
+            if stats["memories_processed"] % 10 == 0:
+                logger.info(
+                    f"[CONSOLIDATION] bank={bank_id} progress: "
+                    f"{stats['memories_processed']}/{total_count} memories processed"
+                )
+
+        batch_time = time.time() - batch_start
+        perf.log(
+            f"[2] Batch {batch_num}: {len(memories)} memories in {batch_time:.3f}s "
+            f"(avg {batch_time / len(memories):.3f}s/memory)"
+        )
+
+    # Build summary
+    perf.log(
+        f"[3] Results: {stats['memories_processed']} memories -> "
+        f"{stats['actions_executed']} actions "
+        f"({stats['observations_created']} created, "
+        f"{stats['observations_updated']} updated, "
+        f"{stats['observations_merged']} merged, "
+        f"{stats['skipped']} skipped)"
+    )
+
+    # Add timing breakdown
+    timing_parts = []
+    if "recall" in perf.timings:
+        timing_parts.append(f"recall={perf.timings['recall']:.3f}s")
+    if "llm" in perf.timings:
+        timing_parts.append(f"llm={perf.timings['llm']:.3f}s")
+    if "embedding" in perf.timings:
+        timing_parts.append(f"embedding={perf.timings['embedding']:.3f}s")
+    if "db_write" in perf.timings:
+        timing_parts.append(f"db_write={perf.timings['db_write']:.3f}s")
+
+    if timing_parts:
+        perf.log(f"[4] Timing breakdown: {', '.join(timing_parts)}")
+
+    # Trigger mental model refreshes for models with refresh_after_consolidation=true
+    mental_models_refreshed = await _trigger_mental_model_refreshes(
+        memory_engine=memory_engine,
+        bank_id=bank_id,
+        request_context=request_context,
+        perf=perf,
+    )
+    stats["mental_models_refreshed"] = mental_models_refreshed
+
+    perf.flush()
+
+    return {"status": "completed", "bank_id": bank_id, **stats}
+
+
+async def _trigger_mental_model_refreshes(
+    memory_engine: "MemoryEngine",
+    bank_id: str,
+    request_context: "RequestContext",
+    perf: ConsolidationPerfLog | None = None,
+) -> int:
+    """
+    Trigger refreshes for mental models with refresh_after_consolidation=true.
+
+    Args:
+        memory_engine: MemoryEngine instance
+        bank_id: Bank identifier
+        request_context: Request context for authentication
+        perf: Performance logging
+
+    Returns:
+        Number of mental models scheduled for refresh
+    """
+    pool = memory_engine._pool
+
+    # Find mental models with refresh_after_consolidation=true
+    async with pool.acquire() as conn:
+        rows = await conn.fetch(
+            f"""
+            SELECT id, name
+            FROM {fq_table("mental_models")}
+            WHERE bank_id = $1
+              AND (trigger->>'refresh_after_consolidation')::boolean = true
+            """,
+            bank_id,
+        )
+
+    if not rows:
+        return 0
+
+    if perf:
+        perf.log(f"[5] Triggering refresh for {len(rows)} mental models with refresh_after_consolidation=true")
+
+    # Submit refresh tasks for each mental model
+    refreshed_count = 0
+    for row in rows:
+        mental_model_id = row["id"]
+        try:
+            await memory_engine.submit_async_refresh_mental_model(
+                bank_id=bank_id,
+                mental_model_id=mental_model_id,
+                request_context=request_context,
+            )
+            refreshed_count += 1
+            logger.info(
+                f"[CONSOLIDATION] Triggered refresh for mental model {mental_model_id} "
+                f"(name: {row['name']}) in bank {bank_id}"
+            )
+        except Exception as e:
+            logger.warning(f"[CONSOLIDATION] Failed to trigger refresh for mental model {mental_model_id}: {e}")
+
+    return refreshed_count
+
+
+async def _process_memory(
+    conn: "Connection",
+    memory_engine: "MemoryEngine",
+    bank_id: str,
+    memory: dict[str, Any],
+    mission: str,
+    request_context: "RequestContext",
+    perf: ConsolidationPerfLog | None = None,
+) -> dict[str, Any]:
+    """
+    Process a single memory for consolidation using a SINGLE LLM call.
+
+    This function:
+    1. Finds related observations (can be empty)
+    2. Uses ONE LLM call to extract durable knowledge AND decide on actions
+    3. Executes array of actions (can be multiple creates/updates)
+
+    The LLM handles all cases:
+    - No related observations: returns create action(s) with extracted durable knowledge
+    - Related observations exist: returns update/create actions based on tag routing
+    - Purely ephemeral fact: returns empty array (skip)
+
+    Returns:
+        Dict with action summary: created/updated/merged counts
+    """
+    fact_text = memory["text"]
+    memory_id = memory["id"]
+    fact_tags = memory.get("tags") or []
+
+    # Find related observations using the full recall system (NO tag filtering)
+    t0 = time.time()
+    related_observations = await _find_related_observations(
+        conn=conn,
+        memory_engine=memory_engine,
+        bank_id=bank_id,
+        query=fact_text,
+        request_context=request_context,
+    )
+    if perf:
+        perf.record_timing("recall", time.time() - t0)
+
+    # Single LLM call handles ALL cases (with or without existing observations)
+    # Note: Tags are NOT passed to LLM - they are handled algorithmically
+    t0 = time.time()
+    actions = await _consolidate_with_llm(
+        memory_engine=memory_engine,
+        fact_text=fact_text,
+        observations=related_observations,  # Can be empty list
+        mission=mission,
+    )
+    if perf:
+        perf.record_timing("llm", time.time() - t0)
+
+    if not actions:
+        # LLM returned empty array - fact is purely ephemeral, skip
+        return {"action": "skipped", "reason": "no_durable_knowledge"}
+
+    # Execute all actions and collect results
+    results = []
+    for action in actions:
+        action_type = action.get("action")
+        if action_type == "update":
+            result = await _execute_update_action(
+                conn=conn,
+                memory_engine=memory_engine,
+                bank_id=bank_id,
+                memory_id=memory_id,
+                action=action,
+                observations=related_observations,
+                source_fact_tags=fact_tags,  # Pass source fact's tags for security
+                source_occurred_start=memory.get("occurred_start"),
+                source_occurred_end=memory.get("occurred_end"),
+                source_mentioned_at=memory.get("mentioned_at"),
+                perf=perf,
+            )
+            results.append(result)
+        elif action_type == "create":
+            result = await _execute_create_action(
+                conn=conn,
+                memory_engine=memory_engine,
+                bank_id=bank_id,
+                memory_id=memory_id,
+                action=action,
+                source_fact_tags=fact_tags,  # Pass source fact's tags for security
+                event_date=memory.get("event_date"),
+                occurred_start=memory.get("occurred_start"),
+                occurred_end=memory.get("occurred_end"),
+                mentioned_at=memory.get("mentioned_at"),
+                perf=perf,
+            )
+            results.append(result)
+
+    if not results:
+        # No valid actions executed
+        return {"action": "skipped", "reason": "no_valid_actions"}
+
+    # Summarize results
+    created = sum(1 for r in results if r.get("action") == "created")
+    updated = sum(1 for r in results if r.get("action") == "updated")
+    merged = sum(1 for r in results if r.get("action") == "merged")
+
+    if len(results) == 1:
+        return results[0]
+
+    return {
+        "action": "multiple",
+        "created": created,
+        "updated": updated,
+        "merged": merged,
+        "total_actions": len(results),
+    }
+
+
+async def _execute_update_action(
+    conn: "Connection",
+    memory_engine: "MemoryEngine",
+    bank_id: str,
+    memory_id: uuid.UUID,
+    action: dict[str, Any],
+    observations: list[dict[str, Any]],
+    source_fact_tags: list[str] | None = None,
+    source_occurred_start: datetime | None = None,
+    source_occurred_end: datetime | None = None,
+    source_mentioned_at: datetime | None = None,
+    perf: ConsolidationPerfLog | None = None,
+) -> dict[str, Any]:
+    """
+    Execute an update action on an existing observation.
+
+    Updates the observation text, adds to history, increments proof_count,
+    and updates temporal fields:
+    - occurred_start: uses LEAST to keep the earliest start time
+    - occurred_end: uses GREATEST to keep the most recent end time
+    - mentioned_at: uses GREATEST to keep the most recent mention time
+
+    SECURITY: Merges source fact's tags into the observation's existing tags.
+    This ensures all contributors can see the observation they contributed to.
+    For example, if Lisa's observation (tags=['user_lisa']) is updated with
+    Mike's fact (tags=['user_mike']), the observation will have both tags.
+    """
+    learning_id = action.get("learning_id")
+    new_text = action.get("text")
+    reason = action.get("reason", "Updated with new fact")
+
+    if not learning_id or not new_text:
+        return {"action": "skipped", "reason": "missing_learning_id_or_text"}
+
+    # Find the observation
+    model = next((m for m in observations if str(m["id"]) == learning_id), None)
+    if not model:
+        return {"action": "skipped", "reason": "learning_not_found"}
+
+    # Build history entry
+    history = list(model.get("history", []))
+    history.append(
+        {
+            "previous_text": model["text"],
+            "changed_at": datetime.now(timezone.utc).isoformat(),
+            "reason": reason,
+            "source_memory_id": str(memory_id),
+        }
+    )
+
+    # Update source_memory_ids
+    source_ids = list(model.get("source_memory_ids", []))
+    source_ids.append(memory_id)
+
+    # SECURITY: Merge source fact's tags into existing observation tags
+    # This ensures all contributors can see the observation they contributed to
+    existing_tags = set(model.get("tags", []) or [])
+    source_tags = set(source_fact_tags or [])
+    merged_tags = list(existing_tags | source_tags)  # Union of both tag sets
+    if source_tags and source_tags != existing_tags:
+        logger.debug(
+            f"Security: Merging tags for observation {learning_id}: "
+            f"existing={list(existing_tags)}, source={list(source_tags)}, merged={merged_tags}"
+        )
+
+    # Generate new embedding for updated text
+    t0 = time.time()
+    embeddings = await embedding_utils.generate_embeddings_batch(memory_engine.embeddings, [new_text])
+    embedding_str = str(embeddings[0]) if embeddings else None
+    if perf:
+        perf.record_timing("embedding", time.time() - t0)
+
+    # Update the observation
+    # - occurred_start: LEAST keeps the earliest start time across all source facts
+    # - occurred_end: GREATEST keeps the most recent end time across all source facts
+    # - mentioned_at: GREATEST keeps the most recent mention time
+    # - tags: merged from existing + source fact (for visibility)
+    t0 = time.time()
+    await conn.execute(
+        f"""
+        UPDATE {fq_table("memory_units")}
+        SET text = $1,
+            embedding = $2::vector,
+            history = $3,
+            source_memory_ids = $4,
+            proof_count = $5,
+            tags = $10,
+            updated_at = now(),
+            occurred_start = LEAST(occurred_start, COALESCE($7, occurred_start)),
+            occurred_end = GREATEST(occurred_end, COALESCE($8, occurred_end)),
+            mentioned_at = GREATEST(mentioned_at, COALESCE($9, mentioned_at))
+        WHERE id = $6
+        """,
+        new_text,
+        embedding_str,
+        json.dumps(history),
+        source_ids,
+        len(source_ids),
+        uuid.UUID(learning_id),
+        source_occurred_start,
+        source_occurred_end,
+        source_mentioned_at,
+        merged_tags,
+    )
+
+    # Create links from memory to observation
+    await _create_memory_links(conn, memory_id, uuid.UUID(learning_id))
+    if perf:
+        perf.record_timing("db_write", time.time() - t0)
+
+    logger.debug(f"Updated observation {learning_id} with memory {memory_id}")
+
+    return {"action": "updated", "observation_id": learning_id}
+
+
+async def _execute_create_action(
+    conn: "Connection",
+    memory_engine: "MemoryEngine",
+    bank_id: str,
+    memory_id: uuid.UUID,
+    action: dict[str, Any],
+    source_fact_tags: list[str] | None = None,
+    event_date: datetime | None = None,
+    occurred_start: datetime | None = None,
+    occurred_end: datetime | None = None,
+    mentioned_at: datetime | None = None,
+    perf: ConsolidationPerfLog | None = None,
+) -> dict[str, Any]:
+    """
+    Execute a create action for a new observation.
+
+    Creates a new observation with the specified text.
+    The text comes directly from the classify LLM - no second LLM call needed.
+
+    Tags are determined algorithmically (not by LLM):
+    - Observations always inherit their source fact's tags
+    - This ensures visibility scope is maintained (security)
+    """
+    text = action.get("text")
+
+    # Tags are determined algorithmically - always use source fact's tags
+    # This ensures private memories create private observations
+    tags = source_fact_tags or []
+
+    if not text:
+        return {"action": "skipped", "reason": "missing_text"}
+
+    # Use text directly from classify - skip the redundant LLM call
+    result = await _create_observation_directly(
+        conn=conn,
+        memory_engine=memory_engine,
+        bank_id=bank_id,
+        source_memory_id=memory_id,
+        observation_text=text,  # Text already processed by classify LLM
+        tags=tags,
+        event_date=event_date,
+        occurred_start=occurred_start,
+        occurred_end=occurred_end,
+        mentioned_at=mentioned_at,
+        perf=perf,
+    )
+
+    logger.debug(f"Created observation {result.get('observation_id')} from memory {memory_id} (tags: {tags})")
+
+    return result
+
+
+async def _create_memory_links(
+    conn: "Connection",
+    memory_id: uuid.UUID,
+    observation_id: uuid.UUID,
+) -> None:
+    """
+    Placeholder for observation link creation.
+
+    Observations do NOT get any memory_links copied from their source facts.
+    Instead, retrieval uses source_memory_ids to traverse:
+    - Entity connections: observation → source_memory_ids → unit_entities
+    - Semantic similarity: observations have their own embeddings
+    - Temporal proximity: observations have their own temporal fields
+
+    This avoids data duplication and ensures observations are always
+    connected via their source facts' relationships.
+
+    The memory_id and observation_id parameters are kept for interface
+    compatibility but no links are created.
+    """
+    # No links are created - observations rely on source_memory_ids for traversal
+    pass
+
+
+async def _find_related_observations(
+    conn: "Connection",
+    memory_engine: "MemoryEngine",
+    bank_id: str,
+    query: str,
+    request_context: "RequestContext",
+) -> list[dict[str, Any]]:
+    """
+    Find observations related to the given query using the full recall system.
+
+    IMPORTANT: We do NOT filter by tags here. Consolidation needs to see ALL
+    potentially related observations regardless of scope, so the LLM can
+    decide on tag routing (same scope update vs cross-scope create).
+
+    This leverages:
+    - Semantic search (embedding similarity)
+    - BM25 text search (keyword matching)
+    - Entity-based retrieval (shared entities)
+    - Graph traversal (connected via entity links)
+
+    Returns:
+        List of related observations with their tags for LLM tag routing
+    """
+    # Use recall to find related observations
+    # NO tags parameter - we want ALL observations regardless of scope
+    # Use low max_tokens since we only need observations, not memories
+    recall_result = await memory_engine.recall_async(
+        bank_id=bank_id,
+        query=query,
+        max_tokens=5000,  # Token budget for observations
+        fact_type=["observation"],  # Only retrieve observations
+        request_context=request_context,
+        _quiet=True,  # Suppress logging
+        # NO tags parameter - intentionally get ALL observations
+    )
+
+    # If no observations returned, return empty list
+    # When fact_type=["observation"], results come back in `results` field
+    if not recall_result.results:
+        return []
+
+    # Trust recall's relevance filtering - fetch full data for each observation
+    results = []
+    for obs in recall_result.results:
+        # Fetch full observation data from DB to get history, source_memory_ids, tags
+        row = await conn.fetchrow(
+            f"""
+            SELECT id, text, proof_count, history, tags, source_memory_ids, created_at, updated_at
+            FROM {fq_table("memory_units")}
+            WHERE id = $1 AND bank_id = $2 AND fact_type = 'observation'
+            """,
+            uuid.UUID(obs.id),
+            bank_id,
+        )
+
+        if row:
+            history = row["history"]
+            if isinstance(history, str):
+                history = json.loads(history)
+            elif history is None:
+                history = []
+
+            results.append(
+                {
+                    "id": row["id"],
+                    "text": row["text"],
+                    "proof_count": row["proof_count"] or 1,
+                    "history": history,
+                    "tags": row["tags"] or [],  # Include tags for LLM tag routing
+                    "source_memory_ids": row["source_memory_ids"] or [],
+                    "similarity": 1.0,  # Retrieved via recall so assumed relevant
+                }
+            )
+
+    return results
+
+
+async def _consolidate_with_llm(
+    memory_engine: "MemoryEngine",
+    fact_text: str,
+    observations: list[dict[str, Any]],
+    mission: str,
+) -> list[dict[str, Any]]:
+    """
+    Single LLM call to extract durable knowledge and decide on consolidation actions.
+
+    This handles ALL cases:
+    - No related observations: extracts durable knowledge, returns create action
+    - Related observations exist: compares and returns update/create actions
+    - Purely ephemeral fact: returns empty array
+
+    Note: Tags are NOT handled by the LLM. They are determined algorithmically:
+    - CREATE: observation inherits source fact's tags
+    - UPDATE: observation merges source fact's tags with existing tags
+
+    Returns:
+        List of actions, each being:
+        - {"action": "update", "learning_id": "uuid", "text": "...", "reason": "..."}
+        - {"action": "create", "text": "...", "reason": "..."}
+        - [] if fact is purely ephemeral (no durable knowledge)
+    """
+    # Format observations WITH their tags (or "None" if empty)
+    if observations:
+        observations_text = "\n".join(
+            f'- ID: {obs["id"]}, Tags: {json.dumps(obs["tags"])}, Text: "{obs["text"]}" (proof_count: {obs["proof_count"]})'
+            for obs in observations
+        )
+    else:
+        observations_text = "None (this is a new topic - create if fact contains durable knowledge)"
+
+    # Only include mission section if mission is set and not the default
+    mission_section = ""
+    if mission and mission != "General memory consolidation":
+        mission_section = f"""
+MISSION CONTEXT: {mission}
+
+Focus on DURABLE knowledge that serves this mission, not ephemeral state.
+"""
+
+    user_prompt = CONSOLIDATION_USER_PROMPT.format(
+        mission_section=mission_section,
+        fact_text=fact_text,
+        observations_text=observations_text,
+    )
+
+    messages = [
+        {"role": "system", "content": CONSOLIDATION_SYSTEM_PROMPT},
+        {"role": "user", "content": user_prompt},
+    ]
+
+    try:
+        result = await memory_engine._consolidation_llm_config.call(
+            messages=messages,
+            skip_validation=True,  # Raw JSON response
+            scope="consolidation",
+        )
+        # Parse JSON response - should be an array
+        if isinstance(result, str):
+            result = json.loads(result)
+        # Ensure result is a list
+        if isinstance(result, list):
+            return result
+        # Handle legacy single-action format for backward compatibility
+        if isinstance(result, dict):
+            if result.get("related_ids") and result.get("consolidated_text"):
+                # Convert old format to new format
+                return [
+                    {
+                        "action": "update",
+                        "learning_id": result["related_ids"][0],
+                        "text": result["consolidated_text"],
+                        "reason": result.get("reason", ""),
+                    }
+                ]
+            return []
+        return []
+    except Exception as e:
+        logger.warning(f"Error in consolidation LLM call: {e}")
+        return []
+
+
+async def _create_observation_directly(
+    conn: "Connection",
+    memory_engine: "MemoryEngine",
+    bank_id: str,
+    source_memory_id: uuid.UUID,
+    observation_text: str,
+    tags: list[str] | None = None,
+    event_date: datetime | None = None,
+    occurred_start: datetime | None = None,
+    occurred_end: datetime | None = None,
+    mentioned_at: datetime | None = None,
+    perf: ConsolidationPerfLog | None = None,
+) -> dict[str, Any]:
+    """
+    Create an observation directly with pre-processed text (no LLM call).
+
+    Used when the classify LLM has already provided the learning text.
+    This avoids the redundant second LLM call.
+    """
+    # Generate embedding for the observation (convert to string for pgvector)
+    t0 = time.time()
+    embeddings = await embedding_utils.generate_embeddings_batch(memory_engine.embeddings, [observation_text])
+    embedding_str = str(embeddings[0]) if embeddings else None
+    if perf:
+        perf.record_timing("embedding", time.time() - t0)
+
+    # Create the observation as a memory_unit
+    now = datetime.now(timezone.utc)
+    obs_event_date = event_date or now
+    obs_occurred_start = occurred_start or now
+    obs_occurred_end = occurred_end or now
+    obs_mentioned_at = mentioned_at or now
+    obs_tags = tags or []
+
+    t0 = time.time()
+    observation_id = uuid.uuid4()
+    row = await conn.fetchrow(
+        f"""
+        INSERT INTO {fq_table("memory_units")} (
+            id, bank_id, text, fact_type, embedding, proof_count, source_memory_ids, history,
+            tags, event_date, occurred_start, occurred_end, mentioned_at
+        )
+        VALUES ($1, $2, $3, 'observation', $4::vector, 1, $5, '[]'::jsonb, $6, $7, $8, $9, $10)
+        RETURNING id
+        """,
+        observation_id,
+        bank_id,
+        observation_text,
+        embedding_str,
+        [source_memory_id],
+        obs_tags,
+        obs_event_date,
+        obs_occurred_start,
+        obs_occurred_end,
+        obs_mentioned_at,
+    )
+
+    # Create links between memory and observation (includes entity links, memory_links)
+    await _create_memory_links(conn, source_memory_id, observation_id)
+    if perf:
+        perf.record_timing("db_write", time.time() - t0)
+
+    logger.debug(f"Created observation {observation_id} from memory {source_memory_id} (tags: {obs_tags})")
+
+    return {"action": "created", "observation_id": str(row["id"]), "tags": obs_tags}

hindsight-api/hindsight_api/engine/consolidation/prompts.py

@@ -0,0 +1,69 @@
+"""Prompts for the consolidation engine."""
+
+CONSOLIDATION_SYSTEM_PROMPT = """You are a memory consolidation system. Your job is to convert facts into durable knowledge (observations) and merge with existing knowledge when appropriate.
+
+You must output ONLY valid JSON with no markdown formatting, no code blocks, and no additional text.
+
+## EXTRACT DURABLE KNOWLEDGE, NOT EPHEMERAL STATE
+Facts often describe events or actions. Extract the DURABLE KNOWLEDGE implied by the fact, not the transient state.
+
+Examples of extracting durable knowledge:
+- "User moved to Room 203" -> "Room 203 exists" (location exists, not where user is now)
+- "User visited Acme Corp at Room 105" -> "Acme Corp is located in Room 105"
+- "User took the elevator to floor 3" -> "Floor 3 is accessible by elevator"
+- "User met Sarah at the lobby" -> "Sarah can be found at the lobby"
+
+DO NOT track current user position/state as knowledge - that changes constantly.
+DO track permanent facts learned from the user's actions.
+
+## PRESERVE SPECIFIC DETAILS
+Keep names, locations, numbers, and other specifics. Do NOT:
+- Abstract into general principles
+- Generate business insights
+- Make knowledge generic
+
+GOOD examples:
+- Fact: "John likes pizza" -> "John likes pizza"
+- Fact: "Alice works at Google" -> "Alice works at Google"
+
+BAD examples:
+- "John likes pizza" -> "Understanding dietary preferences helps..." (TOO ABSTRACT)
+- "User is at Room 203" -> "User is currently at Room 203" (EPHEMERAL STATE)
+
+## MERGE RULES (when comparing to existing observations):
+1. REDUNDANT: Same information worded differently → update existing
+2. CONTRADICTION: Opposite information about same topic → update with history (e.g., "used to X, now Y")
+3. UPDATE: New state replacing old state → update with history
+
+## CRITICAL RULES:
+- NEVER merge facts about DIFFERENT people
+- NEVER merge unrelated topics (food preferences vs work vs hobbies)
+- When merging contradictions, capture the CHANGE (before → after)
+- Keep observations focused on ONE specific topic per person
+- The "text" field MUST contain durable knowledge, not ephemeral state
+- Do NOT include "tags" in output - tags are handled automatically"""
+
+CONSOLIDATION_USER_PROMPT = """Analyze this new fact and consolidate into knowledge.
+{mission_section}
+NEW FACT: {fact_text}
+
+EXISTING OBSERVATIONS:
+{observations_text}
+
+Instructions:
+1. First, extract the DURABLE KNOWLEDGE from the fact (not ephemeral state like "user is at X")
+2. Then compare with existing observations:
+   - If an observation covers the same topic: UPDATE it with the new knowledge
+   - If no observation covers the topic: CREATE a new one
+
+Output JSON array of actions (ALWAYS an array, even for single action):
+[
+  {{"action": "update", "learning_id": "uuid", "text": "updated durable knowledge", "reason": "..."}},
+  {{"action": "create", "text": "new durable knowledge", "reason": "..."}}
+]
+
+If NO consolidation is needed (fact is purely ephemeral with no durable knowledge):
+[]
+
+If no observations exist and fact contains durable knowledge:
+[{{"action": "create", "text": "durable knowledge text", "reason": "new topic"}}]"""

hindsight-api/hindsight_api/engine/cross_encoder.py

@@ -6,17 +6,38 @@ Provides an interface for reranking with different backends.
 Configuration via environment variables - see hindsight_api.config for all env var names.
 """
 
+import asyncio
 import logging
 import os
 from abc import ABC, abstractmethod
+from concurrent.futures import ThreadPoolExecutor
 
 import httpx
 
 from ..config import (
+    DEFAULT_LITELLM_API_BASE,
+    DEFAULT_RERANKER_COHERE_MODEL,
+    DEFAULT_RERANKER_FLASHRANK_CACHE_DIR,
+    DEFAULT_RERANKER_FLASHRANK_MODEL,
+    DEFAULT_RERANKER_LITELLM_MODEL,
+    DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT,
     DEFAULT_RERANKER_LOCAL_MODEL,
     DEFAULT_RERANKER_PROVIDER,
+    DEFAULT_RERANKER_TEI_BATCH_SIZE,
+    DEFAULT_RERANKER_TEI_MAX_CONCURRENT,
+    ENV_COHERE_API_KEY,
+    ENV_LITELLM_API_BASE,
+    ENV_LITELLM_API_KEY,
+    ENV_RERANKER_COHERE_BASE_URL,
+    ENV_RERANKER_COHERE_MODEL,
+    ENV_RERANKER_FLASHRANK_CACHE_DIR,
+    ENV_RERANKER_FLASHRANK_MODEL,
+    ENV_RERANKER_LITELLM_MODEL,
+    ENV_RERANKER_LOCAL_MAX_CONCURRENT,
     ENV_RERANKER_LOCAL_MODEL,
     ENV_RERANKER_PROVIDER,
+    ENV_RERANKER_TEI_BATCH_SIZE,
+    ENV_RERANKER_TEI_MAX_CONCURRENT,
     ENV_RERANKER_TEI_URL,
 )
 
@@ -47,7 +68,7 @@ class CrossEncoderModel(ABC):
         pass
 
     @abstractmethod
-    def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
         """
         Score query-document pairs for relevance.
 
@@ -70,25 +91,34 @@ class LocalSTCrossEncoder(CrossEncoderModel):
     - Fast inference (~80ms for 100 pairs on CPU)
     - Small model (80MB)
     - Trained for passage re-ranking
+
+    Uses a dedicated thread pool to limit concurrent CPU-bound work.
     """
 
-    def __init__(self, model_name: str | None = None):
+    # Shared executor across all instances (one model loaded anyway)
+    _executor: ThreadPoolExecutor | None = None
+    _max_concurrent: int = 4  # Limit concurrent CPU-bound reranking calls
+
+    def __init__(self, model_name: str | None = None, max_concurrent: int = 4):
         """
         Initialize local SentenceTransformers cross-encoder.
 
         Args:
             model_name: Name of the CrossEncoder model to use.
                        Default: cross-encoder/ms-marco-MiniLM-L-6-v2
+            max_concurrent: Maximum concurrent reranking calls (default: 2).
+                           Higher values may cause CPU thrashing under load.
         """
         self.model_name = model_name or DEFAULT_RERANKER_LOCAL_MODEL
         self._model = None
+        LocalSTCrossEncoder._max_concurrent = max_concurrent
 
     @property
     def provider_name(self) -> str:
         return "local"
 
     async def initialize(self) -> None:
-        """Load the cross-encoder model."""
+        """Load the cross-encoder model and initialize the executor."""
         if self._model is not None:
             return
 
@@ -101,13 +131,134 @@ class LocalSTCrossEncoder(CrossEncoderModel):
             )
 
         logger.info(f"Reranker: initializing local provider with model {self.model_name}")
-        self._model = CrossEncoder(self.model_name)
-        logger.info("Reranker: local provider initialized")
 
-    def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+        # Determine device based on hardware availability.
+        # We always set low_cpu_mem_usage=False to prevent lazy loading (meta tensors)
+        # which can cause issues when accelerate is installed but no GPU is available.
+        # Note: We do NOT use device_map because CrossEncoder internally calls .to(device)
+        # after loading, which conflicts with accelerate's device_map handling.
+        import torch
+
+        # Check for GPU (CUDA) or Apple Silicon (MPS)
+        has_gpu = torch.cuda.is_available() or (hasattr(torch.backends, "mps") and torch.backends.mps.is_available())
+
+        if has_gpu:
+            device = None  # Let sentence-transformers auto-detect GPU/MPS
+        else:
+            device = "cpu"
+
+        self._model = CrossEncoder(
+            self.model_name,
+            device=device,
+            model_kwargs={"low_cpu_mem_usage": False},
+        )
+
+        # Initialize shared executor (limited workers naturally limits concurrency)
+        if LocalSTCrossEncoder._executor is None:
+            LocalSTCrossEncoder._executor = ThreadPoolExecutor(
+                max_workers=LocalSTCrossEncoder._max_concurrent,
+                thread_name_prefix="reranker",
+            )
+            logger.info(f"Reranker: local provider initialized (max_concurrent={LocalSTCrossEncoder._max_concurrent})")
+        else:
+            logger.info("Reranker: local provider initialized (using existing executor)")
+
+    def _is_xpc_error(self, error: Exception) -> bool:
+        """
+        Check if an error is an XPC connection error (macOS daemon issue).
+
+        On macOS, long-running daemons can lose XPC connections to system services
+        when the process is idle for extended periods.
+        """
+        error_str = str(error).lower()
+        return "xpc_error_connection_invalid" in error_str or "xpc error" in error_str
+
+    def _reinitialize_model_sync(self) -> None:
+        """
+        Clear and reinitialize the cross-encoder model synchronously.
+
+        This is used to recover from XPC errors on macOS where the
+        PyTorch/MPS backend loses its connection to system services.
+        """
+        logger.warning(f"Reinitializing reranker model {self.model_name} due to backend error")
+
+        # Clear existing model
+        self._model = None
+
+        # Force garbage collection to free resources
+        import gc
+
+        import torch
+
+        gc.collect()
+
+        # If using CUDA/MPS, clear the cache
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+            try:
+                torch.mps.empty_cache()
+            except AttributeError:
+                pass  # Method might not exist in all PyTorch versions
+
+        # Reinitialize the model
+        try:
+            from sentence_transformers import CrossEncoder
+        except ImportError:
+            raise ImportError(
+                "sentence-transformers is required for LocalSTCrossEncoder. "
+                "Install it with: pip install sentence-transformers"
+            )
+
+        # Determine device based on hardware availability
+        has_gpu = torch.cuda.is_available() or (hasattr(torch.backends, "mps") and torch.backends.mps.is_available())
+
+        if has_gpu:
+            device = None  # Let sentence-transformers auto-detect GPU/MPS
+        else:
+            device = "cpu"
+
+        self._model = CrossEncoder(
+            self.model_name,
+            device=device,
+            model_kwargs={"low_cpu_mem_usage": False},
+        )
+
+        logger.info("Reranker: local provider reinitialized successfully")
+
+    def _predict_with_recovery(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """
+        Predict with automatic recovery from XPC errors.
+
+        This runs synchronously in the thread pool.
+        """
+        max_retries = 1
+        for attempt in range(max_retries + 1):
+            try:
+                scores = self._model.predict(pairs, show_progress_bar=False)
+                return scores.tolist() if hasattr(scores, "tolist") else list(scores)
+            except Exception as e:
+                # Check if this is an XPC error (macOS daemon issue)
+                if self._is_xpc_error(e) and attempt < max_retries:
+                    logger.warning(f"XPC error detected in reranker (attempt {attempt + 1}): {e}")
+                    try:
+                        self._reinitialize_model_sync()
+                        logger.info("Reranker reinitialized successfully, retrying prediction")
+                        continue
+                    except Exception as reinit_error:
+                        logger.error(f"Failed to reinitialize reranker: {reinit_error}")
+                        raise Exception(f"Failed to recover from XPC error: {str(e)}")
+                else:
+                    # Not an XPC error or out of retries
+                    raise
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
         """
         Score query-document pairs for relevance.
 
+        Uses a dedicated thread pool with limited workers to prevent CPU thrashing.
+        Automatically recovers from XPC errors on macOS by reinitializing the model.
+
         Args:
             pairs: List of (query, document) tuples to score
 
@@ -116,8 +267,14 @@ class LocalSTCrossEncoder(CrossEncoderModel):
         """
         if self._model is None:
             raise RuntimeError("Reranker not initialized. Call initialize() first.")
-        scores = self._model.predict(pairs, show_progress_bar=False)
-        return scores.tolist() if hasattr(scores, "tolist") else list(scores)
+
+        # Use dedicated executor - limited workers naturally limits concurrency
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(
+            LocalSTCrossEncoder._executor,
+            self._predict_with_recovery,
+            pairs,
+        )
 
 
 class RemoteTEICrossEncoder(CrossEncoderModel):
@@ -128,13 +285,21 @@ class RemoteTEICrossEncoder(CrossEncoderModel):
     See: https://github.com/huggingface/text-embeddings-inference
 
     Note: The TEI server must be running a cross-encoder/reranker model.
+
+    Requests are made in parallel with configurable batch size and max concurrency (backpressure).
+    Uses a GLOBAL semaphore to limit concurrent requests across ALL recall operations.
     """
 
+    # Global semaphore shared across all instances and calls to prevent thundering herd
+    _global_semaphore: asyncio.Semaphore | None = None
+    _global_max_concurrent: int = DEFAULT_RERANKER_TEI_MAX_CONCURRENT
+
     def __init__(
         self,
         base_url: str,
         timeout: float = 30.0,
-        batch_size: int = 32,
+        batch_size: int = DEFAULT_RERANKER_TEI_BATCH_SIZE,
+        max_concurrent: int = DEFAULT_RERANKER_TEI_MAX_CONCURRENT,
         max_retries: int = 3,
         retry_delay: float = 0.5,
     ):
@@ -144,80 +309,246 @@ class RemoteTEICrossEncoder(CrossEncoderModel):
         Args:
             base_url: Base URL of the TEI server (e.g., "http://localhost:8080")
             timeout: Request timeout in seconds (default: 30.0)
-            batch_size: Maximum batch size for rerank requests (default: 32)
+            batch_size: Maximum batch size for rerank requests (default: 128)
+            max_concurrent: Maximum concurrent requests for backpressure (default: 8).
+                           This is a GLOBAL limit across all parallel recall operations.
             max_retries: Maximum number of retries for failed requests (default: 3)
             retry_delay: Initial delay between retries in seconds, doubles each retry (default: 0.5)
         """
         self.base_url = base_url.rstrip("/")
         self.timeout = timeout
         self.batch_size = batch_size
+        self.max_concurrent = max_concurrent
         self.max_retries = max_retries
         self.retry_delay = retry_delay
-        self._client: httpx.Client | None = None
+        self._async_client: httpx.AsyncClient | None = None
         self._model_id: str | None = None
 
+        # Update global semaphore if max_concurrent changed
+        if (
+            RemoteTEICrossEncoder._global_semaphore is None
+            or RemoteTEICrossEncoder._global_max_concurrent != max_concurrent
+        ):
+            RemoteTEICrossEncoder._global_max_concurrent = max_concurrent
+            RemoteTEICrossEncoder._global_semaphore = asyncio.Semaphore(max_concurrent)
+
     @property
     def provider_name(self) -> str:
         return "tei"
 
-    def _request_with_retry(self, method: str, url: str, **kwargs) -> httpx.Response:
-        """Make an HTTP request with automatic retries on transient errors."""
-        import time
-
+    async def _async_request_with_retry(
+        self,
+        client: httpx.AsyncClient,
+        semaphore: asyncio.Semaphore,
+        method: str,
+        url: str,
+        **kwargs,
+    ) -> httpx.Response:
+        """Make an async HTTP request with automatic retries on transient errors and semaphore for backpressure."""
         last_error = None
         delay = self.retry_delay
 
-        for attempt in range(self.max_retries + 1):
-            try:
-                if method == "GET":
-                    response = self._client.get(url, **kwargs)
-                else:
-                    response = self._client.post(url, **kwargs)
-                response.raise_for_status()
-                return response
-            except (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout) as e:
-                last_error = e
-                if attempt < self.max_retries:
-                    logger.warning(
-                        f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. Retrying in {delay}s..."
-                    )
-                    time.sleep(delay)
-                    delay *= 2  # Exponential backoff
-            except httpx.HTTPStatusError as e:
-                # Retry on 5xx server errors
-                if e.response.status_code >= 500 and attempt < self.max_retries:
+        async with semaphore:
+            for attempt in range(self.max_retries + 1):
+                try:
+                    if method == "GET":
+                        response = await client.get(url, **kwargs)
+                    else:
+                        response = await client.post(url, **kwargs)
+                    response.raise_for_status()
+                    return response
+                except (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout) as e:
                     last_error = e
-                    logger.warning(
-                        f"TEI server error (attempt {attempt + 1}/{self.max_retries + 1}): {e}. Retrying in {delay}s..."
-                    )
-                    time.sleep(delay)
-                    delay *= 2
-                else:
-                    raise
+                    if attempt < self.max_retries:
+                        logger.warning(
+                            f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. "
+                            f"Retrying in {delay}s..."
+                        )
+                        await asyncio.sleep(delay)
+                        delay *= 2  # Exponential backoff
+                except httpx.HTTPStatusError as e:
+                    # Retry on 5xx server errors
+                    if e.response.status_code >= 500 and attempt < self.max_retries:
+                        last_error = e
+                        logger.warning(
+                            f"TEI server error (attempt {attempt + 1}/{self.max_retries + 1}): {e}. "
+                            f"Retrying in {delay}s..."
+                        )
+                        await asyncio.sleep(delay)
+                        delay *= 2
+                    else:
+                        raise
 
         raise last_error
 
     async def initialize(self) -> None:
         """Initialize the HTTP client and verify server connectivity."""
-        if self._client is not None:
+        if self._async_client is not None:
             return
 
-        logger.info(f"Reranker: initializing TEI provider at {self.base_url}")
-        self._client = httpx.Client(timeout=self.timeout)
+        logger.info(
+            f"Reranker: initializing TEI provider at {self.base_url} "
+            f"(batch_size={self.batch_size}, max_concurrent={self.max_concurrent})"
+        )
+        self._async_client = httpx.AsyncClient(timeout=self.timeout)
 
         # Verify server is reachable and get model info
+        # Use a temporary semaphore for initialization
+        init_semaphore = asyncio.Semaphore(1)
         try:
-            response = self._request_with_retry("GET", f"{self.base_url}/info")
+            response = await self._async_request_with_retry(
+                self._async_client, init_semaphore, "GET", f"{self.base_url}/info"
+            )
             info = response.json()
             self._model_id = info.get("model_id", "unknown")
             logger.info(f"Reranker: TEI provider initialized (model: {self._model_id})")
         except httpx.HTTPError as e:
+            self._async_client = None
             raise RuntimeError(f"Failed to connect to TEI server at {self.base_url}: {e}")
 
-    def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+    async def _rerank_query_group(
+        self,
+        client: httpx.AsyncClient,
+        semaphore: asyncio.Semaphore,
+        query: str,
+        texts: list[str],
+    ) -> list[tuple[int, float]]:
+        """Rerank a single query group and return list of (original_index, score) tuples."""
+        try:
+            response = await self._async_request_with_retry(
+                client,
+                semaphore,
+                "POST",
+                f"{self.base_url}/rerank",
+                json={
+                    "query": query,
+                    "texts": texts,
+                    "return_text": False,
+                },
+            )
+            results = response.json()
+            # TEI returns results sorted by score descending, with original index
+            return [(result["index"], result["score"]) for result in results]
+        except httpx.HTTPError as e:
+            raise RuntimeError(f"TEI rerank request failed: {e}")
+
+    async def _predict_async(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """Async implementation of predict that runs requests in parallel with backpressure."""
+        if not pairs:
+            return []
+
+        # Group all pairs by query
+        query_groups: dict[str, list[tuple[int, str]]] = {}
+        for idx, (query, text) in enumerate(pairs):
+            if query not in query_groups:
+                query_groups[query] = []
+            query_groups[query].append((idx, text))
+
+        # Split each query group into batches
+        tasks_info: list[tuple[str, list[int], list[str]]] = []  # (query, indices, texts)
+        for query, indexed_texts in query_groups.items():
+            indices = [idx for idx, _ in indexed_texts]
+            texts = [text for _, text in indexed_texts]
+
+            # Split into batches
+            for i in range(0, len(texts), self.batch_size):
+                batch_indices = indices[i : i + self.batch_size]
+                batch_texts = texts[i : i + self.batch_size]
+                tasks_info.append((query, batch_indices, batch_texts))
+
+        # Run all requests in parallel with GLOBAL semaphore for backpressure
+        # This ensures max_concurrent is respected across ALL parallel recall operations
+        all_scores = [0.0] * len(pairs)
+        semaphore = RemoteTEICrossEncoder._global_semaphore
+
+        tasks = [
+            self._rerank_query_group(self._async_client, semaphore, query, texts) for query, _, texts in tasks_info
+        ]
+        results = await asyncio.gather(*tasks)
+
+        # Map scores back to original positions
+        for (_, indices, _), result_scores in zip(tasks_info, results):
+            for original_idx_in_batch, score in result_scores:
+                global_idx = indices[original_idx_in_batch]
+                all_scores[global_idx] = score
+
+        return all_scores
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
         """
         Score query-document pairs using the remote TEI reranker.
 
+        Requests are made in parallel with configurable backpressure.
+
+        Args:
+            pairs: List of (query, document) tuples to score
+
+        Returns:
+            List of relevance scores
+        """
+        if self._async_client is None:
+            raise RuntimeError("Reranker not initialized. Call initialize() first.")
+
+        return await self._predict_async(pairs)
+
+
+class CohereCrossEncoder(CrossEncoderModel):
+    """
+    Cohere cross-encoder implementation using the Cohere Rerank API.
+
+    Supports rerank-english-v3.0 and rerank-multilingual-v3.0 models.
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        model: str = DEFAULT_RERANKER_COHERE_MODEL,
+        base_url: str | None = None,
+        timeout: float = 60.0,
+    ):
+        """
+        Initialize Cohere cross-encoder client.
+
+        Args:
+            api_key: Cohere API key
+            model: Cohere rerank model name (default: rerank-english-v3.0)
+            base_url: Custom base URL for Cohere-compatible API (e.g., Azure-hosted endpoint)
+            timeout: Request timeout in seconds (default: 60.0)
+        """
+        self.api_key = api_key
+        self.model = model
+        self.base_url = base_url
+        self.timeout = timeout
+        self._client = None
+
+    @property
+    def provider_name(self) -> str:
+        return "cohere"
+
+    async def initialize(self) -> None:
+        """Initialize the Cohere client."""
+        if self._client is not None:
+            return
+
+        try:
+            import cohere
+        except ImportError:
+            raise ImportError("cohere is required for CohereCrossEncoder. Install it with: pip install cohere")
+
+        base_url_msg = f" at {self.base_url}" if self.base_url else ""
+        logger.info(f"Reranker: initializing Cohere provider with model {self.model}{base_url_msg}")
+
+        # Build client kwargs, only including base_url if set (for Azure or custom endpoints)
+        client_kwargs = {"api_key": self.api_key, "timeout": self.timeout}
+        if self.base_url:
+            client_kwargs["base_url"] = self.base_url
+        self._client = cohere.Client(**client_kwargs)
+        logger.info("Reranker: Cohere provider initialized")
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """
+        Score query-document pairs using the Cohere Rerank API.
+
         Args:
             pairs: List of (query, document) tuples to score
 
@@ -230,50 +561,312 @@ class RemoteTEICrossEncoder(CrossEncoderModel):
         if not pairs:
             return []
 
-        all_scores = []
+        # Run sync Cohere API calls in thread pool
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, self._predict_sync, pairs)
 
-        # Process in batches
-        for i in range(0, len(pairs), self.batch_size):
-            batch = pairs[i : i + self.batch_size]
+    def _predict_sync(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """Synchronous predict implementation for Cohere API."""
+        # Group pairs by query for efficient batching
+        # Cohere rerank expects one query with multiple documents
+        query_groups: dict[str, list[tuple[int, str]]] = {}
+        for idx, (query, text) in enumerate(pairs):
+            if query not in query_groups:
+                query_groups[query] = []
+            query_groups[query].append((idx, text))
 
-            # TEI rerank endpoint expects query and texts separately
-            # All pairs in a batch should have the same query for optimal performance
-            # but we handle mixed queries by making separate requests per unique query
-            query_groups: dict[str, list[tuple[int, str]]] = {}
-            for idx, (query, text) in enumerate(batch):
-                if query not in query_groups:
-                    query_groups[query] = []
-                query_groups[query].append((idx, text))
+        all_scores = [0.0] * len(pairs)
 
-            batch_scores = [0.0] * len(batch)
+        for query, indexed_texts in query_groups.items():
+            texts = [text for _, text in indexed_texts]
+            indices = [idx for idx, _ in indexed_texts]
 
-            for query, indexed_texts in query_groups.items():
-                texts = [text for _, text in indexed_texts]
-                indices = [idx for idx, _ in indexed_texts]
+            response = self._client.rerank(
+                query=query,
+                documents=texts,
+                model=self.model,
+                return_documents=False,
+            )
 
-                try:
-                    response = self._request_with_retry(
-                        "POST",
-                        f"{self.base_url}/rerank",
-                        json={
-                            "query": query,
-                            "texts": texts,
-                            "return_text": False,
-                        },
-                    )
-                    results = response.json()
+            # Map scores back to original positions
+            for result in response.results:
+                original_idx = result.index
+                score = result.relevance_score
+                all_scores[indices[original_idx]] = score
 
-                    # TEI returns results sorted by score descending, with original index
-                    for result in results:
-                        original_idx = result["index"]
-                        score = result["score"]
-                        # Map back to batch position
-                        batch_scores[indices[original_idx]] = score
+        return all_scores
 
-                except httpx.HTTPError as e:
-                    raise RuntimeError(f"TEI rerank request failed: {e}")
 
-            all_scores.extend(batch_scores)
+class RRFPassthroughCrossEncoder(CrossEncoderModel):
+    """
+    Passthrough cross-encoder that preserves RRF scores without neural reranking.
+
+    This is useful for:
+    - Testing retrieval quality without reranking overhead
+    - Deployments where reranking latency is unacceptable
+    - Debugging to isolate retrieval vs reranking issues
+    """
+
+    def __init__(self):
+        """Initialize RRF passthrough cross-encoder."""
+        pass
+
+    @property
+    def provider_name(self) -> str:
+        return "rrf"
+
+    async def initialize(self) -> None:
+        """No initialization needed."""
+        logger.info("Reranker: RRF passthrough provider initialized (neural reranking disabled)")
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """
+        Return neutral scores - actual ranking uses RRF scores from retrieval.
+
+        Args:
+            pairs: List of (query, document) tuples (ignored)
+
+        Returns:
+            List of 0.5 scores (neutral, lets RRF scores dominate)
+        """
+        # Return neutral scores so RRF ranking is preserved
+        return [0.5] * len(pairs)
+
+
+class FlashRankCrossEncoder(CrossEncoderModel):
+    """
+    FlashRank cross-encoder implementation.
+
+    FlashRank is an ultra-lite reranking library that runs on CPU without
+    requiring PyTorch or Transformers. It's ideal for serverless deployments
+    with minimal cold-start overhead.
+
+    Available models:
+    - ms-marco-TinyBERT-L-2-v2: Fastest, ~4MB
+    - ms-marco-MiniLM-L-12-v2: Best quality, ~34MB (default)
+    - rank-T5-flan: Best zero-shot, ~110MB
+    - ms-marco-MultiBERT-L-12: Multi-lingual, ~150MB
+    """
+
+    # Shared executor for CPU-bound reranking
+    _executor: ThreadPoolExecutor | None = None
+    _max_concurrent: int = 4
+
+    def __init__(
+        self,
+        model_name: str | None = None,
+        cache_dir: str | None = None,
+        max_length: int = 512,
+        max_concurrent: int = 4,
+    ):
+        """
+        Initialize FlashRank cross-encoder.
+
+        Args:
+            model_name: FlashRank model name. Default: ms-marco-MiniLM-L-12-v2
+            cache_dir: Directory to cache downloaded models. Default: system cache
+            max_length: Maximum sequence length for reranking. Default: 512
+            max_concurrent: Maximum concurrent reranking calls. Default: 4
+        """
+        self.model_name = model_name or DEFAULT_RERANKER_FLASHRANK_MODEL
+        self.cache_dir = cache_dir or DEFAULT_RERANKER_FLASHRANK_CACHE_DIR
+        self.max_length = max_length
+        self._ranker = None
+        FlashRankCrossEncoder._max_concurrent = max_concurrent
+
+    @property
+    def provider_name(self) -> str:
+        return "flashrank"
+
+    async def initialize(self) -> None:
+        """Load the FlashRank model."""
+        if self._ranker is not None:
+            return
+
+        try:
+            from flashrank import Ranker  # type: ignore[import-untyped]
+        except ImportError:
+            raise ImportError("flashrank is required for FlashRankCrossEncoder. Install it with: pip install flashrank")
+
+        logger.info(f"Reranker: initializing FlashRank provider with model {self.model_name}")
+
+        # Initialize ranker with optional cache directory
+        ranker_kwargs = {"model_name": self.model_name, "max_length": self.max_length}
+        if self.cache_dir:
+            ranker_kwargs["cache_dir"] = self.cache_dir
+
+        self._ranker = Ranker(**ranker_kwargs)
+
+        # Initialize shared executor
+        if FlashRankCrossEncoder._executor is None:
+            FlashRankCrossEncoder._executor = ThreadPoolExecutor(
+                max_workers=FlashRankCrossEncoder._max_concurrent,
+                thread_name_prefix="flashrank",
+            )
+            logger.info(
+                f"Reranker: FlashRank provider initialized (max_concurrent={FlashRankCrossEncoder._max_concurrent})"
+            )
+        else:
+            logger.info("Reranker: FlashRank provider initialized (using existing executor)")
+
+    def _predict_sync(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """Synchronous predict - processes each query group."""
+        from flashrank import RerankRequest  # type: ignore[import-untyped]
+
+        if not pairs:
+            return []
+
+        # Group pairs by query
+        query_groups: dict[str, list[tuple[int, str]]] = {}
+        for idx, (query, text) in enumerate(pairs):
+            if query not in query_groups:
+                query_groups[query] = []
+            query_groups[query].append((idx, text))
+
+        all_scores = [0.0] * len(pairs)
+
+        for query, indexed_texts in query_groups.items():
+            # Build passages list for FlashRank
+            passages = [{"id": i, "text": text} for i, (_, text) in enumerate(indexed_texts)]
+            global_indices = [idx for idx, _ in indexed_texts]
+
+            # Create rerank request
+            request = RerankRequest(query=query, passages=passages)
+            results = self._ranker.rerank(request)
+
+            # Map scores back to original positions
+            for result in results:
+                local_idx = result["id"]
+                score = result["score"]
+                global_idx = global_indices[local_idx]
+                all_scores[global_idx] = score
+
+        return all_scores
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """
+        Score query-document pairs using FlashRank.
+
+        Args:
+            pairs: List of (query, document) tuples to score
+
+        Returns:
+            List of relevance scores (higher = more relevant)
+        """
+        if self._ranker is None:
+            raise RuntimeError("Reranker not initialized. Call initialize() first.")
+
+        # Run in thread pool to avoid blocking event loop
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(FlashRankCrossEncoder._executor, self._predict_sync, pairs)
+
+
+class LiteLLMCrossEncoder(CrossEncoderModel):
+    """
+    LiteLLM cross-encoder implementation using LiteLLM proxy's /rerank endpoint.
+
+    LiteLLM provides a unified interface for multiple reranking providers via
+    the Cohere-compatible /rerank endpoint.
+    See: https://docs.litellm.ai/docs/rerank
+
+    Supported providers via LiteLLM:
+    - Cohere (rerank-english-v3.0, etc.) - prefix with cohere/
+    - Together AI - prefix with together_ai/
+    - Azure AI - prefix with azure_ai/
+    - Jina AI - prefix with jina_ai/
+    - AWS Bedrock - prefix with bedrock/
+    - Voyage AI - prefix with voyage/
+    """
+
+    def __init__(
+        self,
+        api_base: str = DEFAULT_LITELLM_API_BASE,
+        api_key: str | None = None,
+        model: str = DEFAULT_RERANKER_LITELLM_MODEL,
+        timeout: float = 60.0,
+    ):
+        """
+        Initialize LiteLLM cross-encoder client.
+
+        Args:
+            api_base: Base URL of the LiteLLM proxy (default: http://localhost:4000)
+            api_key: API key for the LiteLLM proxy (optional, depends on proxy config)
+            model: Reranking model name (default: cohere/rerank-english-v3.0)
+                   Use provider prefix (e.g., cohere/, together_ai/, voyage/)
+            timeout: Request timeout in seconds (default: 60.0)
+        """
+        self.api_base = api_base.rstrip("/")
+        self.api_key = api_key
+        self.model = model
+        self.timeout = timeout
+        self._async_client: httpx.AsyncClient | None = None
+
+    @property
+    def provider_name(self) -> str:
+        return "litellm"
+
+    async def initialize(self) -> None:
+        """Initialize the async HTTP client."""
+        if self._async_client is not None:
+            return
+
+        logger.info(f"Reranker: initializing LiteLLM provider at {self.api_base} with model {self.model}")
+
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+
+        self._async_client = httpx.AsyncClient(timeout=self.timeout, headers=headers)
+        logger.info("Reranker: LiteLLM provider initialized")
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """
+        Score query-document pairs using the LiteLLM proxy's /rerank endpoint.
+
+        Args:
+            pairs: List of (query, document) tuples to score
+
+        Returns:
+            List of relevance scores
+        """
+        if self._async_client is None:
+            raise RuntimeError("Reranker not initialized. Call initialize() first.")
+
+        if not pairs:
+            return []
+
+        # Group pairs by query (LiteLLM rerank expects one query with multiple documents)
+        query_groups: dict[str, list[tuple[int, str]]] = {}
+        for idx, (query, text) in enumerate(pairs):
+            if query not in query_groups:
+                query_groups[query] = []
+            query_groups[query].append((idx, text))
+
+        all_scores = [0.0] * len(pairs)
+
+        for query, indexed_texts in query_groups.items():
+            texts = [text for _, text in indexed_texts]
+            indices = [idx for idx, _ in indexed_texts]
+
+            # LiteLLM /rerank follows Cohere API format
+            response = await self._async_client.post(
+                f"{self.api_base}/rerank",
+                json={
+                    "model": self.model,
+                    "query": query,
+                    "documents": texts,
+                    "top_n": len(texts),  # Return all scores
+                },
+            )
+            response.raise_for_status()
+            result = response.json()
+
+            # Map scores back to original positions
+            # Response format: {"results": [{"index": 0, "relevance_score": 0.9}, ...]}
+            for item in result.get("results", []):
+                original_idx = item["index"]
+                score = item.get("relevance_score", item.get("score", 0.0))
+                all_scores[indices[original_idx]] = score
 
         return all_scores
 
@@ -293,10 +886,35 @@ def create_cross_encoder_from_env() -> CrossEncoderModel:
         url = os.environ.get(ENV_RERANKER_TEI_URL)
         if not url:
             raise ValueError(f"{ENV_RERANKER_TEI_URL} is required when {ENV_RERANKER_PROVIDER} is 'tei'")
-        return RemoteTEICrossEncoder(base_url=url)
+        batch_size = int(os.environ.get(ENV_RERANKER_TEI_BATCH_SIZE, str(DEFAULT_RERANKER_TEI_BATCH_SIZE)))
+        max_concurrent = int(os.environ.get(ENV_RERANKER_TEI_MAX_CONCURRENT, str(DEFAULT_RERANKER_TEI_MAX_CONCURRENT)))
+        return RemoteTEICrossEncoder(base_url=url, batch_size=batch_size, max_concurrent=max_concurrent)
     elif provider == "local":
         model = os.environ.get(ENV_RERANKER_LOCAL_MODEL)
         model_name = model or DEFAULT_RERANKER_LOCAL_MODEL
-        return LocalSTCrossEncoder(model_name=model_name)
+        max_concurrent = int(
+            os.environ.get(ENV_RERANKER_LOCAL_MAX_CONCURRENT, str(DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT))
+        )
+        return LocalSTCrossEncoder(model_name=model_name, max_concurrent=max_concurrent)
+    elif provider == "cohere":
+        api_key = os.environ.get(ENV_COHERE_API_KEY)
+        if not api_key:
+            raise ValueError(f"{ENV_COHERE_API_KEY} is required when {ENV_RERANKER_PROVIDER} is 'cohere'")
+        model = os.environ.get(ENV_RERANKER_COHERE_MODEL, DEFAULT_RERANKER_COHERE_MODEL)
+        base_url = os.environ.get(ENV_RERANKER_COHERE_BASE_URL) or None
+        return CohereCrossEncoder(api_key=api_key, model=model, base_url=base_url)
+    elif provider == "flashrank":
+        model = os.environ.get(ENV_RERANKER_FLASHRANK_MODEL, DEFAULT_RERANKER_FLASHRANK_MODEL)
+        cache_dir = os.environ.get(ENV_RERANKER_FLASHRANK_CACHE_DIR, DEFAULT_RERANKER_FLASHRANK_CACHE_DIR)
+        return FlashRankCrossEncoder(model_name=model, cache_dir=cache_dir)
+    elif provider == "litellm":
+        api_base = os.environ.get(ENV_LITELLM_API_BASE, DEFAULT_LITELLM_API_BASE)
+        api_key = os.environ.get(ENV_LITELLM_API_KEY)
+        model = os.environ.get(ENV_RERANKER_LITELLM_MODEL, DEFAULT_RERANKER_LITELLM_MODEL)
+        return LiteLLMCrossEncoder(api_base=api_base, api_key=api_key, model=model)
+    elif provider == "rrf":
+        return RRFPassthroughCrossEncoder()
     else:
-        raise ValueError(f"Unknown reranker provider: {provider}. Supported: 'local', 'tei'")
+        raise ValueError(
+            f"Unknown reranker provider: {provider}. Supported: 'local', 'tei', 'cohere', 'flashrank', 'litellm', 'rrf'"
+        )

hindsight-api/hindsight_api/engine/db_budget.py

@@ -0,0 +1,284 @@
+"""
+Database connection budget management.
+
+Limits concurrent database connections per operation to prevent
+a single operation (e.g., recall with parallel queries) from
+exhausting the connection pool.
+"""
+
+import asyncio
+import logging
+import uuid
+from contextlib import asynccontextmanager
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, AsyncIterator
+
+if TYPE_CHECKING:
+    import asyncpg
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class OperationBudget:
+    """
+    Tracks connection budget for a single operation.
+
+    Each operation gets a semaphore limiting its concurrent connections.
+    """
+
+    operation_id: str
+    max_connections: int
+    semaphore: asyncio.Semaphore = field(init=False)
+    active_count: int = field(default=0, init=False)
+
+    def __post_init__(self):
+        self.semaphore = asyncio.Semaphore(self.max_connections)
+
+
+class ConnectionBudgetManager:
+    """
+    Manages per-operation connection budgets.
+
+    Usage:
+        manager = ConnectionBudgetManager(default_budget=4)
+
+        # Start an operation
+        async with manager.operation(max_connections=2) as op:
+            # Acquire connections within the budget
+            async with op.acquire(pool) as conn:
+                await conn.fetch(...)
+
+            # Multiple connections respect the budget
+            async with op.acquire(pool) as conn1, op.acquire(pool) as conn2:
+                # At most 2 concurrent connections for this operation
+                ...
+    """
+
+    def __init__(self, default_budget: int = 4):
+        """
+        Initialize the budget manager.
+
+        Args:
+            default_budget: Default max connections per operation
+        """
+        self.default_budget = default_budget
+        self._operations: dict[str, OperationBudget] = {}
+        self._lock = asyncio.Lock()
+
+    @asynccontextmanager
+    async def operation(
+        self,
+        max_connections: int | None = None,
+        operation_id: str | None = None,
+    ) -> AsyncIterator["BudgetedOperation"]:
+        """
+        Create a budgeted operation context.
+
+        Args:
+            max_connections: Max concurrent connections for this operation.
+                           Defaults to manager's default_budget.
+            operation_id: Optional custom operation ID. Auto-generated if not provided.
+
+        Yields:
+            BudgetedOperation context for acquiring connections
+        """
+        op_id = operation_id or f"op-{uuid.uuid4().hex[:12]}"
+        budget = max_connections or self.default_budget
+
+        async with self._lock:
+            if op_id in self._operations:
+                raise ValueError(f"Operation {op_id} already exists")
+            self._operations[op_id] = OperationBudget(op_id, budget)
+
+        try:
+            yield BudgetedOperation(self, op_id)
+        finally:
+            async with self._lock:
+                self._operations.pop(op_id, None)
+
+    def _get_budget(self, operation_id: str) -> OperationBudget:
+        """Get budget for an operation (internal use)."""
+        budget = self._operations.get(operation_id)
+        if not budget:
+            raise ValueError(f"Operation {operation_id} not found")
+        return budget
+
+
+class BudgetedOperation:
+    """
+    A single operation with connection budget.
+
+    Provides methods to acquire connections within the budget.
+    """
+
+    def __init__(self, manager: ConnectionBudgetManager, operation_id: str):
+        self._manager = manager
+        self.operation_id = operation_id
+
+    @property
+    def budget(self) -> OperationBudget:
+        """Get the budget for this operation."""
+        return self._manager._get_budget(self.operation_id)
+
+    @asynccontextmanager
+    async def acquire(self, pool: "asyncpg.Pool") -> AsyncIterator["asyncpg.Connection"]:
+        """
+        Acquire a connection within the operation's budget.
+
+        Blocks if the operation has reached its connection limit.
+
+        Args:
+            pool: asyncpg connection pool
+
+        Yields:
+            Database connection
+        """
+        budget = self.budget
+        async with budget.semaphore:
+            budget.active_count += 1
+            conn = await pool.acquire()
+            try:
+                yield conn
+            finally:
+                budget.active_count -= 1
+                await pool.release(conn)
+
+    def wrap_pool(self, pool: "asyncpg.Pool") -> "BudgetedPool":
+        """
+        Wrap a pool with this operation's budget.
+
+        The returned BudgetedPool can be passed to functions expecting a pool,
+        and all acquire() calls will be limited by this operation's budget.
+
+        Args:
+            pool: asyncpg connection pool to wrap
+
+        Returns:
+            BudgetedPool that limits connections to this operation's budget
+        """
+        return BudgetedPool(pool, self)
+
+    async def acquire_many(
+        self,
+        pool: "asyncpg.Pool",
+        count: int,
+    ) -> AsyncIterator[list["asyncpg.Connection"]]:
+        """
+        Acquire multiple connections within the budget.
+
+        Note: This acquires connections sequentially to respect the budget.
+        For parallel acquisition, use multiple acquire() calls with asyncio.gather().
+
+        Args:
+            pool: asyncpg connection pool
+            count: Number of connections to acquire
+
+        Yields:
+            List of database connections
+        """
+        connections = []
+        try:
+            for _ in range(count):
+                conn = await pool.acquire()
+                connections.append(conn)
+            yield connections
+        finally:
+            for conn in connections:
+                await pool.release(conn)
+
+
+# Global default manager instance
+_default_manager: ConnectionBudgetManager | None = None
+
+
+def get_budget_manager(default_budget: int = 4) -> ConnectionBudgetManager:
+    """
+    Get or create the global budget manager.
+
+    Args:
+        default_budget: Default max connections per operation
+
+    Returns:
+        Global ConnectionBudgetManager instance
+    """
+    global _default_manager
+    if _default_manager is None:
+        _default_manager = ConnectionBudgetManager(default_budget=default_budget)
+    return _default_manager
+
+
+@asynccontextmanager
+async def budgeted_operation(
+    max_connections: int | None = None,
+    operation_id: str | None = None,
+    default_budget: int = 4,
+) -> AsyncIterator[BudgetedOperation]:
+    """
+    Convenience function to create a budgeted operation.
+
+    Args:
+        max_connections: Max concurrent connections for this operation
+        operation_id: Optional custom operation ID
+        default_budget: Default budget if manager not yet created
+
+    Yields:
+        BudgetedOperation context
+
+    Example:
+        async with budgeted_operation(max_connections=2) as op:
+            async with op.acquire(pool) as conn:
+                await conn.fetch(...)
+    """
+    manager = get_budget_manager(default_budget)
+    async with manager.operation(max_connections, operation_id) as op:
+        yield op
+
+
+class BudgetedPool:
+    """
+    A pool wrapper that limits concurrent connection acquisitions.
+
+    This can be passed to functions expecting a pool, and acquire()
+    calls will be limited by the budget semaphore.
+
+    Usage:
+        async with budgeted_operation(max_connections=4) as op:
+            budgeted_pool = op.wrap_pool(pool)
+            # Pass budgeted_pool to functions that expect a pool
+            await some_function(budgeted_pool, ...)
+    """
+
+    def __init__(self, pool: "asyncpg.Pool", operation: BudgetedOperation):
+        self._pool = pool
+        self._operation = operation
+
+    async def acquire(self) -> "asyncpg.Connection":
+        """
+        Acquire a connection within the budget.
+
+        Note: Caller must release the connection when done.
+        Prefer using as context manager via acquire_with_retry or op.acquire().
+        """
+        budget = self._operation.budget
+        await budget.semaphore.acquire()
+        budget.active_count += 1
+        try:
+            return await self._pool.acquire()
+        except Exception:
+            budget.active_count -= 1
+            budget.semaphore.release()
+            raise
+
+    async def release(self, conn: "asyncpg.Connection") -> None:
+        """Release a connection back to the pool."""
+        budget = self._operation.budget
+        try:
+            await self._pool.release(conn)
+        finally:
+            budget.active_count -= 1
+            budget.semaphore.release()
+
+    def __getattr__(self, name):
+        """Proxy other attributes to the underlying pool."""
+        return getattr(self._pool, name)

hindsight-api/hindsight_api/engine/db_utils.py

@@ -83,11 +83,22 @@ async def acquire_with_retry(pool: asyncpg.Pool, max_retries: int = DEFAULT_MAX_
     Yields:
         An asyncpg connection
     """
+    import time
+
+    start = time.time()
 
     async def acquire():
         return await pool.acquire()
 
     conn = await retry_with_backoff(acquire, max_retries=max_retries)
+    acquire_time = time.time() - start
+
+    # Log slow connection acquisitions (indicates pool contention)
+    if acquire_time > 0.05:  # 50ms threshold
+        pool_size = pool.get_size()
+        pool_free = pool.get_idle_size()
+        logger.warning(f"[DB POOL] Slow acquire: {acquire_time:.3f}s | size={pool_size}, idle={pool_free}")
+
     try:
         yield conn
     finally:

hindsight-api/hindsight_api/engine/directives/__init__.py

@@ -0,0 +1,5 @@
+"""Directives module for hard rules injected into prompts."""
+
+from .models import Directive
+
+__all__ = ["Directive"]

hindsight-api/hindsight_api/engine/directives/models.py

@@ -0,0 +1,37 @@
+"""Pydantic models for directives."""
+
+from datetime import datetime, timezone
+from uuid import UUID
+
+from pydantic import BaseModel, Field
+
+
+class Directive(BaseModel):
+    """A directive is a hard rule injected into prompts.
+
+    Directives are user-defined rules that guide agent behavior. Unlike mental models
+    which are automatically consolidated from memories, directives are explicit
+    instructions that are always included in relevant prompts.
+
+    Examples:
+    - "Always respond in formal English"
+    - "Never share personal data with third parties"
+    - "Prefer conservative investment recommendations"
+    """
+
+    id: UUID = Field(description="Unique identifier")
+    bank_id: str = Field(description="Bank this directive belongs to")
+    name: str = Field(description="Human-readable name")
+    content: str = Field(description="The directive text to inject into prompts")
+    priority: int = Field(default=0, description="Higher priority directives are injected first")
+    is_active: bool = Field(default=True, description="Whether this directive is currently active")
+    tags: list[str] = Field(default_factory=list, description="Tags for filtering")
+    created_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc), description="When this directive was created"
+    )
+    updated_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc), description="When this directive was last updated"
+    )
+
+    class Config:
+        from_attributes = True

hindsight-api/hindsight_api/engine/embeddings.py

@@ -16,14 +16,24 @@ from abc import ABC, abstractmethod
 import httpx
 
 from ..config import (
+    DEFAULT_EMBEDDINGS_COHERE_MODEL,
+    DEFAULT_EMBEDDINGS_LITELLM_MODEL,
     DEFAULT_EMBEDDINGS_LOCAL_MODEL,
     DEFAULT_EMBEDDINGS_OPENAI_MODEL,
     DEFAULT_EMBEDDINGS_PROVIDER,
+    DEFAULT_LITELLM_API_BASE,
+    ENV_COHERE_API_KEY,
+    ENV_EMBEDDINGS_COHERE_BASE_URL,
+    ENV_EMBEDDINGS_COHERE_MODEL,
+    ENV_EMBEDDINGS_LITELLM_MODEL,
     ENV_EMBEDDINGS_LOCAL_MODEL,
     ENV_EMBEDDINGS_OPENAI_API_KEY,
+    ENV_EMBEDDINGS_OPENAI_BASE_URL,
     ENV_EMBEDDINGS_OPENAI_MODEL,
     ENV_EMBEDDINGS_PROVIDER,
     ENV_EMBEDDINGS_TEI_URL,
+    ENV_LITELLM_API_BASE,
+    ENV_LITELLM_API_KEY,
     ENV_LLM_API_KEY,
 )
 
@@ -118,20 +128,98 @@ class LocalSTEmbeddings(Embeddings):
             )
 
         logger.info(f"Embeddings: initializing local provider with model {self.model_name}")
-        # Disable lazy loading (meta tensors) which causes issues with newer transformers/accelerate
-        # Setting low_cpu_mem_usage=False and device_map=None ensures tensors are fully materialized
+
+        # Determine device based on hardware availability.
+        # We always set low_cpu_mem_usage=False to prevent lazy loading (meta tensors)
+        # which can cause issues when accelerate is installed but no GPU is available.
+        import torch
+
+        # Check for GPU (CUDA) or Apple Silicon (MPS)
+        has_gpu = torch.cuda.is_available() or (hasattr(torch.backends, "mps") and torch.backends.mps.is_available())
+
+        if has_gpu:
+            device = None  # Let sentence-transformers auto-detect GPU/MPS
+        else:
+            device = "cpu"
+
         self._model = SentenceTransformer(
             self.model_name,
-            model_kwargs={"low_cpu_mem_usage": False, "device_map": None},
+            device=device,
+            model_kwargs={"low_cpu_mem_usage": False},
         )
 
         self._dimension = self._model.get_sentence_embedding_dimension()
         logger.info(f"Embeddings: local provider initialized (dim: {self._dimension})")
 
+    def _is_xpc_error(self, error: Exception) -> bool:
+        """
+        Check if an error is an XPC connection error (macOS daemon issue).
+
+        On macOS, long-running daemons can lose XPC connections to system services
+        when the process is idle for extended periods.
+        """
+        error_str = str(error).lower()
+        return "xpc_error_connection_invalid" in error_str or "xpc error" in error_str
+
+    def _reinitialize_model_sync(self) -> None:
+        """
+        Clear and reinitialize the embedding model synchronously.
+
+        This is used to recover from XPC errors on macOS where the
+        PyTorch/MPS backend loses its connection to system services.
+        """
+        logger.warning(f"Reinitializing embedding model {self.model_name} due to backend error")
+
+        # Clear existing model
+        self._model = None
+
+        # Force garbage collection to free resources
+        import gc
+
+        import torch
+
+        gc.collect()
+
+        # If using CUDA/MPS, clear the cache
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+            try:
+                torch.mps.empty_cache()
+            except AttributeError:
+                pass  # Method might not exist in all PyTorch versions
+
+        # Reinitialize the model (inline version of initialize() but synchronous)
+        try:
+            from sentence_transformers import SentenceTransformer
+        except ImportError:
+            raise ImportError(
+                "sentence-transformers is required for LocalSTEmbeddings. "
+                "Install it with: pip install sentence-transformers"
+            )
+
+        # Determine device based on hardware availability
+        has_gpu = torch.cuda.is_available() or (hasattr(torch.backends, "mps") and torch.backends.mps.is_available())
+
+        if has_gpu:
+            device = None  # Let sentence-transformers auto-detect GPU/MPS
+        else:
+            device = "cpu"
+
+        self._model = SentenceTransformer(
+            self.model_name,
+            device=device,
+            model_kwargs={"low_cpu_mem_usage": False},
+        )
+
+        logger.info("Embeddings: local provider reinitialized successfully")
+
     def encode(self, texts: list[str]) -> list[list[float]]:
         """
         Generate embeddings for a list of texts.
 
+        Automatically recovers from XPC errors on macOS by reinitializing the model.
+
         Args:
             texts: List of text strings to encode
 
@@ -140,8 +228,27 @@ class LocalSTEmbeddings(Embeddings):
         """
         if self._model is None:
             raise RuntimeError("Embeddings not initialized. Call initialize() first.")
-        embeddings = self._model.encode(texts, convert_to_numpy=True, show_progress_bar=False)
-        return [emb.tolist() for emb in embeddings]
+
+        # Try encoding with automatic recovery from XPC errors
+        max_retries = 1
+        for attempt in range(max_retries + 1):
+            try:
+                embeddings = self._model.encode(texts, convert_to_numpy=True, show_progress_bar=False)
+                return [emb.tolist() for emb in embeddings]
+            except Exception as e:
+                # Check if this is an XPC error (macOS daemon issue)
+                if self._is_xpc_error(e) and attempt < max_retries:
+                    logger.warning(f"XPC error detected in embedding generation (attempt {attempt + 1}): {e}")
+                    try:
+                        self._reinitialize_model_sync()
+                        logger.info("Model reinitialized successfully, retrying embedding generation")
+                        continue
+                    except Exception as reinit_error:
+                        logger.error(f"Failed to reinitialize model: {reinit_error}")
+                        raise Exception(f"Failed to recover from XPC error: {str(e)}")
+                else:
+                    # Not an XPC error or out of retries
+                    raise
 
 
 class RemoteTEIEmbeddings(Embeddings):
@@ -319,6 +426,7 @@ class OpenAIEmbeddings(Embeddings):
         self,
         api_key: str,
         model: str = DEFAULT_EMBEDDINGS_OPENAI_MODEL,
+        base_url: str | None = None,
         batch_size: int = 100,
         max_retries: int = 3,
     ):
@@ -328,11 +436,13 @@ class OpenAIEmbeddings(Embeddings):
         Args:
             api_key: OpenAI API key
             model: OpenAI embedding model name (default: text-embedding-3-small)
+            base_url: Custom base URL for OpenAI-compatible API (e.g., Azure OpenAI endpoint)
             batch_size: Maximum batch size for embedding requests (default: 100)
             max_retries: Maximum number of retries for failed requests (default: 3)
         """
         self.api_key = api_key
         self.model = model
+        self.base_url = base_url
         self.batch_size = batch_size
         self.max_retries = max_retries
         self._client = None
@@ -358,8 +468,14 @@ class OpenAIEmbeddings(Embeddings):
         except ImportError:
             raise ImportError("openai is required for OpenAIEmbeddings. Install it with: pip install openai")
 
-        logger.info(f"Embeddings: initializing OpenAI provider with model {self.model}")
-        self._client = OpenAI(api_key=self.api_key, max_retries=self.max_retries)
+        base_url_msg = f" at {self.base_url}" if self.base_url else ""
+        logger.info(f"Embeddings: initializing OpenAI provider with model {self.model}{base_url_msg}")
+
+        # Build client kwargs, only including base_url if set (for Azure or custom endpoints)
+        client_kwargs = {"api_key": self.api_key, "max_retries": self.max_retries}
+        if self.base_url:
+            client_kwargs["base_url"] = self.base_url
+        self._client = OpenAI(**client_kwargs)
 
         # Try to get dimension from known models, otherwise do a test embedding
         if self.model in self.MODEL_DIMENSIONS:
@@ -409,6 +525,249 @@ class OpenAIEmbeddings(Embeddings):
         return all_embeddings
 
 
+class CohereEmbeddings(Embeddings):
+    """
+    Cohere embeddings implementation using the Cohere API.
+
+    Supports embed-english-v3.0 (1024 dims) and embed-multilingual-v3.0 (1024 dims).
+
+    The embedding dimension is auto-detected from the model at initialization.
+    """
+
+    # Known dimensions for Cohere embedding models
+    MODEL_DIMENSIONS = {
+        "embed-english-v3.0": 1024,
+        "embed-multilingual-v3.0": 1024,
+        "embed-english-light-v3.0": 384,
+        "embed-multilingual-light-v3.0": 384,
+        "embed-english-v2.0": 4096,
+        "embed-multilingual-v2.0": 768,
+    }
+
+    def __init__(
+        self,
+        api_key: str,
+        model: str = DEFAULT_EMBEDDINGS_COHERE_MODEL,
+        base_url: str | None = None,
+        batch_size: int = 96,
+        timeout: float = 60.0,
+        input_type: str = "search_document",
+    ):
+        """
+        Initialize Cohere embeddings client.
+
+        Args:
+            api_key: Cohere API key
+            model: Cohere embedding model name (default: embed-english-v3.0)
+            base_url: Custom base URL for Cohere-compatible API (e.g., Azure-hosted endpoint)
+            batch_size: Maximum batch size for embedding requests (default: 96, Cohere's limit)
+            timeout: Request timeout in seconds (default: 60.0)
+            input_type: Input type for embeddings (default: search_document).
+                       Options: search_document, search_query, classification, clustering
+        """
+        self.api_key = api_key
+        self.model = model
+        self.base_url = base_url
+        self.batch_size = batch_size
+        self.timeout = timeout
+        self.input_type = input_type
+        self._client = None
+        self._dimension: int | None = None
+
+    @property
+    def provider_name(self) -> str:
+        return "cohere"
+
+    @property
+    def dimension(self) -> int:
+        if self._dimension is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+        return self._dimension
+
+    async def initialize(self) -> None:
+        """Initialize the Cohere client and detect dimension."""
+        if self._client is not None:
+            return
+
+        try:
+            import cohere
+        except ImportError:
+            raise ImportError("cohere is required for CohereEmbeddings. Install it with: pip install cohere")
+
+        base_url_msg = f" at {self.base_url}" if self.base_url else ""
+        logger.info(f"Embeddings: initializing Cohere provider with model {self.model}{base_url_msg}")
+
+        # Build client kwargs, only including base_url if set (for Azure or custom endpoints)
+        client_kwargs = {"api_key": self.api_key, "timeout": self.timeout}
+        if self.base_url:
+            client_kwargs["base_url"] = self.base_url
+        self._client = cohere.Client(**client_kwargs)
+
+        # Try to get dimension from known models, otherwise do a test embedding
+        if self.model in self.MODEL_DIMENSIONS:
+            self._dimension = self.MODEL_DIMENSIONS[self.model]
+        else:
+            # Do a test embedding to detect dimension
+            response = self._client.embed(
+                texts=["test"],
+                model=self.model,
+                input_type=self.input_type,
+            )
+            if response.embeddings:
+                self._dimension = len(response.embeddings[0])
+
+        logger.info(f"Embeddings: Cohere provider initialized (model: {self.model}, dim: {self._dimension})")
+
+    def encode(self, texts: list[str]) -> list[list[float]]:
+        """
+        Generate embeddings using the Cohere API.
+
+        Args:
+            texts: List of text strings to encode
+
+        Returns:
+            List of embedding vectors
+        """
+        if self._client is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+
+        if not texts:
+            return []
+
+        all_embeddings = []
+
+        # Process in batches
+        for i in range(0, len(texts), self.batch_size):
+            batch = texts[i : i + self.batch_size]
+
+            response = self._client.embed(
+                texts=batch,
+                model=self.model,
+                input_type=self.input_type,
+            )
+
+            all_embeddings.extend(response.embeddings)
+
+        return all_embeddings
+
+
+class LiteLLMEmbeddings(Embeddings):
+    """
+    LiteLLM embeddings implementation using LiteLLM proxy's /embeddings endpoint.
+
+    LiteLLM provides a unified interface for multiple embedding providers.
+    The proxy exposes an OpenAI-compatible /embeddings endpoint.
+    See: https://docs.litellm.ai/docs/embedding/supported_embedding
+
+    Supported providers via LiteLLM:
+    - OpenAI (text-embedding-3-small, text-embedding-ada-002, etc.)
+    - Cohere (embed-english-v3.0, etc.) - prefix with cohere/
+    - Vertex AI (textembedding-gecko, etc.) - prefix with vertex_ai/
+    - HuggingFace, Mistral, Voyage AI, etc.
+
+    The embedding dimension is auto-detected from the model at initialization.
+    """
+
+    def __init__(
+        self,
+        api_base: str = DEFAULT_LITELLM_API_BASE,
+        api_key: str | None = None,
+        model: str = DEFAULT_EMBEDDINGS_LITELLM_MODEL,
+        batch_size: int = 100,
+        timeout: float = 60.0,
+    ):
+        """
+        Initialize LiteLLM embeddings client.
+
+        Args:
+            api_base: Base URL of the LiteLLM proxy (default: http://localhost:4000)
+            api_key: API key for the LiteLLM proxy (optional, depends on proxy config)
+            model: Embedding model name (default: text-embedding-3-small)
+                   Use provider prefix for non-OpenAI models (e.g., cohere/embed-english-v3.0)
+            batch_size: Maximum batch size for embedding requests (default: 100)
+            timeout: Request timeout in seconds (default: 60.0)
+        """
+        self.api_base = api_base.rstrip("/")
+        self.api_key = api_key
+        self.model = model
+        self.batch_size = batch_size
+        self.timeout = timeout
+        self._client: httpx.Client | None = None
+        self._dimension: int | None = None
+
+    @property
+    def provider_name(self) -> str:
+        return "litellm"
+
+    @property
+    def dimension(self) -> int:
+        if self._dimension is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+        return self._dimension
+
+    async def initialize(self) -> None:
+        """Initialize the HTTP client and detect embedding dimension."""
+        if self._client is not None:
+            return
+
+        logger.info(f"Embeddings: initializing LiteLLM provider at {self.api_base} with model {self.model}")
+
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+
+        self._client = httpx.Client(timeout=self.timeout, headers=headers)
+
+        # Do a test embedding to detect dimension
+        try:
+            response = self._client.post(
+                f"{self.api_base}/embeddings",
+                json={"model": self.model, "input": ["test"]},
+            )
+            response.raise_for_status()
+            result = response.json()
+            if result.get("data") and len(result["data"]) > 0:
+                self._dimension = len(result["data"][0]["embedding"])
+            logger.info(f"Embeddings: LiteLLM provider initialized (model: {self.model}, dim: {self._dimension})")
+        except httpx.HTTPError as e:
+            raise RuntimeError(f"Failed to connect to LiteLLM proxy at {self.api_base}: {e}")
+
+    def encode(self, texts: list[str]) -> list[list[float]]:
+        """
+        Generate embeddings using the LiteLLM proxy.
+
+        Args:
+            texts: List of text strings to encode
+
+        Returns:
+            List of embedding vectors
+        """
+        if self._client is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+
+        if not texts:
+            return []
+
+        all_embeddings = []
+
+        # Process in batches
+        for i in range(0, len(texts), self.batch_size):
+            batch = texts[i : i + self.batch_size]
+
+            response = self._client.post(
+                f"{self.api_base}/embeddings",
+                json={"model": self.model, "input": batch},
+            )
+            response.raise_for_status()
+            result = response.json()
+
+            # Sort by index to ensure correct order
+            batch_embeddings = sorted(result["data"], key=lambda x: x["index"])
+            all_embeddings.extend([e["embedding"] for e in batch_embeddings])
+
+        return all_embeddings
+
+
 def create_embeddings_from_env() -> Embeddings:
     """
     Create an Embeddings instance based on environment variables.
@@ -438,6 +797,21 @@ def create_embeddings_from_env() -> Embeddings:
                 f"when {ENV_EMBEDDINGS_PROVIDER} is 'openai'"
             )
         model = os.environ.get(ENV_EMBEDDINGS_OPENAI_MODEL, DEFAULT_EMBEDDINGS_OPENAI_MODEL)
-        return OpenAIEmbeddings(api_key=api_key, model=model)
+        base_url = os.environ.get(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None
+        return OpenAIEmbeddings(api_key=api_key, model=model, base_url=base_url)
+    elif provider == "cohere":
+        api_key = os.environ.get(ENV_COHERE_API_KEY)
+        if not api_key:
+            raise ValueError(f"{ENV_COHERE_API_KEY} is required when {ENV_EMBEDDINGS_PROVIDER} is 'cohere'")
+        model = os.environ.get(ENV_EMBEDDINGS_COHERE_MODEL, DEFAULT_EMBEDDINGS_COHERE_MODEL)
+        base_url = os.environ.get(ENV_EMBEDDINGS_COHERE_BASE_URL) or None
+        return CohereEmbeddings(api_key=api_key, model=model, base_url=base_url)
+    elif provider == "litellm":
+        api_base = os.environ.get(ENV_LITELLM_API_BASE, DEFAULT_LITELLM_API_BASE)
+        api_key = os.environ.get(ENV_LITELLM_API_KEY)
+        model = os.environ.get(ENV_EMBEDDINGS_LITELLM_MODEL, DEFAULT_EMBEDDINGS_LITELLM_MODEL)
+        return LiteLLMEmbeddings(api_base=api_base, api_key=api_key, model=model)
     else:
-        raise ValueError(f"Unknown embeddings provider: {provider}. Supported: 'local', 'tei', 'openai'")
+        raise ValueError(
+            f"Unknown embeddings provider: {provider}. Supported: 'local', 'tei', 'openai', 'cohere', 'litellm'"
+        )

hindsight-api/hindsight_api/engine/entity_resolver.py

@@ -209,7 +209,7 @@ class EntityResolver:
         # This handles duplicates via ON CONFLICT and returns all IDs
         if entities_to_create:
             # Group entities by canonical name (lowercase) to handle duplicates within batch
-            # For duplicates, we only insert once and reuse the ID
+            # For duplicates, we only insert once and reuse the ID, but track the count
             unique_entities = {}  # lowercase_name -> (entity_data, event_date, [indices])
             for idx, entity_data, event_date in entities_to_create:
                 name_lower = entity_data["text"].lower()
@@ -223,29 +223,32 @@ class EntityResolver:
             # Use a single query with unnest for speed
             entity_names = []
             entity_dates = []
+            entity_counts = []  # Track how many times each entity appears in this batch
             indices_map = []  # Maps result index -> list of original indices
 
             for name_lower, (entity_data, event_date, indices) in unique_entities.items():
                 entity_names.append(entity_data["text"])
                 entity_dates.append(event_date)
+                entity_counts.append(len(indices))  # Count of occurrences in this batch
                 indices_map.append(indices)
 
             # Batch INSERT ... ON CONFLICT with RETURNING
-            # This is much faster than individual inserts
+            # Uses the batch count for mention_count instead of always 1
             rows = await conn.fetch(
                 f"""
                 INSERT INTO {fq_table("entities")} (bank_id, canonical_name, first_seen, last_seen, mention_count)
-                SELECT $1, name, event_date, event_date, 1
-                FROM unnest($2::text[], $3::timestamptz[]) AS t(name, event_date)
+                SELECT $1, name, event_date, event_date, cnt
+                FROM unnest($2::text[], $3::timestamptz[], $4::int[]) AS t(name, event_date, cnt)
                 ON CONFLICT (bank_id, LOWER(canonical_name))
                 DO UPDATE SET
-                    mention_count = {fq_table("entities")}.mention_count + 1,
+                    mention_count = {fq_table("entities")}.mention_count + EXCLUDED.mention_count,
                     last_seen = EXCLUDED.last_seen
                 RETURNING id
                 """,
                 bank_id,
                 entity_names,
                 entity_dates,
+                entity_counts,
             )
 
             # Map returned IDs back to original indices

hindsight-api/hindsight_api/engine/interface.py

@@ -160,14 +160,14 @@ class MemoryEngineInterface(ABC):
         request_context: "RequestContext",
     ) -> dict[str, Any]:
         """
-        Get bank profile including disposition and background.
+        Get bank profile including disposition and mission.
 
         Args:
             bank_id: The memory bank ID.
             request_context: Request context for authentication.
 
         Returns:
-            Bank profile dict.
+            Bank profile dict with bank_id, name, disposition, and mission.
         """
         ...
 
@@ -190,25 +190,44 @@ class MemoryEngineInterface(ABC):
         ...
 
     @abstractmethod
-    async def merge_bank_background(
+    async def merge_bank_mission(
         self,
         bank_id: str,
         new_info: str,
         *,
-        update_disposition: bool = True,
         request_context: "RequestContext",
     ) -> dict[str, Any]:
         """
-        Merge new background information into bank profile.
+        Merge new mission information into bank profile.
 
         Args:
             bank_id: The memory bank ID.
-            new_info: New background information to merge.
-            update_disposition: Whether to infer disposition from background.
+            new_info: New mission information to merge.
             request_context: Request context for authentication.
 
         Returns:
-            Updated background info.
+            Updated mission info.
+        """
+        ...
+
+    @abstractmethod
+    async def set_bank_mission(
+        self,
+        bank_id: str,
+        mission: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        Set the bank's mission (replaces existing).
+
+        Args:
+            bank_id: The memory bank ID.
+            mission: The mission text.
+            request_context: Request context for authentication.
+
+        Returns:
+            Dict with bank_id and mission.
         """
         ...
 
@@ -289,6 +308,7 @@ class MemoryEngineInterface(ABC):
         bank_id: str,
         *,
         fact_type: str | None = None,
+        limit: int = 1000,
         request_context: "RequestContext",
     ) -> dict[str, Any]:
         """
@@ -297,10 +317,11 @@ class MemoryEngineInterface(ABC):
         Args:
             bank_id: The memory bank ID.
             fact_type: Filter by fact type.
+            limit: Maximum number of items to return (default: 1000).
             request_context: Request context for authentication.
 
         Returns:
-            Dict with nodes, edges, table_rows, total_units.
+            Dict with nodes, edges, table_rows, total_units, limit.
         """
         ...
 
@@ -404,18 +425,20 @@ class MemoryEngineInterface(ABC):
         bank_id: str,
         *,
         limit: int = 100,
+        offset: int = 0,
         request_context: "RequestContext",
-    ) -> list[dict[str, Any]]:
+    ) -> dict[str, Any]:
         """
-        List entities for a bank.
+        List entities for a bank with pagination.
 
         Args:
             bank_id: The memory bank ID.
             limit: Maximum results.
+            offset: Offset for pagination.
             request_context: Request context for authentication.
 
         Returns:
-            List of entity dicts.
+            Dict with items, total, limit, offset.
         """
         ...
 
@@ -514,7 +537,7 @@ class MemoryEngineInterface(ABC):
         bank_id: str,
         *,
         request_context: "RequestContext",
-    ) -> list[dict[str, Any]]:
+    ) -> dict[str, Any]:
         """
         List async operations for a bank.
 
@@ -523,7 +546,7 @@ class MemoryEngineInterface(ABC):
             request_context: Request context for authentication.
 
         Returns:
-            List of operation dicts with id, task_type, status, etc.
+            Dict with 'total' (int) and 'operations' (list of operation dicts).
         """
         ...
 
@@ -557,16 +580,16 @@ class MemoryEngineInterface(ABC):
         bank_id: str,
         *,
         name: str | None = None,
-        background: str | None = None,
+        mission: str | None = None,
         request_context: "RequestContext",
     ) -> dict[str, Any]:
         """
-        Update bank name and/or background.
+        Update bank name and/or mission.
 
         Args:
             bank_id: The memory bank ID.
             name: New bank name (optional).
-            background: New background text (optional, replaces existing).
+            mission: New mission text (optional, replaces existing).
             request_context: Request context for authentication.
 
         Returns:

hindsight-api/hindsight_api/engine/llm_wrapper.py

@@ -23,6 +23,8 @@ from ..config import (
     ENV_LLM_MAX_CONCURRENT,
     ENV_LLM_TIMEOUT,
 )
+from ..metrics import get_metrics_collector
+from .response_models import TokenUsage
 
 # Seed applied to every Groq request for deterministic behavior.
 DEFAULT_LLM_SEED = 4242
@@ -86,10 +88,14 @@ class LLMProvider:
         self.groq_service_tier = groq_service_tier or os.getenv(ENV_LLM_GROQ_SERVICE_TIER, "auto")
 
         # Validate provider
-        valid_providers = ["openai", "groq", "ollama", "gemini", "anthropic", "lmstudio"]
+        valid_providers = ["openai", "groq", "ollama", "gemini", "anthropic", "lmstudio", "mock"]
         if self.provider not in valid_providers:
             raise ValueError(f"Invalid LLM provider: {self.provider}. Must be one of: {', '.join(valid_providers)}")
 
+        # Mock provider tracking (for testing)
+        self._mock_calls: list[dict] = []
+        self._mock_response: Any = None
+
         # Set default base URLs
         if not self.base_url:
             if self.provider == "groq":
@@ -99,8 +105,8 @@ class LLMProvider:
             elif self.provider == "lmstudio":
                 self.base_url = "http://localhost:1234/v1"
 
-        # Validate API key (not needed for ollama or lmstudio)
-        if self.provider not in ("ollama", "lmstudio") and not self.api_key:
+        # Validate API key (not needed for ollama, lmstudio, or mock)
+        if self.provider not in ("ollama", "lmstudio", "mock") and not self.api_key:
             raise ValueError(f"API key not found for {self.provider}")
 
         # Get timeout config (set HINDSIGHT_API_LLM_TIMEOUT for local LLMs that need longer timeouts)
@@ -111,7 +117,10 @@ class LLMProvider:
         self._gemini_client = None
         self._anthropic_client = None
 
-        if self.provider == "gemini":
+        if self.provider == "mock":
+            # Mock provider - no client needed
+            pass
+        elif self.provider == "gemini":
             self._gemini_client = genai.Client(api_key=self.api_key)
         elif self.provider == "anthropic":
             from anthropic import AsyncAnthropic
@@ -174,6 +183,7 @@ class LLMProvider:
         max_backoff: float = 60.0,
         skip_validation: bool = False,
         strict_schema: bool = False,
+        return_usage: bool = False,
     ) -> Any:
         """
         Make an LLM API call with retry logic.
@@ -189,21 +199,43 @@ class LLMProvider:
             max_backoff: Maximum backoff time in seconds.
             skip_validation: Return raw JSON without Pydantic validation.
             strict_schema: Use strict JSON schema enforcement (OpenAI only). Guarantees all required fields.
+            return_usage: If True, return tuple (result, TokenUsage) instead of just result.
 
         Returns:
-            Parsed response if response_format is provided, otherwise text content.
+            If return_usage=False: Parsed response if response_format is provided, otherwise text content.
+            If return_usage=True: Tuple of (result, TokenUsage) with token counts from the LLM call.
 
         Raises:
             OutputTooLongError: If output exceeds token limits.
             Exception: Re-raises API errors after retries exhausted.
         """
+        semaphore_start = time.time()
         async with _global_llm_semaphore:
+            semaphore_wait_time = time.time() - semaphore_start
             start_time = time.time()
 
+            # Handle Mock provider (for testing)
+            if self.provider == "mock":
+                return await self._call_mock(
+                    messages,
+                    response_format,
+                    scope,
+                    return_usage,
+                )
+
             # Handle Gemini provider separately
             if self.provider == "gemini":
                 return await self._call_gemini(
-                    messages, response_format, max_retries, initial_backoff, max_backoff, skip_validation, start_time
+                    messages,
+                    response_format,
+                    max_retries,
+                    initial_backoff,
+                    max_backoff,
+                    skip_validation,
+                    start_time,
+                    scope,
+                    return_usage,
+                    semaphore_wait_time,
                 )
 
             # Handle Anthropic provider separately
@@ -217,6 +249,9 @@ class LLMProvider:
                     max_backoff,
                     skip_validation,
                     start_time,
+                    scope,
+                    return_usage,
+                    semaphore_wait_time,
                 )
 
             # Handle Ollama with native API for structured output (better schema enforcement)
@@ -231,6 +266,9 @@ class LLMProvider:
                     max_backoff,
                     skip_validation,
                     start_time,
+                    scope,
+                    return_usage,
+                    semaphore_wait_time,
                 )
 
             call_params = {
@@ -280,43 +318,44 @@ class LLMProvider:
 
             last_exception = None
 
+            # Prepare response format ONCE before the retry loop
+            # (to avoid appending schema to messages on every retry)
+            if response_format is not None:
+                schema = None
+                if hasattr(response_format, "model_json_schema"):
+                    schema = response_format.model_json_schema()
+
+                if strict_schema and schema is not None:
+                    # Use OpenAI's strict JSON schema enforcement
+                    # This guarantees all required fields are returned
+                    call_params["response_format"] = {
+                        "type": "json_schema",
+                        "json_schema": {
+                            "name": "response",
+                            "strict": True,
+                            "schema": schema,
+                        },
+                    }
+                else:
+                    # Soft enforcement: add schema to prompt and use json_object mode
+                    if schema is not None:
+                        schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}"
+
+                        if call_params["messages"] and call_params["messages"][0].get("role") == "system":
+                            call_params["messages"][0]["content"] += schema_msg
+                        elif call_params["messages"]:
+                            call_params["messages"][0]["content"] = (
+                                schema_msg + "\n\n" + call_params["messages"][0]["content"]
+                            )
+                    if self.provider not in ("lmstudio", "ollama"):
+                        # LM Studio and Ollama don't support json_object response format reliably
+                        # We rely on the schema in the system message instead
+                        call_params["response_format"] = {"type": "json_object"}
+
             for attempt in range(max_retries + 1):
                 try:
                     if response_format is not None:
-                        schema = None
-                        if hasattr(response_format, "model_json_schema"):
-                            schema = response_format.model_json_schema()
-
-                        if strict_schema and schema is not None:
-                            # Use OpenAI's strict JSON schema enforcement
-                            # This guarantees all required fields are returned
-                            call_params["response_format"] = {
-                                "type": "json_schema",
-                                "json_schema": {
-                                    "name": "response",
-                                    "strict": True,
-                                    "schema": schema,
-                                },
-                            }
-                        else:
-                            # Soft enforcement: add schema to prompt and use json_object mode
-                            if schema is not None:
-                                schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}"
-
-                                if call_params["messages"] and call_params["messages"][0].get("role") == "system":
-                                    call_params["messages"][0]["content"] += schema_msg
-                                elif call_params["messages"]:
-                                    call_params["messages"][0]["content"] = (
-                                        schema_msg + "\n\n" + call_params["messages"][0]["content"]
-                                    )
-                            if self.provider not in ("lmstudio", "ollama"):
-                                # LM Studio and Ollama don't support json_object response format reliably
-                                # We rely on the schema in the system message instead
-                                call_params["response_format"] = {"type": "json_object"}
-
-                        logger.debug(f"Sending request to {self.provider}/{self.model} (timeout={self.timeout})")
                         response = await self._client.chat.completions.create(**call_params)
-                        logger.debug(f"Received response from {self.provider}/{self.model}")
 
                         content = response.choices[0].message.content
 
@@ -379,21 +418,46 @@ class LLMProvider:
                         response = await self._client.chat.completions.create(**call_params)
                         result = response.choices[0].message.content
 
-                    # Log slow calls
+                    # Record token usage metrics
                     duration = time.time() - start_time
                     usage = response.usage
-                    if duration > 10.0:
-                        ratio = max(1, usage.completion_tokens) / usage.prompt_tokens
+                    input_tokens = usage.prompt_tokens or 0 if usage else 0
+                    output_tokens = usage.completion_tokens or 0 if usage else 0
+                    total_tokens = usage.total_tokens or 0 if usage else 0
+
+                    # Record LLM metrics
+                    metrics = get_metrics_collector()
+                    metrics.record_llm_call(
+                        provider=self.provider,
+                        model=self.model,
+                        scope=scope,
+                        duration=duration,
+                        input_tokens=input_tokens,
+                        output_tokens=output_tokens,
+                        success=True,
+                    )
+
+                    # Log slow calls
+                    if duration > 10.0 and usage:
+                        ratio = max(1, output_tokens) / max(1, input_tokens)
                         cached_tokens = 0
                         if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details:
                             cached_tokens = getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0
                         cache_info = f", cached_tokens={cached_tokens}" if cached_tokens > 0 else ""
+                        wait_info = f", wait={semaphore_wait_time:.3f}s" if semaphore_wait_time > 0.1 else ""
                         logger.info(
-                            f"slow llm call: model={self.provider}/{self.model}, "
-                            f"input_tokens={usage.prompt_tokens}, output_tokens={usage.completion_tokens}, "
-                            f"total_tokens={usage.total_tokens}{cache_info}, time={duration:.3f}s, ratio out/in={ratio:.2f}"
+                            f"slow llm call: scope={scope}, model={self.provider}/{self.model}, "
+                            f"input_tokens={input_tokens}, output_tokens={output_tokens}, "
+                            f"total_tokens={total_tokens}{cache_info}, time={duration:.3f}s{wait_info}, ratio out/in={ratio:.2f}"
                         )
 
+                    if return_usage:
+                        token_usage = TokenUsage(
+                            input_tokens=input_tokens,
+                            output_tokens=output_tokens,
+                            total_tokens=total_tokens,
+                        )
+                        return result, token_usage
                     return result
 
                 except LengthFinishReasonError as e:
@@ -404,13 +468,11 @@ class LLMProvider:
 
                 except APIConnectionError as e:
                     last_exception = e
+                    status_code = getattr(e, "status_code", None) or getattr(
+                        getattr(e, "response", None), "status_code", None
+                    )
+                    logger.warning(f"APIConnectionError (HTTP {status_code}), attempt {attempt + 1}: {str(e)[:200]}")
                     if attempt < max_retries:
-                        status_code = getattr(e, "status_code", None) or getattr(
-                            getattr(e, "response", None), "status_code", None
-                        )
-                        logger.warning(
-                            f"Connection error, retrying... (attempt {attempt + 1}/{max_retries + 1}) - status_code={status_code}, message={e}"
-                        )
                         backoff = min(initial_backoff * (2**attempt), max_backoff)
                         await asyncio.sleep(backoff)
                         continue
@@ -424,6 +486,45 @@ class LLMProvider:
                         logger.error(f"Auth error (HTTP {e.status_code}), not retrying: {str(e)}")
                         raise
 
+                    # Handle tool_use_failed error - model outputted in tool call format
+                    # Convert to expected JSON format and continue
+                    if e.status_code == 400 and response_format is not None:
+                        try:
+                            error_body = e.body if hasattr(e, "body") else {}
+                            if isinstance(error_body, dict):
+                                error_info: dict[str, Any] = error_body.get("error") or {}
+                                if error_info.get("code") == "tool_use_failed":
+                                    failed_gen = error_info.get("failed_generation", "")
+                                    if failed_gen:
+                                        # Parse the tool call format and convert to actions format
+                                        tool_call = json.loads(failed_gen)
+                                        tool_name = tool_call.get("name", "")
+                                        tool_args = tool_call.get("arguments", {})
+                                        # Convert to actions format: {"actions": [{"tool": "name", ...args}]}
+                                        converted = {"actions": [{"tool": tool_name, **tool_args}]}
+                                        if skip_validation:
+                                            result = converted
+                                        else:
+                                            result = response_format.model_validate(converted)
+
+                                        # Record metrics for this successful recovery
+                                        duration = time.time() - start_time
+                                        metrics = get_metrics_collector()
+                                        metrics.record_llm_call(
+                                            provider=self.provider,
+                                            model=self.model,
+                                            scope=scope,
+                                            duration=duration,
+                                            input_tokens=0,
+                                            output_tokens=0,
+                                            success=True,
+                                        )
+                                        if return_usage:
+                                            return result, TokenUsage(input_tokens=0, output_tokens=0, total_tokens=0)
+                                        return result
+                        except (json.JSONDecodeError, KeyError, TypeError):
+                            pass  # Failed to parse tool_use_failed, continue with normal retry
+
                     last_exception = e
                     if attempt < max_retries:
                         backoff = min(initial_backoff * (2**attempt), max_backoff)
@@ -434,14 +535,438 @@ class LLMProvider:
                         logger.error(f"API error after {max_retries + 1} attempts: {str(e)}")
                         raise
 
-                except Exception as e:
-                    logger.error(f"Unexpected error during LLM call: {type(e).__name__}: {str(e)}")
+                except Exception:
                     raise
 
             if last_exception:
                 raise last_exception
             raise RuntimeError("LLM call failed after all retries with no exception captured")
 
+    async def call_with_tools(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]],
+        max_completion_tokens: int | None = None,
+        temperature: float | None = None,
+        scope: str = "tools",
+        max_retries: int = 5,
+        initial_backoff: float = 1.0,
+        max_backoff: float = 30.0,
+        tool_choice: str | dict[str, Any] = "auto",
+    ) -> "LLMToolCallResult":
+        """
+        Make an LLM API call with tool/function calling support.
+
+        Args:
+            messages: List of message dicts. Can include tool results with role='tool'.
+            tools: List of tool definitions in OpenAI format.
+            max_completion_tokens: Maximum tokens in response.
+            temperature: Sampling temperature (0.0-2.0).
+            scope: Scope identifier for tracking.
+            max_retries: Maximum retry attempts.
+            initial_backoff: Initial backoff time in seconds.
+            max_backoff: Maximum backoff time in seconds.
+            tool_choice: How to choose tools - "auto", "none", "required", or {"type": "function", "function": {"name": "..."}}
+
+        Returns:
+            LLMToolCallResult with content and/or tool_calls.
+        """
+        from .response_models import LLMToolCall, LLMToolCallResult
+
+        async with _global_llm_semaphore:
+            start_time = time.time()
+
+            # Handle Mock provider
+            if self.provider == "mock":
+                return await self._call_with_tools_mock(messages, tools, scope)
+
+            # Handle Anthropic separately (uses different tool format)
+            if self.provider == "anthropic":
+                return await self._call_with_tools_anthropic(
+                    messages, tools, max_completion_tokens, max_retries, initial_backoff, max_backoff, start_time, scope
+                )
+
+            # Handle Gemini (convert to Gemini tool format)
+            if self.provider == "gemini":
+                return await self._call_with_tools_gemini(
+                    messages, tools, max_retries, initial_backoff, max_backoff, start_time, scope
+                )
+
+            # OpenAI-compatible providers (OpenAI, Groq, Ollama, LMStudio)
+            call_params: dict[str, Any] = {
+                "model": self.model,
+                "messages": messages,
+                "tools": tools,
+                "tool_choice": tool_choice,
+            }
+
+            if max_completion_tokens is not None:
+                call_params["max_completion_tokens"] = max_completion_tokens
+            if temperature is not None:
+                call_params["temperature"] = temperature
+
+            # Provider-specific parameters
+            if self.provider == "groq":
+                call_params["seed"] = DEFAULT_LLM_SEED
+
+            last_exception = None
+
+            for attempt in range(max_retries + 1):
+                try:
+                    response = await self._client.chat.completions.create(**call_params)
+
+                    message = response.choices[0].message
+                    finish_reason = response.choices[0].finish_reason
+
+                    # Extract tool calls if present
+                    tool_calls: list[LLMToolCall] = []
+                    if message.tool_calls:
+                        for tc in message.tool_calls:
+                            try:
+                                args = json.loads(tc.function.arguments) if tc.function.arguments else {}
+                            except json.JSONDecodeError:
+                                args = {"_raw": tc.function.arguments}
+                            tool_calls.append(LLMToolCall(id=tc.id, name=tc.function.name, arguments=args))
+
+                    content = message.content
+
+                    # Record metrics
+                    duration = time.time() - start_time
+                    usage = response.usage
+                    input_tokens = usage.prompt_tokens or 0 if usage else 0
+                    output_tokens = usage.completion_tokens or 0 if usage else 0
+
+                    metrics = get_metrics_collector()
+                    metrics.record_llm_call(
+                        provider=self.provider,
+                        model=self.model,
+                        scope=scope,
+                        duration=duration,
+                        input_tokens=input_tokens,
+                        output_tokens=output_tokens,
+                        success=True,
+                    )
+
+                    return LLMToolCallResult(
+                        content=content,
+                        tool_calls=tool_calls,
+                        finish_reason=finish_reason,
+                        input_tokens=input_tokens,
+                        output_tokens=output_tokens,
+                    )
+
+                except APIConnectionError as e:
+                    last_exception = e
+                    if attempt < max_retries:
+                        await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff))
+                        continue
+                    raise
+
+                except APIStatusError as e:
+                    if e.status_code in (401, 403):
+                        raise
+                    last_exception = e
+                    if attempt < max_retries:
+                        await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff))
+                        continue
+                    raise
+
+                except Exception:
+                    raise
+
+            if last_exception:
+                raise last_exception
+            raise RuntimeError("Tool call failed after all retries")
+
+    async def _call_with_tools_mock(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]],
+        scope: str,
+    ) -> "LLMToolCallResult":
+        """Handle mock tool calls for testing."""
+        from .response_models import LLMToolCallResult
+
+        call_record = {
+            "provider": self.provider,
+            "model": self.model,
+            "messages": messages,
+            "tools": [t.get("function", {}).get("name") for t in tools],
+            "scope": scope,
+        }
+        self._mock_calls.append(call_record)
+
+        if self._mock_response is not None:
+            if isinstance(self._mock_response, LLMToolCallResult):
+                return self._mock_response
+            # Allow setting just tool calls as a list
+            if isinstance(self._mock_response, list):
+                from .response_models import LLMToolCall
+
+                return LLMToolCallResult(
+                    tool_calls=[
+                        LLMToolCall(id=f"mock_{i}", name=tc["name"], arguments=tc.get("arguments", {}))
+                        for i, tc in enumerate(self._mock_response)
+                    ],
+                    finish_reason="tool_calls",
+                )
+
+        return LLMToolCallResult(content="mock response", finish_reason="stop")
+
+    async def _call_with_tools_anthropic(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]],
+        max_completion_tokens: int | None,
+        max_retries: int,
+        initial_backoff: float,
+        max_backoff: float,
+        start_time: float,
+        scope: str,
+    ) -> "LLMToolCallResult":
+        """Handle Anthropic tool calling."""
+        from anthropic import APIConnectionError, APIStatusError
+
+        from .response_models import LLMToolCall, LLMToolCallResult
+
+        # Convert OpenAI tool format to Anthropic format
+        anthropic_tools = []
+        for tool in tools:
+            func = tool.get("function", {})
+            anthropic_tools.append(
+                {
+                    "name": func.get("name", ""),
+                    "description": func.get("description", ""),
+                    "input_schema": func.get("parameters", {"type": "object", "properties": {}}),
+                }
+            )
+
+        # Convert messages - handle tool results
+        system_prompt = None
+        anthropic_messages = []
+        for msg in messages:
+            role = msg.get("role", "user")
+            content = msg.get("content", "")
+
+            if role == "system":
+                system_prompt = (system_prompt + "\n\n" + content) if system_prompt else content
+            elif role == "tool":
+                # Anthropic uses tool_result blocks
+                anthropic_messages.append(
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": content}
+                        ],
+                    }
+                )
+            elif role == "assistant" and msg.get("tool_calls"):
+                # Convert assistant tool calls
+                tool_use_blocks = []
+                for tc in msg["tool_calls"]:
+                    tool_use_blocks.append(
+                        {
+                            "type": "tool_use",
+                            "id": tc.get("id", ""),
+                            "name": tc.get("function", {}).get("name", ""),
+                            "input": json.loads(tc.get("function", {}).get("arguments", "{}")),
+                        }
+                    )
+                anthropic_messages.append({"role": "assistant", "content": tool_use_blocks})
+            else:
+                anthropic_messages.append({"role": role, "content": content})
+
+        call_params: dict[str, Any] = {
+            "model": self.model,
+            "messages": anthropic_messages,
+            "tools": anthropic_tools,
+            "max_tokens": max_completion_tokens or 4096,
+        }
+        if system_prompt:
+            call_params["system"] = system_prompt
+
+        last_exception = None
+        for attempt in range(max_retries + 1):
+            try:
+                response = await self._anthropic_client.messages.create(**call_params)
+
+                # Extract content and tool calls
+                content_parts = []
+                tool_calls: list[LLMToolCall] = []
+
+                for block in response.content:
+                    if block.type == "text":
+                        content_parts.append(block.text)
+                    elif block.type == "tool_use":
+                        tool_calls.append(LLMToolCall(id=block.id, name=block.name, arguments=block.input or {}))
+
+                content = "".join(content_parts) if content_parts else None
+                finish_reason = "tool_calls" if tool_calls else "stop"
+
+                # Extract token usage
+                input_tokens = response.usage.input_tokens or 0
+                output_tokens = response.usage.output_tokens or 0
+
+                # Record metrics
+                metrics = get_metrics_collector()
+                metrics.record_llm_call(
+                    provider=self.provider,
+                    model=self.model,
+                    scope=scope,
+                    duration=time.time() - start_time,
+                    input_tokens=input_tokens,
+                    output_tokens=output_tokens,
+                    success=True,
+                )
+
+                return LLMToolCallResult(
+                    content=content,
+                    tool_calls=tool_calls,
+                    finish_reason=finish_reason,
+                    input_tokens=input_tokens,
+                    output_tokens=output_tokens,
+                )
+
+            except (APIConnectionError, APIStatusError) as e:
+                if isinstance(e, APIStatusError) and e.status_code in (401, 403):
+                    raise
+                last_exception = e
+                if attempt < max_retries:
+                    await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff))
+                    continue
+                raise
+
+        if last_exception:
+            raise last_exception
+        raise RuntimeError("Anthropic tool call failed")
+
+    async def _call_with_tools_gemini(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]],
+        max_retries: int,
+        initial_backoff: float,
+        max_backoff: float,
+        start_time: float,
+        scope: str,
+    ) -> "LLMToolCallResult":
+        """Handle Gemini tool calling."""
+        from .response_models import LLMToolCall, LLMToolCallResult
+
+        # Convert tools to Gemini format
+        gemini_tools = []
+        for tool in tools:
+            func = tool.get("function", {})
+            gemini_tools.append(
+                genai_types.Tool(
+                    function_declarations=[
+                        genai_types.FunctionDeclaration(
+                            name=func.get("name", ""),
+                            description=func.get("description", ""),
+                            parameters=func.get("parameters"),
+                        )
+                    ]
+                )
+            )
+
+        # Convert messages
+        system_instruction = None
+        gemini_contents = []
+        for msg in messages:
+            role = msg.get("role", "user")
+            content = msg.get("content", "")
+
+            if role == "system":
+                system_instruction = (system_instruction + "\n\n" + content) if system_instruction else content
+            elif role == "tool":
+                # Gemini uses function_response
+                gemini_contents.append(
+                    genai_types.Content(
+                        role="user",
+                        parts=[
+                            genai_types.Part(
+                                function_response=genai_types.FunctionResponse(
+                                    name=msg.get("name", ""),
+                                    response={"result": content},
+                                )
+                            )
+                        ],
+                    )
+                )
+            elif role == "assistant":
+                gemini_contents.append(genai_types.Content(role="model", parts=[genai_types.Part(text=content)]))
+            else:
+                gemini_contents.append(genai_types.Content(role="user", parts=[genai_types.Part(text=content)]))
+
+        config = genai_types.GenerateContentConfig(
+            system_instruction=system_instruction,
+            tools=gemini_tools,
+        )
+
+        last_exception = None
+        for attempt in range(max_retries + 1):
+            try:
+                response = await self._gemini_client.aio.models.generate_content(
+                    model=self.model,
+                    contents=gemini_contents,
+                    config=config,
+                )
+
+                # Extract content and tool calls
+                content = None
+                tool_calls: list[LLMToolCall] = []
+
+                if response.candidates and response.candidates[0].content:
+                    for part in response.candidates[0].content.parts:
+                        if hasattr(part, "text") and part.text:
+                            content = part.text
+                        if hasattr(part, "function_call") and part.function_call:
+                            fc = part.function_call
+                            tool_calls.append(
+                                LLMToolCall(
+                                    id=f"gemini_{len(tool_calls)}",
+                                    name=fc.name,
+                                    arguments=dict(fc.args) if fc.args else {},
+                                )
+                            )
+
+                finish_reason = "tool_calls" if tool_calls else "stop"
+
+                # Record metrics
+                metrics = get_metrics_collector()
+                input_tokens = response.usage_metadata.prompt_token_count if response.usage_metadata else 0
+                output_tokens = response.usage_metadata.candidates_token_count if response.usage_metadata else 0
+                metrics.record_llm_call(
+                    provider=self.provider,
+                    model=self.model,
+                    scope=scope,
+                    duration=time.time() - start_time,
+                    input_tokens=input_tokens,
+                    output_tokens=output_tokens,
+                    success=True,
+                )
+
+                return LLMToolCallResult(
+                    content=content,
+                    tool_calls=tool_calls,
+                    finish_reason=finish_reason,
+                    input_tokens=input_tokens,
+                    output_tokens=output_tokens,
+                )
+
+            except genai_errors.APIError as e:
+                if e.code in (401, 403):
+                    raise
+                last_exception = e
+                if attempt < max_retries:
+                    await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff))
+                    continue
+                raise
+
+        if last_exception:
+            raise last_exception
+        raise RuntimeError("Gemini tool call failed")
+
     async def _call_anthropic(
         self,
         messages: list[dict[str, str]],
@@ -452,6 +977,9 @@ class LLMProvider:
         max_backoff: float,
         skip_validation: bool,
         start_time: float,
+        scope: str = "memory",
+        return_usage: bool = False,
+        semaphore_wait_time: float = 0.0,
     ) -> Any:
         """Handle Anthropic-specific API calls."""
         from anthropic import APIConnectionError, APIStatusError, RateLimitError
@@ -524,17 +1052,40 @@ class LLMProvider:
                 else:
                     result = content
 
-                # Log slow calls
+                # Record metrics and log slow calls
                 duration = time.time() - start_time
+                input_tokens = response.usage.input_tokens or 0 if response.usage else 0
+                output_tokens = response.usage.output_tokens or 0 if response.usage else 0
+                total_tokens = input_tokens + output_tokens
+
+                # Record LLM metrics
+                metrics = get_metrics_collector()
+                metrics.record_llm_call(
+                    provider=self.provider,
+                    model=self.model,
+                    scope=scope,
+                    duration=duration,
+                    input_tokens=input_tokens,
+                    output_tokens=output_tokens,
+                    success=True,
+                )
+
+                # Log slow calls
                 if duration > 10.0:
-                    input_tokens = response.usage.input_tokens
-                    output_tokens = response.usage.output_tokens
+                    wait_info = f", wait={semaphore_wait_time:.3f}s" if semaphore_wait_time > 0.1 else ""
                     logger.info(
-                        f"slow llm call: model={self.provider}/{self.model}, "
+                        f"slow llm call: scope={scope}, model={self.provider}/{self.model}, "
                         f"input_tokens={input_tokens}, output_tokens={output_tokens}, "
-                        f"time={duration:.3f}s"
+                        f"time={duration:.3f}s{wait_info}"
                     )
 
+                if return_usage:
+                    token_usage = TokenUsage(
+                        input_tokens=input_tokens,
+                        output_tokens=output_tokens,
+                        total_tokens=total_tokens,
+                    )
+                    return result, token_usage
                 return result
 
             except json.JSONDecodeError as e:
@@ -589,6 +1140,9 @@ class LLMProvider:
         max_backoff: float,
         skip_validation: bool,
         start_time: float,
+        scope: str = "memory",
+        return_usage: bool = False,
+        semaphore_wait_time: float = 0.0,
     ) -> Any:
         """
         Call Ollama using native API with JSON schema enforcement.
@@ -663,11 +1217,39 @@ class LLMProvider:
                         else:
                             raise
 
+                    # Extract token usage from Ollama response
+                    # Ollama returns prompt_eval_count (input) and eval_count (output)
+                    duration = time.time() - start_time
+                    input_tokens = result.get("prompt_eval_count", 0) or 0
+                    output_tokens = result.get("eval_count", 0) or 0
+                    total_tokens = input_tokens + output_tokens
+
+                    # Record LLM metrics
+                    metrics = get_metrics_collector()
+                    metrics.record_llm_call(
+                        provider=self.provider,
+                        model=self.model,
+                        scope=scope,
+                        duration=duration,
+                        input_tokens=input_tokens,
+                        output_tokens=output_tokens,
+                        success=True,
+                    )
+
                     # Validate against Pydantic model or return raw JSON
                     if skip_validation:
-                        return json_data
+                        validated_result = json_data
                     else:
-                        return response_format.model_validate(json_data)
+                        validated_result = response_format.model_validate(json_data)
+
+                    if return_usage:
+                        token_usage = TokenUsage(
+                            input_tokens=input_tokens,
+                            output_tokens=output_tokens,
+                            total_tokens=total_tokens,
+                        )
+                        return validated_result, token_usage
+                    return validated_result
 
                 except httpx.HTTPStatusError as e:
                     last_exception = e
@@ -710,6 +1292,9 @@ class LLMProvider:
         max_backoff: float,
         skip_validation: bool,
         start_time: float,
+        scope: str = "memory",
+        return_usage: bool = False,
+        semaphore_wait_time: float = 0.0,
     ) -> Any:
         """Handle Gemini-specific API calls."""
         # Convert OpenAI-style messages to Gemini format
@@ -786,16 +1371,43 @@ class LLMProvider:
                 else:
                     result = content
 
-                # Log slow calls
+                # Record metrics and log slow calls
                 duration = time.time() - start_time
-                if duration > 10.0 and hasattr(response, "usage_metadata") and response.usage_metadata:
+                input_tokens = 0
+                output_tokens = 0
+                if hasattr(response, "usage_metadata") and response.usage_metadata:
                     usage = response.usage_metadata
+                    input_tokens = usage.prompt_token_count or 0
+                    output_tokens = usage.candidates_token_count or 0
+
+                # Record LLM metrics
+                metrics = get_metrics_collector()
+                metrics.record_llm_call(
+                    provider=self.provider,
+                    model=self.model,
+                    scope=scope,
+                    duration=duration,
+                    input_tokens=input_tokens,
+                    output_tokens=output_tokens,
+                    success=True,
+                )
+
+                # Log slow calls
+                if duration > 10.0 and input_tokens > 0:
+                    wait_info = f", wait={semaphore_wait_time:.3f}s" if semaphore_wait_time > 0.1 else ""
                     logger.info(
-                        f"slow llm call: model={self.provider}/{self.model}, "
-                        f"input_tokens={usage.prompt_token_count}, output_tokens={usage.candidates_token_count}, "
-                        f"time={duration:.3f}s"
+                        f"slow llm call: scope={scope}, model={self.provider}/{self.model}, "
+                        f"input_tokens={input_tokens}, output_tokens={output_tokens}, "
+                        f"time={duration:.3f}s{wait_info}"
                     )
 
+                if return_usage:
+                    token_usage = TokenUsage(
+                        input_tokens=input_tokens,
+                        output_tokens=output_tokens,
+                        total_tokens=input_tokens + output_tokens,
+                    )
+                    return result, token_usage
                 return result
 
             except json.JSONDecodeError as e:
@@ -837,6 +1449,61 @@ class LLMProvider:
             raise last_exception
         raise RuntimeError("Gemini call failed after all retries")
 
+    async def _call_mock(
+        self,
+        messages: list[dict[str, str]],
+        response_format: Any | None,
+        scope: str,
+        return_usage: bool,
+    ) -> Any:
+        """
+        Handle mock provider calls for testing.
+
+        Records the call and returns a configurable mock response.
+        """
+        # Record the call for test verification
+        call_record = {
+            "provider": self.provider,
+            "model": self.model,
+            "messages": messages,
+            "response_format": response_format.__name__
+            if response_format and hasattr(response_format, "__name__")
+            else str(response_format),
+            "scope": scope,
+        }
+        self._mock_calls.append(call_record)
+        logger.debug(f"Mock LLM call recorded: scope={scope}, model={self.model}")
+
+        # Return mock response
+        if self._mock_response is not None:
+            result = self._mock_response
+        elif response_format is not None:
+            # Try to create a minimal valid instance of the response format
+            try:
+                # For Pydantic models, try to create with minimal valid data
+                result = {"mock": True}
+            except Exception:
+                result = {"mock": True}
+        else:
+            result = "mock response"
+
+        if return_usage:
+            token_usage = TokenUsage(input_tokens=10, output_tokens=5, total_tokens=15)
+            return result, token_usage
+        return result
+
+    def set_mock_response(self, response: Any) -> None:
+        """Set the response to return from mock calls."""
+        self._mock_response = response
+
+    def get_mock_calls(self) -> list[dict]:
+        """Get the list of recorded mock calls."""
+        return self._mock_calls
+
+    def clear_mock_calls(self) -> None:
+        """Clear the recorded mock calls."""
+        self._mock_calls = []
+
     @classmethod
     def for_memory(cls) -> "LLMProvider":
         """Create provider for memory operations from environment variables."""

hindsight-api/hindsight_api/engine/mental_models/__init__.py

@@ -0,0 +1,14 @@
+"""
+Mental models module for Hindsight.
+
+Mental models contain directives - hard rules that are injected into reflect prompts.
+Directives are user-defined and their observations are user-provided (not LLM-generated).
+
+Other types of consolidated knowledge are handled by:
+- Learnings: Automatic bottom-up consolidation from facts
+- Pinned Reflections: User-curated living documents
+"""
+
+from .models import MentalModel, MentalModelSubtype
+
+__all__ = ["MentalModel", "MentalModelSubtype"]

hindsight-api/hindsight_api/engine/mental_models/models.py

@@ -0,0 +1,53 @@
+"""
+Pydantic models for mental models.
+"""
+
+from datetime import datetime, timezone
+from enum import Enum
+
+from pydantic import BaseModel, Field
+
+
+class MentalModelSubtype(str, Enum):
+    """Subtype of mental model.
+
+    Currently only DIRECTIVE is supported. Other types of consolidated knowledge
+    are handled by:
+    - Learnings: Automatic bottom-up consolidation from facts
+    - Pinned Reflections: User-curated living documents
+    """
+
+    DIRECTIVE = "directive"  # User-defined hard rules, observations user-provided
+
+
+class MentalModel(BaseModel):
+    """
+    A mental model representing synthesized understanding.
+
+    Mental models are the agent's consolidated knowledge. Unlike raw facts,
+    mental models provide:
+    - A one-liner description for quick scanning/retrieval
+    - A full summary for deep understanding
+    - Links to related mental models
+    """
+
+    id: str = Field(description="Unique identifier within the bank")
+    bank_id: str = Field(description="Bank this mental model belongs to")
+    subtype: MentalModelSubtype = Field(description="How this model was created")
+    name: str = Field(description="Human-readable name")
+    description: str = Field(description="One-liner for quick scanning and retrieval matching")
+    summary: str | None = Field(default=None, description="Full synthesized understanding")
+
+    # References
+    entity_id: str | None = Field(default=None, description="Reference to entities table when type=entity")
+    source_facts: list[str] = Field(default_factory=list, description="Fact IDs used to generate summary")
+    links: list[str] = Field(default_factory=list, description="Related mental model IDs")
+
+    # Tags for scoped visibility (similar to document tags)
+    tags: list[str] = Field(default_factory=list, description="Tags for scoped visibility filtering")
+
+    # Timestamps
+    last_updated: datetime | None = Field(default=None, description="When summary was last regenerated")
+    created_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc), description="When this model was created"
+    )

hindsight-api/hindsight_api/engine/query_analyzer.py

@@ -84,7 +84,7 @@ class DateparserQueryAnalyzer(QueryAnalyzer):
 
     Performance:
     - ~10-50ms per query
-    - No model loading required
+    - No model loading required (lazy import on first use)
     """
 
     def __init__(self):
@@ -112,8 +112,6 @@ class DateparserQueryAnalyzer(QueryAnalyzer):
         Returns:
             QueryAnalysis with temporal_constraint if found
         """
-        self.load()
-
         if reference_date is None:
             reference_date = datetime.now()
 
@@ -123,6 +121,9 @@ class DateparserQueryAnalyzer(QueryAnalyzer):
         if period_result is not None:
             return QueryAnalysis(temporal_constraint=period_result)
 
+        # Lazy load dateparser (only imports on first call, then cached)
+        self.load()
+
         # Use dateparser's search_dates to find temporal expressions
         settings = {
             "RELATIVE_BASE": reference_date,

hindsight-api/hindsight_api/engine/reflect/__init__.py

@@ -0,0 +1,18 @@
+"""
+Reflect agent module for agentic reflection with tools.
+
+The reflect agent uses an iterative loop with tools to:
+1. Lookup mental models (existing knowledge)
+2. Recall facts (semantic + temporal search)
+3. Expand memories (get chunk/document context)
+"""
+
+from .agent import ReflectAgentResult, run_reflect_agent
+from .models import ReflectAction, ReflectActionBatch
+
+__all__ = [
+    "run_reflect_agent",
+    "ReflectAgentResult",
+    "ReflectAction",
+    "ReflectActionBatch",
+]

hindsight-api/hindsight_api/engine/reflect/agent.py

@@ -0,0 +1,927 @@
+"""
+Reflect agent - agentic loop for reflection with native tool calling.
+
+Uses hierarchical retrieval:
+1. search_mental_models - User-curated summaries (highest quality)
+2. search_observations - Consolidated knowledge with freshness
+3. recall - Raw facts as ground truth
+"""
+
+import asyncio
+import json
+import logging
+import re
+import time
+from typing import TYPE_CHECKING, Any, Awaitable, Callable
+
+from .models import DirectiveInfo, LLMCall, ReflectAgentResult, TokenUsageSummary, ToolCall
+from .prompts import FINAL_SYSTEM_PROMPT, _extract_directive_rules, build_final_prompt, build_system_prompt_for_tools
+from .tools_schema import get_reflect_tools
+
+
+def _build_directives_applied(directives: list[dict[str, Any]] | None) -> list[DirectiveInfo]:
+    """Build list of DirectiveInfo from directive mental models.
+
+    Handles multiple directive formats:
+    1. New format: directives have direct 'content' field
+    2. Fallback: directives have 'description' field
+    """
+    if not directives:
+        return []
+
+    result = []
+    for directive in directives:
+        directive_id = directive.get("id", "")
+        directive_name = directive.get("name", "")
+
+        # Get content from 'content' field or fallback to 'description'
+        content = directive.get("content", "") or directive.get("description", "")
+
+        result.append(DirectiveInfo(id=directive_id, name=directive_name, content=content))
+
+    return result
+
+
+if TYPE_CHECKING:
+    from ..llm_wrapper import LLMProvider
+    from ..response_models import LLMToolCall
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_MAX_ITERATIONS = 10
+
+
+def _normalize_tool_name(name: str) -> str:
+    """Normalize tool name from various LLM output formats.
+
+    Some LLMs output tool names in non-standard formats:
+    - 'functions.done' (OpenAI-style prefix)
+    - 'call=functions.done' (some models)
+    - 'call=done' (some models)
+
+    Returns the normalized tool name (e.g., 'done', 'recall', etc.)
+    """
+    # Handle 'call=functions.name' or 'call=name' format
+    if name.startswith("call="):
+        name = name[len("call=") :]
+
+    # Handle 'functions.name' format
+    if name.startswith("functions."):
+        name = name[len("functions.") :]
+
+    return name
+
+
+def _is_done_tool(name: str) -> bool:
+    """Check if the tool name represents the 'done' tool."""
+    return _normalize_tool_name(name) == "done"
+
+
+# Pattern to match done() call as text - handles done({...}) with nested JSON
+_DONE_CALL_PATTERN = re.compile(r"done\s*\(\s*\{.*$", re.DOTALL)
+
+# Patterns for leaked structured output in the answer field
+_LEAKED_JSON_SUFFIX = re.compile(
+    r'\s*```(?:json)?\s*\{[^}]*(?:"(?:observation_ids|memory_ids|mental_model_ids)"|\})\s*```\s*$',
+    re.DOTALL | re.IGNORECASE,
+)
+_LEAKED_JSON_OBJECT = re.compile(
+    r'\s*\{[^{]*"(?:observation_ids|memory_ids|mental_model_ids|answer)"[^}]*\}\s*$', re.DOTALL
+)
+_TRAILING_IDS_PATTERN = re.compile(
+    r"\s*(?:observation_ids|memory_ids|mental_model_ids)\s*[=:]\s*\[.*?\]\s*$", re.DOTALL | re.IGNORECASE
+)
+
+
+def _clean_answer_text(text: str) -> str:
+    """Clean up answer text by removing any done() tool call syntax.
+
+    Some LLMs output the done() call as text instead of a proper tool call.
+    This strips out patterns like: done({"answer": "...", ...})
+    """
+    # Remove done() call pattern from the end of the text
+    cleaned = _DONE_CALL_PATTERN.sub("", text).strip()
+    return cleaned if cleaned else text
+
+
+def _clean_done_answer(text: str) -> str:
+    """Clean up the answer field from a done() tool call.
+
+    Some LLMs leak structured output patterns into the answer text, such as:
+    - JSON code blocks with observation_ids/memory_ids at the end
+    - Raw JSON objects with these fields
+    - Plain text like "observation_ids: [...]"
+
+    This cleans those patterns while preserving the actual answer content.
+    """
+    if not text:
+        return text
+
+    cleaned = text
+
+    # Remove leaked JSON in code blocks at the end
+    cleaned = _LEAKED_JSON_SUFFIX.sub("", cleaned).strip()
+
+    # Remove leaked raw JSON objects at the end
+    cleaned = _LEAKED_JSON_OBJECT.sub("", cleaned).strip()
+
+    # Remove trailing ID patterns
+    cleaned = _TRAILING_IDS_PATTERN.sub("", cleaned).strip()
+
+    return cleaned if cleaned else text
+
+
+async def _generate_structured_output(
+    answer: str,
+    response_schema: dict,
+    llm_config: "LLMProvider",
+    reflect_id: str,
+) -> tuple[dict[str, Any] | None, int, int]:
+    """Generate structured output from an answer using the provided JSON schema.
+
+    Args:
+        answer: The text answer to extract structured data from
+        response_schema: JSON Schema for the expected output structure
+        llm_config: LLM provider for making the extraction call
+        reflect_id: Reflect ID for logging
+
+    Returns:
+        Tuple of (structured_output, input_tokens, output_tokens).
+        structured_output is None if generation fails.
+    """
+    try:
+        from typing import Any as TypingAny
+
+        from pydantic import create_model
+
+        def _json_schema_type_to_python(field_schema: dict) -> type:
+            """Map JSON schema type to Python type for better LLM guidance."""
+            json_type = field_schema.get("type", "string")
+            if json_type == "array":
+                return list
+            elif json_type == "object":
+                return dict
+            elif json_type == "integer":
+                return int
+            elif json_type == "number":
+                return float
+            elif json_type == "boolean":
+                return bool
+            else:
+                return str
+
+        # Build fields from JSON schema properties
+        schema_props = response_schema.get("properties", {})
+        required_fields = set(response_schema.get("required", []))
+        fields: dict[str, TypingAny] = {}
+        for field_name, field_schema in schema_props.items():
+            field_type = _json_schema_type_to_python(field_schema)
+            default = ... if field_name in required_fields else None
+            fields[field_name] = (field_type, default)
+
+        if not fields:
+            logger.warning(f"[REFLECT {reflect_id}] No fields found in response_schema, skipping structured output")
+            return None, 0, 0
+
+        DynamicModel = create_model("StructuredResponse", **fields)
+
+        # Include the full schema in the prompt for better LLM guidance
+        schema_str = json.dumps(response_schema, indent=2)
+
+        # Build field descriptions for the prompt
+        field_descriptions = []
+        for field_name, field_schema in schema_props.items():
+            field_type = field_schema.get("type", "string")
+            field_desc = field_schema.get("description", "")
+            is_required = field_name in required_fields
+            req_marker = " (REQUIRED)" if is_required else " (optional)"
+            field_descriptions.append(f"- {field_name} ({field_type}){req_marker}: {field_desc}")
+        fields_text = "\n".join(field_descriptions)
+
+        # Call LLM with the answer to extract structured data
+        structured_prompt = f"""Your task is to extract specific information from the answer below and format it as JSON.
+
+ANSWER TO EXTRACT FROM:
+\"\"\"
+{answer}
+\"\"\"
+
+REQUIRED OUTPUT FORMAT - Extract the following fields from the answer above:
+{fields_text}
+
+JSON Schema:
+```json
+{schema_str}
+```
+
+INSTRUCTIONS:
+1. Read the answer carefully and identify the information that matches each field
+2. Extract the ACTUAL content from the answer - do NOT leave fields empty if information is present
+3. For string fields: use the exact text or a clear summary from the answer
+4. For array fields: return a JSON array (e.g., ["item1", "item2"]), NOT a string
+5. For required fields: you MUST provide a value extracted from the answer
+6. Return ONLY the JSON object, no explanation
+
+OUTPUT:"""
+
+        structured_result, usage = await llm_config.call(
+            messages=[
+                {
+                    "role": "system",
+                    "content": "You are a precise data extraction assistant. Extract information from text and return it as valid JSON matching the provided schema. Always extract actual content - never return empty strings for required fields if information is available.",
+                },
+                {"role": "user", "content": structured_prompt},
+            ],
+            response_format=DynamicModel,
+            scope="reflect_structured",
+            skip_validation=True,  # We'll handle the dict ourselves
+            return_usage=True,
+        )
+
+        # Convert to dict
+        if hasattr(structured_result, "model_dump"):
+            structured_output = structured_result.model_dump()
+        elif isinstance(structured_result, dict):
+            structured_output = structured_result
+        else:
+            # Try to parse as JSON
+            structured_output = json.loads(str(structured_result))
+
+        # Validate that required fields have non-empty values
+        for field_name in required_fields:
+            value = structured_output.get(field_name)
+            if value is None or value == "" or value == []:
+                logger.warning(f"[REFLECT {reflect_id}] Required field '{field_name}' is empty in structured output")
+
+        logger.info(f"[REFLECT {reflect_id}] Generated structured output with {len(structured_output)} fields")
+        return structured_output, usage.input_tokens, usage.output_tokens
+
+    except Exception as e:
+        logger.warning(f"[REFLECT {reflect_id}] Failed to generate structured output: {e}")
+        return None, 0, 0
+
+
+async def run_reflect_agent(
+    llm_config: "LLMProvider",
+    bank_id: str,
+    query: str,
+    bank_profile: dict[str, Any],
+    search_mental_models_fn: Callable[[str, int], Awaitable[dict[str, Any]]],
+    search_observations_fn: Callable[[str, int], Awaitable[dict[str, Any]]],
+    recall_fn: Callable[[str, int], Awaitable[dict[str, Any]]],
+    expand_fn: Callable[[list[str], str], Awaitable[dict[str, Any]]],
+    context: str | None = None,
+    max_iterations: int = DEFAULT_MAX_ITERATIONS,
+    max_tokens: int | None = None,
+    response_schema: dict | None = None,
+    directives: list[dict[str, Any]] | None = None,
+    has_mental_models: bool = False,
+    budget: str | None = None,
+) -> ReflectAgentResult:
+    """
+    Execute the reflect agent loop using native tool calling.
+
+    The agent uses hierarchical retrieval:
+    1. search_mental_models - User-curated summaries (try first)
+    2. search_observations - Consolidated knowledge with freshness
+    3. recall - Raw facts as ground truth
+
+    Args:
+        llm_config: LLM provider for agent calls
+        bank_id: Bank identifier
+        query: Question to answer
+        bank_profile: Bank profile with name and mission
+        search_mental_models_fn: Tool callback for searching mental models (query, max_results) -> result
+        search_observations_fn: Tool callback for searching observations (query, max_results) -> result
+        recall_fn: Tool callback for recall (query, max_tokens) -> result
+        expand_fn: Tool callback for expand (memory_ids, depth) -> result
+        context: Optional additional context
+        max_iterations: Maximum number of iterations before forcing response
+        max_tokens: Maximum tokens for the final response
+        response_schema: Optional JSON Schema for structured output in final response
+        directives: Optional list of directive mental models to inject as hard rules
+
+    Returns:
+        ReflectAgentResult with final answer and metadata
+    """
+    reflect_id = f"{bank_id[:8]}-{int(time.time() * 1000) % 100000}"
+    start_time = time.time()
+
+    # Build directives_applied for the trace
+    directives_applied = _build_directives_applied(directives)
+
+    # Extract directive rules for tool schema (if any)
+    directive_rules = _extract_directive_rules(directives) if directives else None
+
+    # Get tools for this agent (with directive compliance field if directives exist)
+    tools = get_reflect_tools(directive_rules=directive_rules)
+
+    # Build initial messages (directives are injected into system prompt at START and END)
+    system_prompt = build_system_prompt_for_tools(
+        bank_profile, context, directives=directives, has_mental_models=has_mental_models, budget=budget
+    )
+    messages: list[dict[str, Any]] = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": query},
+    ]
+
+    # Tracking
+    total_tools_called = 0
+    tool_trace: list[ToolCall] = []
+    tool_trace_summary: list[dict[str, Any]] = []
+    llm_trace: list[dict[str, Any]] = []
+    context_history: list[dict[str, Any]] = []  # For final prompt fallback
+
+    # Token usage tracking - accumulate across all LLM calls
+    total_input_tokens = 0
+    total_output_tokens = 0
+
+    # Track available IDs for validation (prevents hallucinated citations)
+    available_memory_ids: set[str] = set()
+    available_mental_model_ids: set[str] = set()
+    available_observation_ids: set[str] = set()
+
+    def _get_llm_trace() -> list[LLMCall]:
+        return [
+            LLMCall(
+                scope=c["scope"],
+                duration_ms=c["duration_ms"],
+                input_tokens=c.get("input_tokens", 0),
+                output_tokens=c.get("output_tokens", 0),
+            )
+            for c in llm_trace
+        ]
+
+    def _get_usage() -> TokenUsageSummary:
+        return TokenUsageSummary(
+            input_tokens=total_input_tokens,
+            output_tokens=total_output_tokens,
+            total_tokens=total_input_tokens + total_output_tokens,
+        )
+
+    def _log_completion(answer: str, iterations: int, forced: bool = False):
+        elapsed_ms = int((time.time() - start_time) * 1000)
+        tools_summary = (
+            ", ".join(
+                f"{t['tool']}({t['input_summary']})={t['duration_ms']}ms/{t.get('output_chars', 0)}c"
+                for t in tool_trace_summary
+            )
+            or "none"
+        )
+        llm_summary = ", ".join(f"{c['scope']}={c['duration_ms']}ms" for c in llm_trace) or "none"
+        total_llm_ms = sum(c["duration_ms"] for c in llm_trace)
+        total_tools_ms = sum(t["duration_ms"] for t in tool_trace_summary)
+
+        answer_preview = answer[:100] + "..." if len(answer) > 100 else answer
+        mode = "forced" if forced else "done"
+        logger.info(
+            f"[REFLECT {reflect_id}] {mode} | "
+            f"query='{query[:50]}...' | "
+            f"iterations={iterations} | "
+            f"llm=[{llm_summary}] ({total_llm_ms}ms) | "
+            f"tools=[{tools_summary}] ({total_tools_ms}ms) | "
+            f"answer='{answer_preview}' | "
+            f"total={elapsed_ms}ms"
+        )
+
+    for iteration in range(max_iterations):
+        is_last = iteration == max_iterations - 1
+
+        if is_last:
+            # Force text response on last iteration - no tools
+            prompt = build_final_prompt(query, context_history, bank_profile, context)
+            llm_start = time.time()
+            response, usage = await llm_config.call(
+                messages=[
+                    {"role": "system", "content": FINAL_SYSTEM_PROMPT},
+                    {"role": "user", "content": prompt},
+                ],
+                scope="reflect_agent_final",
+                max_completion_tokens=max_tokens,
+                return_usage=True,
+            )
+            llm_duration = int((time.time() - llm_start) * 1000)
+            total_input_tokens += usage.input_tokens
+            total_output_tokens += usage.output_tokens
+            llm_trace.append(
+                {
+                    "scope": "final",
+                    "duration_ms": llm_duration,
+                    "input_tokens": usage.input_tokens,
+                    "output_tokens": usage.output_tokens,
+                }
+            )
+            answer = _clean_answer_text(response.strip())
+
+            # Generate structured output if schema provided
+            structured_output = None
+            if response_schema and answer:
+                structured_output, struct_in, struct_out = await _generate_structured_output(
+                    answer, response_schema, llm_config, reflect_id
+                )
+                total_input_tokens += struct_in
+                total_output_tokens += struct_out
+
+            _log_completion(answer, iteration + 1, forced=True)
+            return ReflectAgentResult(
+                text=answer,
+                structured_output=structured_output,
+                iterations=iteration + 1,
+                tools_called=total_tools_called,
+                tool_trace=tool_trace,
+                llm_trace=_get_llm_trace(),
+                usage=_get_usage(),
+                directives_applied=directives_applied,
+            )
+
+        # Call LLM with tools
+        llm_start = time.time()
+
+        try:
+            result = await llm_config.call_with_tools(
+                messages=messages,
+                tools=tools,
+                scope="reflect_agent",
+                tool_choice="required" if iteration == 0 else "auto",  # Force tool use on first iteration
+            )
+            llm_duration = int((time.time() - llm_start) * 1000)
+            total_input_tokens += result.input_tokens
+            total_output_tokens += result.output_tokens
+            llm_trace.append(
+                {
+                    "scope": f"agent_{iteration + 1}",
+                    "duration_ms": llm_duration,
+                    "input_tokens": result.input_tokens,
+                    "output_tokens": result.output_tokens,
+                }
+            )
+
+        except Exception as e:
+            err_duration = int((time.time() - llm_start) * 1000)
+            logger.warning(f"[REFLECT {reflect_id}] LLM error on iteration {iteration + 1}: {e} ({err_duration}ms)")
+            llm_trace.append({"scope": f"agent_{iteration + 1}_err", "duration_ms": err_duration})
+            # Guardrail: If no evidence gathered yet, retry
+            has_gathered_evidence = (
+                bool(available_memory_ids) or bool(available_mental_model_ids) or bool(available_observation_ids)
+            )
+            if not has_gathered_evidence and iteration < max_iterations - 1:
+                continue
+            prompt = build_final_prompt(query, context_history, bank_profile, context)
+            llm_start = time.time()
+            response, usage = await llm_config.call(
+                messages=[
+                    {"role": "system", "content": FINAL_SYSTEM_PROMPT},
+                    {"role": "user", "content": prompt},
+                ],
+                scope="reflect_agent_final",
+                max_completion_tokens=max_tokens,
+                return_usage=True,
+            )
+            llm_duration = int((time.time() - llm_start) * 1000)
+            total_input_tokens += usage.input_tokens
+            total_output_tokens += usage.output_tokens
+            llm_trace.append(
+                {
+                    "scope": "final",
+                    "duration_ms": llm_duration,
+                    "input_tokens": usage.input_tokens,
+                    "output_tokens": usage.output_tokens,
+                }
+            )
+            answer = _clean_answer_text(response.strip())
+
+            # Generate structured output if schema provided
+            structured_output = None
+            if response_schema and answer:
+                structured_output, struct_in, struct_out = await _generate_structured_output(
+                    answer, response_schema, llm_config, reflect_id
+                )
+                total_input_tokens += struct_in
+                total_output_tokens += struct_out
+
+            _log_completion(answer, iteration + 1, forced=True)
+            return ReflectAgentResult(
+                text=answer,
+                structured_output=structured_output,
+                iterations=iteration + 1,
+                tools_called=total_tools_called,
+                tool_trace=tool_trace,
+                llm_trace=_get_llm_trace(),
+                usage=_get_usage(),
+                directives_applied=directives_applied,
+            )
+
+        # No tool calls - LLM wants to respond with text
+        if not result.tool_calls:
+            if result.content:
+                answer = _clean_answer_text(result.content.strip())
+
+                # Generate structured output if schema provided
+                structured_output = None
+                if response_schema and answer:
+                    structured_output, struct_in, struct_out = await _generate_structured_output(
+                        answer, response_schema, llm_config, reflect_id
+                    )
+                    total_input_tokens += struct_in
+                    total_output_tokens += struct_out
+
+                _log_completion(answer, iteration + 1)
+                return ReflectAgentResult(
+                    text=answer,
+                    structured_output=structured_output,
+                    iterations=iteration + 1,
+                    tools_called=total_tools_called,
+                    tool_trace=tool_trace,
+                    llm_trace=_get_llm_trace(),
+                    usage=_get_usage(),
+                    directives_applied=directives_applied,
+                )
+            # Empty response, force final
+            prompt = build_final_prompt(query, context_history, bank_profile, context)
+            llm_start = time.time()
+            response, usage = await llm_config.call(
+                messages=[
+                    {"role": "system", "content": FINAL_SYSTEM_PROMPT},
+                    {"role": "user", "content": prompt},
+                ],
+                scope="reflect_agent_final",
+                max_completion_tokens=max_tokens,
+                return_usage=True,
+            )
+            llm_duration = int((time.time() - llm_start) * 1000)
+            total_input_tokens += usage.input_tokens
+            total_output_tokens += usage.output_tokens
+            llm_trace.append(
+                {
+                    "scope": "final",
+                    "duration_ms": llm_duration,
+                    "input_tokens": usage.input_tokens,
+                    "output_tokens": usage.output_tokens,
+                }
+            )
+            answer = _clean_answer_text(response.strip())
+
+            # Generate structured output if schema provided
+            structured_output = None
+            if response_schema and answer:
+                structured_output, struct_in, struct_out = await _generate_structured_output(
+                    answer, response_schema, llm_config, reflect_id
+                )
+                total_input_tokens += struct_in
+                total_output_tokens += struct_out
+
+            _log_completion(answer, iteration + 1, forced=True)
+            return ReflectAgentResult(
+                text=answer,
+                structured_output=structured_output,
+                iterations=iteration + 1,
+                tools_called=total_tools_called,
+                tool_trace=tool_trace,
+                llm_trace=_get_llm_trace(),
+                usage=_get_usage(),
+                directives_applied=directives_applied,
+            )
+
+        # Check for done tool call (handle various LLM output formats)
+        done_call = next((tc for tc in result.tool_calls if _is_done_tool(tc.name)), None)
+        if done_call:
+            # Guardrail: Require evidence before done
+            has_gathered_evidence = (
+                bool(available_memory_ids) or bool(available_mental_model_ids) or bool(available_observation_ids)
+            )
+            if not has_gathered_evidence and iteration < max_iterations - 1:
+                # Add assistant message and fake tool result asking for evidence
+                messages.append(
+                    {
+                        "role": "assistant",
+                        "tool_calls": [_tool_call_to_dict(done_call)],
+                    }
+                )
+                messages.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": done_call.id,
+                        "name": done_call.name,  # Required by Gemini
+                        "content": json.dumps(
+                            {
+                                "error": "You must search for information first. Use search_mental_models(), search_observations(), or recall() before providing your final answer."
+                            }
+                        ),
+                    }
+                )
+                continue
+
+            # Process done tool
+            return await _process_done_tool(
+                done_call,
+                available_memory_ids,
+                available_mental_model_ids,
+                available_observation_ids,
+                iteration + 1,
+                total_tools_called,
+                tool_trace,
+                _get_llm_trace(),
+                _get_usage(),
+                _log_completion,
+                reflect_id,
+                directives_applied=directives_applied,
+                llm_config=llm_config,
+                response_schema=response_schema,
+            )
+
+        # Execute other tools in parallel (exclude done tool in all its format variants)
+        other_tools = [tc for tc in result.tool_calls if not _is_done_tool(tc.name)]
+        if other_tools:
+            # Add assistant message with tool calls
+            messages.append(
+                {
+                    "role": "assistant",
+                    "tool_calls": [_tool_call_to_dict(tc) for tc in other_tools],
+                }
+            )
+
+            # Execute tools in parallel
+            tool_tasks = [
+                _execute_tool_with_timing(
+                    tc,
+                    search_mental_models_fn,
+                    search_observations_fn,
+                    recall_fn,
+                    expand_fn,
+                )
+                for tc in other_tools
+            ]
+            tool_results = await asyncio.gather(*tool_tasks, return_exceptions=True)
+            total_tools_called += len(other_tools)
+
+            # Process results and add to messages
+            for tc, result_data in zip(other_tools, tool_results):
+                if isinstance(result_data, Exception):
+                    # Tool execution failed - send error back to LLM so it can try again
+                    logger.warning(f"[REFLECT {reflect_id}] Tool {tc.name} failed with exception: {result_data}")
+                    output = {"error": f"Tool execution failed: {result_data}"}
+                    duration_ms = 0
+                else:
+                    output, duration_ms = result_data
+
+                # Normalize tool name for consistent tracking
+                normalized_tool_name = _normalize_tool_name(tc.name)
+
+                # Check if tool returned an error response - log but continue (LLM will see the error)
+                if isinstance(output, dict) and "error" in output:
+                    logger.warning(
+                        f"[REFLECT {reflect_id}] Tool {normalized_tool_name} returned error: {output['error']}"
+                    )
+
+                # Track available IDs from tool results (only for successful responses)
+                if (
+                    normalized_tool_name == "search_mental_models"
+                    and isinstance(output, dict)
+                    and "mental_models" in output
+                ):
+                    for mm in output["mental_models"]:
+                        if "id" in mm:
+                            available_mental_model_ids.add(mm["id"])
+
+                if (
+                    normalized_tool_name == "search_observations"
+                    and isinstance(output, dict)
+                    and "observations" in output
+                ):
+                    for obs in output["observations"]:
+                        if "id" in obs:
+                            available_observation_ids.add(obs["id"])
+
+                if normalized_tool_name == "recall" and isinstance(output, dict) and "memories" in output:
+                    for memory in output["memories"]:
+                        if "id" in memory:
+                            available_memory_ids.add(memory["id"])
+
+                # Add tool result message
+                messages.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": tc.id,
+                        "name": tc.name,  # Required by Gemini
+                        "content": json.dumps(output, default=str),
+                    }
+                )
+
+                # Track for logging and context history
+                input_dict = {"tool": tc.name, **tc.arguments}
+                input_summary = _summarize_input(tc.name, tc.arguments)
+
+                # Extract reason from tool arguments (if provided)
+                tool_reason = tc.arguments.get("reason")
+
+                tool_trace.append(
+                    ToolCall(
+                        tool=tc.name,
+                        reason=tool_reason,
+                        input=input_dict,
+                        output=output,
+                        duration_ms=duration_ms,
+                        iteration=iteration + 1,
+                    )
+                )
+
+                try:
+                    output_chars = len(json.dumps(output))
+                except (TypeError, ValueError):
+                    output_chars = len(str(output))
+
+                tool_trace_summary.append(
+                    {
+                        "tool": tc.name,
+                        "input_summary": input_summary,
+                        "duration_ms": duration_ms,
+                        "output_chars": output_chars,
+                    }
+                )
+
+                # Keep context history for fallback final prompt
+                context_history.append({"tool": tc.name, "input": input_dict, "output": output})
+
+    # Should not reach here
+    answer = "I was unable to formulate a complete answer within the iteration limit."
+    _log_completion(answer, max_iterations, forced=True)
+    return ReflectAgentResult(
+        text=answer,
+        iterations=max_iterations,
+        tools_called=total_tools_called,
+        tool_trace=tool_trace,
+        llm_trace=_get_llm_trace(),
+        usage=_get_usage(),
+        directives_applied=directives_applied,
+    )
+
+
+def _tool_call_to_dict(tc: "LLMToolCall") -> dict[str, Any]:
+    """Convert LLMToolCall to OpenAI message format."""
+    return {
+        "id": tc.id,
+        "type": "function",
+        "function": {
+            "name": tc.name,
+            "arguments": json.dumps(tc.arguments),
+        },
+    }
+
+
+async def _process_done_tool(
+    done_call: "LLMToolCall",
+    available_memory_ids: set[str],
+    available_mental_model_ids: set[str],
+    available_observation_ids: set[str],
+    iterations: int,
+    total_tools_called: int,
+    tool_trace: list[ToolCall],
+    llm_trace: list[LLMCall],
+    usage: TokenUsageSummary,
+    log_completion: Callable,
+    reflect_id: str,
+    directives_applied: list[DirectiveInfo],
+    llm_config: "LLMProvider | None" = None,
+    response_schema: dict | None = None,
+) -> ReflectAgentResult:
+    """Process the done tool call and return the result."""
+    args = done_call.arguments
+
+    # Extract and clean the answer - some LLMs leak structured output into the answer text
+    raw_answer = args.get("answer", "").strip()
+    answer = _clean_done_answer(raw_answer) if raw_answer else ""
+    if not answer:
+        answer = "No answer provided."
+
+    # Validate IDs (only include IDs that were actually retrieved)
+    used_memory_ids = [mid for mid in args.get("memory_ids", []) if mid in available_memory_ids]
+    used_mental_model_ids = [mid for mid in args.get("mental_model_ids", []) if mid in available_mental_model_ids]
+    used_observation_ids = [oid for oid in args.get("observation_ids", []) if oid in available_observation_ids]
+
+    # Generate structured output if schema provided
+    structured_output = None
+    final_usage = usage
+    if response_schema and llm_config and answer:
+        structured_output, struct_in, struct_out = await _generate_structured_output(
+            answer, response_schema, llm_config, reflect_id
+        )
+        # Add structured output tokens to usage
+        final_usage = TokenUsageSummary(
+            input_tokens=usage.input_tokens + struct_in,
+            output_tokens=usage.output_tokens + struct_out,
+            total_tokens=usage.total_tokens + struct_in + struct_out,
+        )
+
+    log_completion(answer, iterations)
+    return ReflectAgentResult(
+        text=answer,
+        structured_output=structured_output,
+        iterations=iterations,
+        tools_called=total_tools_called,
+        tool_trace=tool_trace,
+        llm_trace=llm_trace,
+        usage=final_usage,
+        used_memory_ids=used_memory_ids,
+        used_mental_model_ids=used_mental_model_ids,
+        used_observation_ids=used_observation_ids,
+        directives_applied=directives_applied,
+    )
+
+
+async def _execute_tool_with_timing(
+    tc: "LLMToolCall",
+    search_mental_models_fn: Callable[[str, int], Awaitable[dict[str, Any]]],
+    search_observations_fn: Callable[[str, int], Awaitable[dict[str, Any]]],
+    recall_fn: Callable[[str, int], Awaitable[dict[str, Any]]],
+    expand_fn: Callable[[list[str], str], Awaitable[dict[str, Any]]],
+) -> tuple[dict[str, Any], int]:
+    """Execute a tool call and return result with timing."""
+    start = time.time()
+    result = await _execute_tool(
+        tc.name,
+        tc.arguments,
+        search_mental_models_fn,
+        search_observations_fn,
+        recall_fn,
+        expand_fn,
+    )
+    duration_ms = int((time.time() - start) * 1000)
+    return result, duration_ms
+
+
+async def _execute_tool(
+    tool_name: str,
+    args: dict[str, Any],
+    search_mental_models_fn: Callable[[str, int], Awaitable[dict[str, Any]]],
+    search_observations_fn: Callable[[str, int], Awaitable[dict[str, Any]]],
+    recall_fn: Callable[[str, int], Awaitable[dict[str, Any]]],
+    expand_fn: Callable[[list[str], str], Awaitable[dict[str, Any]]],
+) -> dict[str, Any]:
+    """Execute a single tool by name."""
+    # Normalize tool name for various LLM output formats
+    tool_name = _normalize_tool_name(tool_name)
+
+    if tool_name == "search_mental_models":
+        query = args.get("query")
+        if not query:
+            return {"error": "search_mental_models requires a query parameter"}
+        max_results = args.get("max_results") or 5
+        return await search_mental_models_fn(query, max_results)
+
+    elif tool_name == "search_observations":
+        query = args.get("query")
+        if not query:
+            return {"error": "search_observations requires a query parameter"}
+        max_tokens = max(args.get("max_tokens") or 5000, 1000)  # Default 5000, min 1000
+        return await search_observations_fn(query, max_tokens)
+
+    elif tool_name == "recall":
+        query = args.get("query")
+        if not query:
+            return {"error": "recall requires a query parameter"}
+        max_tokens = max(args.get("max_tokens") or 2048, 1000)  # Default 2048, min 1000
+        return await recall_fn(query, max_tokens)
+
+    elif tool_name == "expand":
+        memory_ids = args.get("memory_ids", [])
+        if not memory_ids:
+            return {"error": "expand requires memory_ids"}
+        depth = args.get("depth", "chunk")
+        return await expand_fn(memory_ids, depth)
+
+    else:
+        return {"error": f"Unknown tool: {tool_name}"}
+
+
+def _summarize_input(tool_name: str, args: dict[str, Any]) -> str:
+    """Create a summary of tool input for logging, showing all params."""
+    if tool_name == "search_mental_models":
+        query = args.get("query", "")
+        query_preview = f"'{query[:30]}...'" if len(query) > 30 else f"'{query}'"
+        max_results = args.get("max_results") or 5
+        return f"(query={query_preview}, max_results={max_results})"
+    elif tool_name == "search_observations":
+        query = args.get("query", "")
+        query_preview = f"'{query[:30]}...'" if len(query) > 30 else f"'{query}'"
+        max_tokens = max(args.get("max_tokens") or 5000, 1000)
+        return f"(query={query_preview}, max_tokens={max_tokens})"
+    elif tool_name == "recall":
+        query = args.get("query", "")
+        query_preview = f"'{query[:30]}...'" if len(query) > 30 else f"'{query}'"
+        # Show actual value used (default 2048, min 1000)
+        max_tokens = max(args.get("max_tokens") or 2048, 1000)
+        return f"(query={query_preview}, max_tokens={max_tokens})"
+    elif tool_name == "expand":
+        memory_ids = args.get("memory_ids", [])
+        depth = args.get("depth", "chunk")
+        return f"(memory_ids=[{len(memory_ids)} ids], depth={depth})"
+    elif tool_name == "done":
+        answer = args.get("answer", "")
+        answer_preview = f"'{answer[:30]}...'" if len(answer) > 30 else f"'{answer}'"
+        memory_ids = args.get("memory_ids", [])
+        mental_model_ids = args.get("mental_model_ids", [])
+        observation_ids = args.get("observation_ids", [])
+        return (
+            f"(answer={answer_preview}, mem={len(memory_ids)}, mm={len(mental_model_ids)}, obs={len(observation_ids)})"
+        )
+    return str(args)

hindsight-api/hindsight_api/engine/reflect/models.py

@@ -0,0 +1,109 @@
+"""
+Pydantic models for the reflect agent.
+"""
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class ObservationSection(BaseModel):
+    """A section within an observation with its supporting memories."""
+
+    title: str = Field(description="Section header (can be empty for intro)")
+    text: str = Field(description="Section content - no headers, use lists/tables/bold")
+    memory_ids: list[str] = Field(default_factory=list, description="Memory IDs supporting this section")
+
+
+class ReflectAction(BaseModel):
+    """Single action the reflect agent can take."""
+
+    tool: Literal["list_observations", "get_observation", "recall", "expand", "done"] = Field(
+        description="Tool to invoke: list_observations, get_observation, recall, expand, or done"
+    )
+    # Tool-specific parameters
+    observation_id: str | None = Field(default=None, description="Observation ID for get_observation")
+    query: str | None = Field(default=None, description="Search query for recall")
+    max_tokens: int | None = Field(default=None, description="Max tokens for recall results (default 2048)")
+    memory_ids: list[str] | None = Field(default=None, description="Memory unit IDs for expand (batched)")
+    depth: Literal["chunk", "document"] | None = Field(default=None, description="Expansion depth for expand")
+    observation_sections: list[ObservationSection] | None = Field(
+        default=None, description="Observation sections for done action (when output_mode=observations)"
+    )
+    # Plain text answer fields (for output_mode=answer)
+    answer: str | None = Field(default=None, description="Plain text answer for done action (no markdown)")
+    answer_memory_ids: list[str] | None = Field(
+        default=None, description="Memory IDs supporting the answer", alias="memory_ids"
+    )
+    answer_model_ids: list[str] | None = Field(
+        default=None, description="Mental model IDs supporting the answer", alias="model_ids"
+    )
+    reasoning: str | None = Field(default=None, description="Brief reasoning for this action")
+
+
+class ReflectActionBatch(BaseModel):
+    """Batch of actions for parallel execution."""
+
+    actions: list[ReflectAction] = Field(description="List of actions to execute in parallel")
+
+
+class ToolCall(BaseModel):
+    """A single tool call made during reflect."""
+
+    tool: str = Field(description="Tool name: lookup, recall, expand")
+    reason: str | None = Field(default=None, description="Agent's reasoning for making this tool call")
+    input: dict = Field(description="Tool input parameters")
+    output: dict = Field(description="Tool output/result")
+    duration_ms: int = Field(description="Execution time in milliseconds")
+    iteration: int = Field(default=0, description="Iteration number (1-based) when this tool was called")
+
+
+class LLMCall(BaseModel):
+    """A single LLM call made during reflect."""
+
+    scope: str = Field(description="Call scope: agent_1, agent_2, final, etc.")
+    duration_ms: int = Field(description="Execution time in milliseconds")
+    input_tokens: int = Field(default=0, description="Input tokens used")
+    output_tokens: int = Field(default=0, description="Output tokens used")
+
+
+class DirectiveInfo(BaseModel):
+    """Information about a directive that was applied during reflect."""
+
+    id: str = Field(description="Directive mental model ID")
+    name: str = Field(description="Directive name")
+    content: str = Field(description="Directive content")
+
+
+class TokenUsageSummary(BaseModel):
+    """Total token usage across all LLM calls."""
+
+    input_tokens: int = Field(default=0, description="Total input tokens used")
+    output_tokens: int = Field(default=0, description="Total output tokens used")
+    total_tokens: int = Field(default=0, description="Total tokens (input + output)")
+
+
+class ReflectAgentResult(BaseModel):
+    """Result from the reflect agent."""
+
+    text: str = Field(description="Final answer text")
+    structured_output: dict[str, Any] | None = Field(
+        default=None, description="Structured output parsed according to provided response_schema"
+    )
+    iterations: int = Field(default=0, description="Number of iterations taken")
+    tools_called: int = Field(default=0, description="Total number of tool calls made")
+    tool_trace: list[ToolCall] = Field(default_factory=list, description="Trace of all tool calls made")
+    llm_trace: list[LLMCall] = Field(default_factory=list, description="Trace of all LLM calls made")
+    usage: TokenUsageSummary = Field(
+        default_factory=TokenUsageSummary, description="Total token usage across all LLM calls"
+    )
+    used_memory_ids: list[str] = Field(default_factory=list, description="Validated memory IDs actually used in answer")
+    used_mental_model_ids: list[str] = Field(
+        default_factory=list, description="Validated mental model IDs actually used in answer"
+    )
+    used_observation_ids: list[str] = Field(
+        default_factory=list, description="Validated observation IDs actually used in answer"
+    )
+    directives_applied: list[DirectiveInfo] = Field(
+        default_factory=list, description="Directive mental models that affected this reflection"
+    )

hindsight-api/hindsight_api/engine/reflect/observations.py

@@ -0,0 +1,186 @@
+"""
+Models and utilities for evidence-grounded observations with computed trends.
+
+Observations are part of mental models and represent patterns/beliefs derived
+from memories. Each observation must be grounded in specific evidence (quotes)
+from memories, and trends are computed algorithmically from evidence timestamps.
+"""
+
+from datetime import datetime, timedelta, timezone
+from enum import Enum
+
+from pydantic import BaseModel, Field, computed_field, field_validator
+
+
+class Trend(str, Enum):
+    """Computed trend for an observation based on evidence timestamps.
+
+    Trends indicate how an observation's evidence is distributed over time:
+    - STABLE: Evidence spread across time, continues to present
+    - STRENGTHENING: More/denser evidence recently than before
+    - WEAKENING: Evidence mostly old, sparse recently
+    - NEW: All evidence within recent window
+    - STALE: No evidence in recent window (may no longer apply)
+    """
+
+    STABLE = "stable"
+    STRENGTHENING = "strengthening"
+    WEAKENING = "weakening"
+    NEW = "new"
+    STALE = "stale"
+
+
+class ObservationEvidence(BaseModel):
+    """A single piece of evidence supporting an observation.
+
+    Each evidence item must include an exact quote from the source memory
+    to ensure observations are grounded and verifiable.
+    """
+
+    memory_id: str = Field(description="ID of the memory unit this evidence comes from")
+    quote: str = Field(description="Exact quote from the memory supporting the observation")
+    relevance: str = Field(default="", description="Brief explanation of how this quote supports the observation")
+    timestamp: datetime = Field(description="When the source memory was created")
+
+    @field_validator("timestamp", mode="before")
+    @classmethod
+    def ensure_timezone_aware(cls, v: datetime | str | None) -> datetime:
+        """Ensure timestamp is always timezone-aware UTC."""
+        if v is None:
+            return datetime.now(timezone.utc)
+        if isinstance(v, str):
+            # Parse ISO format string, handling 'Z' suffix
+            v = datetime.fromisoformat(v.replace("Z", "+00:00"))
+        if isinstance(v, datetime):
+            if v.tzinfo is None:
+                return v.replace(tzinfo=timezone.utc)
+            return v
+        raise ValueError(f"Invalid timestamp type: {type(v)}")
+
+
+class Observation(BaseModel):
+    """A single observation within a mental model.
+
+    Observations represent patterns, preferences, beliefs, or other insights
+    derived from memories. Each observation must be grounded in evidence
+    with exact quotes from source memories.
+    """
+
+    title: str = Field(description="Short summary title for the observation (5-10 words)")
+    content: str = Field(description="The observation content - detailed explanation of what we believe to be true")
+    evidence: list[ObservationEvidence] = Field(default_factory=list, description="Supporting evidence with quotes")
+    created_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc), description="When this observation was first created"
+    )
+
+    @field_validator("created_at", mode="before")
+    @classmethod
+    def ensure_created_at_timezone_aware(cls, v: datetime | str | None) -> datetime:
+        """Ensure created_at is always timezone-aware UTC."""
+        if v is None:
+            return datetime.now(timezone.utc)
+        if isinstance(v, str):
+            v = datetime.fromisoformat(v.replace("Z", "+00:00"))
+        if isinstance(v, datetime):
+            if v.tzinfo is None:
+                return v.replace(tzinfo=timezone.utc)
+            return v
+        raise ValueError(f"Invalid created_at type: {type(v)}")
+
+    @computed_field
+    @property
+    def trend(self) -> Trend:
+        """Compute trend from evidence timestamps."""
+        return compute_trend(self.evidence)
+
+    @computed_field
+    @property
+    def evidence_span(self) -> dict[str, str | None]:
+        """Get the time span covered by evidence."""
+        if not self.evidence:
+            return {"from": None, "to": None}
+        timestamps = [e.timestamp for e in self.evidence]
+        return {
+            "from": min(timestamps).isoformat(),
+            "to": max(timestamps).isoformat(),
+        }
+
+    @computed_field
+    @property
+    def evidence_count(self) -> int:
+        """Number of evidence items supporting this observation."""
+        return len(self.evidence)
+
+
+def compute_trend(
+    evidence: list[ObservationEvidence],
+    now: datetime | None = None,
+    recent_days: int = 30,
+    old_days: int = 90,
+) -> Trend:
+    """Compute the trend for an observation based on evidence timestamps.
+
+    The trend indicates how the evidence is distributed over time:
+    - STABLE: Evidence spread across time, continues to present
+    - STRENGTHENING: More evidence recently than historically
+    - WEAKENING: Evidence mostly old, sparse recently
+    - NEW: All evidence is recent (within recent_days)
+    - STALE: No evidence in recent window
+
+    Args:
+        evidence: List of evidence items with timestamps
+        now: Reference time for calculations (defaults to current UTC time)
+        recent_days: Number of days to consider "recent" (default 30)
+        old_days: Number of days to consider "old" (default 90)
+
+    Returns:
+        Computed Trend enum value
+    """
+    if now is None:
+        now = datetime.now(timezone.utc)
+
+    # Ensure now is timezone-aware
+    if now.tzinfo is None:
+        now = now.replace(tzinfo=timezone.utc)
+
+    if not evidence:
+        return Trend.STALE
+
+    recent_cutoff = now - timedelta(days=recent_days)
+    old_cutoff = now - timedelta(days=old_days)
+
+    # Normalize timestamps to UTC for comparison
+    def normalize_ts(ts: datetime) -> datetime:
+        if ts.tzinfo is None:
+            return ts.replace(tzinfo=timezone.utc)
+        return ts
+
+    recent = [e for e in evidence if normalize_ts(e.timestamp) > recent_cutoff]
+    old = [e for e in evidence if normalize_ts(e.timestamp) < old_cutoff]
+    middle = [e for e in evidence if old_cutoff <= normalize_ts(e.timestamp) <= recent_cutoff]
+
+    # No recent evidence = stale
+    if not recent:
+        return Trend.STALE
+
+    # All evidence is recent = new
+    if not old and not middle:
+        return Trend.NEW
+
+    # Compare density (evidence per day)
+    recent_density = len(recent) / recent_days if recent_days > 0 else 0
+    older_period = old_days - recent_days
+    older_density = (len(old) + len(middle)) / older_period if older_period > 0 else 0
+
+    # Avoid division by zero
+    if older_density == 0:
+        return Trend.NEW
+
+    ratio = recent_density / older_density
+
+    if ratio > 1.5:
+        return Trend.STRENGTHENING
+    elif ratio < 0.5:
+        return Trend.WEAKENING
+    else:
+        return Trend.STABLE

hindsight-api/hindsight_api/engine/reflect/prompts.py

@@ -0,0 +1,483 @@
+"""
+System prompts for the reflect agent.
+
+The reflect agent uses hierarchical retrieval:
+1. search_mental_models - User-curated summaries (highest quality)
+2. search_observations - Consolidated knowledge with freshness awareness
+3. recall - Raw facts as ground truth fallback
+"""
+
+import json
+from typing import Any
+
+
+def _extract_directive_rules(directives: list[dict[str, Any]]) -> list[str]:
+    """
+    Extract directive rules as a list of strings.
+
+    Args:
+        directives: List of directives with name and content
+
+    Returns:
+        List of directive rule strings
+    """
+    rules = []
+    for directive in directives:
+        directive_name = directive.get("name", "")
+        # New format: directives have direct content field
+        content = directive.get("content", "")
+        if content:
+            if directive_name:
+                rules.append(f"**{directive_name}**: {content}")
+            else:
+                rules.append(content)
+        else:
+            # Legacy format: check for observations
+            observations = directive.get("observations", [])
+            if observations:
+                for obs in observations:
+                    # Support both Pydantic Observation objects and dicts
+                    if hasattr(obs, "title"):
+                        title = obs.title
+                        obs_content = obs.content
+                    else:
+                        title = obs.get("title", "")
+                        obs_content = obs.get("content", "")
+                    if title and obs_content:
+                        rules.append(f"**{title}**: {obs_content}")
+                    elif obs_content:
+                        rules.append(obs_content)
+            elif directive_name:
+                # Fallback to description
+                desc = directive.get("description", "")
+                if desc:
+                    rules.append(f"**{directive_name}**: {desc}")
+    return rules
+
+
+def build_directives_section(directives: list[dict[str, Any]]) -> str:
+    """
+    Build the directives section for the system prompt.
+
+    Directives are hard rules that MUST be followed in all responses.
+
+    Args:
+        directives: List of directive mental models with observations
+    """
+    if not directives:
+        return ""
+
+    rules = _extract_directive_rules(directives)
+    if not rules:
+        return ""
+
+    parts = [
+        "## DIRECTIVES (MANDATORY)",
+        "These are hard rules you MUST follow in ALL responses:",
+        "",
+    ]
+
+    for rule in rules:
+        parts.append(f"- {rule}")
+
+    parts.extend(
+        [
+            "",
+            "NEVER violate these directives, even if other context suggests otherwise.",
+            "IMPORTANT: Do NOT explain or justify how you handled directives in your answer. Just follow them silently.",
+            "",
+        ]
+    )
+    return "\n".join(parts)
+
+
+def build_directives_reminder(directives: list[dict[str, Any]]) -> str:
+    """
+    Build a reminder section for directives to place at the end of the prompt.
+
+    Args:
+        directives: List of directive mental models with observations
+    """
+    if not directives:
+        return ""
+
+    rules = _extract_directive_rules(directives)
+    if not rules:
+        return ""
+
+    parts = [
+        "",
+        "## REMINDER: MANDATORY DIRECTIVES",
+        "Before responding, ensure your answer complies with ALL of these directives:",
+        "",
+    ]
+
+    for i, rule in enumerate(rules, 1):
+        parts.append(f"{i}. {rule}")
+
+    parts.append("")
+    parts.append("Your response will be REJECTED if it violates any directive above.")
+    parts.append("Do NOT include any commentary about how you handled directives - just follow them.")
+    return "\n".join(parts)
+
+
+def build_system_prompt_for_tools(
+    bank_profile: dict[str, Any],
+    context: str | None = None,
+    directives: list[dict[str, Any]] | None = None,
+    has_mental_models: bool = False,
+    budget: str | None = None,
+) -> str:
+    """
+    Build the system prompt for tool-calling reflect agent.
+
+    The agent uses hierarchical retrieval:
+    1. search_mental_models - User-curated summaries (try first, if available)
+    2. search_observations - Consolidated knowledge with freshness
+    3. recall - Raw facts as ground truth
+
+    Args:
+        bank_profile: Bank profile with name and mission
+        context: Optional additional context
+        directives: Optional list of directive mental models to inject as hard rules
+        has_mental_models: Whether the bank has any mental models (skip if not)
+        budget: Search depth budget - "low", "mid", or "high". Controls exploration thoroughness.
+    """
+    name = bank_profile.get("name", "Assistant")
+    mission = bank_profile.get("mission", "")
+
+    parts = []
+
+    # Inject directives at the VERY START for maximum prominence
+    if directives:
+        parts.append(build_directives_section(directives))
+
+    parts.extend(
+        [
+            "You are a reflection agent that answers questions by reasoning over retrieved memories.",
+            "",
+        ]
+    )
+
+    parts.extend(
+        [
+            "## CRITICAL RULES",
+            "- You must NEVER fabricate information that has no basis in retrieved data",
+            "- You SHOULD synthesize, infer, and reason from the retrieved memories",
+            "- You MUST search before saying you don't have information",
+            "",
+            "## How to Reason",
+            "- If memories mention someone did an activity, you can infer they likely enjoyed it",
+            "- Synthesize a coherent narrative from related memories",
+            "- Be a thoughtful interpreter, not just a literal repeater",
+            "- When the exact answer isn't stated, use what IS stated to give the best answer",
+            "",
+            "## HIERARCHICAL RETRIEVAL STRATEGY",
+            "",
+        ]
+    )
+
+    # Build retrieval levels based on what's available
+    if has_mental_models:
+        parts.extend(
+            [
+                "You have access to THREE levels of knowledge. Use them in this order:",
+                "",
+                "### 1. MENTAL MODELS (search_mental_models) - Try First",
+                "- User-curated summaries about specific topics",
+                "- HIGHEST quality - manually created and maintained",
+                "- If a relevant mental model exists and is FRESH, it may fully answer the question",
+                "- Check `is_stale` field - if stale, also verify with lower levels",
+                "",
+                "### 2. OBSERVATIONS (search_observations) - Second Priority",
+                "- Auto-consolidated knowledge from memories",
+                "- Check `is_stale` field - if stale, ALSO use recall() to verify",
+                "- Good for understanding patterns and summaries",
+                "",
+                "### 3. RAW FACTS (recall) - Ground Truth",
+                "- Individual memories (world facts and experiences)",
+                "- Use when: no mental models/observations exist, they're stale, or you need specific details",
+                "- This is the source of truth that other levels are built from",
+                "",
+            ]
+        )
+    else:
+        parts.extend(
+            [
+                "You have access to TWO levels of knowledge. Use them in this order:",
+                "",
+                "### 1. OBSERVATIONS (search_observations) - Try First",
+                "- Auto-consolidated knowledge from memories",
+                "- Check `is_stale` field - if stale, ALSO use recall() to verify",
+                "- Good for understanding patterns and summaries",
+                "",
+                "### 2. RAW FACTS (recall) - Ground Truth",
+                "- Individual memories (world facts and experiences)",
+                "- Use when: no observations exist, they're stale, or you need specific details",
+                "- This is the source of truth that observations are built from",
+                "",
+            ]
+        )
+
+    parts.extend(
+        [
+            "## Query Strategy",
+            "recall() uses semantic search. NEVER just echo the user's question - decompose it into targeted searches:",
+            "",
+            "BAD: User asks 'recurring lesson themes between students' → recall('recurring lesson themes between students')",
+            "GOOD: Break it down into component searches:",
+            "  1. recall('lessons') - find all lesson-related memories",
+            "  2. recall('teaching sessions') - alternative phrasing",
+            "  3. recall('student progress') - find student-related memories",
+            "",
+            "Think: What ENTITIES and CONCEPTS does this question involve? Search for each separately.",
+            "",
+        ]
+    )
+
+    # Add budget guidance
+    if budget:
+        budget_lower = budget.lower()
+        if budget_lower == "low":
+            parts.extend(
+                [
+                    "## RESEARCH DEPTH: SHALLOW (Quick Response)",
+                    "- Prioritize speed over completeness",
+                    "- If mental models or observations provide a reasonable answer, stop there",
+                    "- Only dig deeper if the initial results are clearly insufficient",
+                    "- Prefer a quick overview rather than exhaustive details",
+                    "- Answer promptly with available information",
+                    "",
+                ]
+            )
+        elif budget_lower == "mid":
+            parts.extend(
+                [
+                    "## RESEARCH DEPTH: MODERATE (Balanced)",
+                    "- Balance thoroughness with efficiency",
+                    "- Check multiple sources when the question warrants it",
+                    "- Verify stale data if it's central to the answer",
+                    "- Don't over-explore, but ensure reasonable coverage",
+                    "",
+                ]
+            )
+        elif budget_lower == "high":
+            parts.extend(
+                [
+                    "## RESEARCH DEPTH: DEEP (Thorough Exploration)",
+                    "- Explore comprehensively before answering",
+                    "- Search across all available knowledge levels",
+                    "- Use multiple query variations to ensure coverage",
+                    "- Verify information across different retrieval levels",
+                    "- Use expand() to get full context on important memories",
+                    "- Take time to synthesize a complete, well-researched answer",
+                    "",
+                ]
+            )
+
+    parts.append("## Workflow")
+
+    if has_mental_models:
+        parts.extend(
+            [
+                "1. First, try search_mental_models() - check if a curated summary exists",
+                "2. If no mental model or it's stale, try search_observations() for consolidated knowledge",
+                "3. If observations are stale OR you need specific details, use recall() for raw facts",
+                "4. Use expand() if you need more context on specific memories",
+                "5. When ready, call done() with your answer and supporting IDs",
+            ]
+        )
+    else:
+        parts.extend(
+            [
+                "1. First, try search_observations() - check for consolidated knowledge",
+                "2. If observations are stale OR you need specific details, use recall() for raw facts",
+                "3. Use expand() if you need more context on specific memories",
+                "4. When ready, call done() with your answer and supporting IDs",
+            ]
+        )
+
+    parts.extend(
+        [
+            "",
+            "## Output Format: Plain Text Answer",
+            "Call done() with a plain text 'answer' field.",
+            "- Do NOT use markdown formatting",
+            "- NEVER include memory IDs, UUIDs, or 'Memory references' in the answer text",
+            "- Put IDs ONLY in the memory_ids/mental_model_ids/observation_ids arrays, not in the answer",
+        ]
+    )
+
+    parts.append("")
+    parts.append(f"## Memory Bank: {name}")
+
+    if mission:
+        parts.append(f"Mission: {mission}")
+
+    # Disposition traits
+    disposition = bank_profile.get("disposition", {})
+    if disposition:
+        traits = []
+        if "skepticism" in disposition:
+            traits.append(f"skepticism={disposition['skepticism']}")
+        if "literalism" in disposition:
+            traits.append(f"literalism={disposition['literalism']}")
+        if "empathy" in disposition:
+            traits.append(f"empathy={disposition['empathy']}")
+        if traits:
+            parts.append(f"Disposition: {', '.join(traits)}")
+
+    if context:
+        parts.append(f"\n## Additional Context\n{context}")
+
+    # Add directive reminder at the END for recency effect
+    if directives:
+        parts.append(build_directives_reminder(directives))
+
+    return "\n".join(parts)
+
+
+def build_agent_prompt(
+    query: str,
+    context_history: list[dict],
+    bank_profile: dict,
+    additional_context: str | None = None,
+) -> str:
+    """Build the user prompt for the reflect agent."""
+    parts = []
+
+    # Bank identity
+    name = bank_profile.get("name", "Assistant")
+    mission = bank_profile.get("mission", "")
+
+    parts.append(f"## Memory Bank Context\nName: {name}")
+    if mission:
+        parts.append(f"Mission: {mission}")
+
+    # Disposition traits if present
+    disposition = bank_profile.get("disposition", {})
+    if disposition:
+        traits = []
+        if "skepticism" in disposition:
+            traits.append(f"skepticism={disposition['skepticism']}")
+        if "literalism" in disposition:
+            traits.append(f"literalism={disposition['literalism']}")
+        if "empathy" in disposition:
+            traits.append(f"empathy={disposition['empathy']}")
+        if traits:
+            parts.append(f"Disposition: {', '.join(traits)}")
+
+    # Additional context from caller
+    if additional_context:
+        parts.append(f"\n## Additional Context\n{additional_context}")
+
+    # Tool call history
+    if context_history:
+        parts.append("\n## Tool Results (synthesize and reason from this data)")
+        for i, entry in enumerate(context_history, 1):
+            tool = entry["tool"]
+            output = entry["output"]
+            # Format as proper JSON for LLM readability
+            try:
+                output_str = json.dumps(output, indent=2, default=str)
+            except (TypeError, ValueError):
+                output_str = str(output)
+            parts.append(f"\n### Call {i}: {tool}\n```json\n{output_str}\n```")
+
+    # The question
+    parts.append(f"\n## Question\n{query}")
+
+    # Instructions
+    if context_history:
+        parts.append(
+            "\n## Instructions\n"
+            "Based on the tool results above, either call more tools or provide your final answer. "
+            "Synthesize and reason from the data - make reasonable inferences when helpful. "
+            "If you have related information, use it to give the best possible answer."
+        )
+    else:
+        parts.append(
+            "\n## Instructions\n"
+            "Start by searching for relevant information using the hierarchical retrieval strategy:\n"
+            "1. Try search_mental_models() first for curated summaries\n"
+            "2. Try search_observations() for consolidated knowledge\n"
+            "3. Use recall() for specific details or to verify stale data"
+        )
+
+    return "\n".join(parts)
+
+
+def build_final_prompt(
+    query: str,
+    context_history: list[dict],
+    bank_profile: dict,
+    additional_context: str | None = None,
+) -> str:
+    """Build the final prompt when forcing a text response (no tools)."""
+    parts = []
+
+    # Bank identity
+    name = bank_profile.get("name", "Assistant")
+    mission = bank_profile.get("mission", "")
+
+    parts.append(f"## Memory Bank Context\nName: {name}")
+    if mission:
+        parts.append(f"Mission: {mission}")
+
+    # Disposition traits if present
+    disposition = bank_profile.get("disposition", {})
+    if disposition:
+        traits = []
+        if "skepticism" in disposition:
+            traits.append(f"skepticism={disposition['skepticism']}")
+        if "literalism" in disposition:
+            traits.append(f"literalism={disposition['literalism']}")
+        if "empathy" in disposition:
+            traits.append(f"empathy={disposition['empathy']}")
+        if traits:
+            parts.append(f"Disposition: {', '.join(traits)}")
+
+    # Additional context from caller
+    if additional_context:
+        parts.append(f"\n## Additional Context\n{additional_context}")
+
+    # Tool call history
+    if context_history:
+        parts.append("\n## Retrieved Data (synthesize and reason from this data)")
+        for entry in context_history:
+            tool = entry["tool"]
+            output = entry["output"]
+            # Format as proper JSON for LLM readability
+            try:
+                output_str = json.dumps(output, indent=2, default=str)
+            except (TypeError, ValueError):
+                output_str = str(output)
+            parts.append(f"\n### From {tool}:\n```json\n{output_str}\n```")
+    else:
+        parts.append("\n## Retrieved Data\nNo data was retrieved.")
+
+    # The question
+    parts.append(f"\n## Question\n{query}")
+
+    # Final instructions
+    parts.append(
+        "\n## Instructions\n"
+        "Provide a thoughtful answer by synthesizing and reasoning from the retrieved data above. "
+        "You can make reasonable inferences from the memories, but don't completely fabricate information."
+        "If the exact answer isn't stated, use what IS stated to give the best possible answer. "
+        "Only say 'I don't have information' if the retrieved data is truly unrelated to the question."
+    )
+
+    return "\n".join(parts)
+
+
+FINAL_SYSTEM_PROMPT = """You are a thoughtful assistant that synthesizes answers from retrieved memories.
+
+Your approach:
+- Reason over the retrieved memories to answer the question
+- Make reasonable inferences when the exact answer isn't explicitly stated
+- Connect related memories to form a complete picture
+- Be helpful - if you have related information, use it to give the best possible answer
+
+Only say "I don't have information" if the retrieved data is truly unrelated to the question.
+Do NOT fabricate information that has no basis in the retrieved data."""

hindsight-api/hindsight_api/engine/reflect/tools.py

@@ -0,0 +1,437 @@
+"""
+Tool implementations for the reflect agent.
+
+Implements hierarchical retrieval:
+1. search_mental_models - User-curated stored reflect responses (highest quality)
+2. search_observations - Consolidated knowledge with freshness
+3. recall - Raw facts as ground truth
+"""
+
+import logging
+import uuid
+from datetime import datetime, timedelta, timezone
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from asyncpg import Connection
+
+    from ...api.http import RequestContext
+    from ..memory_engine import MemoryEngine
+
+logger = logging.getLogger(__name__)
+
+# Observation is considered stale if not updated in this many days
+STALE_THRESHOLD_DAYS = 7
+
+
+async def tool_search_mental_models(
+    conn: "Connection",
+    bank_id: str,
+    query: str,
+    query_embedding: list[float],
+    max_results: int = 5,
+    tags: list[str] | None = None,
+    tags_match: str = "any",
+    exclude_ids: list[str] | None = None,
+) -> dict[str, Any]:
+    """
+    Search user-curated mental models by semantic similarity.
+
+    Mental models are high-quality, manually created summaries about specific topics.
+    They should be searched FIRST as they represent the most reliable synthesized knowledge.
+
+    Args:
+        conn: Database connection
+        bank_id: Bank identifier
+        query: Search query (for logging/tracing)
+        query_embedding: Pre-computed embedding for semantic search
+        max_results: Maximum number of mental models to return
+        tags: Optional tags to filter mental models
+        tags_match: How to match tags - "any" (OR), "all" (AND)
+        exclude_ids: Optional list of mental model IDs to exclude (e.g., when refreshing a mental model)
+
+    Returns:
+        Dict with matching mental models including content and freshness info
+    """
+    from ..memory_engine import fq_table
+
+    # Build filters dynamically
+    filters = ""
+    params: list[Any] = [bank_id, str(query_embedding), max_results]
+    next_param = 4
+
+    if tags:
+        if tags_match == "all":
+            filters += f" AND tags @> ${next_param}::varchar[]"
+        else:
+            filters += f" AND (tags && ${next_param}::varchar[] OR tags IS NULL OR tags = '{{}}')"
+        params.append(tags)
+        next_param += 1
+
+    if exclude_ids:
+        filters += f" AND id != ALL(${next_param}::uuid[])"
+        params.append(exclude_ids)
+        next_param += 1
+
+    # Search mental models by embedding similarity
+    rows = await conn.fetch(
+        f"""
+        SELECT
+            id, name, content,
+            tags, created_at, last_refreshed_at,
+            1 - (embedding <=> $2::vector) as relevance
+        FROM {fq_table("mental_models")}
+        WHERE bank_id = $1 AND embedding IS NOT NULL {filters}
+        ORDER BY embedding <=> $2::vector
+        LIMIT $3
+        """,
+        *params,
+    )
+
+    now = datetime.now(timezone.utc)
+    mental_models = []
+
+    for row in rows:
+        last_refreshed_at = row["last_refreshed_at"]
+        if last_refreshed_at and last_refreshed_at.tzinfo is None:
+            last_refreshed_at = last_refreshed_at.replace(tzinfo=timezone.utc)
+
+        # Calculate freshness
+        is_stale = False
+        if last_refreshed_at:
+            age = now - last_refreshed_at
+            is_stale = age > timedelta(days=STALE_THRESHOLD_DAYS)
+
+        mental_models.append(
+            {
+                "id": str(row["id"]),
+                "name": row["name"],
+                "content": row["content"],
+                "tags": row["tags"] or [],
+                "relevance": round(row["relevance"], 4),
+                "updated_at": last_refreshed_at.isoformat() if last_refreshed_at else None,
+                "is_stale": is_stale,
+            }
+        )
+
+    return {
+        "query": query,
+        "count": len(mental_models),
+        "mental_models": mental_models,
+    }
+
+
+async def tool_search_observations(
+    memory_engine: "MemoryEngine",
+    bank_id: str,
+    query: str,
+    request_context: "RequestContext",
+    max_tokens: int = 5000,
+    tags: list[str] | None = None,
+    tags_match: str = "any",
+    last_consolidated_at: datetime | None = None,
+    pending_consolidation: int = 0,
+) -> dict[str, Any]:
+    """
+    Search consolidated observations using recall with include_observations.
+
+    Observations are auto-generated from memories. Returns freshness info
+    so the agent knows if it should also verify with recall().
+
+    Args:
+        memory_engine: Memory engine instance
+        bank_id: Bank identifier
+        query: Search query
+        request_context: Request context for authentication
+        max_tokens: Maximum tokens for results (default 5000)
+        tags: Optional tags to filter observations
+        tags_match: How to match tags - "any" (OR), "all" (AND)
+        last_consolidated_at: When consolidation last ran (for staleness check)
+        pending_consolidation: Number of memories waiting to be consolidated
+
+    Returns:
+        Dict with matching observations including freshness info
+    """
+    from ..memory_engine import fq_table
+
+    # Use recall to search observations (they come back in results field when fact_type=["observation"])
+    result = await memory_engine.recall_async(
+        bank_id=bank_id,
+        query=query,
+        fact_type=["observation"],  # Only retrieve observations
+        max_tokens=max_tokens,  # Token budget controls how many observations are returned
+        enable_trace=False,
+        request_context=request_context,
+        tags=tags,
+        tags_match=tags_match,
+        _connection_budget=1,
+        _quiet=True,
+    )
+
+    observations = []
+
+    # When fact_type=["observation"], results come back in `results` field as MemoryFact objects
+    # We need to fetch additional fields (proof_count, source_memory_ids) from the database
+    if result.results:
+        obs_ids = [m.id for m in result.results]
+
+        # Fetch proof_count and source_memory_ids for these observations
+        pool = await memory_engine._get_pool()
+        async with pool.acquire() as conn:
+            obs_rows = await conn.fetch(
+                f"""
+                SELECT id, proof_count, source_memory_ids
+                FROM {fq_table("memory_units")}
+                WHERE id = ANY($1::uuid[])
+                """,
+                obs_ids,
+            )
+            obs_data = {str(row["id"]): row for row in obs_rows}
+
+        for m in result.results:
+            # Get additional data from DB lookup
+            extra = obs_data.get(m.id, {})
+            proof_count = extra.get("proof_count", 1) if extra else 1
+            source_ids = extra.get("source_memory_ids", []) if extra else []
+            # Convert UUIDs to strings
+            source_memory_ids = [str(sid) for sid in (source_ids or [])]
+
+            # Determine staleness
+            is_stale = False
+            staleness_reason = None
+            if pending_consolidation > 0:
+                is_stale = True
+                staleness_reason = f"{pending_consolidation} memories pending consolidation"
+
+            observations.append(
+                {
+                    "id": str(m.id),
+                    "text": m.text,
+                    "proof_count": proof_count,
+                    "source_memory_ids": source_memory_ids,
+                    "tags": m.tags or [],
+                    "is_stale": is_stale,
+                    "staleness_reason": staleness_reason,
+                }
+            )
+
+    # Return freshness info (more understandable than raw pending_consolidation count)
+    if pending_consolidation == 0:
+        freshness = "up_to_date"
+    elif pending_consolidation < 10:
+        freshness = "slightly_stale"
+    else:
+        freshness = "stale"
+
+    return {
+        "query": query,
+        "count": len(observations),
+        "observations": observations,
+        "freshness": freshness,
+    }
+
+
+async def tool_recall(
+    memory_engine: "MemoryEngine",
+    bank_id: str,
+    query: str,
+    request_context: "RequestContext",
+    max_tokens: int = 2048,
+    max_results: int = 50,
+    tags: list[str] | None = None,
+    tags_match: str = "any",
+    connection_budget: int = 1,
+) -> dict[str, Any]:
+    """
+    Search memories using TEMPR retrieval.
+
+    This is the ground truth - raw facts and experiences.
+    Use when mental models/observations don't exist, are stale, or need verification.
+
+    Args:
+        memory_engine: Memory engine instance
+        bank_id: Bank identifier
+        query: Search query
+        request_context: Request context for authentication
+        max_tokens: Maximum tokens for results (default 2048)
+        max_results: Maximum number of results
+        tags: Filter by tags (includes untagged memories)
+        tags_match: How to match tags - "any" (OR), "all" (AND), or "exact"
+        connection_budget: Max DB connections for this recall (default 1 for internal ops)
+
+    Returns:
+        Dict with list of matching memories
+    """
+    result = await memory_engine.recall_async(
+        bank_id=bank_id,
+        query=query,
+        fact_type=["experience", "world"],  # Exclude opinions and observations
+        max_tokens=max_tokens,
+        enable_trace=False,
+        request_context=request_context,
+        tags=tags,
+        tags_match=tags_match,
+        _connection_budget=connection_budget,
+        _quiet=True,  # Suppress logging for internal operations
+    )
+
+    memories = []
+    for m in result.results[:max_results]:
+        memories.append(
+            {
+                "id": str(m.id),
+                "text": m.text,
+                "type": m.fact_type,
+                "entities": m.entities or [],
+                "occurred": m.occurred_start,  # Already ISO format string
+            }
+        )
+
+    return {
+        "query": query,
+        "count": len(memories),
+        "memories": memories,
+    }
+
+
+async def tool_expand(
+    conn: "Connection",
+    bank_id: str,
+    memory_ids: list[str],
+    depth: str,
+) -> dict[str, Any]:
+    """
+    Expand multiple memories to get chunk or document context.
+
+    Args:
+        conn: Database connection
+        bank_id: Bank identifier
+        memory_ids: List of memory unit IDs
+        depth: "chunk" or "document"
+
+    Returns:
+        Dict with results array, each containing memory, chunk, and optionally document data
+    """
+    from ..memory_engine import fq_table
+
+    if not memory_ids:
+        return {"error": "memory_ids is required and must not be empty"}
+
+    # Validate and convert UUIDs
+    valid_uuids: list[uuid.UUID] = []
+    errors: dict[str, str] = {}
+    for mid in memory_ids:
+        try:
+            valid_uuids.append(uuid.UUID(mid))
+        except ValueError:
+            errors[mid] = f"Invalid memory_id format: {mid}"
+
+    if not valid_uuids:
+        return {"error": "No valid memory IDs provided", "details": errors}
+
+    # Batch fetch all memory units
+    memories = await conn.fetch(
+        f"""
+        SELECT id, text, chunk_id, document_id, fact_type, context
+        FROM {fq_table("memory_units")}
+        WHERE id = ANY($1) AND bank_id = $2
+        """,
+        valid_uuids,
+        bank_id,
+    )
+    memory_map = {row["id"]: row for row in memories}
+
+    # Collect chunk_ids and document_ids for batch fetching
+    chunk_ids = [m["chunk_id"] for m in memories if m["chunk_id"]]
+    doc_ids_from_chunks: set[str] = set()
+    doc_ids_direct: set[str] = set()
+
+    # Batch fetch all chunks
+    chunk_map: dict[str, Any] = {}
+    if chunk_ids:
+        chunks = await conn.fetch(
+            f"""
+            SELECT chunk_id, chunk_text, chunk_index, document_id
+            FROM {fq_table("chunks")}
+            WHERE chunk_id = ANY($1)
+            """,
+            chunk_ids,
+        )
+        chunk_map = {row["chunk_id"]: row for row in chunks}
+        if depth == "document":
+            doc_ids_from_chunks = {c["document_id"] for c in chunks if c["document_id"]}
+
+    # Collect direct document IDs (memories without chunks)
+    if depth == "document":
+        for m in memories:
+            if not m["chunk_id"] and m["document_id"]:
+                doc_ids_direct.add(m["document_id"])
+
+    # Batch fetch all documents
+    doc_map: dict[str, Any] = {}
+    all_doc_ids = list(doc_ids_from_chunks | doc_ids_direct)
+    if all_doc_ids:
+        docs = await conn.fetch(
+            f"""
+            SELECT id, original_text, metadata, retain_params
+            FROM {fq_table("documents")}
+            WHERE id = ANY($1) AND bank_id = $2
+            """,
+            all_doc_ids,
+            bank_id,
+        )
+        doc_map = {row["id"]: row for row in docs}
+
+    # Build results
+    results: list[dict[str, Any]] = []
+    for mid, mem_uuid in zip(memory_ids, valid_uuids):
+        if mid in errors:
+            results.append({"memory_id": mid, "error": errors[mid]})
+            continue
+
+        memory = memory_map.get(mem_uuid)
+        if not memory:
+            results.append({"memory_id": mid, "error": f"Memory not found: {mid}"})
+            continue
+
+        item: dict[str, Any] = {
+            "memory_id": mid,
+            "memory": {
+                "id": str(memory["id"]),
+                "text": memory["text"],
+                "type": memory["fact_type"],
+                "context": memory["context"],
+            },
+        }
+
+        # Add chunk if available
+        if memory["chunk_id"] and memory["chunk_id"] in chunk_map:
+            chunk = chunk_map[memory["chunk_id"]]
+            item["chunk"] = {
+                "id": chunk["chunk_id"],
+                "text": chunk["chunk_text"],
+                "index": chunk["chunk_index"],
+                "document_id": chunk["document_id"],
+            }
+            # Add document if depth=document
+            if depth == "document" and chunk["document_id"] in doc_map:
+                doc = doc_map[chunk["document_id"]]
+                item["document"] = {
+                    "id": doc["id"],
+                    "full_text": doc["original_text"],
+                    "metadata": doc["metadata"],
+                    "retain_params": doc["retain_params"],
+                }
+        elif memory["document_id"] and depth == "document" and memory["document_id"] in doc_map:
+            # No chunk, but has document_id
+            doc = doc_map[memory["document_id"]]
+            item["document"] = {
+                "id": doc["id"],
+                "full_text": doc["original_text"],
+                "metadata": doc["metadata"],
+                "retain_params": doc["retain_params"],
+            }
+
+        results.append(item)
+
+    return {"results": results, "count": len(results)}

hindsight-api/hindsight_api/engine/reflect/tools_schema.py

@@ -0,0 +1,250 @@
+"""
+Tool schema definitions for the reflect agent.
+
+These are OpenAI-format tool definitions used with native tool calling.
+The reflect agent uses a hierarchical retrieval strategy:
+1. search_mental_models - User-curated stored reflect responses (highest quality, if applicable)
+2. search_observations - Consolidated knowledge with freshness awareness
+3. recall - Raw facts (world/experience) as ground truth fallback
+"""
+
+# Tool definitions in OpenAI format
+
+TOOL_SEARCH_MENTAL_MODELS = {
+    "type": "function",
+    "function": {
+        "name": "search_mental_models",
+        "description": (
+            "Search user-curated mental models (stored reflect responses). These are high-quality, manually created "
+            "summaries about specific topics. Use FIRST when the question might be covered by an "
+            "existing mental model. Returns mental models with their content and last refresh time."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "reason": {
+                    "type": "string",
+                    "description": "Brief explanation of why you're making this search (for debugging)",
+                },
+                "query": {
+                    "type": "string",
+                    "description": "Search query to find relevant mental models",
+                },
+                "max_results": {
+                    "type": "integer",
+                    "description": "Maximum number of mental models to return (default 5)",
+                },
+            },
+            "required": ["reason", "query"],
+        },
+    },
+}
+
+TOOL_SEARCH_OBSERVATIONS = {
+    "type": "function",
+    "function": {
+        "name": "search_observations",
+        "description": (
+            "Search consolidated observations (auto-generated knowledge). These are automatically "
+            "synthesized from memories. Returns observations with freshness info (updated_at, is_stale). "
+            "If an observation is STALE, you should ALSO use recall() to verify with current facts."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "reason": {
+                    "type": "string",
+                    "description": "Brief explanation of why you're making this search (for debugging)",
+                },
+                "query": {
+                    "type": "string",
+                    "description": "Search query to find relevant observations",
+                },
+                "max_tokens": {
+                    "type": "integer",
+                    "description": "Maximum tokens for results (default 5000). Use higher values for broader searches.",
+                },
+            },
+            "required": ["reason", "query"],
+        },
+    },
+}
+
+TOOL_RECALL = {
+    "type": "function",
+    "function": {
+        "name": "recall",
+        "description": (
+            "Search raw memories (facts and experiences). This is the ground truth data. "
+            "Use when: (1) no reflections/mental models exist, (2) mental models are stale, "
+            "(3) you need specific details not in synthesized knowledge. "
+            "Returns individual memory facts with their timestamps."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "reason": {
+                    "type": "string",
+                    "description": "Brief explanation of why you're making this search (for debugging)",
+                },
+                "query": {
+                    "type": "string",
+                    "description": "Search query string",
+                },
+                "max_tokens": {
+                    "type": "integer",
+                    "description": "Optional limit on result size (default 2048). Use higher values for broader searches.",
+                },
+            },
+            "required": ["reason", "query"],
+        },
+    },
+}
+
+TOOL_EXPAND = {
+    "type": "function",
+    "function": {
+        "name": "expand",
+        "description": "Get more context for one or more memories. Memory hierarchy: memory -> chunk -> document.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "reason": {
+                    "type": "string",
+                    "description": "Brief explanation of why you need more context (for debugging)",
+                },
+                "memory_ids": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Array of memory IDs from recall results (batch multiple for efficiency)",
+                },
+                "depth": {
+                    "type": "string",
+                    "enum": ["chunk", "document"],
+                    "description": "chunk: surrounding text chunk, document: full source document",
+                },
+            },
+            "required": ["reason", "memory_ids", "depth"],
+        },
+    },
+}
+
+TOOL_DONE_ANSWER = {
+    "type": "function",
+    "function": {
+        "name": "done",
+        "description": "Signal completion with your final answer. Use this when you have gathered enough information to answer the question.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "answer": {
+                    "type": "string",
+                    "description": "Your response as plain text. Do NOT use markdown formatting. NEVER include memory IDs, UUIDs, or 'Memory references' in this text - put IDs only in memory_ids array.",
+                },
+                "memory_ids": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Array of memory IDs that support your answer (put IDs here, NOT in answer text)",
+                },
+                "mental_model_ids": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Array of mental model IDs that support your answer",
+                },
+                "observation_ids": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Array of observation IDs that support your answer",
+                },
+            },
+            "required": ["answer"],
+        },
+    },
+}
+
+
+def _build_done_tool_with_directives(directive_rules: list[str]) -> dict:
+    """
+    Build the done tool schema with directive compliance field.
+
+    When directives are present, adds a required field that forces the agent
+    to confirm compliance with each directive before submitting.
+
+    Args:
+        directive_rules: List of directive rule strings
+    """
+    # Build rules list for description
+    rules_list = "\n".join(f"  {i + 1}. {rule}" for i, rule in enumerate(directive_rules))
+
+    # Build the tool with directive compliance field
+    return {
+        "type": "function",
+        "function": {
+            "name": "done",
+            "description": (
+                "Signal completion with your final answer. IMPORTANT: You must confirm directive compliance before submitting. "
+                "Your answer will be REJECTED if it violates any directive."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "answer": {
+                        "type": "string",
+                        "description": "Your response as plain text. Do NOT use markdown formatting. NEVER include memory IDs, UUIDs, or 'Memory references' in this text - put IDs only in memory_ids array.",
+                    },
+                    "memory_ids": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": "Array of memory IDs that support your answer (put IDs here, NOT in answer text)",
+                    },
+                    "mental_model_ids": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": "Array of mental model IDs that support your answer",
+                    },
+                    "observation_ids": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": "Array of observation IDs that support your answer",
+                    },
+                    "directive_compliance": {
+                        "type": "string",
+                        "description": f"REQUIRED: Confirm your answer complies with ALL directives. List each directive and how your answer follows it:\n{rules_list}\n\nFormat: 'Directive 1: [how answer complies]. Directive 2: [how answer complies]...'",
+                    },
+                },
+                "required": ["answer", "directive_compliance"],
+            },
+        },
+    }
+
+
+def get_reflect_tools(directive_rules: list[str] | None = None) -> list[dict]:
+    """
+    Get the list of tools for the reflect agent.
+
+    The tools support a hierarchical retrieval strategy:
+    1. search_mental_models - User-curated stored reflect responses (try first)
+    2. search_observations - Consolidated knowledge with freshness
+    3. recall - Raw facts as ground truth
+
+    Args:
+        directive_rules: Optional list of directive rule strings. If provided,
+                        the done() tool will require directive compliance confirmation.
+
+    Returns:
+        List of tool definitions in OpenAI format
+    """
+    tools = [
+        TOOL_SEARCH_MENTAL_MODELS,
+        TOOL_SEARCH_OBSERVATIONS,
+        TOOL_RECALL,
+        TOOL_EXPAND,
+    ]
+
+    # Use directive-aware done tool if directives are present
+    if directive_rules:
+        tools.append(_build_done_tool_with_directives(directive_rules))
+    else:
+        tools.append(TOOL_DONE_ANSWER)
+
+    return tools

hindsight-api/hindsight_api/engine/response_models.py

@@ -10,8 +10,94 @@ from typing import Any
 
 from pydantic import BaseModel, ConfigDict, Field
 
-# Valid fact types for recall operations (excludes 'observation' which is internal)
-VALID_RECALL_FACT_TYPES = frozenset(["world", "experience", "opinion"])
+# Valid fact types for recall operations (excludes 'opinion' which is deprecated)
+VALID_RECALL_FACT_TYPES = frozenset(["world", "experience", "observation"])
+
+
+class LLMToolCall(BaseModel):
+    """A tool call requested by the LLM."""
+
+    id: str = Field(description="Unique identifier for this tool call")
+    name: str = Field(description="Name of the tool to call")
+    arguments: dict[str, Any] = Field(description="Arguments to pass to the tool")
+
+
+class LLMToolCallResult(BaseModel):
+    """Result from an LLM call that may include tool calls."""
+
+    content: str | None = Field(default=None, description="Text content if any")
+    tool_calls: list[LLMToolCall] = Field(default_factory=list, description="Tool calls requested by the LLM")
+    finish_reason: str | None = Field(default=None, description="Reason the LLM stopped: 'stop', 'tool_calls', etc.")
+    input_tokens: int = Field(default=0, description="Input tokens used in this call")
+    output_tokens: int = Field(default=0, description="Output tokens used in this call")
+
+
+class ToolCallTrace(BaseModel):
+    """A single tool call made during reflect."""
+
+    tool: str = Field(description="Tool name: lookup, recall, learn, expand")
+    reason: str | None = Field(default=None, description="Agent's reasoning for making this tool call")
+    input: dict = Field(description="Tool input parameters")
+    output: dict = Field(description="Tool output/result")
+    duration_ms: int = Field(description="Execution time in milliseconds")
+    iteration: int = Field(default=0, description="Iteration number (1-based) when this tool was called")
+
+
+class LLMCallTrace(BaseModel):
+    """A single LLM call made during reflect."""
+
+    scope: str = Field(description="Call scope: agent_1, agent_2, final, etc.")
+    duration_ms: int = Field(description="Execution time in milliseconds")
+
+
+class ObservationRef(BaseModel):
+    """Reference to an observation accessed during reflect."""
+
+    id: str = Field(description="Observation ID")
+    name: str = Field(description="Observation name")
+    type: str = Field(description="Observation type: entity, concept, event")
+    subtype: str = Field(description="Observation subtype: structural, emergent, learned")
+    description: str = Field(description="Brief description")
+    summary: str | None = Field(default=None, description="Full summary (when looked up in detail)")
+
+
+class DirectiveRef(BaseModel):
+    """Reference to a directive that was applied during reflect."""
+
+    id: str = Field(description="Directive mental model ID")
+    name: str = Field(description="Directive name")
+    content: str = Field(description="Directive content")
+
+
+class TokenUsage(BaseModel):
+    """
+    Token usage metrics for LLM calls.
+
+    Tracks input/output tokens for a single request to enable
+    per-request cost tracking and monitoring.
+    """
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "input_tokens": 1500,
+                "output_tokens": 500,
+                "total_tokens": 2000,
+            }
+        }
+    )
+
+    input_tokens: int = Field(default=0, description="Number of input/prompt tokens consumed")
+    output_tokens: int = Field(default=0, description="Number of output/completion tokens generated")
+    total_tokens: int = Field(default=0, description="Total tokens (input + output)")
+
+    def __add__(self, other: "TokenUsage") -> "TokenUsage":
+        """Allow aggregating token usage from multiple calls."""
+        return TokenUsage(
+            input_tokens=self.input_tokens + other.input_tokens,
+            output_tokens=self.output_tokens + other.output_tokens,
+            total_tokens=self.total_tokens + other.total_tokens,
+        )
 
 
 class DispositionTraits(BaseModel):
@@ -54,6 +140,7 @@ class MemoryFact(BaseModel):
                 "metadata": {"source": "slack"},
                 "chunk_id": "bank123_session_abc123_0",
                 "activation": 0.95,
+                "tags": ["user_a", "session_123"],
             }
         }
     )
@@ -71,6 +158,7 @@ class MemoryFact(BaseModel):
     chunk_id: str | None = Field(
         None, description="ID of the chunk this fact was extracted from (format: bank_id_document_id_chunk_index)"
     )
+    tags: list[str] | None = Field(None, description="Visibility scope tags associated with this fact")
 
 
 class ChunkInfo(BaseModel):
@@ -81,6 +169,28 @@ class ChunkInfo(BaseModel):
     truncated: bool = Field(default=False, description="Whether the chunk was truncated due to token limits")
 
 
+class ObservationResult(BaseModel):
+    """An observation result from recall (consolidated knowledge synthesized from facts)."""
+
+    id: str = Field(description="Unique observation ID")
+    text: str = Field(description="The observation text")
+    proof_count: int = Field(description="Number of facts supporting this observation")
+    relevance: float = Field(default=0.0, description="Relevance score to the query")
+    tags: list[str] | None = Field(default=None, description="Tags for visibility scoping")
+    source_memory_ids: list[str] = Field(
+        default_factory=list, description="IDs of facts that contribute to this observation"
+    )
+
+
+class MentalModelResult(BaseModel):
+    """A mental model result from recall (stored reflect response)."""
+
+    id: str = Field(description="Unique mental model ID")
+    name: str = Field(description="Human-readable name")
+    content: str = Field(description="The synthesized content")
+    relevance: float = Field(default=0.0, description="Relevance score to the query")
+
+
 class RecallResult(BaseModel):
     """
     Result from a recall operation.
@@ -144,22 +254,47 @@ class ReflectResult(BaseModel):
                     ],
                     "experience": [],
                     "opinion": [],
+                    "mental_models": [],
+                    "directives": [
+                        {
+                            "id": "directive-123",
+                            "name": "Response Style",
+                            "rules": ["Always be concise"],
+                        }
+                    ],
                 },
                 "new_opinions": ["Machine learning has great potential in healthcare"],
                 "structured_output": {"summary": "ML in healthcare", "confidence": 0.9},
+                "usage": {"input_tokens": 1500, "output_tokens": 500, "total_tokens": 2000},
             }
         }
     )
 
     text: str = Field(description="The formulated answer text")
-    based_on: dict[str, list[MemoryFact]] = Field(
-        description="Facts used to formulate the answer, organized by type (world, experience, opinion)"
+    based_on: dict[str, Any] = Field(
+        description="Facts used to formulate the answer, organized by type (world, experience, opinion, mental_models, directives)"
     )
     new_opinions: list[str] = Field(default_factory=list, description="List of newly formed opinions during reflection")
     structured_output: dict[str, Any] | None = Field(
         default=None,
         description="Structured output parsed according to the provided response schema. Only present when response_schema was provided.",
     )
+    usage: TokenUsage | None = Field(
+        default=None,
+        description="Token usage metrics for the LLM calls made during this reflect operation.",
+    )
+    tool_trace: list[ToolCallTrace] = Field(
+        default_factory=list,
+        description="Trace of tool calls made during reflection. Only present when include.tool_calls is enabled.",
+    )
+    llm_trace: list[LLMCallTrace] = Field(
+        default_factory=list,
+        description="Trace of LLM calls made during reflection. Only present when include.tool_calls is enabled.",
+    )
+    directives_applied: list[DirectiveRef] = Field(
+        default_factory=list,
+        description="Directive mental models that were applied during this reflection.",
+    )
 
 
 class Opinion(BaseModel):
@@ -223,3 +358,32 @@ class EntityState(BaseModel):
     observations: list[EntityObservation] = Field(
         default_factory=list, description="List of observations about this entity"
     )
+
+
+class MentalModel(BaseModel):
+    """
+    A manually configured mental model for tracking specific topics/areas.
+
+    Mental models are user-defined focus areas that the agent should track
+    and maintain summaries for, unlike auto-extracted entities.
+    """
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "id": "team-dynamics",
+                "name": "Team Dynamics",
+                "description": "Track how the team collaborates, communication patterns, conflicts, and resolutions",
+                "summary": "The team has strong collaboration...",
+                "summary_updated_at": "2024-01-15T10:30:00Z",
+                "created_at": "2024-01-10T08:00:00Z",
+            }
+        }
+    )
+
+    id: str = Field(description="Unique identifier (alphanumeric lowercase)")
+    name: str = Field(description="Display name for the mental model")
+    description: str = Field(description="Prompt/directions for what to track and summarize")
+    summary: str | None = Field(None, description="Generated summary based on relevant facts")
+    summary_updated_at: str | None = Field(None, description="ISO format date when summary was last updated")
+    created_at: str = Field(description="ISO format date when the mental model was created")

hindsight-api/hindsight_api/engine/retain/bank_utils.py

@@ -1,5 +1,5 @@
 """
-bank profile utilities for disposition and background management.
+bank profile utilities for disposition and mission management.
 """
 
 import json
@@ -27,19 +27,18 @@ class BankProfile(TypedDict):
 
     name: str
     disposition: DispositionTraits
-    background: str
+    mission: str
 
 
-class BackgroundMergeResponse(BaseModel):
-    """LLM response for background merge with disposition inference."""
+class MissionMergeResponse(BaseModel):
+    """LLM response for mission merge."""
 
-    background: str = Field(description="Merged background in first person perspective")
-    disposition: DispositionTraits = Field(description="Inferred disposition traits (skepticism, literalism, empathy)")
+    mission: str = Field(description="Merged mission in first person perspective")
 
 
 async def get_bank_profile(pool, bank_id: str) -> BankProfile:
     """
-    Get bank profile (name, disposition + background).
+    Get bank profile (name, disposition + mission).
     Auto-creates bank with default values if not exists.
 
     Args:
@@ -47,13 +46,13 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
         bank_id: bank IDentifier
 
     Returns:
-        BankProfile with name, typed DispositionTraits, and background
+        BankProfile with name, typed DispositionTraits, and mission
     """
     async with acquire_with_retry(pool) as conn:
         # Try to get existing bank
         row = await conn.fetchrow(
             f"""
-            SELECT name, disposition, background
+            SELECT name, disposition, mission
             FROM {fq_table("banks")} WHERE bank_id = $1
             """,
             bank_id,
@@ -66,13 +65,15 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
                 disposition_data = json.loads(disposition_data)
 
             return BankProfile(
-                name=row["name"], disposition=DispositionTraits(**disposition_data), background=row["background"]
+                name=row["name"],
+                disposition=DispositionTraits(**disposition_data),
+                mission=row["mission"] or "",
             )
 
         # Bank doesn't exist, create with defaults
         await conn.execute(
             f"""
-            INSERT INTO {fq_table("banks")} (bank_id, name, disposition, background)
+            INSERT INTO {fq_table("banks")} (bank_id, name, disposition, mission)
             VALUES ($1, $2, $3::jsonb, $4)
             ON CONFLICT (bank_id) DO NOTHING
             """,
@@ -82,7 +83,7 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
             "",
         )
 
-        return BankProfile(name=bank_id, disposition=DispositionTraits(**DEFAULT_DISPOSITION), background="")
+        return BankProfile(name=bank_id, disposition=DispositionTraits(**DEFAULT_DISPOSITION), mission="")
 
 
 async def update_bank_disposition(pool, bank_id: str, disposition: dict[str, int]) -> None:
@@ -110,244 +111,121 @@ async def update_bank_disposition(pool, bank_id: str, disposition: dict[str, int
         )
 
 
-async def merge_bank_background(pool, llm_config, bank_id: str, new_info: str, update_disposition: bool = True) -> dict:
+async def set_bank_mission(pool, bank_id: str, mission: str) -> None:
     """
-    Merge new background information with existing background using LLM.
-    Normalizes to first person ("I") and resolves conflicts.
-    Optionally infers disposition traits from the merged background.
+    Set bank mission (replacing any existing mission).
 
     Args:
         pool: Database connection pool
-        llm_config: LLM configuration for background merging
         bank_id: bank IDentifier
-        new_info: New background information to add/merge
-        update_disposition: If True, infer Big Five traits from background (default: True)
+        mission: The mission text
+    """
+    # Ensure bank exists first
+    await get_bank_profile(pool, bank_id)
+
+    async with acquire_with_retry(pool) as conn:
+        await conn.execute(
+            f"""
+            UPDATE {fq_table("banks")}
+            SET mission = $2,
+                updated_at = NOW()
+            WHERE bank_id = $1
+            """,
+            bank_id,
+            mission,
+        )
+
+
+async def merge_bank_mission(pool, llm_config, bank_id: str, new_info: str) -> dict:
+    """
+    Merge new mission information with existing mission using LLM.
+    Normalizes to first person ("I") and resolves conflicts.
+
+    Args:
+        pool: Database connection pool
+        llm_config: LLM configuration for mission merging
+        bank_id: bank IDentifier
+        new_info: New mission information to add/merge
 
     Returns:
-        Dict with 'background' (str) and optionally 'disposition' (dict) keys
+        Dict with 'mission' (str) key
     """
     # Get current profile
     profile = await get_bank_profile(pool, bank_id)
-    current_background = profile["background"]
+    current_mission = profile["mission"]
 
-    # Use LLM to merge backgrounds and optionally infer disposition
-    result = await _llm_merge_background(llm_config, current_background, new_info, infer_disposition=update_disposition)
+    # Use LLM to merge missions
+    result = await _llm_merge_mission(llm_config, current_mission, new_info)
 
-    merged_background = result["background"]
-    inferred_disposition = result.get("disposition")
+    merged_mission = result["mission"]
 
     # Update in database
     async with acquire_with_retry(pool) as conn:
-        if inferred_disposition:
-            # Update both background and disposition
-            await conn.execute(
-                f"""
-                UPDATE {fq_table("banks")}
-                SET background = $2,
-                    disposition = $3::jsonb,
-                    updated_at = NOW()
-                WHERE bank_id = $1
-                """,
-                bank_id,
-                merged_background,
-                json.dumps(inferred_disposition),
-            )
-        else:
-            # Update only background
-            await conn.execute(
-                f"""
-                UPDATE {fq_table("banks")}
-                SET background = $2,
-                    updated_at = NOW()
-                WHERE bank_id = $1
-                """,
-                bank_id,
-                merged_background,
-            )
+        await conn.execute(
+            f"""
+            UPDATE {fq_table("banks")}
+            SET mission = $2,
+                updated_at = NOW()
+            WHERE bank_id = $1
+            """,
+            bank_id,
+            merged_mission,
+        )
 
-    response = {"background": merged_background}
-    if inferred_disposition:
-        response["disposition"] = inferred_disposition
-
-    return response
+    return {"mission": merged_mission}
 
 
-async def _llm_merge_background(llm_config, current: str, new_info: str, infer_disposition: bool = False) -> dict:
+async def _llm_merge_mission(llm_config, current: str, new_info: str) -> dict:
     """
-    Use LLM to intelligently merge background information.
-    Optionally infer Big Five disposition traits from the merged background.
+    Use LLM to intelligently merge mission information.
 
     Args:
         llm_config: LLM configuration to use
-        current: Current background text
+        current: Current mission text
         new_info: New information to merge
-        infer_disposition: If True, also infer disposition traits
 
     Returns:
-        Dict with 'background' (str) and optionally 'disposition' (dict) keys
+        Dict with 'mission' (str) key
     """
-    if infer_disposition:
-        prompt = f"""You are helping maintain a memory bank's background/profile and infer their disposition. You MUST respond with ONLY valid JSON.
+    prompt = f"""You are helping maintain an agent's mission statement.
 
-Current background: {current if current else "(empty)"}
+Current mission: {current if current else "(empty)"}
 
 New information to add: {new_info}
 
 Instructions:
-1. Merge the new information with the current background
-2. If there are conflicts (e.g., different birthplaces), the NEW information overwrites the old
-3. Keep additions that don't conflict
-4. Output in FIRST PERSON ("I") perspective
-5. Be concise - keep merged background under 500 characters
-6. Infer disposition traits from the merged background (each 1-5 integer):
-   - Skepticism: 1-5 (1=trusting, takes things at face value; 5=skeptical, questions everything)
-   - Literalism: 1-5 (1=flexible interpretation, reads between lines; 5=literal, exact interpretation)
-   - Empathy: 1-5 (1=detached, focuses on facts; 5=empathetic, considers emotional context)
-
-CRITICAL: You MUST respond with ONLY a valid JSON object. No markdown, no code blocks, no explanations. Just the JSON.
-
-Format:
-{{
-  "background": "the merged background text in first person",
-  "disposition": {{
-    "skepticism": 3,
-    "literalism": 3,
-    "empathy": 3
-  }}
-}}
-
-Trait inference examples:
-- "I'm a lawyer" → skepticism: 4, literalism: 5, empathy: 2
-- "I'm a therapist" → skepticism: 2, literalism: 2, empathy: 5
-- "I'm an engineer" → skepticism: 3, literalism: 4, empathy: 3
-- "I've been burned before by trusting people" → skepticism: 5, literalism: 3, empathy: 3
-- "I try to understand what people really mean" → skepticism: 3, literalism: 2, empathy: 4
-- "I take contracts very seriously" → skepticism: 4, literalism: 5, empathy: 2"""
-    else:
-        prompt = f"""You are helping maintain a memory bank's background/profile.
-
-Current background: {current if current else "(empty)"}
-
-New information to add: {new_info}
-
-Instructions:
-1. Merge the new information with the current background
-2. If there are conflicts (e.g., different birthplaces), the NEW information overwrites the old
+1. Merge the new information with the current mission
+2. If there are conflicts, the NEW information overwrites the old
 3. Keep additions that don't conflict
 4. Output in FIRST PERSON ("I") perspective
 5. Be concise - keep it under 500 characters
-6. Return ONLY the merged background text, no explanations
+6. Return ONLY the merged mission text, no explanations
 
-Merged background:"""
+Merged mission:"""
 
     try:
-        # Prepare messages
         messages = [{"role": "user", "content": prompt}]
 
-        if infer_disposition:
-            # Use structured output with Pydantic model for disposition inference
-            try:
-                parsed = await llm_config.call(
-                    messages=messages,
-                    response_format=BackgroundMergeResponse,
-                    scope="bank_background",
-                    temperature=0.3,
-                    max_completion_tokens=8192,
-                )
-                logger.info(f"Successfully got structured response: background={parsed.background[:100]}")
-
-                # Convert Pydantic model to dict format
-                return {"background": parsed.background, "disposition": parsed.disposition.model_dump()}
-            except Exception as e:
-                logger.warning(f"Structured output failed, falling back to manual parsing: {e}")
-                # Fall through to manual parsing below
-
-        # Manual parsing fallback or non-disposition merge
         content = await llm_config.call(
-            messages=messages, scope="bank_background", temperature=0.3, max_completion_tokens=8192
+            messages=messages, scope="bank_mission", temperature=0.3, max_completion_tokens=8192
         )
 
-        logger.info(f"LLM response for background merge (first 500 chars): {content[:500]}")
+        logger.info(f"LLM response for mission merge (first 500 chars): {content[:500]}")
 
-        if infer_disposition:
-            # Parse JSON response - try multiple extraction methods
-            result = None
-
-            # Method 1: Direct parse
-            try:
-                result = json.loads(content)
-                logger.info("Successfully parsed JSON directly")
-            except json.JSONDecodeError:
-                pass
-
-            # Method 2: Extract from markdown code blocks
-            if result is None:
-                # Remove markdown code blocks
-                code_block_match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", content, re.DOTALL)
-                if code_block_match:
-                    try:
-                        result = json.loads(code_block_match.group(1))
-                        logger.info("Successfully extracted JSON from markdown code block")
-                    except json.JSONDecodeError:
-                        pass
-
-            # Method 3: Find nested JSON structure
-            if result is None:
-                # Look for JSON object with nested structure
-                json_match = re.search(
-                    r'\{[^{}]*"background"[^{}]*"disposition"[^{}]*\{[^{}]*\}[^{}]*\}', content, re.DOTALL
-                )
-                if json_match:
-                    try:
-                        result = json.loads(json_match.group())
-                        logger.info("Successfully extracted JSON using nested pattern")
-                    except json.JSONDecodeError:
-                        pass
-
-            # All parsing methods failed - use fallback
-            if result is None:
-                logger.warning(f"Failed to extract JSON from LLM response. Raw content: {content[:200]}")
-                # Fallback: use new_info as background with default disposition
-                return {
-                    "background": new_info if new_info else current if current else "",
-                    "disposition": DEFAULT_DISPOSITION.copy(),
-                }
-
-            # Validate disposition values
-            disposition = result.get("disposition", {})
-            for key in ["skepticism", "literalism", "empathy"]:
-                if key not in disposition:
-                    disposition[key] = 3  # Default to neutral
-                else:
-                    # Clamp to [1, 5] and convert to int
-                    disposition[key] = max(1, min(5, int(disposition[key])))
-
-            result["disposition"] = disposition
-
-            # Ensure background exists
-            if "background" not in result or not result["background"]:
-                result["background"] = new_info if new_info else ""
-
-            return result
-        else:
-            # Just background merge
-            merged = content
-            if not merged or merged.lower() in ["(empty)", "none", "n/a"]:
-                merged = new_info if new_info else ""
-            return {"background": merged}
+        merged = content.strip()
+        if not merged or merged.lower() in ["(empty)", "none", "n/a"]:
+            merged = new_info if new_info else ""
+        return {"mission": merged}
 
     except Exception as e:
-        logger.error(f"Error merging background with LLM: {e}")
+        logger.error(f"Error merging mission with LLM: {e}")
         # Fallback: just append new info
         if current:
             merged = f"{current} {new_info}".strip()
         else:
             merged = new_info
 
-        result = {"background": merged}
-        if infer_disposition:
-            result["disposition"] = DEFAULT_DISPOSITION.copy()
-        return result
+        return {"mission": merged}
 
 
 async def list_banks(pool) -> list:
@@ -358,12 +236,12 @@ async def list_banks(pool) -> list:
         pool: Database connection pool
 
     Returns:
-        List of dicts with bank_id, name, disposition, background, created_at, updated_at
+        List of dicts with bank_id, name, disposition, mission, created_at, updated_at
     """
     async with acquire_with_retry(pool) as conn:
         rows = await conn.fetch(
             f"""
-            SELECT bank_id, name, disposition, background, created_at, updated_at
+            SELECT bank_id, name, disposition, mission, created_at, updated_at
             FROM {fq_table("banks")}
             ORDER BY updated_at DESC
             """
@@ -381,7 +259,7 @@ async def list_banks(pool) -> list:
                     "bank_id": row["bank_id"],
                     "name": row["name"],
                     "disposition": disposition_data,
-                    "background": row["background"],
+                    "mission": row["mission"] or "",
                     "created_at": row["created_at"].isoformat() if row["created_at"] else None,
                     "updated_at": row["updated_at"].isoformat() if row["updated_at"] else None,
                 }

hindsight-api/hindsight_api/engine/retain/fact_storage.py

@@ -41,10 +41,10 @@ async def insert_facts_batch(
     contexts = []
     fact_types = []
     confidence_scores = []
-    access_counts = []
     metadata_jsons = []
     chunk_ids = []
     document_ids = []
+    tags_list = []
 
     for fact in facts:
         fact_texts.append(fact.fact_text)
@@ -60,21 +60,35 @@ async def insert_facts_batch(
         fact_types.append(fact.fact_type)
         # confidence_score is only for opinion facts
         confidence_scores.append(1.0 if fact.fact_type == "opinion" else None)
-        access_counts.append(0)  # Initial access count
         metadata_jsons.append(json.dumps(fact.metadata))
         chunk_ids.append(fact.chunk_id)
         # Use per-fact document_id if available, otherwise fallback to batch-level document_id
         document_ids.append(fact.document_id if fact.document_id else document_id)
+        # Convert tags to JSON string for proper batch insertion (PostgreSQL unnest doesn't handle 2D arrays well)
+        tags_list.append(json.dumps(fact.tags if fact.tags else []))
 
     # Batch insert all facts
+    # Note: tags are passed as JSON strings and converted back to varchar[] via jsonb_array_elements_text + array_agg
     results = await conn.fetch(
         f"""
-        INSERT INTO {fq_table("memory_units")} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
-                                 context, fact_type, confidence_score, access_count, metadata, chunk_id, document_id)
-        SELECT $1, * FROM unnest(
-            $2::text[], $3::vector[], $4::timestamptz[], $5::timestamptz[], $6::timestamptz[], $7::timestamptz[],
-            $8::text[], $9::text[], $10::float[], $11::int[], $12::jsonb[], $13::text[], $14::text[]
+        WITH input_data AS (
+            SELECT * FROM unnest(
+                $2::text[], $3::vector[], $4::timestamptz[], $5::timestamptz[], $6::timestamptz[], $7::timestamptz[],
+                $8::text[], $9::text[], $10::float[], $11::jsonb[], $12::text[], $13::text[], $14::jsonb[]
+            ) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
+                   context, fact_type, confidence_score, metadata, chunk_id, document_id, tags_json)
         )
+        INSERT INTO {fq_table("memory_units")} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
+                                 context, fact_type, confidence_score, metadata, chunk_id, document_id, tags)
+        SELECT
+            $1,
+            text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
+            context, fact_type, confidence_score, metadata, chunk_id, document_id,
+            COALESCE(
+                (SELECT array_agg(elem) FROM jsonb_array_elements_text(tags_json) AS elem),
+                '{{}}'::varchar[]
+            )
+        FROM input_data
         RETURNING id
         """,
         bank_id,
@@ -87,10 +101,10 @@ async def insert_facts_batch(
         contexts,
         fact_types,
         confidence_scores,
-        access_counts,
         metadata_jsons,
         chunk_ids,
         document_ids,
+        tags_list,
     )
 
     unit_ids = [str(row["id"]) for row in results]
@@ -109,7 +123,7 @@ async def ensure_bank_exists(conn, bank_id: str) -> None:
     """
     await conn.execute(
         f"""
-        INSERT INTO {fq_table("banks")} (bank_id, disposition, background)
+        INSERT INTO {fq_table("banks")} (bank_id, disposition, mission)
         VALUES ($1, $2::jsonb, $3)
         ON CONFLICT (bank_id) DO UPDATE
         SET updated_at = NOW()
@@ -121,7 +135,13 @@ async def ensure_bank_exists(conn, bank_id: str) -> None:
 
 
 async def handle_document_tracking(
-    conn, bank_id: str, document_id: str, combined_content: str, is_first_batch: bool, retain_params: dict | None = None
+    conn,
+    bank_id: str,
+    document_id: str,
+    combined_content: str,
+    is_first_batch: bool,
+    retain_params: dict | None = None,
+    document_tags: list[str] | None = None,
 ) -> None:
     """
     Handle document tracking in the database.
@@ -133,6 +153,7 @@ async def handle_document_tracking(
         combined_content: Combined content text from all content items
         is_first_batch: Whether this is the first batch (for chunked operations)
         retain_params: Optional parameters passed during retain (context, event_date, etc.)
+        document_tags: Optional list of tags to associate with the document
     """
     import hashlib
 
@@ -149,13 +170,14 @@ async def handle_document_tracking(
     # Insert document (or update if exists from concurrent operations)
     await conn.execute(
         f"""
-        INSERT INTO {fq_table("documents")} (id, bank_id, original_text, content_hash, metadata, retain_params)
-        VALUES ($1, $2, $3, $4, $5, $6)
+        INSERT INTO {fq_table("documents")} (id, bank_id, original_text, content_hash, metadata, retain_params, tags)
+        VALUES ($1, $2, $3, $4, $5, $6, $7)
         ON CONFLICT (id, bank_id) DO UPDATE
         SET original_text = EXCLUDED.original_text,
             content_hash = EXCLUDED.content_hash,
             metadata = EXCLUDED.metadata,
             retain_params = EXCLUDED.retain_params,
+            tags = EXCLUDED.tags,
             updated_at = NOW()
         """,
         document_id,
@@ -164,4 +186,5 @@ async def handle_document_tracking(
         content_hash,
         json.dumps({}),  # Empty metadata dict
         json.dumps(retain_params) if retain_params else None,
+        document_tags or [],
     )

hindsight-api/hindsight_api/engine/retain/link_utils.py

@@ -479,14 +479,18 @@ async def create_temporal_links_batch_per_fact(
 
         if links:
             insert_start = time_mod.time()
-            await conn.executemany(
-                f"""
-                INSERT INTO {fq_table("memory_links")} (from_unit_id, to_unit_id, link_type, weight, entity_id)
-                VALUES ($1, $2, $3, $4, $5)
-                ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
-                """,
-                links,
-            )
+            # Batch inserts to avoid timeout on large batches
+            BATCH_SIZE = 1000
+            for batch_start in range(0, len(links), BATCH_SIZE):
+                batch = links[batch_start : batch_start + BATCH_SIZE]
+                await conn.executemany(
+                    f"""
+                    INSERT INTO {fq_table("memory_links")} (from_unit_id, to_unit_id, link_type, weight, entity_id)
+                    VALUES ($1, $2, $3, $4, $5)
+                    ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
+                    """,
+                    batch,
+                )
             _log(log_buffer, f"      [7.4] Insert {len(links)} temporal links: {time_mod.time() - insert_start:.3f}s")
 
         return len(links)
@@ -644,14 +648,18 @@ async def create_semantic_links_batch(
 
         if all_links:
             insert_start = time_mod.time()
-            await conn.executemany(
-                f"""
-                INSERT INTO {fq_table("memory_links")} (from_unit_id, to_unit_id, link_type, weight, entity_id)
-                VALUES ($1, $2, $3, $4, $5)
-                ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
-                """,
-                all_links,
-            )
+            # Batch inserts to avoid timeout on large batches
+            BATCH_SIZE = 1000
+            for batch_start in range(0, len(all_links), BATCH_SIZE):
+                batch = all_links[batch_start : batch_start + BATCH_SIZE]
+                await conn.executemany(
+                    f"""
+                    INSERT INTO {fq_table("memory_links")} (from_unit_id, to_unit_id, link_type, weight, entity_id)
+                    VALUES ($1, $2, $3, $4, $5)
+                    ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
+                    """,
+                    batch,
+                )
             _log(
                 log_buffer, f"      [8.3] Insert {len(all_links)} semantic links: {time_mod.time() - insert_start:.3f}s"
             )
@@ -746,17 +754,14 @@ async def create_causal_links_batch(
         causal_relations_per_fact: List of causal relations for each fact.
             Each element is a list of dicts with:
             - target_fact_index: Index into unit_ids for the target fact
-            - relation_type: "causes", "caused_by", "enables", or "prevents"
+            - relation_type: "caused_by"
             - strength: Float in [0.0, 1.0] representing relationship strength
 
     Returns:
         Number of causal links created
 
-    Causal link types:
-    - "causes": This fact directly causes the target fact (forward causation)
-    - "caused_by": This fact was caused by the target fact (backward causation)
-    - "enables": This fact enables/allows the target fact (enablement)
-    - "prevents": This fact prevents/blocks the target fact (prevention)
+    Causal link type:
+    - "caused_by": This fact was caused by the target fact
     """
     if not unit_ids or not causal_relations_per_fact:
         return 0
@@ -779,8 +784,8 @@ async def create_causal_links_batch(
                 relation_type = relation["relation_type"]
                 strength = relation.get("strength", 1.0)
 
-                # Validate relation_type - must match database constraint
-                valid_types = {"causes", "caused_by", "enables", "prevents"}
+                # Validate relation_type - only "caused_by" is supported (DB constraint)
+                valid_types = {"caused_by"}
                 if relation_type not in valid_types:
                     logger.error(
                         f"Invalid relation_type '{relation_type}' (type: {type(relation_type).__name__}) "

hindsight-api/hindsight_api/engine/retain/observation_regeneration.py

@@ -1,254 +0,0 @@
-"""
-Observation regeneration for retain pipeline.
-
-Regenerates entity observations as part of the retain transaction.
-"""
-
-import logging
-import time
-import uuid
-from datetime import UTC, datetime
-
-from ...config import get_config
-from ..memory_engine import fq_table
-from ..search import observation_utils
-from . import embedding_utils
-from .types import EntityLink
-
-logger = logging.getLogger(__name__)
-
-
-def utcnow():
-    """Get current UTC time."""
-    return datetime.now(UTC)
-
-
-# Simple dataclass-like container for facts (avoid importing from memory_engine)
-class MemoryFactForObservation:
-    def __init__(self, id: str, text: str, fact_type: str, context: str, occurred_start: str | None):
-        self.id = id
-        self.text = text
-        self.fact_type = fact_type
-        self.context = context
-        self.occurred_start = occurred_start
-
-
-async def regenerate_observations_batch(
-    conn, embeddings_model, llm_config, bank_id: str, entity_links: list[EntityLink], log_buffer: list[str] = None
-) -> None:
-    """
-    Regenerate observations for top entities in this batch.
-
-    Called INSIDE the retain transaction for atomicity - if observations
-    fail, the entire retain batch is rolled back.
-
-    Args:
-        conn: Database connection (from the retain transaction)
-        embeddings_model: Embeddings model for generating observation embeddings
-        llm_config: LLM configuration for observation extraction
-        bank_id: Bank identifier
-        entity_links: Entity links from this batch
-        log_buffer: Optional log buffer for timing
-    """
-    config = get_config()
-    TOP_N_ENTITIES = config.observation_top_entities
-    MIN_FACTS_THRESHOLD = config.observation_min_facts
-
-    if not entity_links:
-        return
-
-    # Count mentions per entity in this batch
-    entity_mention_counts: dict[str, int] = {}
-    for link in entity_links:
-        if link.entity_id:
-            entity_id = str(link.entity_id)
-            entity_mention_counts[entity_id] = entity_mention_counts.get(entity_id, 0) + 1
-
-    if not entity_mention_counts:
-        return
-
-    # Sort by mention count descending and take top N
-    sorted_entities = sorted(entity_mention_counts.items(), key=lambda x: x[1], reverse=True)
-    entities_to_process = [e[0] for e in sorted_entities[:TOP_N_ENTITIES]]
-
-    obs_start = time.time()
-
-    # Convert to UUIDs
-    entity_uuids = [uuid.UUID(eid) if isinstance(eid, str) else eid for eid in entities_to_process]
-
-    # Batch query for entity names
-    entity_rows = await conn.fetch(
-        f"""
-        SELECT id, canonical_name FROM {fq_table("entities")}
-        WHERE id = ANY($1) AND bank_id = $2
-        """,
-        entity_uuids,
-        bank_id,
-    )
-    entity_names = {row["id"]: row["canonical_name"] for row in entity_rows}
-
-    # Batch query for fact counts
-    fact_counts = await conn.fetch(
-        f"""
-        SELECT ue.entity_id, COUNT(*) as cnt
-        FROM {fq_table("unit_entities")} ue
-        JOIN {fq_table("memory_units")} mu ON ue.unit_id = mu.id
-        WHERE ue.entity_id = ANY($1) AND mu.bank_id = $2
-        GROUP BY ue.entity_id
-        """,
-        entity_uuids,
-        bank_id,
-    )
-    entity_fact_counts = {row["entity_id"]: row["cnt"] for row in fact_counts}
-
-    # Filter entities that meet the threshold
-    entities_with_names = []
-    for entity_id in entities_to_process:
-        entity_uuid = uuid.UUID(entity_id) if isinstance(entity_id, str) else entity_id
-        if entity_uuid not in entity_names:
-            continue
-        fact_count = entity_fact_counts.get(entity_uuid, 0)
-        if fact_count >= MIN_FACTS_THRESHOLD:
-            entities_with_names.append((entity_id, entity_names[entity_uuid]))
-
-    if not entities_with_names:
-        return
-
-    # Process entities SEQUENTIALLY (asyncpg doesn't allow concurrent queries on same connection)
-    # We must use the same connection to stay in the retain transaction
-    total_observations = 0
-
-    for entity_id, entity_name in entities_with_names:
-        try:
-            obs_ids = await _regenerate_entity_observations(
-                conn, embeddings_model, llm_config, bank_id, entity_id, entity_name
-            )
-            total_observations += len(obs_ids)
-        except Exception as e:
-            logger.error(f"[OBSERVATIONS] Error processing entity {entity_id}: {e}")
-
-    obs_time = time.time() - obs_start
-    if log_buffer is not None:
-        log_buffer.append(
-            f"[11] Observations: {total_observations} observations for {len(entities_with_names)} entities in {obs_time:.3f}s"
-        )
-
-
-async def _regenerate_entity_observations(
-    conn, embeddings_model, llm_config, bank_id: str, entity_id: str, entity_name: str
-) -> list[str]:
-    """
-    Regenerate observations for a single entity.
-
-    Uses the provided connection (part of retain transaction).
-
-    Args:
-        conn: Database connection (from the retain transaction)
-        embeddings_model: Embeddings model
-        llm_config: LLM configuration
-        bank_id: Bank identifier
-        entity_id: Entity UUID
-        entity_name: Canonical name of the entity
-
-    Returns:
-        List of created observation IDs
-    """
-    entity_uuid = uuid.UUID(entity_id) if isinstance(entity_id, str) else entity_id
-
-    # Get all facts mentioning this entity (exclude observations themselves)
-    rows = await conn.fetch(
-        f"""
-        SELECT mu.id, mu.text, mu.context, mu.occurred_start, mu.fact_type
-        FROM {fq_table("memory_units")} mu
-        JOIN {fq_table("unit_entities")} ue ON mu.id = ue.unit_id
-        WHERE mu.bank_id = $1
-          AND ue.entity_id = $2
-          AND mu.fact_type IN ('world', 'experience')
-        ORDER BY mu.occurred_start DESC
-        LIMIT 50
-        """,
-        bank_id,
-        entity_uuid,
-    )
-
-    if not rows:
-        return []
-
-    # Convert to fact objects for observation extraction
-    facts = []
-    for row in rows:
-        occurred_start = row["occurred_start"].isoformat() if row["occurred_start"] else None
-        facts.append(
-            MemoryFactForObservation(
-                id=str(row["id"]),
-                text=row["text"],
-                fact_type=row["fact_type"],
-                context=row["context"],
-                occurred_start=occurred_start,
-            )
-        )
-
-    # Extract observations using LLM
-    observations = await observation_utils.extract_observations_from_facts(llm_config, entity_name, facts)
-
-    if not observations:
-        return []
-
-    # Delete old observations for this entity
-    await conn.execute(
-        f"""
-        DELETE FROM {fq_table("memory_units")}
-        WHERE id IN (
-            SELECT mu.id
-            FROM {fq_table("memory_units")} mu
-            JOIN {fq_table("unit_entities")} ue ON mu.id = ue.unit_id
-            WHERE mu.bank_id = $1
-              AND mu.fact_type = 'observation'
-              AND ue.entity_id = $2
-        )
-        """,
-        bank_id,
-        entity_uuid,
-    )
-
-    # Generate embeddings for new observations
-    embeddings = await embedding_utils.generate_embeddings_batch(embeddings_model, observations)
-
-    # Insert new observations
-    current_time = utcnow()
-    created_ids = []
-
-    for obs_text, embedding in zip(observations, embeddings):
-        result = await conn.fetchrow(
-            f"""
-            INSERT INTO {fq_table("memory_units")} (
-                bank_id, text, embedding, context, event_date,
-                occurred_start, occurred_end, mentioned_at,
-                fact_type, access_count
-            )
-            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'observation', 0)
-            RETURNING id
-            """,
-            bank_id,
-            obs_text,
-            str(embedding),
-            f"observation about {entity_name}",
-            current_time,
-            current_time,
-            current_time,
-            current_time,
-        )
-        obs_id = str(result["id"])
-        created_ids.append(obs_id)
-
-        # Link observation to entity
-        await conn.execute(
-            f"""
-            INSERT INTO {fq_table("unit_entities")} (unit_id, entity_id)
-            VALUES ($1, $2)
-            """,
-            uuid.UUID(obs_id),
-            entity_uuid,
-        )
-
-    return created_ids

hindsight-api/hindsight_api/engine/retain/orchestrator.py

@@ -18,6 +18,7 @@ def utcnow():
     return datetime.now(UTC)
 
 
+from ..response_models import TokenUsage
 from . import (
     chunk_storage,
     deduplication,
@@ -26,9 +27,8 @@ from . import (
     fact_extraction,
     fact_storage,
     link_creation,
-    observation_regeneration,
 )
-from .types import ExtractedFact, ProcessedFact, RetainContent, RetainContentDict
+from .types import EntityLink, ExtractedFact, ProcessedFact, RetainContent, RetainContentDict
 
 logger = logging.getLogger(__name__)
 
@@ -38,7 +38,6 @@ async def retain_batch(
     embeddings_model,
     llm_config,
     entity_resolver,
-    task_backend,
     format_date_fn,
     duplicate_checker_fn,
     bank_id: str,
@@ -47,7 +46,8 @@ async def retain_batch(
     is_first_batch: bool = True,
     fact_type_override: str | None = None,
     confidence_score: float | None = None,
-) -> list[list[str]]:
+    document_tags: list[str] | None = None,
+) -> tuple[list[list[str]], TokenUsage]:
     """
     Process a batch of content through the retain pipeline.
 
@@ -56,7 +56,6 @@ async def retain_batch(
         embeddings_model: Embeddings model for generating embeddings
         llm_config: LLM configuration for fact extraction
         entity_resolver: Entity resolver for entity processing
-        task_backend: Task backend for background jobs
         format_date_fn: Function to format datetime to readable string
         duplicate_checker_fn: Function to check for duplicate facts
         bank_id: Bank identifier
@@ -65,9 +64,10 @@ async def retain_batch(
         is_first_batch: Whether this is the first batch
         fact_type_override: Override fact type for all facts
         confidence_score: Confidence score for opinions
+        document_tags: Tags applied to all items in this batch
 
     Returns:
-        List of unit ID lists (one list per content item)
+        Tuple of (unit ID lists, token usage for fact extraction)
     """
     start_time = time.time()
     total_chars = sum(len(item.get("content", "")) for item in contents_dicts)
@@ -86,12 +86,16 @@ async def retain_batch(
     # Convert dicts to RetainContent objects
     contents = []
     for item in contents_dicts:
+        # Merge item-level tags with document-level tags
+        item_tags = item.get("tags", []) or []
+        merged_tags = list(set(item_tags + (document_tags or [])))
         content = RetainContent(
             content=item["content"],
             context=item.get("context", ""),
             event_date=item.get("event_date") or utcnow(),
             metadata=item.get("metadata", {}),
             entities=item.get("entities", []),
+            tags=merged_tags,
         )
         contents.append(content)
 
@@ -99,7 +103,7 @@ async def retain_batch(
     step_start = time.time()
     extract_opinions = fact_type_override == "opinion"
 
-    extracted_facts, chunks = await fact_extraction.extract_facts_from_contents(
+    extracted_facts, chunks, usage = await fact_extraction.extract_facts_from_contents(
         contents, llm_config, agent_name, extract_opinions
     )
     log_buffer.append(
@@ -129,7 +133,7 @@ async def retain_batch(
                         if first_item.get("metadata"):
                             retain_params["metadata"] = first_item["metadata"]
                     await fact_storage.handle_document_tracking(
-                        conn, bank_id, document_id, combined_content, is_first_batch, retain_params
+                        conn, bank_id, document_id, combined_content, is_first_batch, retain_params, document_tags
                     )
                 else:
                     # Check for per-item document_ids
@@ -157,14 +161,14 @@ async def retain_batch(
                             if first_item.get("metadata"):
                                 retain_params["metadata"] = first_item["metadata"]
                         await fact_storage.handle_document_tracking(
-                            conn, bank_id, doc_id, combined_content, is_first_batch, retain_params
+                            conn, bank_id, doc_id, combined_content, is_first_batch, retain_params, document_tags
                         )
 
         total_time = time.time() - start_time
         logger.info(
             f"RETAIN_BATCH COMPLETE: 0 facts extracted from {len(contents)} contents in {total_time:.3f}s (document tracked, no facts)"
         )
-        return [[] for _ in contents]
+        return [[] for _ in contents], usage
 
     # Apply fact_type_override if provided
     if fact_type_override:
@@ -223,7 +227,7 @@ async def retain_batch(
                         retain_params["metadata"] = first_item["metadata"]
 
                 await fact_storage.handle_document_tracking(
-                    conn, bank_id, document_id, combined_content, is_first_batch, retain_params
+                    conn, bank_id, document_id, combined_content, is_first_batch, retain_params, document_tags
                 )
                 document_ids_added.append(document_id)
                 doc_id_mapping[None] = document_id  # For backwards compatibility
@@ -267,7 +271,13 @@ async def retain_batch(
                                     retain_params["metadata"] = first_item["metadata"]
 
                             await fact_storage.handle_document_tracking(
-                                conn, bank_id, actual_doc_id, combined_content, is_first_batch, retain_params
+                                conn,
+                                bank_id,
+                                actual_doc_id,
+                                combined_content,
+                                is_first_batch,
+                                retain_params,
+                                document_tags,
                             )
                             document_ids_added.append(actual_doc_id)
 
@@ -344,7 +354,7 @@ async def retain_batch(
             non_duplicate_facts = deduplication.filter_duplicates(processed_facts, is_duplicate_flags)
 
             if not non_duplicate_facts:
-                return [[] for _ in contents]
+                return [[] for _ in contents], usage
 
             # Insert facts (document_id is now stored per-fact)
             step_start = time.time()
@@ -394,17 +404,9 @@ async def retain_batch(
             causal_link_count = await link_creation.create_causal_links_batch(conn, unit_ids, non_duplicate_facts)
             log_buffer.append(f"[10] Causal links: {causal_link_count} links in {time.time() - step_start:.3f}s")
 
-            # Regenerate observations INSIDE transaction for atomicity
-            await observation_regeneration.regenerate_observations_batch(
-                conn, embeddings_model, llm_config, bank_id, entity_links, log_buffer
-            )
-
             # Map results back to original content items
             result_unit_ids = _map_results_to_contents(contents, extracted_facts, is_duplicate_flags, unit_ids)
 
-        # Trigger background tasks AFTER transaction commits (opinion reinforcement only)
-        await _trigger_background_tasks(task_backend, bank_id, unit_ids, non_duplicate_facts)
-
         # Log final summary
         total_time = time.time() - start_time
         log_buffer.append(f"{'=' * 60}")
@@ -415,7 +417,7 @@ async def retain_batch(
 
         logger.info("\n" + "\n".join(log_buffer) + "\n")
 
-        return result_unit_ids
+        return result_unit_ids, usage
 
 
 def _map_results_to_contents(
@@ -446,24 +448,3 @@ def _map_results_to_contents(
         result_unit_ids.append(content_unit_ids)
 
     return result_unit_ids
-
-
-async def _trigger_background_tasks(
-    task_backend,
-    bank_id: str,
-    unit_ids: list[str],
-    facts: list[ProcessedFact],
-) -> None:
-    """Trigger opinion reinforcement as background task (after transaction commits)."""
-    # Trigger opinion reinforcement if there are entities
-    fact_entities = [[e.name for e in fact.entities] for fact in facts]
-    if any(fact_entities):
-        await task_backend.submit_task(
-            {
-                "type": "reinforce_opinion",
-                "bank_id": bank_id,
-                "created_unit_ids": unit_ids,
-                "unit_texts": [fact.fact_text for fact in facts],
-                "unit_entities": fact_entities,
-            }
-        )

hindsight-api/hindsight_api/engine/retain/types.py

@@ -21,6 +21,7 @@ class RetainContentDict(TypedDict, total=False):
         metadata: Custom key-value metadata (optional)
         document_id: Document ID for this content item (optional)
         entities: User-provided entities to merge with extracted entities (optional)
+        tags: Visibility scope tags for this content item (optional)
     """
 
     content: str  # Required
@@ -29,6 +30,7 @@ class RetainContentDict(TypedDict, total=False):
     metadata: dict[str, str]
     document_id: str
     entities: list[dict[str, str]]  # [{"text": "...", "type": "..."}]
+    tags: list[str]  # Visibility scope tags
 
 
 def _now_utc() -> datetime:
@@ -49,6 +51,7 @@ class RetainContent:
     event_date: datetime = field(default_factory=_now_utc)
     metadata: dict[str, str] = field(default_factory=dict)
     entities: list[dict[str, str]] = field(default_factory=list)  # User-provided entities
+    tags: list[str] = field(default_factory=list)  # Visibility scope tags
 
 
 @dataclass
@@ -83,10 +86,10 @@ class CausalRelation:
     """
     Causal relationship between facts.
 
-    Represents how one fact causes, enables, or prevents another.
+    Represents how one fact was caused by another.
     """
 
-    relation_type: str  # "causes", "enables", "prevents", "caused_by"
+    relation_type: str  # "caused_by"
     target_fact_index: int  # Index of the target fact in the batch
     strength: float = 1.0  # Strength of the causal relationship
 
@@ -113,6 +116,7 @@ class ExtractedFact:
     context: str = ""
     mentioned_at: datetime | None = None
     metadata: dict[str, str] = field(default_factory=dict)
+    tags: list[str] = field(default_factory=list)  # Visibility scope tags
 
 
 @dataclass
@@ -158,6 +162,9 @@ class ProcessedFact:
     # Track which content this fact came from (for user entity merging)
     content_index: int = 0
 
+    # Visibility scope tags
+    tags: list[str] = field(default_factory=list)
+
     @property
     def is_duplicate(self) -> bool:
         """Check if this fact was marked as a duplicate."""
@@ -201,6 +208,7 @@ class ProcessedFact:
             causal_relations=extracted_fact.causal_relations,
             chunk_id=chunk_id,
             content_index=extracted_fact.content_index,
+            tags=extracted_fact.tags,
         )
 
 
@@ -232,6 +240,7 @@ class RetainBatch:
     document_id: str | None = None
     fact_type_override: str | None = None
     confidence_score: float | None = None
+    document_tags: list[str] = field(default_factory=list)  # Tags applied to all items
 
     # Extracted data (populated during processing)
     extracted_facts: list[ExtractedFact] = field(default_factory=list)

hindsight-api/hindsight_api/engine/search/graph_retrieval.py

@@ -11,7 +11,8 @@ from abc import ABC, abstractmethod
 
 from ..db_utils import acquire_with_retry
 from ..memory_engine import fq_table
-from .types import RetrievalResult
+from .tags import TagsMatch, filter_results_by_tags
+from .types import MPFPTimings, RetrievalResult
 
 logger = logging.getLogger(__name__)
 
@@ -42,7 +43,10 @@ class GraphRetriever(ABC):
         query_text: str | None = None,
         semantic_seeds: list[RetrievalResult] | None = None,
         temporal_seeds: list[RetrievalResult] | None = None,
-    ) -> list[RetrievalResult]:
+        adjacency=None,  # TypedAdjacency, optional pre-loaded graph
+        tags: list[str] | None = None,  # Visibility scope tags for filtering
+        tags_match: TagsMatch = "any",  # How to match tags: 'any' (OR) or 'all' (AND)
+    ) -> tuple[list[RetrievalResult], MPFPTimings | None]:
         """
         Retrieve relevant facts via graph traversal.
 
@@ -55,9 +59,11 @@ class GraphRetriever(ABC):
             query_text: Original query text (optional, for some strategies)
             semantic_seeds: Pre-computed semantic entry points (from semantic retrieval)
             temporal_seeds: Pre-computed temporal entry points (from temporal retrieval)
+            adjacency: Pre-loaded typed adjacency graph (optional, for MPFP)
+            tags: Optional list of tags for visibility filtering (OR matching)
 
         Returns:
-            List of RetrievalResult objects with activation scores set
+            Tuple of (List of RetrievalResult with activation scores, optional timing info)
         """
         pass
 
@@ -111,7 +117,10 @@ class BFSGraphRetriever(GraphRetriever):
         query_text: str | None = None,
         semantic_seeds: list[RetrievalResult] | None = None,
         temporal_seeds: list[RetrievalResult] | None = None,
-    ) -> list[RetrievalResult]:
+        adjacency=None,  # Not used by BFS
+        tags: list[str] | None = None,
+        tags_match: TagsMatch = "any",
+    ) -> tuple[list[RetrievalResult], MPFPTimings | None]:
         """
         Retrieve facts using BFS spreading activation.
 
@@ -122,11 +131,14 @@ class BFSGraphRetriever(GraphRetriever):
         4. Return visited nodes up to budget
 
         Note: BFS finds its own entry points via embedding search.
-        The semantic_seeds and temporal_seeds parameters are accepted
+        The semantic_seeds, temporal_seeds, and adjacency parameters are accepted
         for interface compatibility but not used.
         """
         async with acquire_with_retry(pool) as conn:
-            return await self._retrieve_with_conn(conn, query_embedding_str, bank_id, fact_type, budget)
+            results = await self._retrieve_with_conn(
+                conn, query_embedding_str, bank_id, fact_type, budget, tags=tags, tags_match=tags_match
+            )
+            return results, None
 
     async def _retrieve_with_conn(
         self,
@@ -135,33 +147,46 @@ class BFSGraphRetriever(GraphRetriever):
         bank_id: str,
         fact_type: str,
         budget: int,
+        tags: list[str] | None = None,
+        tags_match: TagsMatch = "any",
     ) -> list[RetrievalResult]:
         """Internal implementation with connection."""
+        from .tags import build_tags_where_clause_simple
+
+        tags_clause = build_tags_where_clause_simple(tags, 6, match=tags_match)
+        params = [query_embedding_str, bank_id, fact_type, self.entry_point_threshold, self.entry_point_limit]
+        if tags:
+            params.append(tags)
 
         # Step 1: Find entry points
         entry_points = await conn.fetch(
             f"""
             SELECT id, text, context, event_date, occurred_start, occurred_end,
-                   mentioned_at, access_count, embedding, fact_type, document_id, chunk_id,
+                   mentioned_at, embedding, fact_type, document_id, chunk_id, tags,
                    1 - (embedding <=> $1::vector) AS similarity
             FROM {fq_table("memory_units")}
             WHERE bank_id = $2
               AND embedding IS NOT NULL
               AND fact_type = $3
               AND (1 - (embedding <=> $1::vector)) >= $4
+              {tags_clause}
             ORDER BY embedding <=> $1::vector
             LIMIT $5
             """,
-            query_embedding_str,
-            bank_id,
-            fact_type,
-            self.entry_point_threshold,
-            self.entry_point_limit,
+            *params,
         )
 
         if not entry_points:
+            logger.debug(
+                f"[BFS] No entry points found for fact_type={fact_type} (tags={tags}, tags_match={tags_match})"
+            )
             return []
 
+        logger.debug(
+            f"[BFS] Found {len(entry_points)} entry points for fact_type={fact_type} "
+            f"(tags={tags}, tags_match={tags_match})"
+        )
+
         # Step 2: BFS spreading activation
         visited = set()
         results = []
@@ -191,8 +216,8 @@ class BFSGraphRetriever(GraphRetriever):
                 neighbors = await conn.fetch(
                     f"""
                     SELECT mu.id, mu.text, mu.context, mu.occurred_start, mu.occurred_end,
-                           mu.mentioned_at, mu.access_count, mu.embedding, mu.fact_type,
-                           mu.document_id, mu.chunk_id,
+                           mu.mentioned_at, mu.embedding, mu.fact_type,
+                           mu.document_id, mu.chunk_id, mu.tags,
                            ml.weight, ml.link_type, ml.from_unit_id
                     FROM {fq_table("memory_links")} ml
                     JOIN {fq_table("memory_units")} mu ON ml.to_unit_id = mu.id
@@ -232,4 +257,8 @@ class BFSGraphRetriever(GraphRetriever):
                             neighbor_result = RetrievalResult.from_db_row(dict(n))
                             queue.append((neighbor_result, new_activation))
 
+        # Apply tags filtering (BFS may traverse into memories that don't match tags criteria)
+        if tags:
+            results = filter_results_by_tags(results, tags, match=tags_match)
+
         return results

hindsight-api/hindsight_api/engine/search/link_expansion_retrieval.py

@@ -0,0 +1,392 @@
+"""
+Link Expansion graph retrieval.
+
+A simple, fast graph retrieval that expands from seeds via:
+1. Entity links: Find facts sharing entities with seeds (filtered by entity frequency)
+2. Causal links: Find facts causally linked to seeds (top-k by weight)
+
+Characteristics:
+- 2-3 DB queries (seed finding + parallel entity/causal expansion)
+- Sublinear: only touches connected facts via indexes
+- No iteration, no propagation, no normalization
+- Target: <100ms
+"""
+
+import logging
+import time
+
+from ..db_utils import acquire_with_retry
+from ..memory_engine import fq_table
+from .graph_retrieval import GraphRetriever
+from .tags import TagsMatch, filter_results_by_tags
+from .types import MPFPTimings, RetrievalResult
+
+logger = logging.getLogger(__name__)
+
+
+async def _find_semantic_seeds(
+    conn,
+    query_embedding_str: str,
+    bank_id: str,
+    fact_type: str,
+    limit: int = 20,
+    threshold: float = 0.3,
+    tags: list[str] | None = None,
+    tags_match: TagsMatch = "any",
+) -> list[RetrievalResult]:
+    """Find semantic seeds via embedding search."""
+    from .tags import build_tags_where_clause_simple
+
+    tags_clause = build_tags_where_clause_simple(tags, 6, match=tags_match)
+    params = [query_embedding_str, bank_id, fact_type, threshold, limit]
+    if tags:
+        params.append(tags)
+
+    rows = await conn.fetch(
+        f"""
+        SELECT id, text, context, event_date, occurred_start, occurred_end,
+               mentioned_at, embedding, fact_type, document_id, chunk_id, tags,
+               1 - (embedding <=> $1::vector) AS similarity
+        FROM {fq_table("memory_units")}
+        WHERE bank_id = $2
+          AND embedding IS NOT NULL
+          AND fact_type = $3
+          AND (1 - (embedding <=> $1::vector)) >= $4
+          {tags_clause}
+        ORDER BY embedding <=> $1::vector
+        LIMIT $5
+        """,
+        *params,
+    )
+    return [RetrievalResult.from_db_row(dict(r)) for r in rows]
+
+
+class LinkExpansionRetriever(GraphRetriever):
+    """
+    Graph retrieval via direct link expansion from seeds.
+
+    Expands through entity co-occurrence and causal links in a single query.
+    Fast and simple alternative to MPFP.
+    """
+
+    def __init__(
+        self,
+        max_entity_frequency: int = 500,
+        causal_weight_threshold: float = 0.3,
+        causal_limit_per_seed: int = 10,
+    ):
+        """
+        Initialize link expansion retriever.
+
+        Args:
+            max_entity_frequency: Skip entities appearing in more than this many facts
+            causal_weight_threshold: Minimum weight for causal links
+            causal_limit_per_seed: Max causal links to follow per seed
+        """
+        self.max_entity_frequency = max_entity_frequency
+        self.causal_weight_threshold = causal_weight_threshold
+        self.causal_limit_per_seed = causal_limit_per_seed
+
+    @property
+    def name(self) -> str:
+        return "link_expansion"
+
+    async def retrieve(
+        self,
+        pool,
+        query_embedding_str: str,
+        bank_id: str,
+        fact_type: str,
+        budget: int,
+        query_text: str | None = None,
+        semantic_seeds: list[RetrievalResult] | None = None,
+        temporal_seeds: list[RetrievalResult] | None = None,
+        adjacency=None,
+        tags: list[str] | None = None,
+        tags_match: TagsMatch = "any",
+    ) -> tuple[list[RetrievalResult], MPFPTimings | None]:
+        """
+        Retrieve facts by expanding links from seeds.
+
+        Args:
+            pool: Database connection pool
+            query_embedding_str: Query embedding (unused, kept for interface)
+            bank_id: Memory bank ID
+            fact_type: Fact type to filter
+            budget: Maximum results to return
+            query_text: Original query text (unused)
+            semantic_seeds: Pre-computed semantic entry points
+            temporal_seeds: Pre-computed temporal entry points
+            adjacency: Unused, kept for interface compatibility
+            tags: Optional list of tags for visibility filtering (OR matching)
+
+        Returns:
+            Tuple of (results, timings)
+        """
+        start_time = time.time()
+        timings = MPFPTimings(fact_type=fact_type)
+
+        # Use single connection for all queries to reduce pool pressure
+        # (queries are fast ~50ms each, connection acquisition is the bottleneck)
+        async with acquire_with_retry(pool) as conn:
+            # Find seeds if not provided
+            if semantic_seeds:
+                all_seeds = list(semantic_seeds)
+            else:
+                seeds_start = time.time()
+                all_seeds = await _find_semantic_seeds(
+                    conn,
+                    query_embedding_str,
+                    bank_id,
+                    fact_type,
+                    limit=20,
+                    threshold=0.3,
+                    tags=tags,
+                    tags_match=tags_match,
+                )
+                timings.seeds_time = time.time() - seeds_start
+                logger.debug(
+                    f"[LinkExpansion] Found {len(all_seeds)} semantic seeds for fact_type={fact_type} "
+                    f"(tags={tags}, tags_match={tags_match})"
+                )
+
+            # Add temporal seeds if provided
+            if temporal_seeds:
+                all_seeds.extend(temporal_seeds)
+
+            if not all_seeds:
+                logger.info("[LinkExpansion] No seeds found, returning empty results")
+                return [], timings
+
+            seed_ids = list({s.id for s in all_seeds})
+            timings.pattern_count = len(seed_ids)
+
+            # Run entity and causal expansion sequentially on same connection
+            query_start = time.time()
+
+            # For observations, traverse through source_memory_ids to find entity connections.
+            # Observations don't have direct unit_entities - they inherit entities via their
+            # source world/experience facts.
+            #
+            # Path: observation → source_memory_ids → world fact → entities →
+            #       ALL world facts with those entities → their observations (excluding seeds)
+            if fact_type == "observation":
+                # Debug: Check what source_memory_ids exist on seed observations
+                debug_sources = await conn.fetch(
+                    f"""
+                    SELECT id, source_memory_ids
+                    FROM {fq_table("memory_units")}
+                    WHERE id = ANY($1::uuid[])
+                    """,
+                    seed_ids,
+                )
+                source_ids_found = []
+                for row in debug_sources:
+                    if row["source_memory_ids"]:
+                        source_ids_found.extend(row["source_memory_ids"])
+                logger.debug(
+                    f"[LinkExpansion] observation graph: {len(seed_ids)} seeds, "
+                    f"{len(source_ids_found)} source_memory_ids found"
+                )
+
+                entity_rows = await conn.fetch(
+                    f"""
+                    WITH seed_sources AS (
+                        -- Get source memory IDs from seed observations
+                        SELECT DISTINCT unnest(source_memory_ids) AS source_id
+                        FROM {fq_table("memory_units")}
+                        WHERE id = ANY($1::uuid[])
+                          AND source_memory_ids IS NOT NULL
+                    ),
+                    source_entities AS (
+                        -- Get entities from those source memories (filtered by frequency)
+                        SELECT DISTINCT ue.entity_id
+                        FROM seed_sources ss
+                        JOIN {fq_table("unit_entities")} ue ON ss.source_id = ue.unit_id
+                        JOIN {fq_table("entities")} e ON ue.entity_id = e.id
+                        WHERE e.mention_count < $2
+                    ),
+                    all_connected_sources AS (
+                        -- Find ALL world facts sharing those entities (don't exclude seed sources)
+                        -- The exclusion happens at the observation level, not the source level
+                        SELECT DISTINCT other_ue.unit_id AS source_id
+                        FROM source_entities se
+                        JOIN {fq_table("unit_entities")} other_ue ON se.entity_id = other_ue.entity_id
+                    )
+                    -- Find observations derived from connected source memories
+                    -- Only exclude the actual seed observations
+                    SELECT
+                        mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
+                        mu.occurred_end, mu.mentioned_at, mu.embedding,
+                        mu.fact_type, mu.document_id, mu.chunk_id, mu.tags,
+                        COUNT(DISTINCT cs.source_id)::float AS score
+                    FROM all_connected_sources cs
+                    JOIN {fq_table("memory_units")} mu
+                        ON mu.source_memory_ids @> ARRAY[cs.source_id]
+                    WHERE mu.fact_type = 'observation'
+                      AND mu.id != ALL($1::uuid[])
+                    GROUP BY mu.id
+                    ORDER BY score DESC
+                    LIMIT $3
+                    """,
+                    seed_ids,
+                    self.max_entity_frequency,
+                    budget,
+                )
+                logger.debug(f"[LinkExpansion] observation graph: found {len(entity_rows)} connected observations")
+            else:
+                # For world/experience facts, use direct entity lookup
+                entity_rows = await conn.fetch(
+                    f"""
+                    SELECT
+                        mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
+                        mu.occurred_end, mu.mentioned_at, mu.embedding,
+                        mu.fact_type, mu.document_id, mu.chunk_id, mu.tags,
+                        COUNT(*)::float AS score
+                    FROM {fq_table("unit_entities")} seed_ue
+                    JOIN {fq_table("entities")} e ON seed_ue.entity_id = e.id
+                    JOIN {fq_table("unit_entities")} other_ue ON seed_ue.entity_id = other_ue.entity_id
+                    JOIN {fq_table("memory_units")} mu ON other_ue.unit_id = mu.id
+                    WHERE seed_ue.unit_id = ANY($1::uuid[])
+                      AND e.mention_count < $2
+                      AND mu.id != ALL($1::uuid[])
+                      AND mu.fact_type = $3
+                    GROUP BY mu.id
+                    ORDER BY score DESC
+                    LIMIT $4
+                    """,
+                    seed_ids,
+                    self.max_entity_frequency,
+                    fact_type,
+                    budget,
+                )
+
+            causal_rows = await conn.fetch(
+                f"""
+                SELECT DISTINCT ON (mu.id)
+                    mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
+                    mu.occurred_end, mu.mentioned_at, mu.embedding,
+                    mu.fact_type, mu.document_id, mu.chunk_id, mu.tags,
+                    ml.weight + 1.0 AS score
+                FROM {fq_table("memory_links")} ml
+                JOIN {fq_table("memory_units")} mu ON ml.to_unit_id = mu.id
+                WHERE ml.from_unit_id = ANY($1::uuid[])
+                  AND ml.link_type IN ('causes', 'caused_by', 'enables', 'prevents')
+                  AND ml.weight >= $2
+                  AND mu.fact_type = $3
+                ORDER BY mu.id, ml.weight DESC
+                LIMIT $4
+                """,
+                seed_ids,
+                self.causal_weight_threshold,
+                fact_type,
+                budget,
+            )
+
+            # Fallback: semantic/temporal/entity links from memory_links table
+            # These are secondary to entity links (via unit_entities) and causal links
+            # Weight is halved (0.5x) to prioritize primary link types
+            # Check both directions: seeds -> others AND others -> seeds
+            fallback_rows = await conn.fetch(
+                f"""
+                WITH outgoing AS (
+                    -- Links FROM seeds TO other facts
+                    SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
+                           mu.occurred_end, mu.mentioned_at, mu.embedding,
+                           mu.fact_type, mu.document_id, mu.chunk_id, mu.tags,
+                           ml.weight
+                    FROM {fq_table("memory_links")} ml
+                    JOIN {fq_table("memory_units")} mu ON ml.to_unit_id = mu.id
+                    WHERE ml.from_unit_id = ANY($1::uuid[])
+                      AND ml.link_type IN ('semantic', 'temporal', 'entity')
+                      AND ml.weight >= $2
+                      AND mu.fact_type = $3
+                      AND mu.id != ALL($1::uuid[])
+                ),
+                incoming AS (
+                    -- Links FROM other facts TO seeds (reverse direction)
+                    SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
+                           mu.occurred_end, mu.mentioned_at, mu.embedding,
+                           mu.fact_type, mu.document_id, mu.chunk_id, mu.tags,
+                           ml.weight
+                    FROM {fq_table("memory_links")} ml
+                    JOIN {fq_table("memory_units")} mu ON ml.from_unit_id = mu.id
+                    WHERE ml.to_unit_id = ANY($1::uuid[])
+                      AND ml.link_type IN ('semantic', 'temporal', 'entity')
+                      AND ml.weight >= $2
+                      AND mu.fact_type = $3
+                      AND mu.id != ALL($1::uuid[])
+                ),
+                combined AS (
+                    SELECT * FROM outgoing
+                    UNION ALL
+                    SELECT * FROM incoming
+                )
+                SELECT DISTINCT ON (id)
+                    id, text, context, event_date, occurred_start,
+                    occurred_end, mentioned_at, embedding,
+                    fact_type, document_id, chunk_id, tags,
+                    (MAX(weight) * 0.5) AS score
+                FROM combined
+                GROUP BY id, text, context, event_date, occurred_start,
+                         occurred_end, mentioned_at, embedding,
+                         fact_type, document_id, chunk_id, tags
+                ORDER BY id, score DESC
+                LIMIT $4
+                """,
+                seed_ids,
+                self.causal_weight_threshold,
+                fact_type,
+                budget,
+            )
+
+            timings.edge_load_time = time.time() - query_start
+            timings.db_queries = 3
+            timings.edge_count = len(entity_rows) + len(causal_rows) + len(fallback_rows)
+
+        # Merge results, taking max score per fact
+        # Priority: entity links (unit_entities) > causal links > fallback links
+        score_map: dict[str, float] = {}
+        row_map: dict[str, dict] = {}
+
+        for row in entity_rows:
+            fact_id = str(row["id"])
+            score_map[fact_id] = max(score_map.get(fact_id, 0), row["score"])
+            row_map[fact_id] = dict(row)
+
+        for row in causal_rows:
+            fact_id = str(row["id"])
+            score_map[fact_id] = max(score_map.get(fact_id, 0), row["score"])
+            if fact_id not in row_map:
+                row_map[fact_id] = dict(row)
+
+        for row in fallback_rows:
+            fact_id = str(row["id"])
+            score_map[fact_id] = max(score_map.get(fact_id, 0), row["score"])
+            if fact_id not in row_map:
+                row_map[fact_id] = dict(row)
+
+        # Sort by score and limit
+        sorted_ids = sorted(score_map.keys(), key=lambda x: score_map[x], reverse=True)[:budget]
+        rows = [row_map[fact_id] for fact_id in sorted_ids]
+
+        # Convert to results
+        results = []
+        for row in rows:
+            result = RetrievalResult.from_db_row(dict(row))
+            result.activation = row["score"]
+            results.append(result)
+
+        # Apply tags filtering (graph expansion may reach untagged memories)
+        if tags:
+            results = filter_results_by_tags(results, tags, match=tags_match)
+
+        timings.result_count = len(results)
+        timings.traverse = time.time() - start_time
+
+        logger.debug(
+            f"LinkExpansion: {len(results)} results from {len(seed_ids)} seeds "
+            f"in {timings.traverse * 1000:.1f}ms (query: {timings.edge_load_time * 1000:.1f}ms)"
+        )
+
+        return results, timings

hindsight-api/hindsight_api/engine/search/mpfp_retrieval.py

@@ -9,6 +9,7 @@ propagation from Approximate PPR.
 
 Key properties:
 - Sublinear in graph size (threshold pruning bounds active nodes)
+- Lazy edge loading: only loads edges for frontier nodes, not entire graph
 - Predefined patterns capture different retrieval intents
 - All patterns run in parallel, results fused via RRF
 - No LLM in the loop during traversal
@@ -22,7 +23,8 @@ from dataclasses import dataclass, field
 from ..db_utils import acquire_with_retry
 from ..memory_engine import fq_table
 from .graph_retrieval import GraphRetriever
-from .types import RetrievalResult
+from .tags import TagsMatch
+from .types import MPFPTimings, RetrievalResult
 
 logger = logging.getLogger(__name__)
 
@@ -41,11 +43,27 @@ class EdgeTarget:
 
 
 @dataclass
-class TypedAdjacency:
-    """Adjacency lists split by edge type."""
+class EdgeCache:
+    """
+    Cache for lazily-loaded edges.
 
-    # edge_type -> from_node_id -> list of (to_node_id, weight)
+    Grows per-hop as edges are loaded for frontier nodes.
+    Shared across patterns to avoid redundant loads.
+    Loads ALL edge types at once to minimize DB queries.
+    Thread-safe via asyncio lock to prevent redundant concurrent loads.
+    """
+
+    # edge_type -> from_node_id -> list of EdgeTarget
     graphs: dict[str, dict[str, list[EdgeTarget]]] = field(default_factory=dict)
+    # Track which nodes have been fully loaded (all edge types)
+    _fully_loaded: set[str] = field(default_factory=set)
+    # Timing stats
+    db_queries: int = 0
+    edge_load_time: float = 0.0
+    # Detailed hop timing for debugging
+    hop_details: list[dict] = field(default_factory=list)
+    # Lock to prevent redundant concurrent loads
+    _lock: asyncio.Lock = field(default_factory=asyncio.Lock)
 
     def get_neighbors(self, edge_type: str, node_id: str) -> list[EdgeTarget]:
         """Get neighbors for a node via a specific edge type."""
@@ -63,6 +81,31 @@ class TypedAdjacency:
 
         return [EdgeTarget(node_id=n.node_id, weight=n.weight / total) for n in neighbors]
 
+    def is_fully_loaded(self, node_id: str) -> bool:
+        """Check if all edges for this node have been loaded."""
+        return node_id in self._fully_loaded
+
+    def get_uncached(self, node_ids: list[str]) -> list[str]:
+        """Get node IDs that haven't been fully loaded yet."""
+        return [n for n in node_ids if not self.is_fully_loaded(n)]
+
+    def add_all_edges(self, edges_by_type: dict[str, dict[str, list[EdgeTarget]]], all_queried: list[str]):
+        """
+        Add loaded edges to the cache (all edge types at once).
+
+        Args:
+            edges_by_type: Dict mapping edge_type -> from_node_id -> list of EdgeTarget
+            all_queried: All node IDs that were queried (marks them as fully loaded)
+        """
+        for edge_type, edges in edges_by_type.items():
+            if edge_type not in self.graphs:
+                self.graphs[edge_type] = {}
+            for node_id, neighbors in edges.items():
+                self.graphs[edge_type][node_id] = neighbors
+
+        # Mark all queried nodes as fully loaded (even if they have no edges)
+        self._fully_loaded.update(all_queried)
+
 
 @dataclass
 class PatternResult:
@@ -109,66 +152,249 @@ class SeedNode:
 
 
 # -----------------------------------------------------------------------------
-# Core Algorithm
+# Lazy Edge Loading
 # -----------------------------------------------------------------------------
 
 
-def mpfp_traverse(
-    seeds: list[SeedNode],
-    pattern: list[str],
-    adjacency: TypedAdjacency,
-    config: MPFPConfig,
-) -> PatternResult:
+async def load_all_edges_for_frontier(
+    pool,
+    node_ids: list[str],
+    top_k_per_type: int = 20,
+) -> dict[str, dict[str, list[EdgeTarget]]]:
     """
-    Forward Push traversal following a meta-path pattern.
+    Load top-k edges per (node, edge_type) for frontier nodes.
+
+    Uses a LATERAL join to efficiently fetch only the top-k edges per type,
+    avoiding loading hundreds of entity edges when only 20 are needed.
+
+    Requires composite index: (from_unit_id, link_type, weight DESC)
 
     Args:
-        seeds: Entry point nodes with initial scores
-        pattern: Sequence of edge types to follow
-        adjacency: Typed adjacency structure
-        config: Algorithm parameters
+        pool: Database connection pool
+        node_ids: Frontier node IDs to load edges for
+        top_k_per_type: Max edges to load per (node, link_type) pair
 
     Returns:
-        PatternResult with accumulated scores per node
+        Dict mapping edge_type -> from_node_id -> list of EdgeTarget
+    """
+    if not node_ids:
+        return {}
+
+    async with acquire_with_retry(pool) as conn:
+        # Use LATERAL join to get top-k per (from_node, link_type)
+        # This leverages the composite index for efficient early termination
+        rows = await conn.fetch(
+            f"""
+            WITH frontier(node_id) AS (SELECT unnest($1::uuid[]))
+            SELECT f.node_id as from_unit_id, lt.link_type, edges.to_unit_id, edges.weight
+            FROM frontier f
+            CROSS JOIN (VALUES ('semantic'), ('temporal'), ('entity'), ('causes'), ('caused_by')) AS lt(link_type)
+            CROSS JOIN LATERAL (
+                SELECT ml.to_unit_id, ml.weight
+                FROM {fq_table("memory_links")} ml
+                WHERE ml.from_unit_id = f.node_id
+                  AND ml.link_type = lt.link_type
+                  AND ml.weight >= 0.1
+                ORDER BY ml.weight DESC
+                LIMIT $2
+            ) edges
+            """,
+            node_ids,
+            top_k_per_type,
+        )
+
+    # Group by edge_type -> from_node -> neighbors
+    result: dict[str, dict[str, list[EdgeTarget]]] = defaultdict(lambda: defaultdict(list))
+    for row in rows:
+        edge_type = row["link_type"]
+        from_id = str(row["from_unit_id"])
+        to_id = str(row["to_unit_id"])
+        weight = row["weight"]
+        result[edge_type][from_id].append(EdgeTarget(node_id=to_id, weight=weight))
+
+    # Convert nested defaultdicts to regular dicts
+    return {edge_type: dict(edges) for edge_type, edges in result.items()}
+
+
+# -----------------------------------------------------------------------------
+# Core Algorithm (Async with Lazy Loading)
+# -----------------------------------------------------------------------------
+
+
+@dataclass
+class PatternState:
+    """State for a pattern traversal between hops."""
+
+    pattern: list[str]
+    hop_index: int
+    scores: dict[str, float]
+    frontier: dict[str, float]
+
+
+def _init_pattern_state(seeds: list[SeedNode], pattern: list[str]) -> PatternState:
+    """Initialize pattern state from seeds."""
+    if not seeds:
+        return PatternState(pattern=pattern, hop_index=0, scores={}, frontier={})
+
+    total_seed_score = sum(s.score for s in seeds)
+    if total_seed_score == 0:
+        total_seed_score = len(seeds)
+
+    frontier = {s.node_id: s.score / total_seed_score for s in seeds}
+    return PatternState(pattern=pattern, hop_index=0, scores={}, frontier=frontier)
+
+
+def _execute_hop(state: PatternState, cache: EdgeCache, config: MPFPConfig) -> set[str]:
+    """
+    Execute ONE hop of traversal, return frontier nodes for next hop.
+
+    This is a pure function that uses cached edges (no DB access).
+    Returns set of uncached nodes needed for next hop.
+    """
+    if state.hop_index >= len(state.pattern):
+        return set()
+
+    edge_type = state.pattern[state.hop_index]
+
+    # Collect active nodes above threshold
+    active_nodes = [node_id for node_id, mass in state.frontier.items() if mass >= config.threshold]
+    if not active_nodes:
+        state.frontier = {}
+        return set()
+
+    # Propagate mass using cached edges
+    next_frontier: dict[str, float] = {}
+    uncached_for_next: set[str] = set()
+
+    for node_id, mass in state.frontier.items():
+        if mass < config.threshold:
+            continue
+
+        # Keep α portion for this node
+        state.scores[node_id] = state.scores.get(node_id, 0) + config.alpha * mass
+
+        # Push (1-α) to neighbors
+        push_mass = (1 - config.alpha) * mass
+        neighbors = cache.get_normalized_neighbors(edge_type, node_id, config.top_k_neighbors)
+
+        for neighbor in neighbors:
+            next_frontier[neighbor.node_id] = next_frontier.get(neighbor.node_id, 0) + push_mass * neighbor.weight
+            # Track if we'll need edges for this node in the next hop
+            if not cache.is_fully_loaded(neighbor.node_id):
+                uncached_for_next.add(neighbor.node_id)
+
+    state.frontier = next_frontier
+    state.hop_index += 1
+
+    return uncached_for_next
+
+
+def _finalize_pattern(state: PatternState, config: MPFPConfig) -> PatternResult:
+    """Finalize pattern by adding remaining frontier mass to scores."""
+    for node_id, mass in state.frontier.items():
+        if mass >= config.threshold:
+            state.scores[node_id] = state.scores.get(node_id, 0) + mass
+
+    return PatternResult(pattern=state.pattern, scores=state.scores)
+
+
+async def mpfp_traverse_hop_synchronized(
+    pool,
+    pattern_jobs: list[tuple[list[SeedNode], list[str]]],
+    config: MPFPConfig,
+    cache: EdgeCache,
+) -> list[PatternResult]:
+    """
+    Execute ALL patterns with hop-synchronized edge loading.
+
+    Instead of running each pattern independently (causing multiple DB queries),
+    this function:
+    1. Runs hop 1 for ALL patterns (using pre-warmed seed edges)
+    2. Collects ALL unique hop-2 frontier nodes across patterns
+    3. Pre-warms hop-2 edges in ONE query
+    4. Runs hop 2 for ALL patterns
+
+    This reduces DB queries from O(patterns * hops) to O(hops).
+
+    Args:
+        pool: Database connection pool
+        pattern_jobs: List of (seeds, pattern) tuples
+        config: Algorithm parameters
+        cache: Shared edge cache (should be pre-warmed with seed edges)
+
+    Returns:
+        List of PatternResult for each pattern
+    """
+    import time
+
+    # Initialize all pattern states
+    states = [_init_pattern_state(seeds, pattern) for seeds, pattern in pattern_jobs]
+
+    # Determine max hops (all patterns should be same length, but be safe)
+    max_hops = max((len(p) for _, p in pattern_jobs), default=0)
+
+    # Detailed timing for debugging
+    hop_times: list[dict] = []
+
+    # Execute hop-by-hop across ALL patterns
+    for hop in range(max_hops):
+        hop_start = time.time()
+        hop_timing = {"hop": hop, "patterns_executed": 0, "uncached_count": 0, "load_time": 0.0}
+
+        # Execute this hop for all patterns, collect uncached nodes for next hop
+        all_uncached: set[str] = set()
+        exec_start = time.time()
+        for state in states:
+            if state.hop_index < len(state.pattern):
+                uncached = _execute_hop(state, cache, config)
+                all_uncached.update(uncached)
+                hop_timing["patterns_executed"] += 1
+        hop_timing["exec_time"] = time.time() - exec_start
+
+        # Pre-warm edges for ALL uncached nodes before next hop
+        hop_timing["uncached_count"] = len(all_uncached)
+        if all_uncached:
+            uncached_list = list(all_uncached - cache._fully_loaded)
+            hop_timing["uncached_after_filter"] = len(uncached_list)
+            if uncached_list:
+                load_start = time.time()
+                edges_by_type = await load_all_edges_for_frontier(pool, uncached_list, config.top_k_neighbors)
+                hop_timing["load_time"] = time.time() - load_start
+                cache.edge_load_time += hop_timing["load_time"]
+                cache.db_queries += 1
+                cache.add_all_edges(edges_by_type, uncached_list)
+                hop_timing["edges_loaded"] = sum(
+                    len(neighbors) for edges in edges_by_type.values() for neighbors in edges.values()
+                )
+
+        hop_timing["total_time"] = time.time() - hop_start
+        hop_times.append(hop_timing)
+
+    # Store hop timing details in cache for logging
+    cache.hop_details = hop_times
+
+    # Finalize all patterns
+    return [_finalize_pattern(state, config) for state in states]
+
+
+async def mpfp_traverse_async(
+    pool,
+    seeds: list[SeedNode],
+    pattern: list[str],
+    config: MPFPConfig,
+    cache: EdgeCache,
+) -> PatternResult:
+    """
+    Async Forward Push traversal with lazy edge loading.
+
+    NOTE: For better performance with multiple patterns, use mpfp_traverse_hop_synchronized().
+    This function is kept for single-pattern use cases.
     """
     if not seeds:
         return PatternResult(pattern=pattern, scores={})
 
-    scores: dict[str, float] = {}
-
-    # Initialize frontier with seed masses (normalized)
-    total_seed_score = sum(s.score for s in seeds)
-    if total_seed_score == 0:
-        total_seed_score = len(seeds)  # fallback to uniform
-
-    frontier: dict[str, float] = {s.node_id: s.score / total_seed_score for s in seeds}
-
-    # Follow pattern hop by hop
-    for edge_type in pattern:
-        next_frontier: dict[str, float] = {}
-
-        for node_id, mass in frontier.items():
-            if mass < config.threshold:
-                continue
-
-            # Keep α portion for this node
-            scores[node_id] = scores.get(node_id, 0) + config.alpha * mass
-
-            # Push (1-α) to neighbors
-            push_mass = (1 - config.alpha) * mass
-            neighbors = adjacency.get_normalized_neighbors(edge_type, node_id, config.top_k_neighbors)
-
-            for neighbor in neighbors:
-                next_frontier[neighbor.node_id] = next_frontier.get(neighbor.node_id, 0) + push_mass * neighbor.weight
-
-        frontier = next_frontier
-
-    # Final frontier nodes get their remaining mass
-    for node_id, mass in frontier.items():
-        if mass >= config.threshold:
-            scores[node_id] = scores.get(node_id, 0) + mass
-
-    return PatternResult(pattern=pattern, scores=scores)
+    results = await mpfp_traverse_hop_synchronized(pool, [(seeds, pattern)], config, cache)
+    return results[0] if results else PatternResult(pattern=pattern, scores={})
 
 
 def rrf_fusion(
@@ -210,38 +436,6 @@ def rrf_fusion(
 # -----------------------------------------------------------------------------
 
 
-async def load_typed_adjacency(pool, bank_id: str) -> TypedAdjacency:
-    """
-    Load all edges for a bank, split by edge type.
-
-    Single query, then organize in-memory for fast traversal.
-    """
-    async with acquire_with_retry(pool) as conn:
-        rows = await conn.fetch(
-            f"""
-            SELECT ml.from_unit_id, ml.to_unit_id, ml.link_type, ml.weight
-            FROM {fq_table("memory_links")} ml
-            JOIN {fq_table("memory_units")} mu ON ml.from_unit_id = mu.id
-            WHERE mu.bank_id = $1
-              AND ml.weight >= 0.1
-            ORDER BY ml.from_unit_id, ml.weight DESC
-            """,
-            bank_id,
-        )
-
-    graphs: dict[str, dict[str, list[EdgeTarget]]] = defaultdict(lambda: defaultdict(list))
-
-    for row in rows:
-        from_id = str(row["from_unit_id"])
-        to_id = str(row["to_unit_id"])
-        link_type = row["link_type"]
-        weight = row["weight"]
-
-        graphs[link_type][from_id].append(EdgeTarget(node_id=to_id, weight=weight))
-
-    return TypedAdjacency(graphs=dict(graphs))
-
-
 async def fetch_memory_units_by_ids(
     pool,
     node_ids: list[str],
@@ -255,7 +449,7 @@ async def fetch_memory_units_by_ids(
         rows = await conn.fetch(
             f"""
             SELECT id, text, context, event_date, occurred_start, occurred_end,
-                   mentioned_at, access_count, embedding, fact_type, document_id, chunk_id
+                   mentioned_at, embedding, fact_type, document_id, chunk_id, tags
             FROM {fq_table("memory_units")}
             WHERE id = ANY($1::uuid[])
               AND fact_type = $2
@@ -274,10 +468,10 @@ async def fetch_memory_units_by_ids(
 
 class MPFPGraphRetriever(GraphRetriever):
     """
-    Graph retrieval using Meta-Path Forward Push.
+    Graph retrieval using Meta-Path Forward Push with lazy edge loading.
 
     Runs predefined patterns in parallel from semantic and temporal seeds,
-    then fuses results via RRF.
+    loading edges on-demand per hop instead of loading entire graph upfront.
     """
 
     def __init__(self, config: MPFPConfig | None = None):
@@ -287,8 +481,13 @@ class MPFPGraphRetriever(GraphRetriever):
         Args:
             config: Algorithm configuration (uses defaults if None)
         """
-        self.config = config or MPFPConfig()
-        self._adjacency_cache: dict[str, TypedAdjacency] = {}
+        if config is None:
+            # Read top_k_neighbors from global config
+            from ...config import get_config
+
+            global_config = get_config()
+            config = MPFPConfig(top_k_neighbors=global_config.mpfp_top_k_neighbors)
+        self.config = config
 
     @property
     def name(self) -> str:
@@ -304,9 +503,12 @@ class MPFPGraphRetriever(GraphRetriever):
         query_text: str | None = None,
         semantic_seeds: list[RetrievalResult] | None = None,
         temporal_seeds: list[RetrievalResult] | None = None,
-    ) -> list[RetrievalResult]:
+        adjacency=None,  # Ignored - kept for interface compatibility
+        tags: list[str] | None = None,
+        tags_match: TagsMatch = "any",
+    ) -> tuple[list[RetrievalResult], MPFPTimings | None]:
         """
-        Retrieve facts using MPFP algorithm.
+        Retrieve facts using MPFP algorithm with lazy edge loading.
 
         Args:
             pool: Database connection pool
@@ -317,12 +519,15 @@ class MPFPGraphRetriever(GraphRetriever):
             query_text: Original query text (optional)
             semantic_seeds: Pre-computed semantic entry points
             temporal_seeds: Pre-computed temporal entry points
+            adjacency: Ignored (kept for interface compatibility)
+            tags: Optional list of tags for visibility filtering (OR matching)
 
         Returns:
-            List of RetrievalResult with activation scores
+            Tuple of (List of RetrievalResult with activation scores, MPFPTimings)
         """
-        # Load typed adjacency (could cache per bank_id with TTL)
-        adjacency = await load_typed_adjacency(pool, bank_id)
+        import time
+
+        timings = MPFPTimings(fact_type=fact_type)
 
         # Convert seeds to SeedNode format
         semantic_seed_nodes = self._convert_seeds(semantic_seeds, "similarity")
@@ -330,54 +535,88 @@ class MPFPGraphRetriever(GraphRetriever):
 
         # If no semantic seeds provided, fall back to finding our own
         if not semantic_seed_nodes:
-            semantic_seed_nodes = await self._find_semantic_seeds(pool, query_embedding_str, bank_id, fact_type)
+            seeds_start = time.time()
+            semantic_seed_nodes = await self._find_semantic_seeds(
+                pool, query_embedding_str, bank_id, fact_type, tags=tags, tags_match=tags_match
+            )
+            timings.seeds_time = time.time() - seeds_start
+            logger.debug(
+                f"[MPFP] Found {len(semantic_seed_nodes)} semantic seeds for fact_type={fact_type} (tags={tags}, tags_match={tags_match})"
+            )
 
-        # Run all patterns in parallel
-        tasks = []
+        # Collect all pattern jobs
+        pattern_jobs = []
 
         # Patterns from semantic seeds
         for pattern in self.config.patterns_semantic:
             if semantic_seed_nodes:
-                tasks.append(
-                    asyncio.to_thread(
-                        mpfp_traverse,
-                        semantic_seed_nodes,
-                        pattern,
-                        adjacency,
-                        self.config,
-                    )
-                )
+                pattern_jobs.append((semantic_seed_nodes, pattern))
 
         # Patterns from temporal seeds
         for pattern in self.config.patterns_temporal:
             if temporal_seed_nodes:
-                tasks.append(
-                    asyncio.to_thread(
-                        mpfp_traverse,
-                        temporal_seed_nodes,
-                        pattern,
-                        adjacency,
-                        self.config,
-                    )
-                )
+                pattern_jobs.append((temporal_seed_nodes, pattern))
 
-        if not tasks:
-            return []
+        if not pattern_jobs:
+            logger.debug(
+                f"[MPFP] No pattern jobs (semantic_seeds={len(semantic_seed_nodes)}, temporal_seeds={len(temporal_seed_nodes)})"
+            )
+            return [], timings
 
-        # Gather pattern results
-        pattern_results = await asyncio.gather(*tasks)
+        timings.pattern_count = len(pattern_jobs)
+
+        # Shared edge cache across all patterns
+        cache = EdgeCache()
+
+        # Pre-warm cache with ALL seed node edges BEFORE running patterns
+        # This prevents redundant DB queries at hop 1
+        all_seed_ids = list({s.node_id for seeds, _ in pattern_jobs for s in seeds})
+        if all_seed_ids:
+            import time as time_module
+
+            prewarm_start = time_module.time()
+            edges_by_type = await load_all_edges_for_frontier(pool, all_seed_ids, self.config.top_k_neighbors)
+            cache.edge_load_time += time_module.time() - prewarm_start
+            cache.db_queries += 1
+            cache.add_all_edges(edges_by_type, all_seed_ids)
+
+        # Run all patterns with HOP-SYNCHRONIZED edge loading
+        # This batches hop-2 edge loads across ALL patterns into ONE query
+        # Reduces DB queries from O(patterns * hops) to O(hops)
+        step_start = time.time()
+        pattern_results = await mpfp_traverse_hop_synchronized(pool, pattern_jobs, self.config, cache)
+        timings.traverse = time.time() - step_start
+
+        # Record edge loading stats from cache
+        timings.edge_count = sum(len(neighbors) for g in cache.graphs.values() for neighbors in g.values())
+        timings.db_queries = cache.db_queries
+        timings.edge_load_time = cache.edge_load_time
+        timings.hop_details = cache.hop_details
 
         # Fuse results
+        step_start = time.time()
         fused = rrf_fusion(pattern_results, top_k=budget)
+        timings.fusion = time.time() - step_start
 
         if not fused:
-            return []
+            logger.debug(f"[MPFP] No fused results after RRF fusion (pattern_count={len(pattern_results)})")
+            return [], timings
 
-        # Get top result IDs (don't exclude seeds - they may be highly relevant)
+        # Get top result IDs
         result_ids = [node_id for node_id, score in fused][:budget]
 
         # Fetch full details
+        step_start = time.time()
         results = await fetch_memory_units_by_ids(pool, result_ids, fact_type)
+        timings.fetch = time.time() - step_start
+
+        # Filter results by tags (graph traversal may have picked up unfiltered memories)
+        if tags:
+            from .tags import filter_results_by_tags
+
+            results = filter_results_by_tags(results, tags, match=tags_match)
+
+        timings.result_count = len(results)
 
         # Add activation scores from fusion
         score_map = {node_id: score for node_id, score in fused}
@@ -387,7 +626,7 @@ class MPFPGraphRetriever(GraphRetriever):
         # Sort by activation
         results.sort(key=lambda r: r.activation or 0, reverse=True)
 
-        return results
+        return results, timings
 
     def _convert_seeds(
         self,
@@ -415,8 +654,17 @@ class MPFPGraphRetriever(GraphRetriever):
         fact_type: str,
         limit: int = 20,
         threshold: float = 0.3,
+        tags: list[str] | None = None,
+        tags_match: TagsMatch = "any",
     ) -> list[SeedNode]:
         """Fallback: find semantic seeds via embedding search."""
+        from .tags import build_tags_where_clause_simple
+
+        tags_clause = build_tags_where_clause_simple(tags, 6, match=tags_match)
+        params = [query_embedding_str, bank_id, fact_type, threshold, limit]
+        if tags:
+            params.append(tags)
+
         async with acquire_with_retry(pool) as conn:
             rows = await conn.fetch(
                 f"""
@@ -426,14 +674,11 @@ class MPFPGraphRetriever(GraphRetriever):
                   AND embedding IS NOT NULL
                   AND fact_type = $3
                   AND (1 - (embedding <=> $1::vector)) >= $4
+                  {tags_clause}
                 ORDER BY embedding <=> $1::vector
                 LIMIT $5
                 """,
-                query_embedding_str,
-                bank_id,
-                fact_type,
-                threshold,
-                limit,
+                *params,
             )
 
         return [SeedNode(node_id=str(r["id"]), score=r["similarity"]) for r in rows]

hindsight-api/hindsight_api/engine/search/observation_utils.py

@@ -1,125 +0,0 @@
-"""
-Observation utilities for generating entity observations from facts.
-
-Observations are objective facts synthesized from multiple memory facts
-about an entity, without personality influence.
-"""
-
-import logging
-
-from pydantic import BaseModel, Field
-
-from ..response_models import MemoryFact
-
-logger = logging.getLogger(__name__)
-
-
-class Observation(BaseModel):
-    """An observation about an entity."""
-
-    observation: str = Field(description="The observation text - a factual statement about the entity")
-
-
-class ObservationExtractionResponse(BaseModel):
-    """Response containing extracted observations."""
-
-    observations: list[Observation] = Field(default_factory=list, description="List of observations about the entity")
-
-
-def format_facts_for_observation_prompt(facts: list[MemoryFact]) -> str:
-    """Format facts as text for observation extraction prompt."""
-    import json
-
-    if not facts:
-        return "[]"
-    formatted = []
-    for fact in facts:
-        fact_obj = {"text": fact.text}
-
-        # Add context if available
-        if fact.context:
-            fact_obj["context"] = fact.context
-
-        # Add occurred_start if available
-        if fact.occurred_start:
-            fact_obj["occurred_at"] = fact.occurred_start
-
-        formatted.append(fact_obj)
-
-    return json.dumps(formatted, indent=2)
-
-
-def build_observation_prompt(
-    entity_name: str,
-    facts_text: str,
-) -> str:
-    """Build the observation extraction prompt for the LLM."""
-    return f"""Based on the following facts about "{entity_name}", generate a list of key observations.
-
-FACTS ABOUT {entity_name.upper()}:
-{facts_text}
-
-Your task: Synthesize the facts into clear, objective observations about {entity_name}.
-
-GUIDELINES:
-1. Each observation should be a factual statement about {entity_name}
-2. Combine related facts into single observations where appropriate
-3. Be objective - do not add opinions, judgments, or interpretations
-4. Focus on what we KNOW about {entity_name}, not what we assume
-5. Include observations about: identity, characteristics, roles, relationships, activities
-6. Write in third person (e.g., "John is..." not "I think John is...")
-7. If there are conflicting facts, note the most recent or most supported one
-
-EXAMPLES of good observations:
-- "John works at Google as a software engineer"
-- "John is detail-oriented and methodical in his approach"
-- "John collaborates frequently with Sarah on the AI project"
-- "John joined the company in 2023"
-
-EXAMPLES of bad observations (avoid these):
-- "John seems like a good person" (opinion/judgment)
-- "John probably likes his job" (assumption)
-- "I believe John is reliable" (first-person opinion)
-
-Generate 3-7 observations based on the available facts. If there are very few facts, generate fewer observations."""
-
-
-def get_observation_system_message() -> str:
-    """Get the system message for observation extraction."""
-    return "You are an objective observer synthesizing facts about an entity. Generate clear, factual observations without opinions or personality influence. Be concise and accurate."
-
-
-async def extract_observations_from_facts(llm_config, entity_name: str, facts: list[MemoryFact]) -> list[str]:
-    """
-    Extract observations from facts about an entity using LLM.
-
-    Args:
-        llm_config: LLM configuration to use
-        entity_name: Name of the entity to generate observations about
-        facts: List of facts mentioning the entity
-
-    Returns:
-        List of observation strings
-    """
-    if not facts:
-        return []
-
-    facts_text = format_facts_for_observation_prompt(facts)
-    prompt = build_observation_prompt(entity_name, facts_text)
-
-    try:
-        result = await llm_config.call(
-            messages=[
-                {"role": "system", "content": get_observation_system_message()},
-                {"role": "user", "content": prompt},
-            ],
-            response_format=ObservationExtractionResponse,
-            scope="memory_extract_observation",
-        )
-
-        observations = [op.observation for op in result.observations]
-        return observations
-
-    except Exception as e:
-        logger.warning(f"Failed to extract observations for {entity_name}: {str(e)}")
-        return []

hindsight-api/hindsight_api/engine/search/reranking.py

@@ -44,7 +44,7 @@ class CrossEncoderReranker:
             await cross_encoder.initialize()
         self._initialized = True
 
-    def rerank(self, query: str, candidates: list[MergedCandidate]) -> list[ScoredResult]:
+    async def rerank(self, query: str, candidates: list[MergedCandidate]) -> list[ScoredResult]:
         """
         Rerank candidates using cross-encoder scores.
 
@@ -85,7 +85,7 @@ class CrossEncoderReranker:
             pairs.append([query, doc_text])
 
         # Get cross-encoder scores
-        scores = self.cross_encoder.predict(pairs)
+        scores = await self.cross_encoder.predict(pairs)
 
         # Normalize scores using sigmoid to [0, 1] range
         # Cross-encoder returns logits which can be negative

hindsight-api/hindsight_api/engine/search/scoring.py

@@ -1,159 +0,0 @@
-"""
-Scoring functions for memory search and retrieval.
-
-Includes recency weighting, frequency weighting, temporal proximity,
-and similarity calculations used in memory activation and ranking.
-"""
-
-from datetime import datetime
-
-
-def cosine_similarity(vec1: list[float], vec2: list[float]) -> float:
-    """
-    Calculate cosine similarity between two vectors.
-
-    Args:
-        vec1: First vector
-        vec2: Second vector
-
-    Returns:
-        Similarity score between 0 and 1
-    """
-    if len(vec1) != len(vec2):
-        raise ValueError("Vectors must have same dimension")
-
-    dot_product = sum(a * b for a, b in zip(vec1, vec2))
-    magnitude1 = sum(a * a for a in vec1) ** 0.5
-    magnitude2 = sum(b * b for b in vec2) ** 0.5
-
-    if magnitude1 == 0 or magnitude2 == 0:
-        return 0.0
-
-    return dot_product / (magnitude1 * magnitude2)
-
-
-def calculate_recency_weight(days_since: float, half_life_days: float = 365.0) -> float:
-    """
-    Calculate recency weight using logarithmic decay.
-
-    This provides much better differentiation over long time periods compared to
-    exponential decay. Uses a log-based decay where the half-life parameter controls
-    when memories reach 50% weight.
-
-    Examples:
-        - Today (0 days): 1.0
-        - 1 year (365 days): ~0.5 (with default half_life=365)
-        - 2 years (730 days): ~0.33
-        - 5 years (1825 days): ~0.17
-        - 10 years (3650 days): ~0.09
-
-    This ensures that 2-year-old and 5-year-old memories have meaningfully
-    different weights, unlike exponential decay which makes them both ~0.
-
-    Args:
-        days_since: Number of days since the memory was created
-        half_life_days: Number of days for weight to reach 0.5 (default: 1 year)
-
-    Returns:
-        Weight between 0 and 1
-    """
-    import math
-
-    # Logarithmic decay: 1 / (1 + log(1 + days_since/half_life))
-    # This decays much slower than exponential, giving better long-term differentiation
-    normalized_age = days_since / half_life_days
-    return 1.0 / (1.0 + math.log1p(normalized_age))
-
-
-def calculate_frequency_weight(access_count: int, max_boost: float = 2.0) -> float:
-    """
-    Calculate frequency weight based on access count.
-
-    Frequently accessed memories are weighted higher.
-    Uses logarithmic scaling to avoid over-weighting.
-
-    Args:
-        access_count: Number of times the memory was accessed
-        max_boost: Maximum multiplier for frequently accessed memories
-
-    Returns:
-        Weight between 1.0 and max_boost
-    """
-    import math
-
-    if access_count <= 0:
-        return 1.0
-
-    # Logarithmic scaling: log(access_count + 1) / log(10)
-    # This gives: 0 accesses = 1.0, 9 accesses ~= 1.5, 99 accesses ~= 2.0
-    normalized = math.log(access_count + 1) / math.log(10)
-    return 1.0 + min(normalized, max_boost - 1.0)
-
-
-def calculate_temporal_anchor(occurred_start: datetime, occurred_end: datetime) -> datetime:
-    """
-    Calculate a single temporal anchor point from a temporal range.
-
-    Used for spreading activation - we need a single representative date
-    to calculate temporal proximity between facts. This simplifies the
-    range-to-range distance problem.
-
-    Strategy: Use midpoint of the range for balanced representation.
-
-    Args:
-        occurred_start: Start of temporal range
-        occurred_end: End of temporal range
-
-    Returns:
-        Single datetime representing the temporal anchor (midpoint)
-
-    Examples:
-        - Point event (July 14): start=July 14, end=July 14 → anchor=July 14
-        - Month range (February): start=Feb 1, end=Feb 28 → anchor=Feb 14
-        - Year range (2023): start=Jan 1, end=Dec 31 → anchor=July 1
-    """
-    # Calculate midpoint
-    time_delta = occurred_end - occurred_start
-    midpoint = occurred_start + (time_delta / 2)
-    return midpoint
-
-
-def calculate_temporal_proximity(anchor_a: datetime, anchor_b: datetime, half_life_days: float = 30.0) -> float:
-    """
-    Calculate temporal proximity between two temporal anchors.
-
-    Used for spreading activation to determine how "close" two facts are
-    in time. Uses logarithmic decay so that temporal similarity doesn't
-    drop off too quickly.
-
-    Args:
-        anchor_a: Temporal anchor of first fact
-        anchor_b: Temporal anchor of second fact
-        half_life_days: Number of days for proximity to reach 0.5
-                       (default: 30 days = 1 month)
-
-    Returns:
-        Proximity score in [0, 1] where:
-        - 1.0 = same day
-        - 0.5 = ~half_life days apart
-        - 0.0 = very distant in time
-
-    Examples:
-        - Same day: 1.0
-        - 1 week apart (half_life=30): ~0.7
-        - 1 month apart (half_life=30): ~0.5
-        - 1 year apart (half_life=30): ~0.2
-    """
-    import math
-
-    days_apart = abs((anchor_a - anchor_b).days)
-
-    if days_apart == 0:
-        return 1.0
-
-    # Logarithmic decay: 1 / (1 + log(1 + days_apart/half_life))
-    # Similar to calculate_recency_weight but for proximity between events
-    normalized_distance = days_apart / half_life_days
-    proximity = 1.0 / (1.0 + math.log1p(normalized_distance))
-
-    return proximity

hindsight-api/hindsight_api/engine/search/tags.py

@@ -0,0 +1,172 @@
+"""
+Tags filtering utilities for retrieval.
+
+Provides SQL building functions for filtering memories by tags.
+Supports four matching modes via TagsMatch enum:
+- "any": OR matching, includes untagged memories (default, backward compatible)
+- "all": AND matching, includes untagged memories
+- "any_strict": OR matching, excludes untagged memories
+- "all_strict": AND matching, excludes untagged memories
+
+OR matching (any/any_strict): Memory matches if ANY of its tags overlap with request tags
+AND matching (all/all_strict): Memory matches if ALL request tags are present in its tags
+"""
+
+from typing import Literal
+
+TagsMatch = Literal["any", "all", "any_strict", "all_strict"]
+
+
+def _parse_tags_match(match: TagsMatch) -> tuple[str, bool]:
+    """
+    Parse TagsMatch into operator and include_untagged flag.
+
+    Returns:
+        Tuple of (operator, include_untagged)
+        - operator: "&&" for any/any_strict, "@>" for all/all_strict
+        - include_untagged: True for any/all, False for any_strict/all_strict
+    """
+    if match == "any":
+        return "&&", True
+    elif match == "all":
+        return "@>", True
+    elif match == "any_strict":
+        return "&&", False
+    elif match == "all_strict":
+        return "@>", False
+    else:
+        # Default to "any" behavior
+        return "&&", True
+
+
+def build_tags_where_clause(
+    tags: list[str] | None,
+    param_offset: int = 1,
+    table_alias: str = "",
+    match: TagsMatch = "any",
+) -> tuple[str, list, int]:
+    """
+    Build a SQL WHERE clause for filtering by tags.
+
+    Supports four matching modes:
+    - "any" (default): OR matching, includes untagged memories
+    - "all": AND matching, includes untagged memories
+    - "any_strict": OR matching, excludes untagged memories
+    - "all_strict": AND matching, excludes untagged memories
+
+    Args:
+        tags: List of tags to filter by. If None or empty, returns empty clause (no filtering).
+        param_offset: Starting parameter number for SQL placeholders (default 1).
+        table_alias: Optional table alias prefix (e.g., "mu." for "memory_units mu").
+        match: Matching mode. Defaults to "any".
+
+    Returns:
+        Tuple of (sql_clause, params, next_param_offset):
+        - sql_clause: SQL WHERE clause string
+        - params: List of parameter values to bind
+        - next_param_offset: Next available parameter number
+
+    Example:
+        >>> clause, params, next_offset = build_tags_where_clause(['user_a'], 3, 'mu.', 'any_strict')
+        >>> print(clause)  # "AND mu.tags IS NOT NULL AND mu.tags != '{}' AND mu.tags && $3"
+    """
+    if not tags:
+        return "", [], param_offset
+
+    column = f"{table_alias}tags" if table_alias else "tags"
+    operator, include_untagged = _parse_tags_match(match)
+
+    if include_untagged:
+        # Include untagged memories (NULL or empty array) OR matching tags
+        clause = f"AND ({column} IS NULL OR {column} = '{{}}' OR {column} {operator} ${param_offset})"
+    else:
+        # Strict: only memories with matching tags (exclude NULL and empty)
+        clause = f"AND {column} IS NOT NULL AND {column} != '{{}}' AND {column} {operator} ${param_offset}"
+
+    return clause, [tags], param_offset + 1
+
+
+def build_tags_where_clause_simple(
+    tags: list[str] | None,
+    param_num: int,
+    table_alias: str = "",
+    match: TagsMatch = "any",
+) -> str:
+    """
+    Build a simple SQL WHERE clause for tags filtering.
+
+    This is a convenience version that returns just the clause string,
+    assuming the caller will add the tags array to their params list.
+
+    Args:
+        tags: List of tags to filter by. If None or empty, returns empty string.
+        param_num: Parameter number to use in the clause.
+        table_alias: Optional table alias prefix.
+        match: Matching mode. Defaults to "any".
+
+    Returns:
+        SQL clause string or empty string.
+    """
+    if not tags:
+        return ""
+
+    column = f"{table_alias}tags" if table_alias else "tags"
+    operator, include_untagged = _parse_tags_match(match)
+
+    if include_untagged:
+        # Include untagged memories (NULL or empty array) OR matching tags
+        return f"AND ({column} IS NULL OR {column} = '{{}}' OR {column} {operator} ${param_num})"
+    else:
+        # Strict: only memories with matching tags (exclude NULL and empty)
+        return f"AND {column} IS NOT NULL AND {column} != '{{}}' AND {column} {operator} ${param_num}"
+
+
+def filter_results_by_tags(
+    results: list,
+    tags: list[str] | None,
+    match: TagsMatch = "any",
+) -> list:
+    """
+    Filter retrieval results by tags in Python (for post-processing).
+
+    Used when SQL filtering isn't possible (e.g., graph traversal results).
+
+    Args:
+        results: List of RetrievalResult objects with a 'tags' attribute.
+        tags: List of tags to filter by. If None or empty, returns all results.
+        match: Matching mode. Defaults to "any".
+
+    Returns:
+        Filtered list of results.
+    """
+    if not tags:
+        return results
+
+    _, include_untagged = _parse_tags_match(match)
+    is_any_match = match in ("any", "any_strict")
+
+    tags_set = set(tags)
+    filtered = []
+
+    for result in results:
+        result_tags = getattr(result, "tags", None)
+
+        # Check if untagged
+        is_untagged = result_tags is None or len(result_tags) == 0
+
+        if is_untagged:
+            if include_untagged:
+                filtered.append(result)
+            # else: skip untagged
+        else:
+            result_tags_set = set(result_tags)
+            if is_any_match:
+                # Any overlap
+                if result_tags_set & tags_set:
+                    filtered.append(result)
+            else:
+                # All tags must be present
+                if tags_set <= result_tags_set:
+                    filtered.append(result)
+
+    return filtered

hindsight-api/hindsight_api/engine/search/think_utils.py

@@ -3,31 +3,13 @@ Think operation utilities for formulating answers based on agent and world facts
 """
 
 import logging
-import re
 from datetime import datetime
 
-from pydantic import BaseModel, Field
-
 from ..response_models import DispositionTraits, MemoryFact
 
 logger = logging.getLogger(__name__)
 
 
-class Opinion(BaseModel):
-    """An opinion formed by the bank."""
-
-    opinion: str = Field(description="The opinion or perspective with reasoning included")
-    confidence: float = Field(description="Confidence score for this opinion (0.0 to 1.0, where 1.0 is very confident)")
-
-
-class OpinionExtractionResponse(BaseModel):
-    """Response containing extracted opinions."""
-
-    opinions: list[Opinion] = Field(
-        default_factory=list, description="List of opinions formed with their supporting reasons and confidence scores"
-    )
-
-
 def describe_trait_level(value: int) -> str:
     """Convert trait value (1-5) to descriptive text."""
     levels = {1: "very low", 2: "low", 3: "moderate", 4: "high", 5: "very high"}
@@ -93,17 +75,46 @@ def format_facts_for_prompt(facts: list[MemoryFact]) -> str:
     return json.dumps(formatted, indent=2)
 
 
+def format_entity_summaries_for_prompt(entities: dict) -> str:
+    """Format entity summaries for inclusion in the reflect prompt.
+
+    Args:
+        entities: Dict mapping entity name to EntityState objects
+
+    Returns:
+        Formatted string with entity summaries, or empty string if no summaries
+    """
+    if not entities:
+        return ""
+
+    summaries = []
+    for name, state in entities.items():
+        # Get summary from observations (summary is stored as single observation)
+        if state.observations:
+            summary_text = state.observations[0].text
+            summaries.append(f"## {name}\n{summary_text}")
+
+    if not summaries:
+        return ""
+
+    return "\n\n".join(summaries)
+
+
 def build_think_prompt(
     agent_facts_text: str,
     world_facts_text: str,
-    opinion_facts_text: str,
     query: str,
     name: str,
     disposition: DispositionTraits,
     background: str,
     context: str | None = None,
+    entity_summaries_text: str | None = None,
 ) -> str:
-    """Build the think prompt for the LLM."""
+    """Build the think prompt for the LLM.
+
+    Note: opinion_facts_text parameter removed - opinions are now stored as mental models
+    and included via entity_summaries_text.
+    """
     disposition_desc = build_disposition_description(disposition)
 
     name_section = f"""
@@ -125,6 +136,14 @@ Your background:
 ADDITIONAL CONTEXT:
 {context}
 
+"""
+
+    entity_section = ""
+    if entity_summaries_text:
+        entity_section = f"""
+KEY PEOPLE, PLACES & THINGS I KNOW ABOUT:
+{entity_summaries_text}
+
 """
 
     return f"""Here's what I know and have experienced:
@@ -135,14 +154,11 @@ MY IDENTITY & EXPERIENCES:
 WHAT I KNOW ABOUT THE WORLD:
 {world_facts_text}
 
-MY EXISTING OPINIONS & BELIEFS:
-{opinion_facts_text}
-
-{context_section}{name_section}{disposition_desc}{background_section}
+{entity_section}{context_section}{name_section}{disposition_desc}{background_section}
 
 QUESTION: {query}
 
-Based on everything I know, believe, and who I am (including my name, disposition and background), here's what I genuinely think about this question. I'll draw on my experiences, knowledge, opinions, and personal traits to give you my honest perspective."""
+Based on everything I know, believe, and who I am (including my name, disposition and background), here's what I genuinely think about this question. I'll draw on my experiences, knowledge, and personal traits to give you my honest perspective."""
 
 
 def get_system_message(disposition: DispositionTraits) -> str:
@@ -172,117 +188,7 @@ def get_system_message(disposition: DispositionTraits) -> str:
         " ".join(instructions) if instructions else "Balance your disposition traits when interpreting information."
     )
 
-    return f"You are a person with your own thoughts, experiences, opinions, and disposition. When asked a question, respond naturally like a human would - share what you genuinely think based on what you know and have experienced. {disposition_instruction} Be direct, express your views confidently, and use 'I think', 'I believe', 'in my view', etc. Respond in plain text without markdown formatting."
-
-
-async def extract_opinions_from_text(llm_config, text: str, query: str) -> list[Opinion]:
-    """
-    Extract opinions with reasons and confidence from text using LLM.
-
-    Args:
-        llm_config: LLM configuration to use
-        text: Text to extract opinions from
-        query: The original query that prompted this response
-
-    Returns:
-        List of Opinion objects with text and confidence
-    """
-    extraction_prompt = f"""Extract any NEW opinions or perspectives from the answer below and rewrite them in FIRST-PERSON as if YOU are stating the opinion directly.
-
-ORIGINAL QUESTION:
-{query}
-
-ANSWER PROVIDED:
-{text}
-
-Your task: Find opinions in the answer and rewrite them AS IF YOU ARE THE ONE SAYING THEM.
-
-An opinion is a judgment, viewpoint, or conclusion that goes beyond just stating facts.
-
-IMPORTANT: Do NOT extract statements like:
-- "I don't have enough information"
-- "The facts don't contain information about X"
-- "I cannot answer because..."
-
-ONLY extract actual opinions about substantive topics.
-
-CRITICAL FORMAT REQUIREMENTS:
-1. **ALWAYS start with first-person phrases**: "I think...", "I believe...", "In my view...", "I've come to believe...", "Previously I thought... but now..."
-2. **NEVER use third-person**: Do NOT say "The speaker thinks..." or "They believe..." - always use "I"
-3. Include the reasoning naturally within the statement
-4. Provide a confidence score (0.0 to 1.0)
-
-CORRECT Examples (✓ FIRST-PERSON):
-- "I think Alice is more reliable because she consistently delivers on time and writes clean code"
-- "Previously I thought all engineers were equal, but now I feel that experience and track record really matter"
-- "I believe reliability is best measured by consistent output over time"
-- "I've come to believe that track records are more important than potential"
-
-WRONG Examples (✗ THIRD-PERSON - DO NOT USE):
-- "The speaker thinks Alice is more reliable"
-- "They believe reliability matters"
-- "It is believed that Alice is better"
-
-If no genuine opinions are expressed (e.g., the response just says "I don't know"), return an empty list."""
-
-    try:
-        result = await llm_config.call(
-            messages=[
-                {
-                    "role": "system",
-                    "content": "You are converting opinions from text into first-person statements. Always use 'I think', 'I believe', 'I feel', etc. NEVER use third-person like 'The speaker' or 'They'.",
-                },
-                {"role": "user", "content": extraction_prompt},
-            ],
-            response_format=OpinionExtractionResponse,
-            scope="memory_extract_opinion",
-        )
-
-        # Format opinions with confidence score and convert to first-person
-        formatted_opinions = []
-        for op in result.opinions:
-            # Convert third-person to first-person if needed
-            opinion_text = op.opinion
-
-            # Replace common third-person patterns with first-person
-            def singularize_verb(verb):
-                if verb.endswith("es"):
-                    return verb[:-1]  # believes -> believe
-                elif verb.endswith("s"):
-                    return verb[:-1]  # thinks -> think
-                return verb
-
-            # Pattern: "The speaker/user [verb]..." -> "I [verb]..."
-            match = re.match(
-                r"^(The speaker|The user|They|It is believed) (believes?|thinks?|feels?|says|asserts?|considers?)(\s+that)?(.*)$",
-                opinion_text,
-                re.IGNORECASE,
-            )
-            if match:
-                verb = singularize_verb(match.group(2))
-                that_part = match.group(3) or ""  # Keep " that" if present
-                rest = match.group(4)
-                opinion_text = f"I {verb}{that_part}{rest}"
-
-            # If still doesn't start with first-person, prepend "I believe that "
-            first_person_starters = [
-                "I think",
-                "I believe",
-                "I feel",
-                "In my view",
-                "I've come to believe",
-                "Previously I",
-            ]
-            if not any(opinion_text.startswith(starter) for starter in first_person_starters):
-                opinion_text = "I believe that " + opinion_text[0].lower() + opinion_text[1:]
-
-            formatted_opinions.append(Opinion(opinion=opinion_text, confidence=op.confidence))
-
-        return formatted_opinions
-
-    except Exception as e:
-        logger.warning(f"Failed to extract opinions: {str(e)}")
-        return []
+    return f"You are a person with your own thoughts, experiences, opinions, and disposition. When asked a question, respond naturally like a human would - share what you genuinely think based on what you know and have experienced. {disposition_instruction} Be direct, express your views confidently, and use 'I think', 'I believe', 'in my view', etc. Respond in plain text without markdown formatting. IMPORTANT: Detect the language of the question and respond in the SAME language. Do not translate to English if the question is in another language."
 
 
 async def reflect(
@@ -290,7 +196,6 @@ async def reflect(
     query: str,
     experience_facts: list[str] = None,
     world_facts: list[str] = None,
-    opinion_facts: list[str] = None,
     name: str = "Assistant",
     disposition: DispositionTraits = None,
     background: str = "",
@@ -307,7 +212,6 @@ async def reflect(
         query: Question to answer
         experience_facts: List of experience/agent fact strings
         world_facts: List of world fact strings
-        opinion_facts: List of opinion fact strings
         name: Name of the agent/persona
         disposition: Disposition traits (defaults to neutral)
         background: Background information
@@ -328,18 +232,15 @@ async def reflect(
 
     agent_results = to_memory_facts(experience_facts or [], "experience")
     world_results = to_memory_facts(world_facts or [], "world")
-    opinion_results = to_memory_facts(opinion_facts or [], "opinion")
 
     # Format facts for prompt
     agent_facts_text = format_facts_for_prompt(agent_results)
     world_facts_text = format_facts_for_prompt(world_results)
-    opinion_facts_text = format_facts_for_prompt(opinion_results)
 
     # Build prompt
     prompt = build_think_prompt(
         agent_facts_text=agent_facts_text,
         world_facts_text=world_facts_text,
-        opinion_facts_text=opinion_facts_text,
         query=query,
         name=name,
         disposition=disposition,

hindsight-api/hindsight_api/engine/search/trace.py

@@ -11,6 +11,13 @@ from typing import Any, Literal
 from pydantic import BaseModel, Field
 
 
+class TemporalConstraint(BaseModel):
+    """Detected temporal constraint from query analysis."""
+
+    start: datetime | None = Field(default=None, description="Start of temporal range")
+    end: datetime | None = Field(default=None, description="End of temporal range")
+
+
 class QueryInfo(BaseModel):
     """Information about the search query."""
 
@@ -19,6 +26,11 @@ class QueryInfo(BaseModel):
     timestamp: datetime = Field(description="When the query was executed")
     budget: int = Field(description="Maximum nodes to explore")
     max_tokens: int = Field(description="Maximum tokens to return in results")
+    tags: list[str] | None = Field(default=None, description="Tags filter applied to recall")
+    tags_match: str | None = Field(default=None, description="Tags matching mode: any, all, any_strict, all_strict")
+    temporal_constraint: TemporalConstraint | None = Field(
+        default=None, description="Detected temporal range from query"
+    )
 
 
 class EntryPoint(BaseModel):
@@ -73,7 +85,6 @@ class NodeVisit(BaseModel):
     text: str = Field(description="Memory unit text content")
     context: str = Field(description="Memory unit context")
     event_date: datetime | None = Field(default=None, description="When the memory occurred")
-    access_count: int = Field(description="Number of times accessed before this search")
 
     # How this node was reached
     is_entry_point: bool = Field(description="Whether this is an entry point")

hindsight-api/hindsight_api/engine/search/tracer.py

@@ -22,6 +22,7 @@ from .trace import (
     SearchPhaseMetrics,
     SearchSummary,
     SearchTrace,
+    TemporalConstraint,
     WeightComponents,
 )
 
@@ -45,7 +46,14 @@ class SearchTracer:
         json_output = trace.to_json()
     """
 
-    def __init__(self, query: str, budget: int, max_tokens: int):
+    def __init__(
+        self,
+        query: str,
+        budget: int,
+        max_tokens: int,
+        tags: list[str] | None = None,
+        tags_match: str | None = None,
+    ):
         """
         Initialize tracer.
 
@@ -53,10 +61,14 @@ class SearchTracer:
             query: Search query text
             budget: Maximum nodes to explore
             max_tokens: Maximum tokens to return in results
+            tags: Tags filter applied to recall
+            tags_match: Tags matching mode (any, all, any_strict, all_strict)
         """
         self.query_text = query
         self.budget = budget
         self.max_tokens = max_tokens
+        self.tags = tags
+        self.tags_match = tags_match
 
         # Trace data
         self.query_embedding: list[float] | None = None
@@ -66,6 +78,9 @@ class SearchTracer:
         self.pruned: list[PruningDecision] = []
         self.phase_metrics: list[SearchPhaseMetrics] = []
 
+        # Temporal constraint detected from query
+        self.temporal_constraint: TemporalConstraint | None = None
+
         # New 4-way retrieval tracking
         self.retrieval_results: list[RetrievalMethodResults] = []
         self.rrf_merged: list[RRFMergeResult] = []
@@ -88,6 +103,11 @@ class SearchTracer:
         """Record the query embedding."""
         self.query_embedding = embedding
 
+    def record_temporal_constraint(self, start: datetime | None, end: datetime | None):
+        """Record the detected temporal constraint from query analysis."""
+        if start is not None or end is not None:
+            self.temporal_constraint = TemporalConstraint(start=start, end=end)
+
     def add_entry_point(self, node_id: str, text: str, similarity: float, rank: int):
         """
         Record an entry point.
@@ -116,7 +136,6 @@ class SearchTracer:
         text: str,
         context: str,
         event_date: datetime | None,
-        access_count: int,
         is_entry_point: bool,
         parent_node_id: str | None,
         link_type: Literal["temporal", "semantic", "entity"] | None,
@@ -135,7 +154,6 @@ class SearchTracer:
             text: Memory unit text
             context: Memory unit context
             event_date: When the memory occurred
-            access_count: Access count before this search
             is_entry_point: Whether this is an entry point
             parent_node_id: Node that led here (None for entry points)
             link_type: Type of link from parent
@@ -174,7 +192,6 @@ class SearchTracer:
             text=text,
             context=context,
             event_date=event_date,
-            access_count=access_count,
             is_entry_point=is_entry_point,
             parent_node_id=parent_node_id,
             link_type=link_type,
@@ -313,8 +330,8 @@ class SearchTracer:
                 RetrievalResult(
                     rank=rank,
                     node_id=doc_id,
-                    text=data.get("text", ""),
-                    context=data.get("context", ""),
+                    text=data.get("text") or "",
+                    context=data.get("context") or "",
                     event_date=data.get("event_date"),
                     fact_type=data.get("fact_type") or fact_type,
                     score=score,
@@ -428,6 +445,9 @@ class SearchTracer:
             timestamp=datetime.now(UTC),
             budget=self.budget,
             max_tokens=self.max_tokens,
+            tags=self.tags,
+            tags_match=self.tags_match,
+            temporal_constraint=self.temporal_constraint,
         )
 
         # Create summary

hindsight-api/hindsight_api/engine/search/types.py

@@ -10,6 +10,24 @@ from datetime import datetime
 from typing import Any
 
 
+@dataclass
+class MPFPTimings:
+    """Timing breakdown for a single MPFP retrieval call."""
+
+    fact_type: str
+    edge_count: int = 0  # Total edges loaded
+    db_queries: int = 0  # Number of DB queries for edge loading
+    edge_load_time: float = 0.0  # Time spent loading edges from DB
+    traverse: float = 0.0  # Total traversal time (includes edge loading)
+    pattern_count: int = 0  # Number of patterns executed
+    fusion: float = 0.0  # Time for RRF fusion
+    fetch: float = 0.0  # Time to fetch memory unit details
+    seeds_time: float = 0.0  # Time to find semantic seeds (if fallback used)
+    result_count: int = 0  # Number of results returned
+    # Detailed per-hop timing: list of {hop, exec_time, uncached, load_time, edges_loaded, total_time}
+    hop_details: list[dict] = field(default_factory=list)
+
+
 @dataclass
 class RetrievalResult:
     """
@@ -28,8 +46,8 @@ class RetrievalResult:
     mentioned_at: datetime | None = None
     document_id: str | None = None
     chunk_id: str | None = None
-    access_count: int = 0
     embedding: list[float] | None = None
+    tags: list[str] | None = None  # Visibility scope tags
 
     # Retrieval-specific scores (only one will be set depending on retrieval method)
     similarity: float | None = None  # Semantic retrieval
@@ -52,8 +70,8 @@ class RetrievalResult:
             mentioned_at=row.get("mentioned_at"),
             document_id=row.get("document_id"),
             chunk_id=row.get("chunk_id"),
-            access_count=row.get("access_count", 0),
             embedding=row.get("embedding"),
+            tags=row.get("tags"),
             similarity=row.get("similarity"),
             bm25_score=row.get("bm25_score"),
             activation=row.get("activation"),
@@ -136,8 +154,8 @@ class ScoredResult:
             "mentioned_at": self.retrieval.mentioned_at,
             "document_id": self.retrieval.document_id,
             "chunk_id": self.retrieval.chunk_id,
-            "access_count": self.retrieval.access_count,
             "embedding": self.retrieval.embedding,
+            "tags": self.retrieval.tags,
             "semantic_similarity": self.retrieval.similarity,
             "bm25_score": self.retrieval.bm25_score,
         }

hindsight-api/hindsight_api/engine/task_backend.py

@@ -1,31 +1,40 @@
 """
-Abstract task backend for running async tasks.
+Task backend for distributed task processing.
 
-This provides an abstraction that can be adapted to different execution models:
-- AsyncIO queue (default implementation)
-- Pub/Sub architectures (future)
-- Message brokers (future)
+This provides an abstraction for task storage and execution:
+- BrokerTaskBackend: Uses PostgreSQL as broker (production)
+- SyncTaskBackend: Executes tasks immediately (testing/embedded)
 """
 
-import asyncio
+import json
 import logging
 from abc import ABC, abstractmethod
 from collections.abc import Awaitable, Callable
-from typing import Any
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import asyncpg
 
 logger = logging.getLogger(__name__)
 
 
+def fq_table(table: str, schema: str | None = None) -> str:
+    """Get fully-qualified table name with optional schema prefix."""
+    if schema:
+        return f'"{schema}".{table}'
+    return table
+
+
 class TaskBackend(ABC):
     """
     Abstract base class for task execution backends.
 
     Implementations must:
     1. Store/publish task events (as serializable dicts)
-    2. Execute tasks through a provided executor callback
+    2. Execute tasks through a provided executor callback (optional)
 
     The backend treats tasks as pure dictionaries that can be serialized
-    and sent over the network. The executor (typically MemoryEngine.execute_task)
+    and stored in the database. The executor (typically MemoryEngine.execute_task)
     receives the dict and routes it to the appropriate handler.
     """
 
@@ -46,7 +55,7 @@ class TaskBackend(ABC):
     @abstractmethod
     async def initialize(self):
         """
-        Initialize the backend (e.g., start workers, connect to broker).
+        Initialize the backend (e.g., connect to database).
         """
         pass
 
@@ -63,7 +72,7 @@ class TaskBackend(ABC):
     @abstractmethod
     async def shutdown(self):
         """
-        Shutdown the backend gracefully (e.g., stop workers, close connections).
+        Shutdown the backend gracefully.
         """
         pass
 
@@ -93,9 +102,8 @@ class SyncTaskBackend(TaskBackend):
     """
     Synchronous task backend that executes tasks immediately.
 
-    This is useful for embedded/CLI usage where we don't want background
-    workers that prevent clean exit. Tasks are executed inline rather than
-    being queued.
+    This is useful for tests and embedded/CLI usage where we don't want
+    background workers. Tasks are executed inline rather than being queued.
     """
 
     async def initialize(self):
@@ -121,130 +129,129 @@ class SyncTaskBackend(TaskBackend):
         logger.debug("SyncTaskBackend shutdown")
 
 
-class AsyncIOQueueBackend(TaskBackend):
+class BrokerTaskBackend(TaskBackend):
     """
-    Task backend implementation using asyncio queues.
+    Task backend using PostgreSQL as broker.
 
-    This is the default implementation that uses in-process asyncio queues
-    and a periodic consumer worker.
+    submit_task() stores task_payload in async_operations table.
+    Actual polling and execution is handled separately by WorkerPoller.
+
+    This backend is used by the API to store tasks. Workers poll
+    the database separately to claim and execute tasks.
     """
 
-    def __init__(self, batch_size: int = 100, batch_interval: float = 1.0):
+    def __init__(
+        self,
+        pool_getter: Callable[[], "asyncpg.Pool"],
+        schema: str | None = None,
+        schema_getter: Callable[[], str | None] | None = None,
+    ):
         """
-        Initialize AsyncIO queue backend.
+        Initialize the broker task backend.
 
         Args:
-            batch_size: Maximum number of tasks to process in one batch
-            batch_interval: Maximum time (seconds) to wait before processing batch
+            pool_getter: Callable that returns the asyncpg connection pool
+            schema: Database schema for multi-tenant support (optional, static)
+            schema_getter: Callable that returns current schema dynamically (optional).
+                          If set, takes precedence over static schema for submit_task.
         """
         super().__init__()
-        self._queue: asyncio.Queue | None = None
-        self._worker_task: asyncio.Task | None = None
-        self._shutdown_event: asyncio.Event | None = None
-        self._batch_size = batch_size
-        self._batch_interval = batch_interval
+        self._pool_getter = pool_getter
+        self._schema = schema
+        self._schema_getter = schema_getter
 
     async def initialize(self):
-        """Initialize the queue and start the worker."""
-        if self._initialized:
-            return
-
-        self._queue = asyncio.Queue()
-        self._shutdown_event = asyncio.Event()
-        self._worker_task = asyncio.create_task(self._worker())
+        """Initialize the backend."""
         self._initialized = True
-        logger.info("AsyncIOQueueBackend initialized")
+        logger.info("BrokerTaskBackend initialized")
 
     async def submit_task(self, task_dict: dict[str, Any]):
         """
-        Submit a task by putting it in the queue.
+        Store task payload in async_operations table.
+
+        The task_dict should contain an 'operation_id' if updating an existing
+        operation record, otherwise a new operation will be created.
 
         Args:
-            task_dict: Task dictionary to execute
+            task_dict: Task dictionary to store (must be JSON serializable)
         """
         if not self._initialized:
             await self.initialize()
 
-        await self._queue.put(task_dict)
+        pool = self._pool_getter()
+        operation_id = task_dict.get("operation_id")
         task_type = task_dict.get("type", "unknown")
-        task_id = task_dict.get("id")
+        bank_id = task_dict.get("bank_id")
+        payload_json = json.dumps(task_dict)
 
-    async def wait_for_pending_tasks(self, timeout: float = 5.0):
+        schema = self._schema_getter() if self._schema_getter else self._schema
+        table = fq_table("async_operations", schema)
+
+        if operation_id:
+            # Update existing operation with task payload
+            await pool.execute(
+                f"""
+                UPDATE {table}
+                SET task_payload = $1::jsonb, updated_at = now()
+                WHERE operation_id = $2
+                """,
+                payload_json,
+                operation_id,
+            )
+            logger.debug(f"Updated task payload for operation {operation_id}")
+        else:
+            # Insert new operation (for tasks without pre-created records)
+            # e.g., access_count_update tasks
+            import uuid
+
+            new_id = uuid.uuid4()
+            await pool.execute(
+                f"""
+                INSERT INTO {table} (operation_id, bank_id, operation_type, status, task_payload)
+                VALUES ($1, $2, $3, 'pending', $4::jsonb)
+                """,
+                new_id,
+                bank_id,
+                task_type,
+                payload_json,
+            )
+            logger.debug(f"Created new operation {new_id} for task type {task_type}")
+
+    async def shutdown(self):
+        """Shutdown the backend."""
+        self._initialized = False
+        logger.info("BrokerTaskBackend shutdown")
+
+    async def wait_for_pending_tasks(self, timeout: float = 120.0):
         """
-        Wait for all pending tasks in the queue to be processed.
+        Wait for pending tasks to be processed.
 
-        This is useful in tests to ensure background tasks complete before assertions.
+        In the broker model, this polls the database to check if tasks
+        for this process have been completed. This is useful in tests
+        when worker_enabled=True (API processes its own tasks).
 
         Args:
             timeout: Maximum time to wait in seconds
         """
-        if not self._initialized or self._queue is None:
-            return
+        import asyncio
+
+        pool = self._pool_getter()
+        schema = self._schema_getter() if self._schema_getter else self._schema
+        table = fq_table("async_operations", schema)
 
-        # Wait for queue to be empty and give worker time to process
         start_time = asyncio.get_event_loop().time()
         while asyncio.get_event_loop().time() - start_time < timeout:
-            if self._queue.empty():
-                # Queue is empty, give worker a bit more time to finish any in-flight task
-                await asyncio.sleep(0.3)
-                # Check again - if still empty, we're done
-                if self._queue.empty():
-                    return
-            else:
-                # Queue not empty, wait a bit
-                await asyncio.sleep(0.1)
+            # Check if there are any pending tasks with payloads
+            count = await pool.fetchval(
+                f"""
+                SELECT COUNT(*) FROM {table}
+                WHERE status = 'pending' AND task_payload IS NOT NULL
+                """
+            )
 
-    async def shutdown(self):
-        """Shutdown the worker and drain the queue."""
-        if not self._initialized:
-            return
+            if count == 0:
+                return
 
-        logger.info("Shutting down AsyncIOQueueBackend...")
+            await asyncio.sleep(0.5)
 
-        # Signal shutdown
-        self._shutdown_event.set()
-
-        # Cancel worker
-        if self._worker_task is not None:
-            self._worker_task.cancel()
-            try:
-                await self._worker_task
-            except asyncio.CancelledError:
-                pass  # Worker cancelled successfully
-
-        self._initialized = False
-        logger.info("AsyncIOQueueBackend shutdown complete")
-
-    async def _worker(self):
-        """
-        Background worker that processes tasks in batches.
-
-        Collects tasks for up to batch_interval seconds or batch_size items,
-        then processes them.
-        """
-        while not self._shutdown_event.is_set():
-            try:
-                # Collect tasks for batching
-                tasks = []
-                deadline = asyncio.get_event_loop().time() + self._batch_interval
-
-                while len(tasks) < self._batch_size and asyncio.get_event_loop().time() < deadline:
-                    try:
-                        remaining_time = max(0.1, deadline - asyncio.get_event_loop().time())
-                        task_dict = await asyncio.wait_for(self._queue.get(), timeout=remaining_time)
-                        tasks.append(task_dict)
-                    except TimeoutError:
-                        break
-
-                # Process batch
-                if tasks:
-                    # Execute tasks concurrently
-                    await asyncio.gather(
-                        *[self._execute_task(task_dict) for task_dict in tasks], return_exceptions=True
-                    )
-
-            except asyncio.CancelledError:
-                break
-            except Exception as e:
-                logger.error(f"Worker error: {e}")
-                await asyncio.sleep(1)  # Backoff on error
+        logger.warning(f"Timeout waiting for pending tasks after {timeout}s")

hindsight-api/hindsight_api/engine/utils.py

@@ -49,7 +49,7 @@ async def extract_facts(
     if not text or not text.strip():
         return [], []
 
-    facts, chunks = await extract_facts_from_text(
+    facts, chunks, _ = await extract_facts_from_text(
         text,
         event_date,
         context=context,
@@ -65,154 +65,3 @@ async def extract_facts(
         return [], chunks
 
     return facts, chunks
-
-
-def cosine_similarity(vec1: list[float], vec2: list[float]) -> float:
-    """
-    Calculate cosine similarity between two vectors.
-
-    Args:
-        vec1: First vector
-        vec2: Second vector
-
-    Returns:
-        Similarity score between 0 and 1
-    """
-    if len(vec1) != len(vec2):
-        raise ValueError("Vectors must have same dimension")
-
-    dot_product = sum(a * b for a, b in zip(vec1, vec2))
-    magnitude1 = sum(a * a for a in vec1) ** 0.5
-    magnitude2 = sum(b * b for b in vec2) ** 0.5
-
-    if magnitude1 == 0 or magnitude2 == 0:
-        return 0.0
-
-    return dot_product / (magnitude1 * magnitude2)
-
-
-def calculate_recency_weight(days_since: float, half_life_days: float = 365.0) -> float:
-    """
-    Calculate recency weight using logarithmic decay.
-
-    This provides much better differentiation over long time periods compared to
-    exponential decay. Uses a log-based decay where the half-life parameter controls
-    when memories reach 50% weight.
-
-    Examples:
-        - Today (0 days): 1.0
-        - 1 year (365 days): ~0.5 (with default half_life=365)
-        - 2 years (730 days): ~0.33
-        - 5 years (1825 days): ~0.17
-        - 10 years (3650 days): ~0.09
-
-    This ensures that 2-year-old and 5-year-old memories have meaningfully
-    different weights, unlike exponential decay which makes them both ~0.
-
-    Args:
-        days_since: Number of days since the memory was created
-        half_life_days: Number of days for weight to reach 0.5 (default: 1 year)
-
-    Returns:
-        Weight between 0 and 1
-    """
-    import math
-
-    # Logarithmic decay: 1 / (1 + log(1 + days_since/half_life))
-    # This decays much slower than exponential, giving better long-term differentiation
-    normalized_age = days_since / half_life_days
-    return 1.0 / (1.0 + math.log1p(normalized_age))
-
-
-def calculate_frequency_weight(access_count: int, max_boost: float = 2.0) -> float:
-    """
-    Calculate frequency weight based on access count.
-
-    Frequently accessed memories are weighted higher.
-    Uses logarithmic scaling to avoid over-weighting.
-
-    Args:
-        access_count: Number of times the memory was accessed
-        max_boost: Maximum multiplier for frequently accessed memories
-
-    Returns:
-        Weight between 1.0 and max_boost
-    """
-    import math
-
-    if access_count <= 0:
-        return 1.0
-
-    # Logarithmic scaling: log(access_count + 1) / log(10)
-    # This gives: 0 accesses = 1.0, 9 accesses ~= 1.5, 99 accesses ~= 2.0
-    normalized = math.log(access_count + 1) / math.log(10)
-    return 1.0 + min(normalized, max_boost - 1.0)
-
-
-def calculate_temporal_anchor(occurred_start: datetime, occurred_end: datetime) -> datetime:
-    """
-    Calculate a single temporal anchor point from a temporal range.
-
-    Used for spreading activation - we need a single representative date
-    to calculate temporal proximity between facts. This simplifies the
-    range-to-range distance problem.
-
-    Strategy: Use midpoint of the range for balanced representation.
-
-    Args:
-        occurred_start: Start of temporal range
-        occurred_end: End of temporal range
-
-    Returns:
-        Single datetime representing the temporal anchor (midpoint)
-
-    Examples:
-        - Point event (July 14): start=July 14, end=July 14 → anchor=July 14
-        - Month range (February): start=Feb 1, end=Feb 28 → anchor=Feb 14
-        - Year range (2023): start=Jan 1, end=Dec 31 → anchor=July 1
-    """
-    # Calculate midpoint
-    time_delta = occurred_end - occurred_start
-    midpoint = occurred_start + (time_delta / 2)
-    return midpoint
-
-
-def calculate_temporal_proximity(anchor_a: datetime, anchor_b: datetime, half_life_days: float = 30.0) -> float:
-    """
-    Calculate temporal proximity between two temporal anchors.
-
-    Used for spreading activation to determine how "close" two facts are
-    in time. Uses logarithmic decay so that temporal similarity doesn't
-    drop off too quickly.
-
-    Args:
-        anchor_a: Temporal anchor of first fact
-        anchor_b: Temporal anchor of second fact
-        half_life_days: Number of days for proximity to reach 0.5
-                       (default: 30 days = 1 month)
-
-    Returns:
-        Proximity score in [0, 1] where:
-        - 1.0 = same day
-        - 0.5 = ~half_life days apart
-        - 0.0 = very distant in time
-
-    Examples:
-        - Same day: 1.0
-        - 1 week apart (half_life=30): ~0.7
-        - 1 month apart (half_life=30): ~0.5
-        - 1 year apart (half_life=30): ~0.2
-    """
-    import math
-
-    days_apart = abs((anchor_a - anchor_b).days)
-
-    if days_apart == 0:
-        return 1.0
-
-    # Logarithmic decay: 1 / (1 + log(1 + days_apart/half_life))
-    # Similar to calculate_recency_weight but for proximity between events
-    normalized_distance = days_apart / half_life_days
-    proximity = 1.0 / (1.0 + math.log1p(normalized_distance))
-
-    return proximity

hindsight-api/hindsight_api/extensions/__init__.py

@@ -21,6 +21,10 @@ from hindsight_api.extensions.context import DefaultExtensionContext, ExtensionC
 from hindsight_api.extensions.http import HttpExtension
 from hindsight_api.extensions.loader import load_extension
 from hindsight_api.extensions.operation_validator import (
+    # Consolidation operation
+    ConsolidateContext,
+    ConsolidateResult,
+    # Core operations
     OperationValidationError,
     OperationValidatorExtension,
     RecallContext,
@@ -33,6 +37,7 @@ from hindsight_api.extensions.operation_validator import (
 )
 from hindsight_api.extensions.tenant import (
     AuthenticationError,
+    Tenant,
     TenantContext,
     TenantExtension,
 )
@@ -47,7 +52,7 @@ __all__ = [
     "DefaultExtensionContext",
     # HTTP Extension
     "HttpExtension",
-    # Operation Validator
+    # Operation Validator - Core
     "OperationValidationError",
     "OperationValidatorExtension",
     "RecallContext",
@@ -57,10 +62,14 @@ __all__ = [
     "RetainContext",
     "RetainResult",
     "ValidationResult",
+    # Operation Validator - Consolidation
+    "ConsolidateContext",
+    "ConsolidateResult",
     # Tenant/Auth
     "ApiKeyTenantExtension",
     "AuthenticationError",
     "RequestContext",
+    "Tenant",
     "TenantContext",
     "TenantExtension",
 ]

hindsight-api/hindsight_api/extensions/builtin/tenant.py

@@ -1,6 +1,6 @@
 """Built-in tenant extension implementations."""
 
-from hindsight_api.extensions.tenant import AuthenticationError, TenantContext, TenantExtension
+from hindsight_api.extensions.tenant import AuthenticationError, Tenant, TenantContext, TenantExtension
 from hindsight_api.models import RequestContext
 
 
@@ -31,3 +31,7 @@ class ApiKeyTenantExtension(TenantExtension):
         if context.api_key != self.expected_api_key:
             raise AuthenticationError("Invalid API key")
         return TenantContext(schema_name="public")
+
+    async def list_tenants(self) -> list[Tenant]:
+        """Return public schema for single-tenant setup."""
+        return [Tenant(schema="public")]

hindsight-api/hindsight_api/extensions/context.py

@@ -96,7 +96,7 @@ class DefaultExtensionContext(ExtensionContext):
 
     async def run_migration(self, schema: str) -> None:
         """Run migrations for a specific schema."""
-        from hindsight_api.migrations import run_migrations
+        from hindsight_api.migrations import ensure_embedding_dimension, run_migrations
 
         # Prefer getting URL from memory engine (handles pg0 case where URL is set after init)
         db_url = self._database_url
@@ -107,6 +107,15 @@ class DefaultExtensionContext(ExtensionContext):
 
         run_migrations(db_url, schema=schema)
 
+        # Ensure embedding column dimension matches the model's dimension
+        # This is needed because migrations create columns with default dimension
+        if self._memory_engine is not None:
+            embeddings = getattr(self._memory_engine, "embeddings", None)
+            if embeddings is not None:
+                dimension = getattr(embeddings, "dimension", None)
+                if dimension is not None:
+                    ensure_embedding_dimension(db_url, dimension, schema=schema)
+
     def get_memory_engine(self) -> "MemoryEngineInterface":
         """Get the memory engine interface."""
         if self._memory_engine is None:

hindsight-api/hindsight_api/extensions/operation_validator.py

@@ -1,4 +1,4 @@
-"""Operation Validator Extension for validating retain/recall/reflect operations."""
+"""Operation Validator Extension for validating retain/recall/reflect/consolidate operations."""
 
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
@@ -97,6 +97,19 @@ class ReflectContext:
     context: str | None = None
 
 
+# =============================================================================
+# Consolidation Pre-operation Context
+# =============================================================================
+
+
+@dataclass
+class ConsolidateContext:
+    """Context for a consolidation operation validation (pre-operation)."""
+
+    bank_id: str
+    request_context: "RequestContext"
+
+
 # =============================================================================
 # Post-operation Contexts (includes results)
 # =============================================================================
@@ -164,9 +177,28 @@ class ReflectResultContext:
     error: str | None = None
 
 
+# =============================================================================
+# Consolidation Post-operation Context
+# =============================================================================
+
+
+@dataclass
+class ConsolidateResult:
+    """Result context for post-consolidation hook."""
+
+    bank_id: str
+    request_context: "RequestContext"
+    # Result
+    processed: int = 0
+    created: int = 0
+    updated: int = 0
+    success: bool = True
+    error: str | None = None
+
+
 class OperationValidatorExtension(Extension, ABC):
     """
-    Validates and hooks into retain/recall/reflect operations.
+    Validates and hooks into retain/recall/reflect/consolidate operations.
 
     This extension allows implementing custom logic such as:
     - Rate limiting (pre-operation)
@@ -185,9 +217,13 @@ class OperationValidatorExtension(Extension, ABC):
         -> config = {"max_requests": "100"}
 
     Hook execution order:
-        1. validate_retain/validate_recall/validate_reflect (pre-operation)
+        1. validate_* (pre-operation)
         2. [operation executes]
-        3. on_retain_complete/on_recall_complete/on_reflect_complete (post-operation)
+        3. on_*_complete (post-operation)
+
+    Supported operations:
+        - retain, recall, reflect (core memory operations)
+        - consolidate (mental models consolidation)
     """
 
     # =========================================================================
@@ -325,3 +361,44 @@ class OperationValidatorExtension(Extension, ABC):
                 - error: Error message (if failed)
         """
         pass
+
+    # =========================================================================
+    # Consolidation - Pre-operation validation hook (optional - override to implement)
+    # =========================================================================
+
+    async def validate_consolidate(self, ctx: ConsolidateContext) -> ValidationResult:
+        """
+        Validate a consolidation operation before execution.
+
+        Override to implement custom validation logic for consolidation.
+
+        Args:
+            ctx: Context containing:
+                - bank_id: Bank identifier
+                - request_context: Request context with auth info
+
+        Returns:
+            ValidationResult indicating whether the operation is allowed.
+        """
+        return ValidationResult.accept()
+
+    # =========================================================================
+    # Consolidation - Post-operation hook (optional - override to implement)
+    # =========================================================================
+
+    async def on_consolidate_complete(self, result: ConsolidateResult) -> None:
+        """
+        Called after a consolidation operation completes (success or failure).
+
+        Override to implement post-operation logic such as usage tracking or audit logging.
+
+        Args:
+            result: Result context containing:
+                - bank_id: Bank identifier
+                - processed: Number of memories processed
+                - created: Number of mental models created
+                - updated: Number of mental models updated
+                - success: Whether the operation succeeded
+                - error: Error message (if failed)
+        """
+        pass

hindsight-api/hindsight_api/extensions/tenant.py

@@ -28,6 +28,18 @@ class TenantContext:
     schema_name: str
 
 
+@dataclass
+class Tenant:
+    """
+    Represents a tenant for worker discovery.
+
+    Used by list_tenants() to return tenant information including
+    the PostgreSQL schema name for database operations.
+    """
+
+    schema: str
+
+
 class TenantExtension(Extension, ABC):
     """
     Extension for multi-tenancy and API key authentication.
@@ -61,3 +73,17 @@ class TenantExtension(Extension, ABC):
             AuthenticationError: If authentication fails.
         """
         ...
+
+    @abstractmethod
+    async def list_tenants(self) -> list[Tenant]:
+        """
+        List all tenants that should be processed by workers.
+
+        This method is used by the worker to discover all tenants that need
+        task polling. Workers will poll for pending tasks in each tenant's schema.
+
+        Returns:
+            List of Tenant objects containing schema information.
+            For single-tenant setups, return [Tenant(schema="public")].
+        """
+        ...

hindsight-api/hindsight_api/main.py

@@ -23,7 +23,7 @@ import uvicorn
 from . import MemoryEngine
 from .api import create_app
 from .banner import print_banner
-from .config import HindsightConfig, get_config
+from .config import DEFAULT_WORKERS, ENV_WORKERS, HindsightConfig, get_config
 from .daemon import (
     DEFAULT_DAEMON_PORT,
     DEFAULT_IDLE_TIMEOUT,
@@ -95,7 +95,12 @@ def main():
 
     # Development options
     parser.add_argument("--reload", action="store_true", help="Enable auto-reload on code changes (development only)")
-    parser.add_argument("--workers", type=int, default=1, help="Number of worker processes (default: 1)")
+    parser.add_argument(
+        "--workers",
+        type=int,
+        default=int(os.getenv(ENV_WORKERS, str(DEFAULT_WORKERS))),
+        help=f"Number of worker processes (env: {ENV_WORKERS}, default: {DEFAULT_WORKERS})",
+    )
 
     # Access log options
     parser.add_argument("--access-log", action="store_true", help="Enable access log")
@@ -171,22 +176,62 @@ def main():
             llm_base_url=config.llm_base_url,
             llm_max_concurrent=config.llm_max_concurrent,
             llm_timeout=config.llm_timeout,
+            retain_llm_provider=config.retain_llm_provider,
+            retain_llm_api_key=config.retain_llm_api_key,
+            retain_llm_model=config.retain_llm_model,
+            retain_llm_base_url=config.retain_llm_base_url,
+            reflect_llm_provider=config.reflect_llm_provider,
+            reflect_llm_api_key=config.reflect_llm_api_key,
+            reflect_llm_model=config.reflect_llm_model,
+            reflect_llm_base_url=config.reflect_llm_base_url,
+            consolidation_llm_provider=config.consolidation_llm_provider,
+            consolidation_llm_api_key=config.consolidation_llm_api_key,
+            consolidation_llm_model=config.consolidation_llm_model,
+            consolidation_llm_base_url=config.consolidation_llm_base_url,
             embeddings_provider=config.embeddings_provider,
             embeddings_local_model=config.embeddings_local_model,
             embeddings_tei_url=config.embeddings_tei_url,
+            embeddings_openai_base_url=config.embeddings_openai_base_url,
+            embeddings_cohere_base_url=config.embeddings_cohere_base_url,
             reranker_provider=config.reranker_provider,
             reranker_local_model=config.reranker_local_model,
             reranker_tei_url=config.reranker_tei_url,
+            reranker_tei_batch_size=config.reranker_tei_batch_size,
+            reranker_tei_max_concurrent=config.reranker_tei_max_concurrent,
+            reranker_max_candidates=config.reranker_max_candidates,
+            reranker_cohere_base_url=config.reranker_cohere_base_url,
             host=args.host,
             port=args.port,
             log_level=args.log_level,
+            log_format=config.log_format,
             mcp_enabled=config.mcp_enabled,
             graph_retriever=config.graph_retriever,
-            observation_min_facts=config.observation_min_facts,
-            observation_top_entities=config.observation_top_entities,
+            mpfp_top_k_neighbors=config.mpfp_top_k_neighbors,
+            recall_max_concurrent=config.recall_max_concurrent,
+            recall_connection_budget=config.recall_connection_budget,
             retain_max_completion_tokens=config.retain_max_completion_tokens,
+            retain_chunk_size=config.retain_chunk_size,
+            retain_extract_causal_links=config.retain_extract_causal_links,
+            retain_extraction_mode=config.retain_extraction_mode,
+            retain_custom_instructions=config.retain_custom_instructions,
+            retain_observations_async=config.retain_observations_async,
+            enable_observations=config.enable_observations,
+            consolidation_batch_size=config.consolidation_batch_size,
             skip_llm_verification=config.skip_llm_verification,
             lazy_reranker=config.lazy_reranker,
+            run_migrations_on_startup=config.run_migrations_on_startup,
+            db_pool_min_size=config.db_pool_min_size,
+            db_pool_max_size=config.db_pool_max_size,
+            db_command_timeout=config.db_command_timeout,
+            db_acquire_timeout=config.db_acquire_timeout,
+            worker_enabled=config.worker_enabled,
+            worker_id=config.worker_id,
+            worker_poll_interval_ms=config.worker_poll_interval_ms,
+            worker_max_retries=config.worker_max_retries,
+            worker_batch_size=config.worker_batch_size,
+            worker_http_port=config.worker_http_port,
+            reflect_max_iterations=config.reflect_max_iterations,
+            mental_model_refresh_concurrency=config.mental_model_refresh_concurrency,
         )
     config.configure_logging()
     if not args.daemon:
@@ -212,7 +257,11 @@ def main():
         logging.info(f"Loaded tenant extension: {tenant_extension.__class__.__name__}")
 
     # Create MemoryEngine (reads configuration from environment)
-    _memory = MemoryEngine(operation_validator=operation_validator, tenant_extension=tenant_extension)
+    _memory = MemoryEngine(
+        operation_validator=operation_validator,
+        tenant_extension=tenant_extension,
+        run_migrations=config.run_migrations_on_startup,
+    )
 
     # Set extension context on tenant extension (needed for schema provisioning)
     if tenant_extension:
@@ -239,14 +288,27 @@ def main():
         app = idle_middleware
 
     # Prepare uvicorn config
+    # When using workers or reload, we must use import string so each worker can import the app
+    use_import_string = args.workers > 1 or args.reload
+    # Check for uvloop availability
+    try:
+        import uvloop  # noqa: F401
+
+        loop_impl = "uvloop"
+        print("uvloop available, will use for event loop")
+    except ImportError:
+        loop_impl = "asyncio"
+        print("uvloop not installed, using default asyncio event loop")
+
     uvicorn_config = {
-        "app": app,
+        "app": "hindsight_api.server:app" if use_import_string else app,
         "host": args.host,
         "port": args.port,
         "log_level": args.log_level,
         "access_log": args.access_log,
         "proxy_headers": args.proxy_headers,
         "ws": "wsproto",  # Use wsproto instead of websockets to avoid deprecation warnings
+        "loop": loop_impl,  # Explicitly set event loop implementation
     }
 
     # Add optional parameters if provided

hindsight-api/hindsight_api/mcp_local.py

@@ -44,7 +44,6 @@ import os
 import sys
 
 from mcp.server.fastmcp import FastMCP
-from mcp.types import Icon
 
 from hindsight_api.config import (
     DEFAULT_MCP_LOCAL_BANK_ID,
@@ -53,6 +52,7 @@ from hindsight_api.config import (
     ENV_MCP_INSTRUCTIONS,
     ENV_MCP_LOCAL_BANK_ID,
 )
+from hindsight_api.mcp_tools import MCPToolsConfig, register_mcp_tools
 
 # Configure logging - default to warning to avoid polluting stderr during MCP init
 # MCP clients interpret stderr output as errors, so we suppress INFO logs by default
@@ -85,9 +85,6 @@ def create_local_mcp_server(bank_id: str, memory=None) -> FastMCP:
     """
     # Import here to avoid slow startup if just checking --help
     from hindsight_api import MemoryEngine
-    from hindsight_api.engine.memory_engine import Budget
-    from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES
-    from hindsight_api.models import RequestContext
 
     # Create memory engine with pg0 embedded database if not provided
     if memory is None:
@@ -105,55 +102,17 @@ def create_local_mcp_server(bank_id: str, memory=None) -> FastMCP:
 
     mcp = FastMCP("hindsight")
 
-    @mcp.tool(description=retain_description)
-    async def retain(content: str, context: str = "general") -> dict:
-        """
-        Args:
-            content: The fact/memory to store (be specific and include relevant details)
-            context: Category for the memory (e.g., 'preferences', 'work', 'hobbies', 'family'). Default: 'general'
-        """
-        import asyncio
+    # Configure and register tools using shared module
+    config = MCPToolsConfig(
+        bank_id_resolver=lambda: bank_id,
+        include_bank_id_param=False,  # Local MCP uses fixed bank_id
+        tools={"retain", "recall"},  # Local MCP only has retain and recall
+        retain_description=retain_description,
+        recall_description=recall_description,
+        retain_fire_and_forget=True,  # Local MCP uses fire-and-forget pattern
+    )
 
-        async def _retain():
-            try:
-                await memory.retain_batch_async(
-                    bank_id=bank_id,
-                    contents=[{"content": content, "context": context}],
-                    request_context=RequestContext(),
-                )
-            except Exception as e:
-                logger.error(f"Error storing memory: {e}", exc_info=True)
-
-        # Fire and forget - don't block on memory storage
-        asyncio.create_task(_retain())
-        return {"status": "accepted", "message": "Memory storage initiated"}
-
-    @mcp.tool(description=recall_description)
-    async def recall(query: str, max_tokens: int = 4096, budget: str = "low") -> dict:
-        """
-        Args:
-            query: Natural language search query (e.g., "user's food preferences", "what projects is user working on")
-            max_tokens: Maximum tokens to return in results (default: 4096)
-            budget: Search budget level - "low", "mid", or "high" (default: "low")
-        """
-        try:
-            # Map string budget to enum
-            budget_map = {"low": Budget.LOW, "mid": Budget.MID, "high": Budget.HIGH}
-            budget_enum = budget_map.get(budget.lower(), Budget.LOW)
-
-            search_result = await memory.recall_async(
-                bank_id=bank_id,
-                query=query,
-                fact_type=list(VALID_RECALL_FACT_TYPES),
-                budget=budget_enum,
-                max_tokens=max_tokens,
-                request_context=RequestContext(),
-            )
-
-            return search_result.model_dump()
-        except Exception as e:
-            logger.error(f"Error searching: {e}", exc_info=True)
-            return {"error": str(e), "results": []}
+    register_mcp_tools(mcp, memory, config)
 
     return mcp
 

hindsight-api/hindsight_api/mcp_tools.py

@@ -0,0 +1,494 @@
+"""Shared MCP tool implementations for Hindsight.
+
+This module provides the core tool logic used by both:
+- mcp_local.py (stdio transport for Claude Code)
+- api/mcp.py (HTTP transport for API server)
+"""
+
+import json
+import logging
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Callable
+
+from fastmcp import FastMCP
+
+from hindsight_api import MemoryEngine
+from hindsight_api.config import (
+    DEFAULT_MCP_RECALL_DESCRIPTION,
+    DEFAULT_MCP_RETAIN_DESCRIPTION,
+)
+from hindsight_api.engine.memory_engine import Budget
+from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES
+from hindsight_api.models import RequestContext
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class MCPToolsConfig:
+    """Configuration for MCP tools registration."""
+
+    # How to resolve bank_id for operations
+    bank_id_resolver: Callable[[], str | None]
+
+    # Whether to include bank_id as a parameter on tools (for multi-bank support)
+    include_bank_id_param: bool = False
+
+    # Which tools to register
+    tools: set[str] | None = None  # None means all tools
+
+    # Custom descriptions (if None, uses defaults)
+    retain_description: str | None = None
+    recall_description: str | None = None
+
+    # Retain behavior
+    retain_fire_and_forget: bool = False  # If True, use asyncio.create_task pattern
+
+
+def parse_timestamp(timestamp: str) -> datetime | None:
+    """Parse an ISO format timestamp string.
+
+    Args:
+        timestamp: ISO format timestamp (e.g., '2024-01-15T10:30:00Z')
+
+    Returns:
+        Parsed datetime or None if invalid
+
+    Raises:
+        ValueError: If timestamp format is invalid
+    """
+    try:
+        return datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
+    except ValueError as e:
+        raise ValueError(
+            f"Invalid timestamp format '{timestamp}'. "
+            "Expected ISO format like '2024-01-15T10:30:00' or '2024-01-15T10:30:00Z'"
+        ) from e
+
+
+def build_content_dict(
+    content: str,
+    context: str,
+    timestamp: str | None = None,
+) -> tuple[dict[str, Any], str | None]:
+    """Build a content dict for retain operations.
+
+    Args:
+        content: The memory content
+        context: Category for the memory
+        timestamp: Optional ISO timestamp
+
+    Returns:
+        Tuple of (content_dict, error_message). error_message is None if successful.
+    """
+    content_dict: dict[str, Any] = {"content": content, "context": context}
+
+    if timestamp:
+        try:
+            parsed_timestamp = parse_timestamp(timestamp)
+            content_dict["event_date"] = parsed_timestamp
+        except ValueError as e:
+            return {}, str(e)
+
+    return content_dict, None
+
+
+def register_mcp_tools(
+    mcp: FastMCP,
+    memory: MemoryEngine,
+    config: MCPToolsConfig,
+) -> None:
+    """Register MCP tools on a FastMCP server.
+
+    Args:
+        mcp: FastMCP server instance
+        memory: MemoryEngine instance
+        config: Tool configuration
+    """
+    tools_to_register = config.tools or {"retain", "recall", "reflect", "list_banks", "create_bank"}
+
+    if "retain" in tools_to_register:
+        _register_retain(mcp, memory, config)
+
+    if "recall" in tools_to_register:
+        _register_recall(mcp, memory, config)
+
+    if "reflect" in tools_to_register:
+        _register_reflect(mcp, memory, config)
+
+    if "list_banks" in tools_to_register:
+        _register_list_banks(mcp, memory, config)
+
+    if "create_bank" in tools_to_register:
+        _register_create_bank(mcp, memory, config)
+
+
+def _register_retain(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig) -> None:
+    """Register the retain tool."""
+    description = config.retain_description or DEFAULT_MCP_RETAIN_DESCRIPTION
+
+    if config.include_bank_id_param:
+        if config.retain_fire_and_forget:
+
+            @mcp.tool(description=description)
+            async def retain(
+                content: str,
+                context: str = "general",
+                timestamp: str | None = None,
+                bank_id: str | None = None,
+            ) -> dict:
+                """
+                Args:
+                    content: The fact/memory to store (be specific and include relevant details)
+                    context: Category for the memory (e.g., 'preferences', 'work', 'hobbies', 'family'). Default: 'general'
+                    timestamp: When this event/fact occurred (ISO format, e.g., '2024-01-15T10:30:00Z'). Useful for timeline tracking.
+                    bank_id: Optional bank to store in (defaults to session bank). Use for cross-bank operations.
+                """
+                import asyncio
+
+                target_bank = bank_id or config.bank_id_resolver()
+                if target_bank is None:
+                    return {"status": "error", "message": "No bank_id configured"}
+
+                content_dict, error = build_content_dict(content, context, timestamp)
+                if error:
+                    return {"status": "error", "message": error}
+
+                async def _retain():
+                    try:
+                        await memory.retain_batch_async(
+                            bank_id=target_bank,
+                            contents=[content_dict],
+                            request_context=RequestContext(),
+                        )
+                    except Exception as e:
+                        logger.error(f"Error storing memory: {e}", exc_info=True)
+
+                asyncio.create_task(_retain())
+                return {"status": "accepted", "message": "Memory storage initiated"}
+
+        else:
+
+            @mcp.tool(description=description)
+            async def retain(
+                content: str,
+                context: str = "general",
+                timestamp: str | None = None,
+                async_processing: bool = True,
+                bank_id: str | None = None,
+            ) -> str:
+                """
+                Args:
+                    content: The fact/memory to store (be specific and include relevant details)
+                    context: Category for the memory (e.g., 'preferences', 'work', 'hobbies', 'family'). Default: 'general'
+                    timestamp: When this event/fact occurred (ISO format, e.g., '2024-01-15T10:30:00Z'). Useful for timeline tracking.
+                    async_processing: If True, queue for background processing and return immediately. If False, wait for completion. Default: True
+                    bank_id: Optional bank to store in (defaults to session bank). Use for cross-bank operations.
+                """
+                try:
+                    target_bank = bank_id or config.bank_id_resolver()
+                    if target_bank is None:
+                        return "Error: No bank_id configured"
+
+                    content_dict, error = build_content_dict(content, context, timestamp)
+                    if error:
+                        return f"Error: {error}"
+
+                    contents = [content_dict]
+                    if async_processing:
+                        result = await memory.submit_async_retain(
+                            bank_id=target_bank, contents=contents, request_context=RequestContext()
+                        )
+                        return f"Memory queued for background processing (operation_id: {result.get('operation_id', 'N/A')})"
+                    else:
+                        await memory.retain_batch_async(
+                            bank_id=target_bank,
+                            contents=contents,
+                            request_context=RequestContext(),
+                        )
+                        return f"Memory stored successfully in bank '{target_bank}'"
+                except Exception as e:
+                    logger.error(f"Error storing memory: {e}", exc_info=True)
+                    return f"Error: {str(e)}"
+
+    else:
+        # No bank_id param - use fixed bank from resolver
+
+        @mcp.tool(description=description)
+        async def retain(
+            content: str,
+            context: str = "general",
+            timestamp: str | None = None,
+        ) -> dict:
+            """
+            Args:
+                content: The fact/memory to store (be specific and include relevant details)
+                context: Category for the memory (e.g., 'preferences', 'work', 'hobbies', 'family'). Default: 'general'
+                timestamp: When this event/fact occurred (ISO format, e.g., '2024-01-15T10:30:00Z'). Useful for timeline tracking.
+            """
+            import asyncio
+
+            target_bank = config.bank_id_resolver()
+            if target_bank is None:
+                return {"status": "error", "message": "No bank_id configured"}
+
+            content_dict, error = build_content_dict(content, context, timestamp)
+            if error:
+                return {"status": "error", "message": error}
+
+            async def _retain():
+                try:
+                    await memory.retain_batch_async(
+                        bank_id=target_bank,
+                        contents=[content_dict],
+                        request_context=RequestContext(),
+                    )
+                except Exception as e:
+                    logger.error(f"Error storing memory: {e}", exc_info=True)
+
+            asyncio.create_task(_retain())
+            return {"status": "accepted", "message": "Memory storage initiated"}
+
+
+def _register_recall(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig) -> None:
+    """Register the recall tool."""
+    description = config.recall_description or DEFAULT_MCP_RECALL_DESCRIPTION
+
+    if config.include_bank_id_param:
+
+        @mcp.tool(description=description)
+        async def recall(
+            query: str,
+            max_tokens: int = 4096,
+            bank_id: str | None = None,
+        ) -> str | dict:
+            """
+            Args:
+                query: Natural language search query (e.g., "user's food preferences", "what projects is user working on")
+                max_tokens: Maximum tokens to return in results (default: 4096)
+                bank_id: Optional bank to search in (defaults to session bank). Use for cross-bank operations.
+            """
+            try:
+                target_bank = bank_id or config.bank_id_resolver()
+                if target_bank is None:
+                    return "Error: No bank_id configured"
+
+                recall_result = await memory.recall_async(
+                    bank_id=target_bank,
+                    query=query,
+                    fact_type=list(VALID_RECALL_FACT_TYPES),
+                    budget=Budget.HIGH,
+                    max_tokens=max_tokens,
+                    request_context=RequestContext(),
+                )
+
+                return recall_result.model_dump_json(indent=2)
+            except Exception as e:
+                logger.error(f"Error searching: {e}", exc_info=True)
+                return f'{{"error": "{e}", "results": []}}'
+
+    else:
+
+        @mcp.tool(description=description)
+        async def recall(
+            query: str,
+            max_tokens: int = 4096,
+        ) -> dict:
+            """
+            Args:
+                query: Natural language search query (e.g., "user's food preferences", "what projects is user working on")
+                max_tokens: Maximum tokens to return in results (default: 4096)
+            """
+            try:
+                target_bank = config.bank_id_resolver()
+                if target_bank is None:
+                    return {"error": "No bank_id configured", "results": []}
+
+                recall_result = await memory.recall_async(
+                    bank_id=target_bank,
+                    query=query,
+                    fact_type=list(VALID_RECALL_FACT_TYPES),
+                    budget=Budget.HIGH,
+                    max_tokens=max_tokens,
+                    request_context=RequestContext(),
+                )
+
+                return recall_result.model_dump()
+            except Exception as e:
+                logger.error(f"Error searching: {e}", exc_info=True)
+                return {"error": str(e), "results": []}
+
+
+def _register_reflect(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig) -> None:
+    """Register the reflect tool."""
+
+    if config.include_bank_id_param:
+
+        @mcp.tool()
+        async def reflect(
+            query: str,
+            context: str | None = None,
+            budget: str = "low",
+            bank_id: str | None = None,
+        ) -> str:
+            """
+            Generate thoughtful analysis by synthesizing stored memories with the bank's personality.
+
+            WHEN TO USE THIS TOOL:
+            Use reflect when you need reasoned analysis, not just fact retrieval. This tool
+            thinks through the question using everything the bank knows and its personality traits.
+
+            EXAMPLES OF GOOD QUERIES:
+            - "What patterns have emerged in how I approach debugging?"
+            - "Based on my past decisions, what architectural style do I prefer?"
+            - "What might be the best approach for this problem given what you know about me?"
+            - "How should I prioritize these tasks based on my goals?"
+
+            HOW IT DIFFERS FROM RECALL:
+            - recall: Returns raw facts matching your search (fast lookup)
+            - reflect: Reasons across memories to form a synthesized answer (deeper analysis)
+
+            Use recall for "what did I say about X?" and reflect for "what should I do about X?"
+
+            Args:
+                query: The question or topic to reflect on
+                context: Optional context about why this reflection is needed
+                budget: Search budget - 'low', 'mid', or 'high' (default: 'low')
+                bank_id: Optional bank to reflect in (defaults to session bank). Use for cross-bank operations.
+            """
+            try:
+                target_bank = bank_id or config.bank_id_resolver()
+                if target_bank is None:
+                    return "Error: No bank_id configured"
+
+                budget_map = {"low": Budget.LOW, "mid": Budget.MID, "high": Budget.HIGH}
+                budget_enum = budget_map.get(budget.lower(), Budget.LOW)
+
+                reflect_result = await memory.reflect_async(
+                    bank_id=target_bank,
+                    query=query,
+                    budget=budget_enum,
+                    context=context,
+                    request_context=RequestContext(),
+                )
+
+                return reflect_result.model_dump_json(indent=2)
+            except Exception as e:
+                logger.error(f"Error reflecting: {e}", exc_info=True)
+                return f'{{"error": "{e}", "text": ""}}'
+
+    else:
+
+        @mcp.tool()
+        async def reflect(
+            query: str,
+            context: str | None = None,
+            budget: str = "low",
+        ) -> dict:
+            """
+            Generate thoughtful analysis by synthesizing stored memories with the bank's personality.
+
+            WHEN TO USE THIS TOOL:
+            Use reflect when you need reasoned analysis, not just fact retrieval. This tool
+            thinks through the question using everything the bank knows and its personality traits.
+
+            EXAMPLES OF GOOD QUERIES:
+            - "What patterns have emerged in how I approach debugging?"
+            - "Based on my past decisions, what architectural style do I prefer?"
+            - "What might be the best approach for this problem given what you know about me?"
+            - "How should I prioritize these tasks based on my goals?"
+
+            HOW IT DIFFERS FROM RECALL:
+            - recall: Returns raw facts matching your search (fast lookup)
+            - reflect: Reasons across memories to form a synthesized answer (deeper analysis)
+
+            Use recall for "what did I say about X?" and reflect for "what should I do about X?"
+
+            Args:
+                query: The question or topic to reflect on
+                context: Optional context about why this reflection is needed
+                budget: Search budget - 'low', 'mid', or 'high' (default: 'low')
+            """
+            try:
+                target_bank = config.bank_id_resolver()
+                if target_bank is None:
+                    return {"error": "No bank_id configured", "text": ""}
+
+                budget_map = {"low": Budget.LOW, "mid": Budget.MID, "high": Budget.HIGH}
+                budget_enum = budget_map.get(budget.lower(), Budget.LOW)
+
+                reflect_result = await memory.reflect_async(
+                    bank_id=target_bank,
+                    query=query,
+                    budget=budget_enum,
+                    context=context,
+                    request_context=RequestContext(),
+                )
+
+                return reflect_result.model_dump()
+            except Exception as e:
+                logger.error(f"Error reflecting: {e}", exc_info=True)
+                return {"error": str(e), "text": ""}
+
+
+def _register_list_banks(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig) -> None:
+    """Register the list_banks tool."""
+
+    @mcp.tool()
+    async def list_banks() -> str:
+        """
+        List all available memory banks.
+
+        Use this tool to discover what memory banks exist in the system.
+        Each bank is an isolated memory store (like a separate "brain").
+
+        Returns:
+            JSON list of banks with their IDs, names, dispositions, and missions.
+        """
+        try:
+            banks = await memory.list_banks(request_context=RequestContext())
+            return json.dumps({"banks": banks}, indent=2)
+        except Exception as e:
+            logger.error(f"Error listing banks: {e}", exc_info=True)
+            return f'{{"error": "{e}", "banks": []}}'
+
+
+def _register_create_bank(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig) -> None:
+    """Register the create_bank tool."""
+
+    @mcp.tool()
+    async def create_bank(bank_id: str, name: str | None = None, mission: str | None = None) -> str:
+        """
+        Create a new memory bank or get an existing one.
+
+        Memory banks are isolated stores - each one is like a separate "brain" for a user/agent.
+        Banks are auto-created with default settings if they don't exist.
+
+        Args:
+            bank_id: Unique identifier for the bank (e.g., 'user-123', 'agent-alpha')
+            name: Optional human-friendly name for the bank
+            mission: Optional mission describing who the agent is and what they're trying to accomplish
+        """
+        try:
+            # get_bank_profile auto-creates bank if it doesn't exist
+            profile = await memory.get_bank_profile(bank_id, request_context=RequestContext())
+
+            # Update name/mission if provided
+            if name is not None or mission is not None:
+                await memory.update_bank(
+                    bank_id,
+                    name=name,
+                    mission=mission,
+                    request_context=RequestContext(),
+                )
+                # Fetch updated profile
+                profile = await memory.get_bank_profile(bank_id, request_context=RequestContext())
+
+            # Serialize disposition if it's a Pydantic model
+            if "disposition" in profile and hasattr(profile["disposition"], "model_dump"):
+                profile["disposition"] = profile["disposition"].model_dump()
+            return json.dumps(profile, indent=2)
+        except Exception as e:
+            logger.error(f"Error creating bank: {e}", exc_info=True)
+            return f'{{"error": "{e}"}}'

hindsight-api/hindsight_api/metrics.py

@@ -5,17 +5,86 @@ This module provides metrics for:
 - Operation latency (retain, recall, reflect) with percentiles
 - Token usage (input/output) per operation
 - Per-bank granularity via labels
+- LLM call latency and token usage with scope dimension
+- HTTP request metrics (latency, count by endpoint/method/status)
+- Process metrics (CPU, memory, file descriptors, threads)
+- Database connection pool metrics
 """
 
 import logging
+import os
+import resource
+import threading
 import time
 from contextlib import contextmanager
+from typing import TYPE_CHECKING, Callable
 
 from opentelemetry import metrics
 from opentelemetry.exporter.prometheus import PrometheusMetricReader
 from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
 from opentelemetry.sdk.resources import Resource
 
+if TYPE_CHECKING:
+    import asyncpg
+
+
+def _get_tenant() -> str:
+    """Get current tenant (schema) from context for metrics labeling."""
+    # Import here to avoid circular imports
+    from hindsight_api.engine.memory_engine import get_current_schema
+
+    return get_current_schema()
+
+
+# Custom bucket boundaries for operation duration (in seconds)
+# Fine granularity in 0-30s range where most operations complete
+DURATION_BUCKETS = (0.1, 0.25, 0.5, 0.75, 1.0, 2.0, 3.0, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, 120.0)
+
+# LLM duration buckets (finer granularity for faster LLM calls)
+LLM_DURATION_BUCKETS = (0.1, 0.25, 0.5, 1.0, 2.0, 3.0, 5.0, 10.0, 15.0, 30.0, 60.0, 120.0)
+
+# HTTP request duration buckets (millisecond-level for fast endpoints)
+HTTP_DURATION_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0)
+
+
+def get_token_bucket(token_count: int) -> str:
+    """
+    Convert a token count to a bucket label for use as a dimension.
+
+    This allows analyzing token usage patterns without high-cardinality issues.
+
+    Buckets:
+    - "0-100": Very small requests/responses
+    - "100-500": Small requests/responses
+    - "500-1k": Medium requests/responses
+    - "1k-5k": Large requests/responses
+    - "5k-10k": Very large requests/responses
+    - "10k-50k": Huge requests/responses
+    - "50k+": Extremely large requests/responses
+
+    Args:
+        token_count: Number of tokens
+
+    Returns:
+        Bucket label string
+    """
+    if token_count < 100:
+        return "0-100"
+    elif token_count < 500:
+        return "100-500"
+    elif token_count < 1000:
+        return "500-1k"
+    elif token_count < 5000:
+        return "1k-5k"
+    elif token_count < 10000:
+        return "5k-10k"
+    elif token_count < 50000:
+        return "10k-50k"
+    else:
+        return "50k+"
+
+
 logger = logging.getLogger(__name__)
 
 # Global meter instance
@@ -48,8 +117,30 @@ def initialize_metrics(service_name: str = "hindsight-api", service_version: str
     # Create Prometheus metric reader
     prometheus_reader = PrometheusMetricReader()
 
-    # Create meter provider with Prometheus exporter
-    provider = MeterProvider(resource=resource, metric_readers=[prometheus_reader])
+    # Create view with custom bucket boundaries for duration histogram
+    duration_view = View(
+        instrument_name="hindsight.operation.duration",
+        aggregation=ExplicitBucketHistogramAggregation(boundaries=DURATION_BUCKETS),
+    )
+
+    # Create view with custom bucket boundaries for LLM duration histogram
+    llm_duration_view = View(
+        instrument_name="hindsight.llm.duration",
+        aggregation=ExplicitBucketHistogramAggregation(boundaries=LLM_DURATION_BUCKETS),
+    )
+
+    # Create view with custom bucket boundaries for HTTP request duration histogram
+    http_duration_view = View(
+        instrument_name="hindsight.http.duration",
+        aggregation=ExplicitBucketHistogramAggregation(boundaries=HTTP_DURATION_BUCKETS),
+    )
+
+    # Create meter provider with Prometheus exporter and custom views
+    provider = MeterProvider(
+        resource=resource,
+        metric_readers=[prometheus_reader],
+        views=[duration_view, llm_duration_view, http_duration_view],
+    )
 
     # Set the global meter provider
     metrics.set_meter_provider(provider)
@@ -71,43 +162,84 @@ class MetricsCollectorBase:
     """Base class for metrics collectors."""
 
     @contextmanager
-    def record_operation(self, operation: str, bank_id: str, budget: str | None = None, max_tokens: int | None = None):
-        """Context manager to record operation duration and status."""
-        raise NotImplementedError
-
-    def record_tokens(
+    def record_operation(
         self,
         operation: str,
         bank_id: str,
-        input_tokens: int = 0,
-        output_tokens: int = 0,
+        source: str = "api",
         budget: str | None = None,
         max_tokens: int | None = None,
     ):
-        """Record token usage for an operation."""
+        """Context manager to record operation duration and status."""
         raise NotImplementedError
 
+    def record_llm_call(
+        self,
+        provider: str,
+        model: str,
+        scope: str,
+        duration: float,
+        input_tokens: int = 0,
+        output_tokens: int = 0,
+        success: bool = True,
+    ):
+        """
+        Record metrics for an LLM call.
+
+        Args:
+            provider: LLM provider name (openai, anthropic, gemini, groq, ollama, lmstudio)
+            model: Model name
+            scope: Scope identifier (e.g., "memory", "reflect", "entity_observation")
+            duration: Call duration in seconds
+            input_tokens: Number of input/prompt tokens
+            output_tokens: Number of output/completion tokens
+            success: Whether the call was successful
+        """
+        raise NotImplementedError
+
+    @contextmanager
+    def record_http_request(self, method: str, endpoint: str, status_code_getter: Callable[[], int]):
+        """Context manager to record HTTP request metrics."""
+        raise NotImplementedError
+
+    def set_db_pool(self, pool: "asyncpg.Pool"):
+        """Set the database pool for metrics collection."""
+        pass
+
 
 class NoOpMetricsCollector(MetricsCollectorBase):
     """No-op metrics collector that does nothing. Used when metrics are disabled."""
 
     @contextmanager
-    def record_operation(self, operation: str, bank_id: str, budget: str | None = None, max_tokens: int | None = None):
-        """No-op context manager."""
-        yield
-
-    def record_tokens(
+    def record_operation(
         self,
         operation: str,
         bank_id: str,
-        input_tokens: int = 0,
-        output_tokens: int = 0,
+        source: str = "api",
         budget: str | None = None,
         max_tokens: int | None = None,
     ):
-        """No-op token recording."""
+        """No-op context manager."""
+        yield
+
+    def record_llm_call(
+        self,
+        provider: str,
+        model: str,
+        scope: str,
+        duration: float,
+        input_tokens: int = 0,
+        output_tokens: int = 0,
+        success: bool = True,
+    ):
+        """No-op LLM call recording."""
         pass
 
+    @contextmanager
+    def record_http_request(self, method: str, endpoint: str, status_code_getter: Callable[[], int]):
+        """No-op HTTP request recording."""
+        yield
+
 
 class MetricsCollector(MetricsCollectorBase):
     """
@@ -125,33 +257,73 @@ class MetricsCollector(MetricsCollectorBase):
             name="hindsight.operation.duration", description="Duration of Hindsight operations in seconds", unit="s"
         )
 
-        # Token usage counters
-        self.tokens_input = self.meter.create_counter(
-            name="hindsight.tokens.input", description="Number of input tokens consumed", unit="tokens"
-        )
-
-        self.tokens_output = self.meter.create_counter(
-            name="hindsight.tokens.output", description="Number of output tokens generated", unit="tokens"
-        )
-
         # Operation counter (success/failure)
         self.operation_total = self.meter.create_counter(
             name="hindsight.operation.total", description="Total number of operations executed", unit="operations"
         )
 
+        # LLM call latency histogram (in seconds)
+        # Records duration of LLM API calls with provider, model, and scope dimensions
+        self.llm_duration = self.meter.create_histogram(
+            name="hindsight.llm.duration", description="Duration of LLM API calls in seconds", unit="s"
+        )
+
+        # LLM token usage counters with bucket labels
+        self.llm_tokens_input = self.meter.create_counter(
+            name="hindsight.llm.tokens.input", description="Number of input tokens for LLM calls", unit="tokens"
+        )
+
+        self.llm_tokens_output = self.meter.create_counter(
+            name="hindsight.llm.tokens.output", description="Number of output tokens from LLM calls", unit="tokens"
+        )
+
+        # LLM call counter (success/failure)
+        self.llm_calls_total = self.meter.create_counter(
+            name="hindsight.llm.calls.total", description="Total number of LLM API calls", unit="calls"
+        )
+
+        # HTTP request metrics
+        self.http_request_duration = self.meter.create_histogram(
+            name="hindsight.http.duration", description="Duration of HTTP requests in seconds", unit="s"
+        )
+
+        self.http_requests_total = self.meter.create_counter(
+            name="hindsight.http.requests.total", description="Total number of HTTP requests", unit="requests"
+        )
+
+        self.http_requests_in_progress = self.meter.create_up_down_counter(
+            name="hindsight.http.requests.in_progress",
+            description="Number of HTTP requests in progress",
+            unit="requests",
+        )
+
+        # Process metrics (observable gauges - collected on scrape)
+        self._setup_process_metrics()
+
+        # DB pool metrics holder (set via set_db_pool)
+        self._db_pool: "asyncpg.Pool | None" = None
+
     @contextmanager
-    def record_operation(self, operation: str, bank_id: str, budget: str | None = None, max_tokens: int | None = None):
+    def record_operation(
+        self,
+        operation: str,
+        bank_id: str,
+        source: str = "api",
+        budget: str | None = None,
+        max_tokens: int | None = None,
+    ):
         """
         Context manager to record operation duration and status.
 
         Usage:
-            with metrics.record_operation("recall", bank_id="user123", budget="mid", max_tokens=4096):
+            with metrics.record_operation("recall", bank_id="user123", source="api", budget="mid", max_tokens=4096):
                 # ... perform operation
                 pass
 
         Args:
-            operation: Operation name (retain, recall, reflect)
+            operation: Operation name (retain, recall, reflect, entity_observation)
             bank_id: Memory bank ID
+            source: Source of the operation (api, reflect, internal)
             budget: Optional budget level (low, mid, high)
             max_tokens: Optional max tokens for the operation
         """
@@ -159,6 +331,8 @@ class MetricsCollector(MetricsCollectorBase):
         attributes = {
             "operation": operation,
             "bank_id": bank_id,
+            "source": source,
+            "tenant": _get_tenant(),
         }
         if budget:
             attributes["budget"] = budget
@@ -181,40 +355,251 @@ class MetricsCollector(MetricsCollectorBase):
             # Record operation count
             self.operation_total.add(1, attributes)
 
-    def record_tokens(
+    def record_llm_call(
         self,
-        operation: str,
-        bank_id: str,
+        provider: str,
+        model: str,
+        scope: str,
+        duration: float,
         input_tokens: int = 0,
         output_tokens: int = 0,
-        budget: str | None = None,
-        max_tokens: int | None = None,
+        success: bool = True,
     ):
         """
-        Record token usage for an operation.
+        Record metrics for an LLM call.
 
         Args:
-            operation: Operation name (retain, recall, reflect)
-            bank_id: Memory bank ID
-            input_tokens: Number of input tokens
-            output_tokens: Number of output tokens
-            budget: Optional budget level
-            max_tokens: Optional max tokens for the operation
+            provider: LLM provider name (openai, anthropic, gemini, groq, ollama, lmstudio)
+            model: Model name
+            scope: Scope identifier (e.g., "memory", "reflect", "entity_observation")
+            duration: Call duration in seconds
+            input_tokens: Number of input/prompt tokens
+            output_tokens: Number of output/completion tokens
+            success: Whether the call was successful
         """
-        attributes = {
-            "operation": operation,
-            "bank_id": bank_id,
+        # Base attributes for all metrics
+        base_attributes = {
+            "provider": provider,
+            "model": model,
+            "scope": scope,
+            "success": str(success).lower(),
+            "tenant": _get_tenant(),
         }
-        if budget:
-            attributes["budget"] = budget
-        if max_tokens:
-            attributes["max_tokens"] = str(max_tokens)
 
+        # Record duration
+        self.llm_duration.record(duration, base_attributes)
+
+        # Record call count
+        self.llm_calls_total.add(1, base_attributes)
+
+        # Record tokens with bucket labels for cardinality control
         if input_tokens > 0:
-            self.tokens_input.add(input_tokens, attributes)
+            input_attributes = {
+                **base_attributes,
+                "token_bucket": get_token_bucket(input_tokens),
+            }
+            self.llm_tokens_input.add(input_tokens, input_attributes)
 
         if output_tokens > 0:
-            self.tokens_output.add(output_tokens, attributes)
+            output_attributes = {
+                **base_attributes,
+                "token_bucket": get_token_bucket(output_tokens),
+            }
+            self.llm_tokens_output.add(output_tokens, output_attributes)
+
+    @contextmanager
+    def record_http_request(self, method: str, endpoint: str, status_code_getter: Callable[[], int]):
+        """
+        Context manager to record HTTP request metrics.
+
+        Usage:
+            status_code = [200]  # Use list for mutability
+            with metrics.record_http_request("GET", "/api/banks", lambda: status_code[0]):
+                # ... handle request
+                status_code[0] = response.status_code
+
+        Args:
+            method: HTTP method (GET, POST, etc.)
+            endpoint: Request endpoint path
+            status_code_getter: Callable that returns the status code after request completes
+        """
+        start_time = time.time()
+        base_attributes = {"method": method, "endpoint": endpoint}
+
+        # Track in-progress
+        self.http_requests_in_progress.add(1, base_attributes)
+
+        try:
+            yield
+        finally:
+            duration = time.time() - start_time
+            status_code = status_code_getter()
+            status_class = f"{status_code // 100}xx"
+
+            # Get tenant from context (may be set during request processing)
+            tenant = _get_tenant()
+
+            attributes = {
+                **base_attributes,
+                "status_code": str(status_code),
+                "status_class": status_class,
+                "tenant": tenant,
+            }
+
+            # Record duration and count
+            self.http_request_duration.record(duration, attributes)
+            self.http_requests_total.add(1, attributes)
+
+            # Decrement in-progress
+            self.http_requests_in_progress.add(-1, base_attributes)
+
+    def _setup_process_metrics(self):
+        """Set up observable gauges for process metrics."""
+
+        def get_cpu_times(_options):
+            """Get process CPU times."""
+            try:
+                rusage = resource.getrusage(resource.RUSAGE_SELF)
+                yield metrics.Observation(rusage.ru_utime, {"type": "user"})
+                yield metrics.Observation(rusage.ru_stime, {"type": "system"})
+            except Exception:
+                pass
+
+        def get_memory_usage(_options):
+            """Get process memory usage in bytes."""
+            try:
+                rusage = resource.getrusage(resource.RUSAGE_SELF)
+                # ru_maxrss is in kilobytes on Linux, bytes on macOS
+                max_rss = rusage.ru_maxrss
+                if os.uname().sysname == "Linux":
+                    max_rss *= 1024  # Convert KB to bytes
+                yield metrics.Observation(max_rss, {"type": "rss_max"})
+            except Exception:
+                pass
+
+        def get_open_file_descriptors(_options):
+            """Get number of open file descriptors."""
+            try:
+                # Try to count open FDs by checking /proc on Linux
+                if os.path.exists("/proc/self/fd"):
+                    count = len(os.listdir("/proc/self/fd"))
+                    yield metrics.Observation(count)
+                else:
+                    # Fallback: use resource limits
+                    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+                    yield metrics.Observation(soft, {"limit": "soft"})
+            except Exception:
+                pass
+
+        def get_thread_count(_options):
+            """Get number of active threads."""
+            try:
+                yield metrics.Observation(threading.active_count())
+            except Exception:
+                pass
+
+        # Create observable gauges
+        self.meter.create_observable_gauge(
+            name="hindsight.process.cpu.seconds",
+            callbacks=[get_cpu_times],
+            description="Process CPU time in seconds",
+            unit="s",
+        )
+
+        self.meter.create_observable_gauge(
+            name="hindsight.process.memory.bytes",
+            callbacks=[get_memory_usage],
+            description="Process memory usage in bytes",
+            unit="By",
+        )
+
+        self.meter.create_observable_gauge(
+            name="hindsight.process.open_fds",
+            callbacks=[get_open_file_descriptors],
+            description="Number of open file descriptors",
+            unit="{fds}",
+        )
+
+        self.meter.create_observable_gauge(
+            name="hindsight.process.threads",
+            callbacks=[get_thread_count],
+            description="Number of active threads",
+            unit="{threads}",
+        )
+
+    def set_db_pool(self, pool: "asyncpg.Pool"):
+        """
+        Set the database pool for metrics collection.
+
+        Args:
+            pool: asyncpg connection pool instance
+        """
+        self._db_pool = pool
+        self._setup_db_pool_metrics()
+
+    def _setup_db_pool_metrics(self):
+        """Set up observable gauges for database pool metrics."""
+
+        def get_pool_size(_options):
+            """Get current pool size."""
+            if self._db_pool is not None:
+                try:
+                    yield metrics.Observation(self._db_pool.get_size())
+                except Exception:
+                    pass
+
+        def get_pool_free_size(_options):
+            """Get number of free connections in pool."""
+            if self._db_pool is not None:
+                try:
+                    yield metrics.Observation(self._db_pool.get_idle_size())
+                except Exception:
+                    pass
+
+        def get_pool_min_size(_options):
+            """Get pool minimum size."""
+            if self._db_pool is not None:
+                try:
+                    yield metrics.Observation(self._db_pool.get_min_size())
+                except Exception:
+                    pass
+
+        def get_pool_max_size(_options):
+            """Get pool maximum size."""
+            if self._db_pool is not None:
+                try:
+                    yield metrics.Observation(self._db_pool.get_max_size())
+                except Exception:
+                    pass
+
+        # Create observable gauges for pool metrics
+        self.meter.create_observable_gauge(
+            name="hindsight.db.pool.size",
+            callbacks=[get_pool_size],
+            description="Current number of connections in the pool",
+            unit="{connections}",
+        )
+
+        self.meter.create_observable_gauge(
+            name="hindsight.db.pool.idle",
+            callbacks=[get_pool_free_size],
+            description="Number of idle connections in the pool",
+            unit="{connections}",
+        )
+
+        self.meter.create_observable_gauge(
+            name="hindsight.db.pool.min",
+            callbacks=[get_pool_min_size],
+            description="Minimum pool size",
+            unit="{connections}",
+        )
+
+        self.meter.create_observable_gauge(
+            name="hindsight.db.pool.max",
+            callbacks=[get_pool_max_size],
+            description="Maximum pool size",
+            unit="{connections}",
+        )
 
 
 # Global metrics collector instance (defaults to no-op)

hindsight-api/hindsight_api/migrations.py

@@ -22,6 +22,7 @@ from pathlib import Path
 
 from alembic import command
 from alembic.config import Config
+from alembic.script.revision import ResolutionError
 from sqlalchemy import create_engine, text
 
 logger = logging.getLogger(__name__)
@@ -78,7 +79,18 @@ def _run_migrations_internal(database_url: str, script_location: str, schema: st
         alembic_cfg.set_main_option("target_schema", schema)
 
     # Run migrations
-    command.upgrade(alembic_cfg, "head")
+    try:
+        command.upgrade(alembic_cfg, "head")
+    except ResolutionError as e:
+        # This happens during rolling deployments when a newer version of the code
+        # has already run migrations, and this older replica doesn't have the new
+        # migration files. The database is already at a newer revision than we know.
+        # This is safe to ignore - the newer code has already applied its migrations.
+        logger.warning(
+            f"Database is at a newer migration revision than this code version knows about. "
+            f"This is expected during rolling deployments. Skipping migrations. Error: {e}"
+        )
+        return
 
     logger.info(f"Database migrations completed successfully for schema '{schema_name}'")
 

hindsight-api/hindsight_api/models.py

@@ -95,7 +95,6 @@ class MemoryUnit(Base):
     mentioned_at: Mapped[datetime | None] = mapped_column(TIMESTAMP(timezone=True))  # When fact was mentioned
     fact_type: Mapped[str] = mapped_column(Text, nullable=False, server_default="world")
     confidence_score: Mapped[float | None] = mapped_column(Float)
-    access_count: Mapped[int] = mapped_column(Integer, server_default="0")
     unit_metadata: Mapped[dict] = mapped_column(
         "metadata", JSONB, server_default=sql_text("'{}'::jsonb")
     )  # User-defined metadata (str->str)
@@ -131,7 +130,6 @@ class MemoryUnit(Base):
         Index("idx_memory_units_document_id", "document_id"),
         Index("idx_memory_units_event_date", "event_date", postgresql_ops={"event_date": "DESC"}),
         Index("idx_memory_units_bank_date", "bank_id", "event_date", postgresql_ops={"event_date": "DESC"}),
-        Index("idx_memory_units_access_count", "access_count", postgresql_ops={"access_count": "DESC"}),
         Index("idx_memory_units_fact_type", "fact_type"),
         Index("idx_memory_units_bank_fact_type", "bank_id", "fact_type"),
         Index(

hindsight-api/hindsight_api/server.py

@@ -7,6 +7,7 @@ This module provides the ASGI app for uvicorn import string usage:
 For CLI usage, use the hindsight-api command instead.
 """
 
+import logging
 import os
 import warnings
 
@@ -17,6 +18,12 @@ warnings.filterwarnings("ignore", message="websockets.server.WebSocketServerProt
 from hindsight_api import MemoryEngine
 from hindsight_api.api import create_app
 from hindsight_api.config import get_config
+from hindsight_api.extensions import (
+    DefaultExtensionContext,
+    OperationValidatorExtension,
+    TenantExtension,
+    load_extension,
+)
 
 # Disable tokenizers parallelism to avoid warnings
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
@@ -25,12 +32,42 @@ os.environ["TOKENIZERS_PARALLELISM"] = "false"
 config = get_config()
 config.configure_logging()
 
+# Load operation validator extension if configured
+operation_validator = load_extension("OPERATION_VALIDATOR", OperationValidatorExtension)
+if operation_validator:
+    logging.info(f"Loaded operation validator: {operation_validator.__class__.__name__}")
+
+# Load tenant extension if configured
+tenant_extension = load_extension("TENANT", TenantExtension)
+if tenant_extension:
+    logging.info(f"Loaded tenant extension: {tenant_extension.__class__.__name__}")
+
 # Create app at module level (required for uvicorn import string)
 # MemoryEngine reads configuration from environment variables automatically
-_memory = MemoryEngine()
+# Note: run_migrations=True by default, but migrations are idempotent so safe with workers
+_memory = MemoryEngine(
+    operation_validator=operation_validator,
+    tenant_extension=tenant_extension,
+    run_migrations=config.run_migrations_on_startup,
+)
+
+# Set extension context on tenant extension (needed for schema provisioning)
+if tenant_extension:
+    extension_context = DefaultExtensionContext(
+        database_url=config.database_url,
+        memory_engine=_memory,
+    )
+    tenant_extension.set_context(extension_context)
+    logging.info("Extension context set on tenant extension")
 
 # Create unified app with both HTTP and optionally MCP
-app = create_app(memory=_memory, http_api_enabled=True, mcp_api_enabled=config.mcp_enabled, mcp_mount_path="/mcp")
+app = create_app(
+    memory=_memory,
+    http_api_enabled=True,
+    mcp_api_enabled=config.mcp_enabled,
+    mcp_mount_path="/mcp",
+    initialize_memory=True,
+)
 
 
 if __name__ == "__main__":

hindsight-api/hindsight_api/worker/__init__.py

@@ -0,0 +1,11 @@
+"""
+Worker package for distributed task processing.
+
+This package provides:
+- WorkerPoller: Polls PostgreSQL for pending tasks and executes them
+- main: CLI entry point for hindsight-worker
+"""
+
+from .poller import WorkerPoller
+
+__all__ = ["WorkerPoller"]

hindsight-api/hindsight_api/worker/main.py

@@ -0,0 +1,296 @@
+"""
+Command-line interface for Hindsight Worker.
+
+Run the worker with:
+    hindsight-worker
+
+Stop with Ctrl+C (graceful shutdown).
+"""
+
+import argparse
+import asyncio
+import atexit
+import logging
+import os
+import signal
+import socket
+import sys
+import warnings
+
+from ..config import get_config
+from ..engine.task_backend import SyncTaskBackend
+from .poller import WorkerPoller
+
+# Filter deprecation warnings from third-party libraries
+warnings.filterwarnings("ignore", message="websockets.legacy is deprecated")
+warnings.filterwarnings("ignore", message="websockets.server.WebSocketServerProtocol is deprecated")
+
+# Disable tokenizers parallelism to avoid warnings
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+logger = logging.getLogger(__name__)
+
+
+def create_worker_app(poller: WorkerPoller, memory):
+    """Create a minimal FastAPI app for worker metrics and health."""
+    from fastapi import FastAPI
+    from fastapi.responses import JSONResponse, Response
+    from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
+
+    from ..metrics import create_metrics_collector, get_metrics_collector, initialize_metrics
+
+    app = FastAPI(
+        title="Hindsight Worker",
+        description="Worker process for distributed task execution",
+    )
+
+    # Initialize OpenTelemetry metrics
+    try:
+        prometheus_reader = initialize_metrics(service_name="hindsight-worker", service_version="1.0.0")
+        create_metrics_collector()
+        app.state.prometheus_reader = prometheus_reader
+        logger.info("Metrics initialized - available at /metrics endpoint")
+    except Exception as e:
+        logger.warning(f"Failed to initialize metrics: {e}. Metrics will be disabled.")
+        app.state.prometheus_reader = None
+
+    # Set up DB pool metrics if available
+    metrics_collector = get_metrics_collector()
+    if memory._pool is not None and hasattr(metrics_collector, "set_db_pool"):
+        metrics_collector.set_db_pool(memory._pool)
+        logger.info("DB pool metrics configured")
+
+    @app.get(
+        "/health",
+        summary="Health check endpoint",
+        description="Returns worker health status including database connectivity",
+        tags=["Monitoring"],
+    )
+    async def health_endpoint():
+        """Health check endpoint."""
+        health = await memory.health_check()
+        health["worker_id"] = poller.worker_id
+        health["is_shutdown"] = poller.is_shutdown
+        status_code = 200 if health.get("status") == "healthy" else 503
+        return JSONResponse(content=health, status_code=status_code)
+
+    @app.get(
+        "/metrics",
+        summary="Prometheus metrics endpoint",
+        description="Exports metrics in Prometheus format for scraping",
+        tags=["Monitoring"],
+    )
+    async def metrics_endpoint():
+        """Return Prometheus metrics."""
+        metrics_data = generate_latest()
+        return Response(content=metrics_data, media_type=CONTENT_TYPE_LATEST)
+
+    @app.get(
+        "/",
+        summary="Worker info",
+        description="Basic worker information",
+        tags=["Info"],
+    )
+    async def root():
+        """Return basic worker info."""
+        return {
+            "service": "hindsight-worker",
+            "worker_id": poller.worker_id,
+            "is_shutdown": poller.is_shutdown,
+        }
+
+    return app
+
+
+def main():
+    """Main entry point for the hindsight-worker CLI."""
+    # Load configuration from environment
+    config = get_config()
+
+    parser = argparse.ArgumentParser(
+        prog="hindsight-worker",
+        description="Hindsight Worker - distributed task processor",
+    )
+
+    # Worker options
+    parser.add_argument(
+        "--worker-id",
+        default=config.worker_id or socket.gethostname(),
+        help="Worker identifier (default: hostname, env: HINDSIGHT_API_WORKER_ID)",
+    )
+    parser.add_argument(
+        "--poll-interval",
+        type=int,
+        default=config.worker_poll_interval_ms,
+        help=f"Poll interval in milliseconds (default: {config.worker_poll_interval_ms}, env: HINDSIGHT_API_WORKER_POLL_INTERVAL_MS)",
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=config.worker_batch_size,
+        help=f"Tasks to claim per poll (default: {config.worker_batch_size}, env: HINDSIGHT_API_WORKER_BATCH_SIZE)",
+    )
+    parser.add_argument(
+        "--max-retries",
+        type=int,
+        default=config.worker_max_retries,
+        help=f"Max retries before marking failed (default: {config.worker_max_retries}, env: HINDSIGHT_API_WORKER_MAX_RETRIES)",
+    )
+
+    # HTTP server options
+    parser.add_argument(
+        "--http-port",
+        type=int,
+        default=config.worker_http_port,
+        help=f"HTTP port for metrics/health endpoints (default: {config.worker_http_port}, env: HINDSIGHT_API_WORKER_HTTP_PORT)",
+    )
+    parser.add_argument(
+        "--http-host",
+        default="0.0.0.0",
+        help="HTTP host to bind (default: 0.0.0.0)",
+    )
+
+    # Logging options
+    parser.add_argument(
+        "--log-level",
+        default=config.log_level,
+        choices=["critical", "error", "warning", "info", "debug", "trace"],
+        help=f"Log level (default: {config.log_level}, env: HINDSIGHT_API_LOG_LEVEL)",
+    )
+
+    args = parser.parse_args()
+
+    # Configure logging
+    config.configure_logging()
+
+    # Import MemoryEngine here to avoid circular imports
+    from .. import MemoryEngine
+
+    print(f"Starting Hindsight Worker: {args.worker_id}")
+    print(f"  Poll interval: {args.poll_interval}ms")
+    print(f"  Batch size: {args.batch_size}")
+    print(f"  Max retries: {args.max_retries}")
+    print(f"  HTTP server: {args.http_host}:{args.http_port}")
+    print()
+
+    # Global references for cleanup
+    memory = None
+    poller = None
+
+    async def run():
+        nonlocal memory, poller
+        import uvicorn
+
+        from ..extensions import TenantExtension, load_extension
+
+        # Initialize MemoryEngine
+        # Workers use SyncTaskBackend because they execute tasks directly,
+        # they don't need to store tasks (they poll from DB)
+        memory = MemoryEngine(
+            run_migrations=False,  # Workers don't run migrations
+            task_backend=SyncTaskBackend(),
+        )
+
+        await memory.initialize()
+
+        print(f"Database connected: {config.database_url}")
+
+        # Load tenant extension for dynamic schema discovery
+        tenant_extension = load_extension("TENANT", TenantExtension)
+
+        if tenant_extension:
+            print("Tenant extension loaded - schemas will be discovered dynamically on each poll")
+        else:
+            print("No tenant extension configured, using public schema only")
+
+        # Create a single poller that handles all schemas dynamically
+        poller = WorkerPoller(
+            pool=memory._pool,
+            worker_id=args.worker_id,
+            executor=memory.execute_task,
+            poll_interval_ms=args.poll_interval,
+            batch_size=args.batch_size,
+            max_retries=args.max_retries,
+            tenant_extension=tenant_extension,
+        )
+
+        # Create the HTTP app for metrics/health
+        app = create_worker_app(poller, memory)
+
+        # Setup signal handlers for graceful shutdown
+        shutdown_requested = asyncio.Event()
+
+        def signal_handler(signum, frame):
+            print(f"\nReceived signal {signum}, initiating graceful shutdown...")
+            shutdown_requested.set()
+
+        signal.signal(signal.SIGINT, signal_handler)
+        signal.signal(signal.SIGTERM, signal_handler)
+
+        # Create uvicorn config and server
+        uvicorn_config = uvicorn.Config(
+            app,
+            host=args.http_host,
+            port=args.http_port,
+            log_level="info",  # Reduce uvicorn noise
+            access_log=False,
+        )
+        server = uvicorn.Server(uvicorn_config)
+
+        # Run the poller and HTTP server concurrently
+        poller_task = asyncio.create_task(poller.run())
+        http_task = asyncio.create_task(server.serve())
+
+        print(f"Worker started. Metrics available at http://{args.http_host}:{args.http_port}/metrics")
+
+        # Wait for shutdown signal
+        await shutdown_requested.wait()
+
+        # Graceful shutdown
+        print("Shutting down HTTP server...")
+        server.should_exit = True
+
+        print("Waiting for poller to finish...")
+        await poller.shutdown_graceful(timeout=30.0)
+        poller_task.cancel()
+        try:
+            await poller_task
+        except asyncio.CancelledError:
+            pass
+
+        # Wait for HTTP server to finish
+        try:
+            await asyncio.wait_for(http_task, timeout=5.0)
+        except asyncio.TimeoutError:
+            http_task.cancel()
+            try:
+                await http_task
+            except asyncio.CancelledError:
+                pass
+
+        # Close memory engine
+        await memory.close()
+        print("Worker shutdown complete")
+
+    def cleanup():
+        """Synchronous cleanup for atexit."""
+        if memory is not None and memory._pg0 is not None:
+            try:
+                loop = asyncio.new_event_loop()
+                loop.run_until_complete(memory._pg0.stop())
+                loop.close()
+                print("\npg0 stopped.")
+            except Exception as e:
+                print(f"\nError stopping pg0: {e}")
+
+    atexit.register(cleanup)
+
+    try:
+        asyncio.run(run())
+    except KeyboardInterrupt:
+        print("\nWorker interrupted")
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

hindsight-api/hindsight_api/worker/poller.py

@@ -0,0 +1,486 @@
+"""
+Worker poller for distributed task execution.
+
+Polls PostgreSQL for pending tasks and executes them using
+FOR UPDATE SKIP LOCKED for safe concurrent claiming.
+"""
+
+import asyncio
+import json
+import logging
+import time
+import traceback
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import asyncpg
+
+    from hindsight_api.extensions.tenant import TenantExtension
+
+logger = logging.getLogger(__name__)
+
+# Progress logging interval in seconds
+PROGRESS_LOG_INTERVAL = 30
+
+
+def fq_table(table: str, schema: str | None = None) -> str:
+    """Get fully-qualified table name with optional schema prefix."""
+    if schema:
+        return f'"{schema}".{table}'
+    return table
+
+
+@dataclass
+class ClaimedTask:
+    """A task claimed from the database with its schema context."""
+
+    operation_id: str
+    task_dict: dict[str, Any]
+    schema: str | None
+
+
+class WorkerPoller:
+    """
+    Polls PostgreSQL for pending tasks and executes them.
+
+    Uses FOR UPDATE SKIP LOCKED for safe distributed claiming,
+    allowing multiple workers to process tasks without conflicts.
+
+    Supports dynamic multi-tenant discovery via tenant_extension.
+    """
+
+    def __init__(
+        self,
+        pool: "asyncpg.Pool",
+        worker_id: str,
+        executor: Callable[[dict[str, Any]], Awaitable[None]],
+        poll_interval_ms: int = 500,
+        batch_size: int = 10,
+        max_retries: int = 3,
+        schema: str | None = None,
+        tenant_extension: "TenantExtension | None" = None,
+    ):
+        """
+        Initialize the worker poller.
+
+        Args:
+            pool: asyncpg connection pool
+            worker_id: Unique identifier for this worker
+            executor: Async function to execute tasks (typically MemoryEngine.execute_task)
+            poll_interval_ms: Interval between polls when no tasks found (milliseconds)
+            batch_size: Maximum number of tasks to claim per poll cycle
+            max_retries: Maximum retry attempts before marking task as failed
+            schema: Database schema for single-tenant support (ignored if tenant_extension is set)
+            tenant_extension: Extension for dynamic multi-tenant discovery. If set, list_tenants()
+                            is called on each poll cycle to discover schemas dynamically.
+        """
+        self._pool = pool
+        self._worker_id = worker_id
+        self._executor = executor
+        self._poll_interval_ms = poll_interval_ms
+        self._batch_size = batch_size
+        self._max_retries = max_retries
+        self._schema = schema
+        self._tenant_extension = tenant_extension
+        self._shutdown = asyncio.Event()
+        self._current_tasks: set[asyncio.Task] = set()
+        self._in_flight_count = 0
+        self._in_flight_lock = asyncio.Lock()
+        self._last_progress_log = 0.0
+        self._tasks_completed_since_log = 0
+        # Track active tasks locally: operation_id -> (op_type, bank_id, schema)
+        self._active_tasks: dict[str, tuple[str, str, str | None]] = {}
+
+    async def _get_schemas(self) -> list[str | None]:
+        """Get list of schemas to poll. Returns [None] for public schema."""
+        if self._tenant_extension is not None:
+            tenants = await self._tenant_extension.list_tenants()
+            # Convert "public" to None for SQL compatibility, keep others as-is
+            return [t.schema if t.schema != "public" else None for t in tenants]
+        # Single schema mode
+        return [self._schema]
+
+    async def claim_batch(self) -> list[ClaimedTask]:
+        """
+        Claim up to batch_size pending tasks atomically across all tenant schemas.
+
+        Uses FOR UPDATE SKIP LOCKED to ensure no conflicts with other workers.
+
+        For consolidation tasks specifically, skips pending tasks if there's already
+        a processing consolidation for the same bank (to avoid duplicate work).
+
+        If tenant_extension is configured, dynamically discovers schemas on each call.
+
+        Returns:
+            List of ClaimedTask objects containing operation_id, task_dict, and schema
+        """
+        schemas = await self._get_schemas()
+        all_tasks: list[ClaimedTask] = []
+        remaining_batch = self._batch_size
+
+        for schema in schemas:
+            if remaining_batch <= 0:
+                break
+
+            tasks = await self._claim_batch_for_schema(schema, remaining_batch)
+            all_tasks.extend(tasks)
+            remaining_batch -= len(tasks)
+
+        return all_tasks
+
+    async def _claim_batch_for_schema(self, schema: str | None, limit: int) -> list[ClaimedTask]:
+        """Claim tasks from a specific schema."""
+        table = fq_table("async_operations", schema)
+
+        async with self._pool.acquire() as conn:
+            async with conn.transaction():
+                # Select and lock pending tasks
+                # For consolidation: skip if same bank already has one processing
+                rows = await conn.fetch(
+                    f"""
+                    SELECT operation_id, task_payload
+                    FROM {table} AS pending
+                    WHERE status = 'pending' AND task_payload IS NOT NULL
+                    AND (
+                        -- Non-consolidation tasks: always claimable
+                        operation_type != 'consolidation'
+                        OR
+                        -- Consolidation: only if no other consolidation processing for same bank
+                        NOT EXISTS (
+                            SELECT 1 FROM {table} AS processing
+                            WHERE processing.bank_id = pending.bank_id
+                            AND processing.operation_type = 'consolidation'
+                            AND processing.status = 'processing'
+                        )
+                    )
+                    ORDER BY created_at
+                    LIMIT $1
+                    FOR UPDATE SKIP LOCKED
+                    """,
+                    limit,
+                )
+
+                if not rows:
+                    return []
+
+                # Claim the tasks by updating status and worker_id
+                operation_ids = [row["operation_id"] for row in rows]
+                await conn.execute(
+                    f"""
+                    UPDATE {table}
+                    SET status = 'processing', worker_id = $1, claimed_at = now(), updated_at = now()
+                    WHERE operation_id = ANY($2)
+                    """,
+                    self._worker_id,
+                    operation_ids,
+                )
+
+                # Parse and return task payloads with schema context
+                return [
+                    ClaimedTask(
+                        operation_id=str(row["operation_id"]),
+                        task_dict=json.loads(row["task_payload"]),
+                        schema=schema,
+                    )
+                    for row in rows
+                ]
+
+    async def _mark_completed(self, operation_id: str, schema: str | None):
+        """Mark a task as completed."""
+        table = fq_table("async_operations", schema)
+        await self._pool.execute(
+            f"""
+            UPDATE {table}
+            SET status = 'completed', completed_at = now(), updated_at = now()
+            WHERE operation_id = $1
+            """,
+            operation_id,
+        )
+
+    async def _mark_failed(self, operation_id: str, error_message: str, schema: str | None):
+        """Mark a task as failed with error message."""
+        table = fq_table("async_operations", schema)
+        # Truncate error message if too long (max 5000 chars in schema)
+        error_message = error_message[:5000] if len(error_message) > 5000 else error_message
+        await self._pool.execute(
+            f"""
+            UPDATE {table}
+            SET status = 'failed', error_message = $2, completed_at = now(), updated_at = now()
+            WHERE operation_id = $1
+            """,
+            operation_id,
+            error_message,
+        )
+
+    async def _retry_or_fail(self, operation_id: str, error_message: str, schema: str | None):
+        """Increment retry count or mark as failed if max retries exceeded."""
+        table = fq_table("async_operations", schema)
+
+        # Get current retry count
+        row = await self._pool.fetchrow(
+            f"SELECT retry_count FROM {table} WHERE operation_id = $1",
+            operation_id,
+        )
+
+        if row is None:
+            logger.warning(f"Operation {operation_id} not found, cannot retry")
+            return
+
+        retry_count = row["retry_count"]
+
+        if retry_count >= self._max_retries:
+            # Max retries exceeded, mark as failed
+            await self._mark_failed(
+                operation_id, f"Max retries ({self._max_retries}) exceeded. Last error: {error_message}", schema
+            )
+            logger.error(f"Task {operation_id} failed after {retry_count} retries")
+        else:
+            # Increment retry and reset to pending
+            await self._pool.execute(
+                f"""
+                UPDATE {table}
+                SET status = 'pending', worker_id = NULL, claimed_at = NULL,
+                    retry_count = retry_count + 1, updated_at = now()
+                WHERE operation_id = $1
+                """,
+                operation_id,
+            )
+            logger.warning(f"Task {operation_id} failed, will retry (attempt {retry_count + 1}/{self._max_retries})")
+
+    async def execute_task(self, task: ClaimedTask):
+        """Execute a single task and update its status."""
+        task_type = task.task_dict.get("type", "unknown")
+        bank_id = task.task_dict.get("bank_id", "unknown")
+
+        # Track this task as active
+        async with self._in_flight_lock:
+            self._active_tasks[task.operation_id] = (task_type, bank_id, task.schema)
+
+        try:
+            schema_info = f", schema={task.schema}" if task.schema else ""
+            logger.debug(f"Executing task {task.operation_id} (type={task_type}, bank={bank_id}{schema_info})")
+            # Pass schema to executor so it can set the correct context
+            if task.schema:
+                task.task_dict["_schema"] = task.schema
+            await self._executor(task.task_dict)
+            await self._mark_completed(task.operation_id, task.schema)
+            logger.debug(f"Task {task.operation_id} completed successfully")
+        except Exception as e:
+            error_msg = f"{type(e).__name__}: {e}\n{traceback.format_exc()}"
+            logger.error(f"Task {task.operation_id} failed: {e}")
+            await self._retry_or_fail(task.operation_id, error_msg, task.schema)
+        finally:
+            # Remove from active tasks
+            async with self._in_flight_lock:
+                self._active_tasks.pop(task.operation_id, None)
+
+    async def recover_own_tasks(self) -> int:
+        """
+        Recover tasks that were assigned to this worker but not completed.
+
+        This handles the case where a worker crashes while processing tasks.
+        On startup, we reset any tasks stuck in 'processing' for this worker_id
+        back to 'pending' so they can be picked up again.
+
+        If tenant_extension is configured, recovers across all tenant schemas.
+
+        Returns:
+            Number of tasks recovered
+        """
+        schemas = await self._get_schemas()
+        total_count = 0
+
+        for schema in schemas:
+            table = fq_table("async_operations", schema)
+
+            result = await self._pool.execute(
+                f"""
+                UPDATE {table}
+                SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
+                WHERE status = 'processing' AND worker_id = $1
+                """,
+                self._worker_id,
+            )
+
+            # Parse "UPDATE N" to get count
+            count = int(result.split()[-1]) if result else 0
+            total_count += count
+
+        if total_count > 0:
+            logger.info(f"Worker {self._worker_id} recovered {total_count} stale tasks from previous run")
+        return total_count
+
+    async def run(self):
+        """
+        Main polling loop.
+
+        Continuously polls for pending tasks, claims them, and executes them
+        until shutdown is signaled.
+
+        If tenant_extension is configured, dynamically discovers schemas on each poll.
+        """
+        # Recover any tasks from a previous crash before starting
+        await self.recover_own_tasks()
+
+        logger.info(f"Worker {self._worker_id} starting polling loop")
+
+        while not self._shutdown.is_set():
+            try:
+                # Claim a batch of tasks (across all tenant schemas if configured)
+                tasks = await self.claim_batch()
+
+                if tasks:
+                    # Log batch info
+                    task_types: dict[str, int] = {}
+                    schemas_seen: set[str | None] = set()
+                    for task in tasks:
+                        t = task.task_dict.get("type", "unknown")
+                        task_types[t] = task_types.get(t, 0) + 1
+                        schemas_seen.add(task.schema)
+                    types_str = ", ".join(f"{k}:{v}" for k, v in task_types.items())
+                    schemas_str = ", ".join(s or "public" for s in schemas_seen)
+                    logger.info(
+                        f"Worker {self._worker_id} claimed {len(tasks)} tasks: {types_str} (schemas: {schemas_str})"
+                    )
+
+                    # Track in-flight tasks
+                    async with self._in_flight_lock:
+                        self._in_flight_count += len(tasks)
+
+                    # Execute tasks concurrently
+                    try:
+                        await asyncio.gather(
+                            *[self.execute_task(task) for task in tasks],
+                            return_exceptions=True,
+                        )
+                    finally:
+                        async with self._in_flight_lock:
+                            self._in_flight_count -= len(tasks)
+                else:
+                    # No tasks found, wait before polling again
+                    try:
+                        await asyncio.wait_for(
+                            self._shutdown.wait(),
+                            timeout=self._poll_interval_ms / 1000,
+                        )
+                    except asyncio.TimeoutError:
+                        pass  # Normal timeout, continue polling
+
+                # Log progress stats periodically
+                await self._log_progress_if_due()
+
+            except asyncio.CancelledError:
+                logger.info(f"Worker {self._worker_id} polling loop cancelled")
+                break
+            except Exception as e:
+                logger.error(f"Worker {self._worker_id} error in polling loop: {e}")
+                traceback.print_exc()
+                # Backoff on error
+                await asyncio.sleep(1)
+
+        logger.info(f"Worker {self._worker_id} polling loop stopped")
+
+    async def shutdown_graceful(self, timeout: float = 30.0):
+        """
+        Signal shutdown and wait for current tasks to complete.
+
+        Args:
+            timeout: Maximum time to wait for in-flight tasks (seconds)
+        """
+        logger.info(f"Worker {self._worker_id} initiating graceful shutdown")
+        self._shutdown.set()
+
+        # Wait for in-flight tasks to complete
+        start_time = asyncio.get_event_loop().time()
+        while asyncio.get_event_loop().time() - start_time < timeout:
+            async with self._in_flight_lock:
+                in_flight = self._in_flight_count
+
+            if in_flight == 0:
+                logger.info(f"Worker {self._worker_id} graceful shutdown complete")
+                return
+
+            logger.info(f"Worker {self._worker_id} waiting for {in_flight} in-flight tasks")
+            await asyncio.sleep(0.5)
+
+        logger.warning(f"Worker {self._worker_id} shutdown timeout after {timeout}s")
+
+    async def _log_progress_if_due(self):
+        """Log progress stats every PROGRESS_LOG_INTERVAL seconds."""
+        now = time.time()
+        if now - self._last_progress_log < PROGRESS_LOG_INTERVAL:
+            return
+
+        self._last_progress_log = now
+
+        try:
+            # Get local active tasks (this worker only)
+            async with self._in_flight_lock:
+                in_flight = self._in_flight_count
+                active_tasks = dict(self._active_tasks)  # Copy to avoid holding lock
+
+            # Build local processing breakdown grouped by (op_type, bank_id)
+            task_groups: dict[tuple[str, str], int] = {}
+            for op_type, bank_id, _ in active_tasks.values():
+                key = (op_type, bank_id)
+                task_groups[key] = task_groups.get(key, 0) + 1
+
+            processing_info = [f"{op}:{bank}({cnt})" for (op, bank), cnt in task_groups.items()]
+            processing_str = ", ".join(processing_info[:10]) if processing_info else "none"
+            if len(processing_info) > 10:
+                processing_str += f" +{len(processing_info) - 10} more"
+
+            # Get global stats from DB across all schemas
+            schemas = await self._get_schemas()
+            global_pending = 0
+            all_worker_counts: dict[str, int] = {}
+
+            async with self._pool.acquire() as conn:
+                for schema in schemas:
+                    table = fq_table("async_operations", schema)
+
+                    row = await conn.fetchrow(f"SELECT COUNT(*) as count FROM {table} WHERE status = 'pending'")
+                    global_pending += row["count"] if row else 0
+
+                    # Get processing breakdown by worker
+                    worker_rows = await conn.fetch(
+                        f"""
+                        SELECT worker_id, COUNT(*) as count
+                        FROM {table}
+                        WHERE status = 'processing'
+                        GROUP BY worker_id
+                        """
+                    )
+                    for wr in worker_rows:
+                        wid = wr["worker_id"] or "unknown"
+                        all_worker_counts[wid] = all_worker_counts.get(wid, 0) + wr["count"]
+
+            # Format other workers' processing counts
+            other_workers = []
+            for wid, cnt in all_worker_counts.items():
+                if wid != self._worker_id:
+                    other_workers.append(f"{wid}:{cnt}")
+            others_str = ", ".join(other_workers) if other_workers else "none"
+
+            schemas_str = ", ".join(s or "public" for s in schemas)
+            logger.info(
+                f"[WORKER_STATS] worker={self._worker_id} in_flight={in_flight} | "
+                f"global: pending={global_pending} (schemas: {schemas_str}) | "
+                f"others: {others_str} | "
+                f"my_active: {processing_str}"
+            )
+
+        except Exception as e:
+            logger.debug(f"Failed to log progress stats: {e}")
+
+    @property
+    def worker_id(self) -> str:
+        """Get the worker ID."""
+        return self._worker_id
+
+    @property
+    def is_shutdown(self) -> bool:
+        """Check if shutdown has been signaled."""
+        return self._shutdown.is_set()

hindsight-api/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-api"
-version = "0.2.1"
+version = "0.3.0"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -14,7 +14,6 @@ dependencies = [
     "openai>=1.0.0",
     "pydantic>=2.0.0",
     "rich>=13.0.0",
-    "sentence-transformers>=3.0.0,<3.3.0",
     "langchain-text-splitters>=0.3.0",
     "fastapi[standard]>=0.120.3",
     "uvicorn>=0.38.0",
@@ -24,11 +23,9 @@ dependencies = [
     "pgvector>=0.4.1",
     "greenlet>=3.2.4",
     "psycopg2-binary>=2.9.11",
-    "transformers>=4.30.0,<4.46.0",
-    "torch>=2.0.0",
     "tiktoken>=0.12.0",
     "httpx>=0.27.0",
-    "fastmcp>=2.3.0",
+    "fastmcp>=2.14.0",  # CVE-2025-66416
     "pg0-embedded>=0.11.0",
     "python-dateutil>=2.8.0",
     "opentelemetry-api>=1.20.0",
@@ -39,6 +36,20 @@ dependencies = [
     "google-genai>=1.0.0",
     "anthropic>=0.40.0",
     "typer>=0.9.0",
+    "cohere>=5.0.0",
+    "flashrank>=0.2.0",
+    # Local ML models for embeddings/reranking - can be excluded in Docker with INCLUDE_LOCAL_MODELS=false
+    "sentence-transformers>=3.3.0",
+    "transformers>=4.53.0",  # Security fixes for ReDoS vulnerabilities
+    "torch>=2.6.0",  # CVE fix for remote code execution
+    "uvloop>=0.22.1",
+    # Transitive dependency security fixes
+    "pyasn1>=0.6.2",  # DoS vulnerability fix
+    "urllib3>=2.6.3",  # Decompression-bomb safeguards bypass fix
+    "langchain-core>=1.2.5",  # Serialization injection vulnerability fix
+    "filelock>=3.20.1",  # TOCTOU race condition fix
+    "authlib>=1.6.6",  # Account takeover vulnerability fix
+    "aiohttp>=3.13.3",  # Multiple DoS vulnerabilities
 ]
 
 [project.optional-dependencies]
@@ -47,11 +58,12 @@ test = [
     "pytest-asyncio>=0.21.0",
     "pytest-timeout>=2.4.0",
     "pytest-xdist>=3.0.0",
-    "filelock>=3.0.0",
+    "filelock>=3.20.1",  # TOCTOU race condition fix
 ]
 
 [project.scripts]
 hindsight-api = "hindsight_api.main:main"
+hindsight-worker = "hindsight_api.worker.main:main"
 hindsight-local-mcp = "hindsight_api.mcp_local:main"
 hindsight-admin = "hindsight_api.admin.cli:main"
 
@@ -93,7 +105,7 @@ dev = [
     "pytest-timeout>=2.4.0",
     "pytest-xdist>=3.8.0",
     "python-dotenv>=1.2.1",
-    "filelock>=3.0.0",
+    "filelock>=3.20.1",  # TOCTOU race condition fix
     "ruff>=0.8.0",
     "ty>=0.0.1",
 ]
@@ -122,6 +134,9 @@ ignore = [
     "F821",   # undefined name (forward references in type hints)
 ]
 
+[tool.ruff.lint.isort]
+known-third-party = ["alembic"]
+
 [tool.ruff.format]
 quote-style = "double"
 indent-style = "space"

hindsight-api/tests/conftest.py

@@ -12,6 +12,7 @@ from hindsight_api import MemoryEngine, LLMConfig, LocalSTEmbeddings, RequestCon
 
 from hindsight_api.engine.cross_encoder import LocalSTCrossEncoder
 from hindsight_api.engine.query_analyzer import DateparserQueryAnalyzer
+from hindsight_api.engine.task_backend import SyncTaskBackend
 from hindsight_api.pg0 import EmbeddedPostgres
 
 # Default pg0 instance configuration for tests
@@ -115,16 +116,65 @@ def llm_config():
 
 
 @pytest.fixture(scope="session")
-def embeddings():
+def embeddings(tmp_path_factory, worker_id):
+    """
+    Session-scoped embeddings fixture with filelock to prevent race conditions.
 
-    return LocalSTEmbeddings()
+    When pytest-xdist runs multiple workers in parallel, they all try to load
+    models from the HuggingFace cache simultaneously, which can cause race
+    conditions and meta tensor errors. We use a filelock to serialize model
+    initialization across workers.
+    """
+    # Get shared temp dir for coordination between xdist workers
+    if worker_id == "master":
+        root_tmp_dir = tmp_path_factory.getbasetemp()
+    else:
+        root_tmp_dir = tmp_path_factory.getbasetemp().parent
 
+    lock_file = root_tmp_dir / "embeddings_init.lock"
+
+    emb = LocalSTEmbeddings()
+
+    # Serialize model initialization across workers
+    with filelock.FileLock(str(lock_file)):
+        loop = asyncio.new_event_loop()
+        try:
+            loop.run_until_complete(emb.initialize())
+        finally:
+            loop.close()
+
+    return emb
 
 
 @pytest.fixture(scope="session")
-def cross_encoder():
+def cross_encoder(tmp_path_factory, worker_id):
+    """
+    Session-scoped cross-encoder fixture with filelock to prevent race conditions.
 
-    return LocalSTCrossEncoder()
+    When pytest-xdist runs multiple workers in parallel, they all try to load
+    models from the HuggingFace cache simultaneously, which can cause race
+    conditions and meta tensor errors. We use a filelock to serialize model
+    initialization across workers.
+    """
+    # Get shared temp dir for coordination between xdist workers
+    if worker_id == "master":
+        root_tmp_dir = tmp_path_factory.getbasetemp()
+    else:
+        root_tmp_dir = tmp_path_factory.getbasetemp().parent
+
+    lock_file = root_tmp_dir / "cross_encoder_init.lock"
+
+    ce = LocalSTCrossEncoder()
+
+    # Serialize model initialization across workers
+    with filelock.FileLock(str(lock_file)):
+        loop = asyncio.new_event_loop()
+        try:
+            loop.run_until_complete(ce.initialize())
+        finally:
+            loop.close()
+
+    return ce
 
 @pytest.fixture(scope="session")
 def query_analyzer():
@@ -147,6 +197,7 @@ async def memory(pg0_db_url, embeddings, cross_encoder, query_analyzer):
     Uses pg0_db_url (a postgresql:// URL) directly, so MemoryEngine won't try to
     manage pg0 lifecycle - that's handled by the session-scoped pg0_db_url fixture.
     Migrations are disabled here since they're run once at session scope in pg0_db_url.
+    Uses SyncTaskBackend so async tasks execute immediately (no worker needed).
     """
     mem = MemoryEngine(
         db_url=pg0_db_url,  # Direct postgresql:// URL, not pg0://
@@ -160,6 +211,7 @@ async def memory(pg0_db_url, embeddings, cross_encoder, query_analyzer):
         pool_min_size=1,
         pool_max_size=5,
         run_migrations=False,  # Migrations already run at session scope
+        task_backend=SyncTaskBackend(),  # Execute tasks immediately in tests
     )
     await mem.initialize()
     yield mem

hindsight-api/tests/test_admin_backup_restore.py

@@ -1,8 +1,9 @@
 """
 Tests for admin backup and restore functionality.
 
-Note: These tests run sequentially (not in parallel) because they all
-manipulate the same database and do full backup/restore operations.
+These tests use an isolated schema to avoid interfering with other tests.
+The backup/restore operations truncate tables, which would cause deadlocks
+and race conditions if run against the shared public schema.
 """
 
 import tempfile
@@ -10,49 +11,106 @@ import uuid
 import zipfile
 from pathlib import Path
 
+import asyncpg
 import pytest
+import pytest_asyncio
 
-from hindsight_api import RequestContext
 from hindsight_api.admin.cli import _backup, _restore, BACKUP_TABLES
+from hindsight_api.migrations import run_migrations
 
 
 # Run these tests sequentially since they do full DB backup/restore
 pytestmark = pytest.mark.xdist_group(name="backup_restore")
 
 
+@pytest_asyncio.fixture(scope="function")
+async def backup_test_schema(pg0_db_url, embeddings):
+    """Create an isolated schema for backup/restore tests.
+
+    Uses a unique schema name per test invocation to avoid conflicts with
+    parallel test runs or leftover state from interrupted runs.
+
+    Returns a tuple of (db_url, schema_name, fq_helper, embeddings).
+    """
+    # Initialize embeddings if not already done
+    await embeddings.initialize()
+
+    # Use unique schema name to avoid conflicts
+    schema_name = f"backup_test_{uuid.uuid4().hex[:8]}"
+
+    def _fq(table: str) -> str:
+        """Get fully-qualified table name in test schema."""
+        return f"{schema_name}.{table}"
+
+    conn = await asyncpg.connect(pg0_db_url)
+    try:
+        await conn.execute(f"CREATE SCHEMA {schema_name}")
+    finally:
+        await conn.close()
+
+    # Run migrations on the isolated schema
+    run_migrations(pg0_db_url, schema=schema_name)
+
+    yield pg0_db_url, schema_name, _fq, embeddings
+
+    # Cleanup after test
+    conn = await asyncpg.connect(pg0_db_url)
+    try:
+        await conn.execute(f"DROP SCHEMA IF EXISTS {schema_name} CASCADE")
+    finally:
+        await conn.close()
+
+
 @pytest.mark.asyncio
-async def test_backup_restore_roundtrip(memory, pg0_db_url, request_context):
+async def test_backup_restore_roundtrip(backup_test_schema):
     """Test that backup and restore preserves all data correctly."""
-    # Use unique bank ID to avoid conflicts
+    db_url, schema_name, _fq, embeddings = backup_test_schema
     bank_id = f"test-backup-{uuid.uuid4().hex[:8]}"
+    conn = await asyncpg.connect(db_url)
 
-    # Create some test data
-    await memory.retain_batch_async(
-        bank_id=bank_id,
-        contents=[
-            {"content": "Alice is a software engineer who loves Python."},
-            {"content": "Bob works with Alice on the backend team."},
-            {"content": "The team uses PostgreSQL for their database."},
-        ],
-        request_context=request_context,
-    )
+    try:
+        # Create a bank
+        await conn.execute(
+            f"INSERT INTO {_fq('banks')} (bank_id) VALUES ($1) ON CONFLICT DO NOTHING",
+            bank_id,
+        )
 
-    # Get counts before backup
-    async with memory._pool.acquire() as conn:
+        # Create some test memory units with embeddings
+        # Convert embedding list to pgvector format string
+        embedding_list = embeddings.encode(["Test content about Alice"])[0]
+        embedding_str = "[" + ",".join(str(x) for x in embedding_list) + "]"
+        for text in [
+            "Alice is a software engineer who loves Python.",
+            "Bob works with Alice on the backend team.",
+            "The team uses PostgreSQL for their database.",
+        ]:
+            await conn.execute(
+                f"""INSERT INTO {_fq('memory_units')}
+                    (bank_id, text, fact_type, embedding, event_date)
+                    VALUES ($1, $2, 'world', $3::vector, NOW())""",
+                bank_id,
+                text,
+                embedding_str,
+            )
+
+        # Get counts before backup
         counts_before = {}
         for table in BACKUP_TABLES:
-            counts_before[table] = await conn.fetchval(f"SELECT COUNT(*) FROM {table}")
+            counts_before[table] = await conn.fetchval(f"SELECT COUNT(*) FROM {_fq(table)}")
 
-    # Verify we have data
-    assert counts_before["banks"] > 0
-    assert counts_before["memory_units"] > 0
+        # Verify we have data
+        assert counts_before["banks"] > 0
+        assert counts_before["memory_units"] > 0
+
+    finally:
+        await conn.close()
 
     # Backup to a temp file
     with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as f:
         backup_path = Path(f.name)
 
     try:
-        manifest = await _backup(pg0_db_url, backup_path)
+        manifest = await _backup(db_url, backup_path, schema=schema_name)
 
         # Verify backup file exists and is valid
         assert backup_path.exists()
@@ -72,33 +130,37 @@ async def test_backup_restore_roundtrip(memory, pg0_db_url, request_context):
                 assert f"{table}.bin" in zf.namelist()
 
         # Clear all data
-        async with memory._pool.acquire() as conn:
+        conn = await asyncpg.connect(db_url)
+        try:
             for table in reversed(BACKUP_TABLES):
-                await conn.execute(f"TRUNCATE TABLE {table} CASCADE")
+                await conn.execute(f"TRUNCATE TABLE {_fq(table)} CASCADE")
 
-        # Verify data is gone
-        async with memory._pool.acquire() as conn:
+            # Verify data is gone
             for table in BACKUP_TABLES:
-                count = await conn.fetchval(f"SELECT COUNT(*) FROM {table}")
+                count = await conn.fetchval(f"SELECT COUNT(*) FROM {_fq(table)}")
                 assert count == 0, f"Table {table} should be empty after truncate"
+        finally:
+            await conn.close()
 
         # Restore from backup
-        await _restore(pg0_db_url, backup_path)
+        await _restore(db_url, backup_path, schema=schema_name)
 
         # Verify counts match original
-        async with memory._pool.acquire() as conn:
+        conn = await asyncpg.connect(db_url)
+        try:
             for table in BACKUP_TABLES:
-                count = await conn.fetchval(f"SELECT COUNT(*) FROM {table}")
+                count = await conn.fetchval(f"SELECT COUNT(*) FROM {_fq(table)}")
                 assert count == counts_before[table], f"Table {table} count mismatch after restore"
 
-        # Verify data content is preserved
-        async with memory._pool.acquire() as conn:
+            # Verify data content is preserved
             texts = await conn.fetch(
-                "SELECT text FROM memory_units WHERE bank_id = $1",
+                f"SELECT text FROM {_fq('memory_units')} WHERE bank_id = $1",
                 bank_id,
             )
             text_content = " ".join(r["text"] for r in texts)
             assert "Alice" in text_content or "software" in text_content
+        finally:
+            await conn.close()
 
     finally:
         # Cleanup
@@ -107,42 +169,60 @@ async def test_backup_restore_roundtrip(memory, pg0_db_url, request_context):
 
 
 @pytest.mark.asyncio
-async def test_backup_restore_preserves_all_column_types(memory, pg0_db_url, request_context):
+async def test_backup_restore_preserves_all_column_types(backup_test_schema):
     """Test that all column types are preserved: vectors, UUIDs, timestamps, JSONB."""
-    # Use unique bank ID
+    db_url, schema_name, _fq, embeddings = backup_test_schema
     bank_id = f"test-types-{uuid.uuid4().hex[:8]}"
+    conn = await asyncpg.connect(db_url)
 
-    # Create data with meaningful content that will produce facts
-    await memory.retain_batch_async(
-        bank_id=bank_id,
-        contents=[
-            {"content": "John Smith is a senior engineer at Acme Corp since 2020."},
-            {"content": "The project deadline is December 15th 2024."},
-        ],
-        request_context=request_context,
-    )
+    try:
+        # Create a bank
+        await conn.execute(
+            f"INSERT INTO {_fq('banks')} (bank_id) VALUES ($1) ON CONFLICT DO NOTHING",
+            bank_id,
+        )
 
-    # Get original data with all important column types
-    async with memory._pool.acquire() as conn:
-        # memory_units: UUID (id), Vector (embedding), Timestamp (event_date, created_at), JSONB (metadata)
+        # Create a memory unit with all column types
+        # Convert embedding list to pgvector format string
+        embedding_list = embeddings.encode(["John Smith engineer"])[0]
+        embedding_str = "[" + ",".join(str(x) for x in embedding_list) + "]"
+        await conn.execute(
+            f"""INSERT INTO {_fq('memory_units')}
+                (bank_id, text, fact_type, embedding, event_date, metadata)
+                VALUES ($1, $2, 'world', $3::vector, NOW(), $4)""",
+            bank_id,
+            "John Smith is a senior engineer at Acme Corp since 2020.",
+            embedding_str,
+            '{"key": "value"}',
+        )
+
+        # Create an entity
+        await conn.execute(
+            f"""INSERT INTO {_fq('entities')}
+                (bank_id, canonical_name, metadata)
+                VALUES ($1, $2, $3)""",
+            bank_id,
+            "John Smith",
+            '{"role": "engineer"}',
+        )
+
+        # Get original data
         original_unit = await conn.fetchrow(
-            """SELECT id, embedding, event_date, created_at, metadata, text
-               FROM memory_units WHERE bank_id = $1 LIMIT 1""",
+            f"""SELECT id, embedding, event_date, created_at, metadata, text
+               FROM {_fq('memory_units')} WHERE bank_id = $1 LIMIT 1""",
             bank_id,
         )
-
-        # entities: UUID (id), Timestamp (first_seen, last_seen), JSONB (metadata)
         original_entity = await conn.fetchrow(
-            """SELECT id, first_seen, last_seen, metadata, canonical_name
-               FROM entities WHERE bank_id = $1 LIMIT 1""",
+            f"""SELECT id, first_seen, last_seen, metadata, canonical_name
+               FROM {_fq('entities')} WHERE bank_id = $1 LIMIT 1""",
             bank_id,
         )
-
-        # banks: JSONB (personality/disposition)
         original_bank = await conn.fetchrow(
-            "SELECT bank_id, created_at, updated_at FROM banks WHERE bank_id = $1",
+            f"SELECT bank_id, created_at, updated_at FROM {_fq('banks')} WHERE bank_id = $1",
             bank_id,
         )
+    finally:
+        await conn.close()
 
     assert original_unit is not None, "Should have created memory units"
     assert original_unit["embedding"] is not None, "Should have embedding"
@@ -153,33 +233,37 @@ async def test_backup_restore_preserves_all_column_types(memory, pg0_db_url, req
         backup_path = Path(f.name)
 
     try:
-        await _backup(pg0_db_url, backup_path)
+        await _backup(db_url, backup_path, schema=schema_name)
 
         # Clear all data
-        async with memory._pool.acquire() as conn:
+        conn = await asyncpg.connect(db_url)
+        try:
             for table in reversed(BACKUP_TABLES):
-                await conn.execute(f"TRUNCATE TABLE {table} CASCADE")
+                await conn.execute(f"TRUNCATE TABLE {_fq(table)} CASCADE")
+        finally:
+            await conn.close()
 
-        await _restore(pg0_db_url, backup_path)
+        await _restore(db_url, backup_path, schema=schema_name)
 
         # Verify all column types are preserved exactly
-        async with memory._pool.acquire() as conn:
+        conn = await asyncpg.connect(db_url)
+        try:
             restored_unit = await conn.fetchrow(
-                """SELECT id, embedding, event_date, created_at, metadata, text
-                   FROM memory_units WHERE bank_id = $1 LIMIT 1""",
+                f"""SELECT id, embedding, event_date, created_at, metadata, text
+                   FROM {_fq('memory_units')} WHERE bank_id = $1 LIMIT 1""",
                 bank_id,
             )
-
             restored_entity = await conn.fetchrow(
-                """SELECT id, first_seen, last_seen, metadata, canonical_name
-                   FROM entities WHERE bank_id = $1 LIMIT 1""",
+                f"""SELECT id, first_seen, last_seen, metadata, canonical_name
+                   FROM {_fq('entities')} WHERE bank_id = $1 LIMIT 1""",
                 bank_id,
             )
-
             restored_bank = await conn.fetchrow(
-                "SELECT bank_id, created_at, updated_at FROM banks WHERE bank_id = $1",
+                f"SELECT bank_id, created_at, updated_at FROM {_fq('banks')} WHERE bank_id = $1",
                 bank_id,
             )
+        finally:
+            await conn.close()
 
         # Verify memory_units
         assert restored_unit is not None, "Should have restored memory unit"

hindsight-api/tests/test_agents_api.py

@@ -1,5 +1,5 @@
 """
-Tests for agent management API (profile, disposition, background).
+Tests for agent management API (profile, disposition).
 """
 import pytest
 import uuid
@@ -25,15 +25,12 @@ class TestAgentProfile:
 
         assert profile is not None
         assert "disposition" in profile
-        assert "background" in profile
 
         disposition = profile["disposition"]
         assert disposition.skepticism == 3
         assert disposition.literalism == 3
         assert disposition.empathy == 3
 
-        assert profile["background"] == ""
-
     @pytest.mark.asyncio
     async def test_update_agent_disposition(self, memory: MemoryEngine, request_context):
         """Test updating agent disposition traits."""
@@ -76,63 +73,10 @@ class TestAgentProfile:
         for agent in agents:
             assert "bank_id" in agent
             assert "disposition" in agent
-            assert "background" in agent
             assert "created_at" in agent
             assert "updated_at" in agent
 
 
-class TestAgentBackground:
-    """Tests for agent background management."""
-
-    @pytest.mark.asyncio
-    async def test_merge_agent_background(self, memory: MemoryEngine, request_context):
-        """Test merging agent background information."""
-        bank_id = unique_agent_id("test_profile_merge")
-
-        profile = await memory.get_bank_profile(bank_id, request_context=request_context)
-        assert profile["background"] == ""
-
-        result1 = await memory.merge_bank_background(
-            bank_id,
-            "I was born in Texas",
-            update_disposition=False,
-            request_context=request_context,
-        )
-        assert "Texas" in result1["background"]
-
-        result2 = await memory.merge_bank_background(
-            bank_id,
-            "I have 10 years of startup experience",
-            update_disposition=False,
-            request_context=request_context,
-        )
-        assert "Texas" in result2["background"] or "startup" in result2["background"]
-
-        final_profile = await memory.get_bank_profile(bank_id, request_context=request_context)
-        assert final_profile["background"] != ""
-
-    @pytest.mark.asyncio
-    async def test_merge_background_handles_conflicts(self, memory: MemoryEngine, request_context):
-        """Test that merging background handles conflicts (new overwrites old)."""
-        bank_id = unique_agent_id("test_profile_conflict")
-
-        result1 = await memory.merge_bank_background(
-            bank_id,
-            "I was born in Colorado",
-            update_disposition=False,
-            request_context=request_context,
-        )
-        assert "Colorado" in result1["background"]
-
-        result2 = await memory.merge_bank_background(
-            bank_id,
-            "You were born in Texas",
-            update_disposition=False,
-            request_context=request_context,
-        )
-        assert "Texas" in result2["background"]
-
-
 class TestAgentEndpoint:
     """Tests for agent PUT endpoint logic."""
 
@@ -147,7 +91,6 @@ class TestAgentEndpoint:
                 literalism=5,
                 empathy=2
             ),
-            background="I am a creative software engineer"
         )
 
         profile = await memory.get_bank_profile(bank_id, request_context=request_context)
@@ -159,55 +102,10 @@ class TestAgentEndpoint:
                 request_context=request_context,
             )
 
-        if request.background is not None:
-            pool = await memory._get_pool()
-            async with pool.acquire() as conn:
-                await conn.execute(
-                    """
-                    UPDATE banks
-                    SET background = $2,
-                        updated_at = NOW()
-                    WHERE bank_id = $1
-                    """,
-                    bank_id,
-                    request.background
-                )
-
         final_profile = await memory.get_bank_profile(bank_id, request_context=request_context)
 
         assert final_profile["disposition"].skepticism == 4
         assert final_profile["disposition"].literalism == 5
-        assert final_profile["background"] == "I am a creative software engineer"
-
-    @pytest.mark.asyncio
-    async def test_put_agent_partial_update(self, memory: MemoryEngine, request_context):
-        """Test updating only background."""
-        bank_id = unique_agent_id("test_put_partial")
-
-        request = CreateBankRequest(
-            background="I am a data scientist"
-        )
-
-        profile = await memory.get_bank_profile(bank_id, request_context=request_context)
-
-        if request.background is not None:
-            pool = await memory._get_pool()
-            async with pool.acquire() as conn:
-                await conn.execute(
-                    """
-                    UPDATE banks
-                    SET background = $2,
-                        updated_at = NOW()
-                    WHERE bank_id = $1
-                    """,
-                    bank_id,
-                    request.background
-                )
-
-        final_profile = await memory.get_bank_profile(bank_id, request_context=request_context)
-
-        assert final_profile["disposition"].skepticism == 3  # Default
-        assert final_profile["background"] == "I am a data scientist"
 
 
 class TestAgentDispositionIntegration:
@@ -225,13 +123,6 @@ class TestAgentDispositionIntegration:
         }
         await memory.update_bank_disposition(bank_id, disposition, request_context=request_context)
 
-        await memory.merge_bank_background(
-            bank_id,
-            "I am a creative artist who values innovation over tradition",
-            update_disposition=False,
-            request_context=request_context,
-        )
-
         await memory.retain_batch_async(
             bank_id=bank_id,
             contents=[

hindsight-api/tests/test_causal_relations.py

@@ -0,0 +1,223 @@
+"""
+Test suite for causal relations extraction and validation.
+
+Tests that:
+1. Causal relations only reference previous facts (target_index < current fact index)
+2. Invalid causal relation indices are rejected
+3. The new per-fact causal relations schema works correctly
+"""
+
+from datetime import datetime
+
+import pytest
+
+from hindsight_api import LLMConfig
+from hindsight_api.engine.retain.fact_extraction import extract_facts_from_text
+
+
+class TestCausalRelationsValidation:
+    """Tests for causal relations index validation."""
+
+    @pytest.mark.asyncio
+    async def test_causal_relations_only_reference_previous_facts(self):
+        """
+        Test that causal relations can only reference facts that appear before them.
+
+        This test verifies the new schema that prevents hallucination of invalid
+        fact indices by constraining target_index to be less than the current fact's index.
+        """
+        # Text with clear causal chain
+        text = """
+        I lost my job in January due to company layoffs.
+        Because I lost my job, I couldn't pay my rent.
+        Since I couldn't afford rent, I had to move to a cheaper apartment.
+        After moving, I started looking for a new job.
+        """
+
+        context = "Personal life update"
+        llm_config = LLMConfig.for_memory()
+        event_date = datetime(2024, 3, 15)
+
+        facts, _, usage = await extract_facts_from_text(
+            text=text,
+            event_date=event_date,
+            context=context,
+            llm_config=llm_config,
+            agent_name="TestUser",
+        )
+
+        assert len(facts) > 0, "Should extract at least one fact"
+
+        # Verify all causal relations reference valid previous facts
+        for i, fact in enumerate(facts):
+            if fact.causal_relations:
+                for rel in fact.causal_relations:
+                    assert rel.target_fact_index < i, (
+                        f"Fact {i} has causal relation to fact {rel.target_fact_index}, "
+                        f"but target_index must be < current index ({i})"
+                    )
+                    assert rel.target_fact_index >= 0, (
+                        f"Fact {i} has negative causal relation index: {rel.target_fact_index}"
+                    )
+                    assert rel.relation_type in ["caused_by", "enabled_by", "prevented_by"], (
+                        f"Invalid relation_type: {rel.relation_type}"
+                    )
+
+    @pytest.mark.asyncio
+    async def test_first_fact_has_no_causal_relations(self):
+        """
+        Test that the first fact (index 0) cannot have causal relations.
+
+        Since causal relations can only reference previous facts,
+        and there are no facts before index 0, the first fact should
+        have no causal relations.
+        """
+        text = """
+        The user started a new machine learning project.
+        The project requires learning TensorFlow.
+        Learning TensorFlow is challenging but rewarding.
+        """
+
+        context = "Project update"
+        llm_config = LLMConfig.for_memory()
+        event_date = datetime(2024, 6, 1)
+
+        facts, _, _ = await extract_facts_from_text(
+            text=text,
+            event_date=event_date,
+            context=context,
+            llm_config=llm_config,
+            agent_name="TestUser",
+        )
+
+        assert len(facts) > 0, "Should extract at least one fact"
+
+        # First fact should have no causal relations (nothing to reference)
+        if facts[0].causal_relations:
+            # If there are causal relations on the first fact, they should be empty
+            # or the validation should have filtered them out
+            for rel in facts[0].causal_relations:
+                # This should never happen due to validation
+                assert False, (
+                    f"First fact should not have causal relations, "
+                    f"but found: target_index={rel.target_fact_index}"
+                )
+
+    @pytest.mark.asyncio
+    async def test_causal_chain_extraction(self):
+        """
+        Test that a clear causal chain is extracted with valid relations.
+        """
+        text = """
+        Emily got promoted to senior engineer last month.
+        Because of her promotion, she received a significant salary increase.
+        With the extra money, she decided to buy a new car.
+        """
+
+        context = "Personal achievement story"
+        llm_config = LLMConfig.for_memory()
+        event_date = datetime(2024, 7, 15)
+
+        facts, _, _ = await extract_facts_from_text(
+            text=text,
+            event_date=event_date,
+            context=context,
+            llm_config=llm_config,
+            agent_name="TestUser",
+        )
+
+        assert len(facts) > 0, "Should extract facts about the causal chain"
+
+        # Collect all causal relations
+        all_relations = []
+        for i, fact in enumerate(facts):
+            if fact.causal_relations:
+                for rel in fact.causal_relations:
+                    all_relations.append({
+                        "from_fact": i,
+                        "to_fact": rel.target_fact_index,
+                        "type": rel.relation_type,
+                    })
+
+        # If causal relations were extracted, verify they form a valid chain
+        if all_relations:
+            for rel in all_relations:
+                assert rel["to_fact"] < rel["from_fact"], (
+                    f"Causal relation from fact {rel['from_fact']} to fact {rel['to_fact']} "
+                    f"is invalid (target must be < source)"
+                )
+
+    @pytest.mark.asyncio
+    async def test_token_efficiency_with_causal_relations(self):
+        """
+        Test that causal relations don't cause excessive output tokens.
+
+        This test verifies that the new schema (per-fact causal relations
+        with index constraints) doesn't waste tokens on invalid relations.
+        """
+        text = """
+        The company announced budget cuts in Q1.
+        Due to the budget cuts, the marketing team was reduced.
+        The reduced team meant fewer campaigns could be run.
+        With fewer campaigns, lead generation dropped.
+        Lower leads resulted in decreased sales.
+        """
+
+        context = "Business impact analysis"
+        llm_config = LLMConfig.for_memory()
+        event_date = datetime(2024, 4, 1)
+
+        facts, _, usage = await extract_facts_from_text(
+            text=text,
+            event_date=event_date,
+            context=context,
+            llm_config=llm_config,
+            agent_name="TestUser",
+        )
+
+        assert len(facts) > 0, "Should extract facts"
+
+        # Calculate output/input ratio
+        if usage.input_tokens > 0:
+            ratio = usage.output_tokens / usage.input_tokens
+            # The ratio should be reasonable (< 5x) with the new schema
+            # Previously it could be 7-10x due to hallucinated indices
+            assert ratio < 6, (
+                f"Output/input token ratio {ratio:.2f}x is too high. "
+                f"Input: {usage.input_tokens}, Output: {usage.output_tokens}"
+            )
+
+    @pytest.mark.asyncio
+    async def test_relation_types_are_backward_looking(self):
+        """
+        Test that all relation types describe how the current fact
+        relates to a previous fact (caused_by, enabled_by, prevented_by).
+        """
+        text = """
+        Alice learned Python programming.
+        Because she knew Python, she got a job as a data scientist.
+        Her data science skills enabled her to lead the analytics team.
+        """
+
+        context = "Career progression"
+        llm_config = LLMConfig.for_memory()
+        event_date = datetime(2024, 5, 1)
+
+        facts, _, _ = await extract_facts_from_text(
+            text=text,
+            event_date=event_date,
+            context=context,
+            llm_config=llm_config,
+            agent_name="TestUser",
+        )
+
+        # Verify relation types are all backward-looking
+        valid_types = {"caused_by", "enabled_by", "prevented_by"}
+
+        for i, fact in enumerate(facts):
+            if fact.causal_relations:
+                for rel in fact.causal_relations:
+                    assert rel.relation_type in valid_types, (
+                        f"Invalid relation_type '{rel.relation_type}'. "
+                        f"Must be one of: {valid_types}"
+                    )