misc: performance improvements

.github/workflows/test.yml

@@ -153,15 +153,8 @@ jobs:
     - name: Build docs
       run: npm run build --workspace=hindsight-docs
 
-  test-rust-cli:
+  build-rust-cli:
     runs-on: ubuntu-latest
-    env:
-      HINDSIGHT_API_LLM_PROVIDER: groq
-      HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
-      HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
-      HINDSIGHT_API_URL: http://localhost:8888
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v4
@@ -178,10 +171,6 @@ jobs:
           hindsight-cli/target
         key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
-    - name: Run unit tests
-      working-directory: hindsight-cli
-      run: cargo test
-
     - name: Build CLI
       working-directory: hindsight-cli
       run: cargo build --release
@@ -193,6 +182,29 @@ jobs:
         path: hindsight-cli/target/release/hindsight
         retention-days: 1
 
+  test-rust-cli:
+    runs-on: ubuntu-latest
+    needs: build-rust-cli
+    env:
+      HINDSIGHT_API_LLM_PROVIDER: groq
+      HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
+      HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
+      HINDSIGHT_API_URL: http://localhost:8888
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Download CLI artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: hindsight-cli
+        path: /tmp/cli
+
+    - name: Make CLI executable
+      run: chmod +x /tmp/cli/hindsight
+
     - name: Install uv
       uses: astral-sh/setup-uv@v5
       with:
@@ -210,7 +222,7 @@ jobs:
 
     - name: Install API dependencies
       working-directory: ./hindsight-api
-      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
+      run: uv sync --no-install-project --index-strategy unsafe-best-match
 
     - name: Create .env file
       run: |
@@ -239,7 +251,7 @@ jobs:
 
     - name: Run CLI smoke test
       run: |
-        HINDSIGHT_CLI=hindsight-cli/target/release/hindsight ./hindsight-cli/smoke-test.sh
+        HINDSIGHT_CLI=/tmp/cli/hindsight ./hindsight-cli/smoke-test.sh
 
     - name: Show API server logs
       if: always()
@@ -340,7 +352,7 @@ jobs:
 
     - name: Install dependencies
       working-directory: ./hindsight-api
-      run: uv sync --frozen --extra test --no-install-project --index-strategy unsafe-best-match
+      run: uv sync --extra test --no-install-project --index-strategy unsafe-best-match
 
     - name: Cache HuggingFace models
       uses: actions/cache@v4
@@ -401,11 +413,11 @@ jobs:
 
     - name: Install client test dependencies
       working-directory: ./hindsight-clients/python
-      run: uv sync --frozen --extra test --index-strategy unsafe-best-match
+      run: uv sync --extra test --index-strategy unsafe-best-match
 
     - name: Install API dependencies
       working-directory: ./hindsight-api
-      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
+      run: uv sync --no-install-project --index-strategy unsafe-best-match
 
     - name: Create .env file
       run: |
@@ -478,7 +490,7 @@ jobs:
 
     - name: Install API dependencies
       working-directory: ./hindsight-api
-      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
+      run: uv sync --no-install-project --index-strategy unsafe-best-match
 
     - name: Install TypeScript client dependencies
       working-directory: ./hindsight-clients/typescript
@@ -566,7 +578,7 @@ jobs:
 
     - name: Install API dependencies
       working-directory: ./hindsight-api
-      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
+      run: uv sync --no-install-project --index-strategy unsafe-best-match
 
     - name: Create .env file
       run: |
@@ -633,11 +645,11 @@ jobs:
 
     - name: Install API dependencies
       working-directory: ./hindsight-api
-      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
+      run: uv sync --no-install-project --index-strategy unsafe-best-match
 
     - name: Install integration test dependencies
       working-directory: ./hindsight-integration-tests
-      run: uv sync --frozen
+      run: uv sync
 
     - name: Cache HuggingFace models
       uses: actions/cache@v4
@@ -717,7 +729,7 @@ jobs:
 
     - name: Install dependencies
       working-directory: ./hindsight-integrations/litellm
-      run: uv sync --frozen --extra dev
+      run: uv sync --extra dev
 
     - name: Run tests
       working-directory: ./hindsight-integrations/litellm
@@ -748,7 +760,7 @@ jobs:
 
     - name: Install dependencies
       working-directory: ./hindsight-embed
-      run: uv sync --frozen --index-strategy unsafe-best-match
+      run: uv sync --index-strategy unsafe-best-match
 
     - name: Cache HuggingFace models
       uses: actions/cache@v4
@@ -765,7 +777,7 @@ jobs:
 
   test-doc-examples:
     runs-on: ubuntu-latest
-    needs: test-rust-cli
+    needs: build-rust-cli
     env:
       HINDSIGHT_API_LLM_PROVIDER: groq
       HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
@@ -808,11 +820,11 @@ jobs:
       working-directory: ./hindsight-api
       run: |
         uv build
-        uv sync --frozen --no-install-project --index-strategy unsafe-best-match
+        uv sync --no-install-project --index-strategy unsafe-best-match
 
     - name: Install Python client dependencies
       working-directory: ./hindsight-clients/python
-      run: uv sync --frozen --extra test --index-strategy unsafe-best-match
+      run: uv sync --extra test --index-strategy unsafe-best-match
 
     - name: Install TypeScript client
       run: |
@@ -916,9 +928,9 @@ jobs:
 
     - name: Install Python dependencies
       run: |
-        cd hindsight-dev && uv sync --frozen --index-strategy unsafe-best-match
-        cd ../hindsight-api && uv sync --frozen --index-strategy unsafe-best-match
-        cd ../hindsight-embed && uv sync --frozen --index-strategy unsafe-best-match
+        cd hindsight-dev && uv sync --index-strategy unsafe-best-match
+        cd ../hindsight-api && uv sync --index-strategy unsafe-best-match
+        cd ../hindsight-embed && uv sync --index-strategy unsafe-best-match
 
     - name: Run generate-openapi
       run: ./scripts/generate-openapi.sh

.gitignore

@@ -29,7 +29,6 @@ nltk_data/
 
 # Monitoring stack (Prometheus/Grafana binaries and data)
 .monitoring/
-.pgbouncer/
 
 # Large benchmark datasets (will be downloaded automatically)
 **/longmemeval_s_cleaned.json

AGENTS.md

@@ -1,3 +1,153 @@
 # AGENTS.md
 
-See [CLAUDE.md](./CLAUDE.md) for project documentation and coding conventions.
+This document captures architectural decisions and coding conventions for the Hindsight project.
+
+## Documentation
+
+- **Main documentation**: [hindsight-docs/docs/developer/](./hindsight-docs/docs/developer/)
+- **Use case patterns**: [hindsight-docs/docs/cookbook/](./hindsight-docs/docs/cookbook/)
+- **API reference**: Auto-generated from OpenAPI spec
+
+## Project Structure
+
+```
+hindsight/              # Python package for embedded usage
+hindsight-api/          # FastAPI server (core memory engine)
+hindsight-cli/          # Rust CLI client
+hindsight-embed/        # Embedded CLI (no server needed)
+hindsight-control-plane/ # Next.js admin UI
+hindsight-docs/         # Docusaurus documentation site
+hindsight-dev/          # Development tools and benchmarks
+hindsight-integrations/ # Framework integrations (LangChain, etc.)
+hindsight-clients/      # Generated API clients (Python, TypeScript, Rust)
+```
+
+## Core Concepts
+
+### Memory Banks
+- Each bank is an isolated memory store (like a "brain" for one user/agent)
+- Banks contain: memory units (facts), entities, documents, entity links
+- Banks have a **disposition** (personality traits) and **background** (context)
+- Bank isolation is strict - no cross-bank data leakage
+
+### Memory Types
+- **World facts**: General knowledge ("The sky is blue")
+- **Experience facts**: Personal experiences ("I visited Paris in 2023")
+- **Opinion facts**: Beliefs with confidence scores ("Paris is beautiful" - 0.9 confidence)
+
+### Operations
+- **Retain**: Store new memories (extracts facts, entities, relationships)
+- **Recall**: Retrieve memories (semantic, BM25, graph, temporal search)
+- **Reflect**: Deep analysis to form new insights/opinions
+
+## API Design Decisions
+
+### Single Bank Per Request
+- All API endpoints (`recall`, `reflect`, `retain`) operate on a single bank
+- Multi-bank queries are the **client/agent's responsibility** to orchestrate
+- This keeps the API simple and the isolation model clear
+
+### Disposition Traits (3-trait system)
+- **Skepticism** (1-5): How skeptical vs trusting when forming opinions
+- **Literalism** (1-5): How literally to interpret information
+- **Empathy** (1-5): How much to consider emotional context
+- These influence the `reflect` operation, not `recall`
+- Background info also only affects `reflect` (opinion formation)
+
+## Multi-Bank Architecture Patterns
+
+See [hindsight-docs/docs/cookbook/](./hindsight-docs/docs/cookbook/) for detailed guides:
+
+- **Per-User Memory**: One bank per user, simplest pattern
+- **Support Agent + Shared Knowledge**: User bank + shared docs bank, client orchestrates
+
+## Developer Guide
+
+### Running the API Server
+
+```bash
+# From project root
+./scripts/dev/start-api.sh
+
+# With options
+./scripts/dev/start-api.sh --reload --port 8888 --log-level debug
+```
+
+### Running Tests
+
+```bash
+# API tests
+cd hindsight-api
+uv run pytest tests/
+
+# Specific test
+uv run pytest tests/test_http_api_integration.py -v
+```
+
+### Generating OpenAPI Spec
+
+After changing API endpoints, regenerate the OpenAPI spec and docs:
+
+```bash
+./scripts/generate-openapi.sh
+```
+
+This will:
+1. Generate `openapi.json` at project root
+2. Copy to `hindsight-docs/openapi.json`
+3. Regenerate API reference documentation
+
+### Generating API Clients
+
+After updating the OpenAPI spec, regenerate all clients:
+
+```bash
+./scripts/generate-clients.sh
+```
+
+This generates:
+- **Rust client**: `hindsight-clients/rust/` (via progenitor in build.rs)
+- **Python client**: `hindsight-clients/python/` (via openapi-generator Docker)
+- **TypeScript client**: `hindsight-clients/typescript/` (via @hey-api/openapi-ts)
+
+Note: The maintained wrapper `hindsight_client.py` and `README.md` are preserved during regeneration.
+
+### Running the Documentation Site
+
+```bash
+./scripts/dev/start-docs.sh
+```
+
+### Running the Control Plane
+
+```bash
+./scripts/dev/start-control-plane.sh
+```
+
+## Code Style
+
+### Python (hindsight-api)
+- Use `uv` for package management
+- Async throughout (asyncpg, async FastAPI endpoints)
+- Pydantic models for request/response validation
+- No py files at project root - maintain clean directory structure
+
+### TypeScript (control-plane, clients)
+- Next.js with App Router for control plane
+- Tailwind CSS with shadcn/ui components
+
+### Rust (CLI)
+- Async with tokio
+- reqwest for HTTP client
+- progenitor for API client generation
+
+## Database
+
+- PostgreSQL with pgvector extension
+- Schema managed via Alembic migrations in `hindsight-api/alembic/`, db migrations happen during api startup, no manual commands
+- Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
+
+# Branding
+## Colors
+- Primary: gradient from #0074d9 to #009296
+

CLAUDE.md

@@ -108,52 +108,6 @@ PostgreSQL with pgvector. Schema managed via Alembic migrations in `hindsight-ap
 
 Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
 
-### Adding Database Migrations
-
-1. **Create a new migration file** in `hindsight-api/hindsight_api/alembic/versions/`:
-   - File name format: `<revision_id>_<description>.py` (e.g., `f1a2b3c4d5e6_add_new_index.py`)
-   - Use a unique hex revision ID (12 chars)
-   - Set `down_revision` to the previous migration's revision ID
-
-2. **Migration template**:
-   ```python
-   """Description of the migration
-
-   Revision ID: f1a2b3c4d5e6
-   Revises: <previous_revision_id>
-   Create Date: YYYY-MM-DD
-   """
-   from collections.abc import Sequence
-   from alembic import context, op
-
-   revision: str = "f1a2b3c4d5e6"
-   down_revision: str | Sequence[str] | None = "<previous_revision_id>"
-   branch_labels: str | Sequence[str] | None = None
-   depends_on: str | Sequence[str] | None = None
-
-   def _get_schema_prefix() -> str:
-       """Get schema prefix for table names (required for multi-tenant support)."""
-       schema = context.config.get_main_option("target_schema")
-       return f'"{schema}".' if schema else ""
-
-   def upgrade() -> None:
-       schema = _get_schema_prefix()
-       op.execute(f"CREATE INDEX ... ON {schema}table_name(...)")
-
-   def downgrade() -> None:
-       schema = _get_schema_prefix()
-       op.execute(f"DROP INDEX IF EXISTS {schema}index_name")
-   ```
-
-3. **Run migrations locally**:
-   ```bash
-   # Set database URL and run migrations
-   uv run hindsight-admin run-db-migration
-
-   # Run on a specific tenant schema
-   uv run hindsight-admin run-db-migration --schema tenant_xyz
-   ```
-
 ## Key Conventions
 
 ### Code Quality
@@ -174,63 +128,12 @@ This runs the same checks as the pre-commit hook (Ruff for Python, ESLint/Pretti
 - Multi-bank queries are client responsibility to orchestrate
 - Disposition traits only affect reflect, not recall
 
-### Control Plane API Routes
-
-When adding or modifying parameters in the dataplane API (hindsight-api), you must also update the control plane routes that proxy to it:
-
-1. **API Routes** (`hindsight-control-plane/src/app/api/`):
-   - `recall/route.ts` - proxies to `/v1/default/banks/{bank_id}/memories/recall`
-   - `reflect/route.ts` - proxies to `/v1/default/banks/{bank_id}/reflect`
-   - `memories/retain/route.ts` - proxies to `/v1/default/banks/{bank_id}/memories/retain`
-   - Other routes follow the same pattern
-
-2. **Client types** (`hindsight-control-plane/src/lib/api.ts`):
-   - Update the TypeScript type definitions for `recall()`, `reflect()`, `retain()` etc.
-
-3. **Checklist when adding new API parameters**:
-   - Add parameter extraction in the route handler (destructure from `body`)
-   - Pass the parameter to the SDK call
-   - Update the client type definition in `lib/api.ts`
-   - Update any UI components that need to use the new parameter
-
 ### Python Style
 - Python 3.11+, type hints required
 - Async throughout (asyncpg, async FastAPI)
 - Pydantic models for request/response
 - Ruff for linting (line-length 120)
 - No Python files at project root - maintain clean directory structure
-- **Never use multi-item tuple return values** - prefer dataclass or Pydantic model for structured returns
-
-### Type Safety with Pydantic Models
-**NEVER use raw `dict` types for structured data.** Always use Pydantic models:
-- Use Pydantic `BaseModel` for all data structures passed between functions
-- Add `@field_validator` for type coercion (e.g., ensuring datetimes are timezone-aware)
-- Avoid `dict.get()` patterns - use typed model attributes instead
-- Parse external data (JSON, API responses) into Pydantic models at the boundary
-- This catches type errors at parse time, not deep in business logic
-
-```python
-# BAD - error-prone dict access
-def process(data: dict) -> str:
-    return data.get("name", "")  # No validation, silent failures
-
-# GOOD - typed and validated
-class UserData(BaseModel):
-    name: str
-    created_at: datetime
-
-    @field_validator("created_at", mode="before")
-    @classmethod
-    def ensure_tz_aware(cls, v):
-        if isinstance(v, str):
-            v = datetime.fromisoformat(v.replace("Z", "+00:00"))
-        if v.tzinfo is None:
-            return v.replace(tzinfo=timezone.utc)
-        return v
-
-def process(data: UserData) -> str:
-    return data.name  # Type-safe, validated at construction
-```
 
 ### TypeScript Style
 - Next.js App Router for control plane

README.md

@@ -5,7 +5,7 @@
 [Documentation](https://hindsight.vectorize.io) • [Paper](https://arxiv.org/abs/2512.12818) • [Cookbook](https://hindsight.vectorize.io/cookbook) • [Hindsight Cloud](https://vectorize.io/hindsight/cloud)
 
 [![CI](https://github.com/vectorize-io/hindsight/actions/workflows/release.yml/badge.svg)](https://github.com/vectorize-io/hindsight/actions/workflows/release.yml)
-[![Slack Community](https://img.shields.io/badge/Slack-Join%20Community-4A154B?logo=slack)](https://join.slack.com/t/hindsight-space/shared_invite/zt-3nhbm4w29-LeSJ5Ixi6j8PdiYOCPlOgg)
+[![Slack Community](https://img.shields.io/badge/Slack-Join%20Community-4A154B?logo=slack)](https://join.slack.com/t/hindsight-space/shared_invite/zt-3klo21kua-VUCC_zHP5rIcXFB1_5yw6A)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 ![PyPI - Downloads](https://img.shields.io/pypi/dm/hindsight-api?label=PyPI)
 ![NPM Downloads](https://img.shields.io/npm/dm/%40vectorize-io%2Fhindsight-client?logoColor=orange&label=NPM&color=blue&link=https%3A%2F%2Fwww.npmjs.com%2Fpackage%2F%40vectorize-io%2Fhindsight-client)
@@ -242,7 +242,7 @@ client.reflect(bank_id="my-bank", query="What should I know about Alice?")
 - [CLI](https://hindsight.vectorize.io/sdks/cli)
 
 **Community:**
-- [Slack](https://join.slack.com/t/hindsight-space/shared_invite/zt-3nhbm4w29-LeSJ5Ixi6j8PdiYOCPlOgg)
+- [Slack](https://join.slack.com/t/hindsight-space/shared_invite/zt-3klo21kua-VUCC_zHP5rIcXFB1_5yw6A)
 - [GitHub Issues](https://github.com/vectorize-io/hindsight/issues)
 
 ---

helm/hindsight/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: hindsight
 description: Hindsight helm chart
 type: application
-version: 0.3.0
-appVersion: "0.3.0"
+version: 0.2.1
+appVersion: "0.2.1"
 keywords:
   - ai
   - memory

helm/hindsight/templates/_helpers.tpl

@@ -80,22 +80,6 @@ Control plane selector labels
 app.kubernetes.io/component: control-plane
 {{- end }}
 
-{{/*
-Worker labels
-*/}}
-{{- define "hindsight.worker.labels" -}}
-{{ include "hindsight.labels" . }}
-app.kubernetes.io/component: worker
-{{- end }}
-
-{{/*
-Worker selector labels
-*/}}
-{{- define "hindsight.worker.selectorLabels" -}}
-{{ include "hindsight.selectorLabels" . }}
-app.kubernetes.io/component: worker
-{{- end }}
-
 {{/*
 Create the name of the service account to use
 */}}

helm/hindsight/templates/api-deployment.yaml

@@ -55,11 +55,6 @@ spec:
         {{- end }}
         - name: HINDSIGHT_API_DATABASE_URL
           value: {{ include "hindsight.databaseUrl" . | quote }}
-        {{- /* Disable internal worker when dedicated workers are enabled */}}
-        {{- if .Values.worker.enabled }}
-        - name: HINDSIGHT_API_WORKER_ENABLED
-          value: "false"
-        {{- end }}
         {{- range $key, $value := .Values.api.env }}
         - name: {{ $key }}
           value: {{ $value | quote }}

helm/hindsight/templates/worker-service.yaml

@@ -1,25 +0,0 @@
-{{- if .Values.worker.enabled }}
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "hindsight.fullname" . }}-worker
-  labels:
-    {{- include "hindsight.worker.labels" . | nindent 4 }}
-  {{- if .Values.podAnnotations }}
-  annotations:
-    {{- /* Common Prometheus annotations for metrics scraping */}}
-    prometheus.io/scrape: "true"
-    prometheus.io/port: {{ .Values.worker.service.port | quote }}
-    prometheus.io/path: "/metrics"
-  {{- end }}
-spec:
-  # Headless service for StatefulSet (enables stable DNS names like worker-0.worker.namespace)
-  clusterIP: None
-  ports:
-  - port: {{ .Values.worker.service.port }}
-    targetPort: {{ .Values.worker.service.targetPort }}
-    protocol: TCP
-    name: http
-  selector:
-    {{- include "hindsight.worker.selectorLabels" . | nindent 4 }}
-{{- end }}

helm/hindsight/templates/worker-statefulset.yaml

@@ -1,110 +0,0 @@
-{{- if .Values.worker.enabled }}
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: {{ include "hindsight.fullname" . }}-worker
-  labels:
-    {{- include "hindsight.worker.labels" . | nindent 4 }}
-spec:
-  serviceName: {{ include "hindsight.fullname" . }}-worker
-  replicas: {{ .Values.worker.replicaCount }}
-  selector:
-    matchLabels:
-      {{- include "hindsight.worker.selectorLabels" . | nindent 6 }}
-  template:
-    metadata:
-      annotations:
-        {{- if not .Values.existingSecret }}
-        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
-        {{- end }}
-        {{- with .Values.podAnnotations }}
-        {{- toYaml . | nindent 8 }}
-        {{- end }}
-      labels:
-        {{- include "hindsight.worker.selectorLabels" . | nindent 8 }}
-    spec:
-      {{- if .Values.serviceAccount.create }}
-      serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
-      {{- end }}
-      securityContext:
-        {{- toYaml .Values.podSecurityContext | nindent 8 }}
-      containers:
-      - name: worker
-        securityContext:
-          {{- toYaml .Values.securityContext | nindent 10 }}
-        image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Values.version }}"
-        imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
-        command: ["hindsight-worker"]
-        ports:
-        - name: http
-          containerPort: {{ .Values.worker.service.targetPort }}
-          protocol: TCP
-        {{- if .Values.existingSecret }}
-        envFrom:
-        - secretRef:
-            name: {{ .Values.existingSecret }}
-        {{- end }}
-        env:
-        {{- /* POSTGRES_PASSWORD must be defined before DATABASE_URL for $(VAR) interpolation */}}
-        {{- if not .Values.postgresql.enabled }}
-        - name: POSTGRES_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "hindsight.secretName" . }}
-              key: postgres-password
-        {{- end }}
-        - name: HINDSIGHT_API_DATABASE_URL
-          value: {{ include "hindsight.databaseUrl" . | quote }}
-        {{- /* Worker ID uses pod name (StatefulSet provides stable names like worker-0, worker-1) */}}
-        - name: HINDSIGHT_API_WORKER_ID
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        {{- /* Inherit LLM config from api.env */}}
-        {{- range $key, $value := .Values.api.env }}
-        - name: {{ $key }}
-          value: {{ $value | quote }}
-        {{- end }}
-        {{- /* Worker-specific env vars */}}
-        {{- range $key, $value := .Values.worker.env }}
-        - name: {{ $key }}
-          value: {{ $value | quote }}
-        {{- end }}
-        {{- /* Only use secrets when not using existingSecret */}}
-        {{- if not .Values.existingSecret }}
-        {{- /* Inherit secrets from api.secrets */}}
-        {{- range $key, $value := .Values.api.secrets }}
-        - name: {{ $key }}
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "hindsight.secretName" $ }}
-              key: {{ $key }}
-        {{- end }}
-        {{- /* Worker-specific secrets (can override api.secrets) */}}
-        {{- range $key, $value := .Values.worker.secrets }}
-        - name: {{ $key }}
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "hindsight.secretName" $ }}
-              key: {{ $key }}
-        {{- end }}
-        {{- end }}
-        livenessProbe:
-          {{- toYaml .Values.worker.livenessProbe | nindent 10 }}
-        readinessProbe:
-          {{- toYaml .Values.worker.readinessProbe | nindent 10 }}
-        resources:
-          {{- toYaml .Values.worker.resources | nindent 10 }}
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-{{- end }}

helm/hindsight/values.yaml

@@ -67,63 +67,6 @@ api:
     # HINDSIGHT_API_LLM_API_KEY: "your-api-key"
     # HINDSIGHT_API_LLM_BASE_URL: "https://api.groq.com/openai/v1"
 
-# Worker settings (distributed task processing)
-# When enabled, dedicated worker pods process tasks and the API's internal worker is disabled
-worker:
-  enabled: false
-  replicaCount: 2
-  image:
-    repository: ghcr.io/vectorize-io/hindsight-api
-    pullPolicy: IfNotPresent
-    # tag defaults to .Values.version if not specified
-
-  service:
-    # Service for metrics scraping (headless for StatefulSet)
-    port: 8889
-    targetPort: 8889
-
-  # Resource limits and requests
-  resources:
-    limits:
-      cpu: 2000m
-      memory: 4Gi
-    requests:
-      cpu: 500m
-      memory: 1Gi
-
-  # Liveness and readiness probes
-  livenessProbe:
-    httpGet:
-      path: /health
-      port: 8889
-    initialDelaySeconds: 30
-    periodSeconds: 10
-    timeoutSeconds: 5
-    failureThreshold: 3
-
-  readinessProbe:
-    httpGet:
-      path: /health
-      port: 8889
-    initialDelaySeconds: 10
-    periodSeconds: 5
-    timeoutSeconds: 3
-    failureThreshold: 3
-
-  # Worker-specific environment variables
-  env:
-    # Poll interval in milliseconds (how often to check for new tasks)
-    HINDSIGHT_API_WORKER_POLL_INTERVAL_MS: "500"
-    # Number of tasks to claim per poll cycle
-    HINDSIGHT_API_WORKER_BATCH_SIZE: "10"
-    # Max retries before marking a task as failed
-    HINDSIGHT_API_WORKER_MAX_RETRIES: "3"
-    # HTTP port for metrics/health (matches service.targetPort)
-    HINDSIGHT_API_WORKER_HTTP_PORT: "8889"
-
-  # Secret environment variables (inherited from api.secrets if not specified)
-  secrets: {}
-
 # Image settings for control plane
 controlPlane:
   enabled: true

hindsight-api/hindsight_api/admin/cli.py

@@ -244,65 +244,6 @@ def run_db_migration(
     typer.echo("Database migrations completed successfully")
 
 
-async def _decommission_worker(db_url: str, worker_id: str, schema: str = "public") -> int:
-    """Release all tasks owned by a worker, setting them back to pending status."""
-    is_pg0, instance_name, _ = parse_pg0_url(db_url)
-    if is_pg0:
-        typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
-    resolved_url = await resolve_database_url(db_url)
-
-    conn = await asyncpg.connect(resolved_url)
-    try:
-        table = _fq_table("async_operations", schema)
-        result = await conn.fetch(
-            f"""
-            UPDATE {table}
-            SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
-            WHERE worker_id = $1 AND status = 'processing'
-            RETURNING operation_id
-            """,
-            worker_id,
-        )
-        return len(result)
-    finally:
-        await conn.close()
-
-
-@app.command(name="decommission-worker")
-def decommission_worker(
-    worker_id: str = typer.Argument(..., help="Worker ID to decommission"),
-    schema: str = typer.Option("public", "--schema", "-s", help="Database schema"),
-    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"),
-):
-    """Release all tasks owned by a worker (sets status back to pending).
-
-    Use this command when a worker has crashed or been removed without graceful shutdown.
-    All tasks that were being processed by the worker will be released back to the queue
-    so other workers can pick them up.
-    """
-    config = HindsightConfig.from_env()
-
-    if not config.database_url:
-        typer.echo("Error: Database URL not configured.", err=True)
-        typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
-        raise typer.Exit(1)
-
-    if not yes:
-        typer.confirm(
-            f"This will release all tasks owned by worker '{worker_id}' back to pending. Continue?",
-            abort=True,
-        )
-
-    typer.echo(f"Decommissioning worker '{worker_id}' (schema: {schema})...")
-
-    count = asyncio.run(_decommission_worker(config.database_url, worker_id, schema))
-
-    if count > 0:
-        typer.echo(f"Released {count} task(s) from worker '{worker_id}'")
-    else:
-        typer.echo(f"No tasks found for worker '{worker_id}'")
-
-
 def main():
     app()
 

hindsight-api/hindsight_api/alembic/versions/f1a2b3c4d5e6_add_memory_links_composite_index.py

@@ -1,44 +0,0 @@
-"""add_memory_links_from_type_weight_index
-
-Revision ID: f1a2b3c4d5e6
-Revises: e0a1b2c3d4e5
-Create Date: 2025-01-12
-
-Add composite index on memory_links (from_unit_id, link_type, weight DESC)
-to optimize MPFP graph traversal queries that need top-k edges per type.
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-# revision identifiers, used by Alembic.
-revision: str = "f1a2b3c4d5e6"
-down_revision: str | Sequence[str] | None = "e0a1b2c3d4e5"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    """Get schema prefix for table names (e.g., 'tenant_x.' or '' for public)."""
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    """Add composite index for efficient MPFP edge loading."""
-    schema = _get_schema_prefix()
-    # Create composite index for efficient top-k per (from_node, link_type) queries
-    # This enables LATERAL joins to use index-only scans with early termination
-    # Note: Not using CONCURRENTLY here as it requires running outside a transaction
-    # For production with large tables, consider running this manually with CONCURRENTLY
-    op.execute(
-        f"CREATE INDEX IF NOT EXISTS idx_memory_links_from_type_weight "
-        f"ON {schema}memory_links(from_unit_id, link_type, weight DESC)"
-    )
-
-
-def downgrade() -> None:
-    """Remove the composite index."""
-    schema = _get_schema_prefix()
-    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_links_from_type_weight")

hindsight-api/hindsight_api/alembic/versions/g2a3b4c5d6e7_add_tags_column.py

@@ -1,48 +0,0 @@
-"""add_tags_column
-
-Revision ID: g2a3b4c5d6e7
-Revises: f1a2b3c4d5e6
-Create Date: 2025-01-13
-
-Add tags column to memory_units and documents tables for visibility scoping.
-Tags enable filtering memories by scope (e.g., user IDs, session IDs) during recall/reflect.
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-# revision identifiers, used by Alembic.
-revision: str = "g2a3b4c5d6e7"
-down_revision: str | Sequence[str] | None = "f1a2b3c4d5e6"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    """Get schema prefix for table names (e.g., 'tenant_x.' or '' for public)."""
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    """Add tags column to memory_units and documents tables."""
-    schema = _get_schema_prefix()
-
-    # Add tags column to memory_units table
-    op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS tags VARCHAR[] NOT NULL DEFAULT '{{}}'")
-
-    # Create GIN index for efficient array containment queries (tags && ARRAY['x'])
-    op.execute(f"CREATE INDEX IF NOT EXISTS idx_memory_units_tags ON {schema}memory_units USING GIN (tags)")
-
-    # Add tags column to documents table for document-level tags
-    op.execute(f"ALTER TABLE {schema}documents ADD COLUMN IF NOT EXISTS tags VARCHAR[] NOT NULL DEFAULT '{{}}'")
-
-
-def downgrade() -> None:
-    """Remove tags columns and index."""
-    schema = _get_schema_prefix()
-
-    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_tags")
-    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS tags")
-    op.execute(f"ALTER TABLE {schema}documents DROP COLUMN IF EXISTS tags")

hindsight-api/hindsight_api/alembic/versions/h3c4d5e6f7g8_mental_models_v4.py

@@ -1,112 +0,0 @@
-"""mental_models_v4
-
-Revision ID: h3c4d5e6f7g8
-Revises: g2a3b4c5d6e7
-Create Date: 2026-01-08 00:00:00.000000
-
-This migration implements the v4 mental models system:
-1. Deletes existing observation memory_units (observations now in mental models)
-2. Adds mission column to banks (replacing background)
-3. Creates mental_models table with final schema
-
-Mental models can reference entities when an entity is "promoted" to a mental model.
-Summary content is stored as JSONB observations with per-observation fact attribution.
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-# revision identifiers, used by Alembic.
-revision: str = "h3c4d5e6f7g8"
-down_revision: str | Sequence[str] | None = "g2a3b4c5d6e7"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    """Get schema prefix for table names (required for multi-tenant support)."""
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    """Apply mental models v4 changes."""
-    schema = _get_schema_prefix()
-
-    # Step 1: Delete observation memory_units (cascades to unit_entities links)
-    # Observations are now handled through mental models, not memory_units
-    op.execute(f"DELETE FROM {schema}memory_units WHERE fact_type = 'observation'")
-
-    # Step 2: Drop observation-specific index (if it exists)
-    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_observation_date")
-
-    # Step 3: Add mission column to banks (replacing background)
-    op.execute(f"ALTER TABLE {schema}banks ADD COLUMN IF NOT EXISTS mission TEXT")
-
-    # Migrate: copy background to mission if background column exists
-    # Use DO block to check column existence first (idempotent for re-runs)
-    schema_name = context.config.get_main_option("target_schema") or "public"
-    op.execute(f"""
-        DO $$
-        BEGIN
-            IF EXISTS (
-                SELECT 1 FROM information_schema.columns
-                WHERE table_schema = '{schema_name}' AND table_name = 'banks' AND column_name = 'background'
-            ) THEN
-                UPDATE {schema}banks
-                SET mission = background
-                WHERE mission IS NULL;
-            END IF;
-        END $$;
-    """)
-
-    # Remove background column (replaced by mission)
-    op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS background")
-
-    # Step 4: Create mental_models table with final v4 schema (if not exists)
-    op.execute(f"""
-        CREATE TABLE IF NOT EXISTS {schema}mental_models (
-            id VARCHAR(64) NOT NULL,
-            bank_id VARCHAR(64) NOT NULL,
-            subtype VARCHAR(32) NOT NULL,
-            name VARCHAR(256) NOT NULL,
-            description TEXT NOT NULL,
-            entity_id UUID,
-            observations JSONB DEFAULT '{{"observations": []}}'::jsonb,
-            links VARCHAR[],
-            tags VARCHAR[] DEFAULT '{{}}',
-            last_updated TIMESTAMP WITH TIME ZONE,
-            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
-            PRIMARY KEY (id, bank_id),
-            FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE,
-            FOREIGN KEY (entity_id) REFERENCES {schema}entities(id) ON DELETE SET NULL,
-            CONSTRAINT ck_mental_models_subtype CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
-        )
-    """)
-
-    # Step 5: Create indexes for efficient queries (if not exist)
-    op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_bank_id ON {schema}mental_models(bank_id)")
-    op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_subtype ON {schema}mental_models(bank_id, subtype)")
-    op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_entity_id ON {schema}mental_models(entity_id)")
-    # GIN index for efficient tags array filtering
-    op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_tags ON {schema}mental_models USING GIN(tags)")
-
-
-def downgrade() -> None:
-    """Revert mental models v4 changes."""
-    schema = _get_schema_prefix()
-
-    # Drop mental_models table (cascades to indexes)
-    op.execute(f"DROP TABLE IF EXISTS {schema}mental_models CASCADE")
-
-    # Add back background column to banks
-    op.execute(f"ALTER TABLE {schema}banks ADD COLUMN IF NOT EXISTS background TEXT")
-
-    # Migrate mission back to background
-    op.execute(f"UPDATE {schema}banks SET background = mission WHERE background IS NULL")
-
-    # Remove mission column
-    op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS mission")
-
-    # Note: Cannot restore deleted observations - they are lost on downgrade

hindsight-api/hindsight_api/alembic/versions/i4d5e6f7g8h9_delete_opinions.py

@@ -1,41 +0,0 @@
-"""delete_opinions
-
-Revision ID: i4d5e6f7g8h9
-Revises: h3c4d5e6f7g8
-Create Date: 2026-01-15 00:00:00.000000
-
-This migration removes opinion facts from memory_units.
-Opinions are no longer a separate fact type - they are now represented
-through mental model observations with confidence scores.
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-# revision identifiers, used by Alembic.
-revision: str = "i4d5e6f7g8h9"
-down_revision: str | Sequence[str] | None = "h3c4d5e6f7g8"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    """Get schema prefix for table names (required for multi-tenant support)."""
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    """Delete opinion memory_units."""
-    schema = _get_schema_prefix()
-
-    # Delete opinion memory_units (cascades to unit_entities links)
-    # Opinions are now handled through mental model observations
-    op.execute(f"DELETE FROM {schema}memory_units WHERE fact_type = 'opinion'")
-
-
-def downgrade() -> None:
-    """Cannot restore deleted opinions."""
-    # Note: Cannot restore deleted opinions - they are lost on downgrade
-    pass

hindsight-api/hindsight_api/alembic/versions/j5e6f7g8h9i0_mental_model_versions.py

@@ -1,95 +0,0 @@
-"""mental_model_versions
-
-Revision ID: j5e6f7g8h9i0
-Revises: i4d5e6f7g8h9
-Create Date: 2026-01-16 00:00:00.000000
-
-This migration adds versioning support for mental models:
-1. Creates mental_model_versions table to store observation snapshots
-2. Adds version column to mental_models for tracking current version
-
-This enables changelog/diff functionality for mental model observations.
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-# revision identifiers, used by Alembic.
-revision: str = "j5e6f7g8h9i0"
-down_revision: str | Sequence[str] | None = "i4d5e6f7g8h9"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    """Get schema prefix for table names (required for multi-tenant support)."""
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    """Create mental_model_versions table and add version tracking."""
-    schema = _get_schema_prefix()
-
-    # Create mental_model_versions table for storing observation snapshots
-    op.execute(f"""
-        CREATE TABLE {schema}mental_model_versions (
-            id SERIAL PRIMARY KEY,
-            mental_model_id VARCHAR(64) NOT NULL,
-            bank_id VARCHAR(64) NOT NULL,
-            version INT NOT NULL,
-            observations JSONB NOT NULL DEFAULT '{{"observations": []}}'::jsonb,
-            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
-            FOREIGN KEY (mental_model_id, bank_id)
-                REFERENCES {schema}mental_models(id, bank_id) ON DELETE CASCADE,
-            UNIQUE (mental_model_id, bank_id, version)
-        )
-    """)
-
-    # Index for efficient version queries (get latest, list versions)
-    op.execute(f"""
-        CREATE INDEX idx_mental_model_versions_lookup
-        ON {schema}mental_model_versions(mental_model_id, bank_id, version DESC)
-    """)
-
-    # Add version column to mental_models to track current version
-    op.execute(f"""
-        ALTER TABLE {schema}mental_models
-        ADD COLUMN IF NOT EXISTS version INT NOT NULL DEFAULT 0
-    """)
-
-    # Migrate existing mental models: create version 1 for any that have observations
-    op.execute(f"""
-        INSERT INTO {schema}mental_model_versions (mental_model_id, bank_id, version, observations, created_at)
-        SELECT id, bank_id, 1, observations, COALESCE(last_updated, created_at)
-        FROM {schema}mental_models
-        WHERE observations IS NOT NULL
-          AND observations != '{{"observations": []}}'::jsonb
-          AND (observations->'observations') IS NOT NULL
-          AND jsonb_array_length(observations->'observations') > 0
-    """)
-
-    # Update version to 1 for migrated mental models
-    op.execute(f"""
-        UPDATE {schema}mental_models
-        SET version = 1
-        WHERE observations IS NOT NULL
-          AND observations != '{{"observations": []}}'::jsonb
-          AND (observations->'observations') IS NOT NULL
-          AND jsonb_array_length(observations->'observations') > 0
-    """)
-
-
-def downgrade() -> None:
-    """Remove mental_model_versions table and version column."""
-    schema = _get_schema_prefix()
-
-    # Drop index
-    op.execute(f"DROP INDEX IF EXISTS {schema}idx_mental_model_versions_lookup")
-
-    # Drop versions table
-    op.execute(f"DROP TABLE IF EXISTS {schema}mental_model_versions")
-
-    # Remove version column from mental_models
-    op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS version")

hindsight-api/hindsight_api/alembic/versions/k6f7g8h9i0j1_add_directive_subtype.py

@@ -1,58 +0,0 @@
-"""add_directive_subtype
-
-Revision ID: k6f7g8h9i0j1
-Revises: j5e6f7g8h9i0
-Create Date: 2026-01-16 00:00:00.000000
-
-This migration adds 'directive' to the mental_models subtype constraint.
-Directives are hard rules with user-provided observations that the reflect agent must follow.
-"""
-
-from collections.abc import Sequence
-
-from alembic import context, op
-
-# revision identifiers, used by Alembic.
-revision: str = "k6f7g8h9i0j1"
-down_revision: str | Sequence[str] | None = "j5e6f7g8h9i0"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    """Get schema prefix for table names (required for multi-tenant support)."""
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    """Add 'directive' to mental_models subtype constraint."""
-    schema = _get_schema_prefix()
-
-    # Drop existing constraint
-    op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
-
-    # Create new constraint with 'directive' added
-    op.execute(f"""
-        ALTER TABLE {schema}mental_models
-        ADD CONSTRAINT ck_mental_models_subtype
-        CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned', 'directive'))
-    """)
-
-
-def downgrade() -> None:
-    """Remove 'directive' from mental_models subtype constraint."""
-    schema = _get_schema_prefix()
-
-    # First delete any directives (cannot downgrade if they exist)
-    op.execute(f"DELETE FROM {schema}mental_models WHERE subtype = 'directive'")
-
-    # Drop constraint with directive
-    op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
-
-    # Recreate original constraint without directive
-    op.execute(f"""
-        ALTER TABLE {schema}mental_models
-        ADD CONSTRAINT ck_mental_models_subtype
-        CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
-    """)

hindsight-api/hindsight_api/alembic/versions/l7g8h9i0j1k2_add_worker_columns.py

@@ -1,109 +0,0 @@
-"""add_worker_columns
-
-Revision ID: l7g8h9i0j1k2
-Revises: k6f7g8h9i0j1
-Create Date: 2026-01-19 00:00:00.000000
-
-This migration adds columns to async_operations for distributed worker support:
-- worker_id: ID of the worker that claimed the task
-- claimed_at: When the task was claimed
-- retry_count: Number of retry attempts
-- task_payload: The serialized task dictionary
-"""
-
-from collections.abc import Sequence
-
-import sqlalchemy as sa
-from alembic import context, op
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision: str = "l7g8h9i0j1k2"
-down_revision: str | Sequence[str] | None = "k6f7g8h9i0j1"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _get_schema_prefix() -> str:
-    """Get schema prefix for table names (required for multi-tenant support)."""
-    schema = context.config.get_main_option("target_schema")
-    return f'"{schema}".' if schema else ""
-
-
-def upgrade() -> None:
-    """Add worker columns to async_operations."""
-    schema = _get_schema_prefix()
-
-    # Add worker_id column (ID of worker that claimed the task)
-    op.add_column(
-        "async_operations",
-        sa.Column("worker_id", sa.Text(), nullable=True),
-        schema=context.config.get_main_option("target_schema") or None,
-    )
-
-    # Add claimed_at column (when task was claimed by worker)
-    op.add_column(
-        "async_operations",
-        sa.Column("claimed_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
-        schema=context.config.get_main_option("target_schema") or None,
-    )
-
-    # Add retry_count column (number of retry attempts)
-    op.add_column(
-        "async_operations",
-        sa.Column("retry_count", sa.Integer(), server_default="0", nullable=False),
-        schema=context.config.get_main_option("target_schema") or None,
-    )
-
-    # Add task_payload column (serialized task dictionary)
-    op.add_column(
-        "async_operations",
-        sa.Column(
-            "task_payload",
-            postgresql.JSONB(astext_type=sa.Text()),
-            nullable=True,
-        ),
-        schema=context.config.get_main_option("target_schema") or None,
-    )
-
-    # Add index for efficient worker polling (pending tasks ordered by creation time)
-    op.execute(
-        f"CREATE INDEX idx_async_operations_pending_claim ON {schema}async_operations (status, created_at) "
-        f"WHERE status = 'pending' AND task_payload IS NOT NULL"
-    )
-
-    # Add index for finding tasks by worker_id (for decommissioning)
-    op.execute(
-        f"CREATE INDEX idx_async_operations_worker_id ON {schema}async_operations (worker_id) WHERE worker_id IS NOT NULL"
-    )
-
-
-def downgrade() -> None:
-    """Remove worker columns from async_operations."""
-    schema = _get_schema_prefix()
-
-    # Drop indexes
-    op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_pending_claim")
-    op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_worker_id")
-
-    # Drop columns
-    op.drop_column(
-        "async_operations",
-        "task_payload",
-        schema=context.config.get_main_option("target_schema") or None,
-    )
-    op.drop_column(
-        "async_operations",
-        "retry_count",
-        schema=context.config.get_main_option("target_schema") or None,
-    )
-    op.drop_column(
-        "async_operations",
-        "claimed_at",
-        schema=context.config.get_main_option("target_schema") or None,
-    )
-    op.drop_column(
-        "async_operations",
-        "worker_id",
-        schema=context.config.get_main_option("target_schema") or None,
-    )

hindsight-api/hindsight_api/api/mcp.py

@@ -196,7 +196,7 @@ def create_mcp_server(memory: MemoryEngine) -> FastMCP:
         Each bank is an isolated memory store (like a separate "brain").
 
         Returns:
-            JSON list of banks with their IDs, names, dispositions, and missions.
+            JSON list of banks with their IDs, names, dispositions, and backgrounds.
         """
         try:
             banks = await memory.list_banks(request_context=RequestContext())
@@ -206,7 +206,7 @@ def create_mcp_server(memory: MemoryEngine) -> FastMCP:
             return f'{{"error": "{e}", "banks": []}}'
 
     @mcp.tool()
-    async def create_bank(bank_id: str, name: str | None = None, mission: str | None = None) -> str:
+    async def create_bank(bank_id: str, name: str | None = None, background: str | None = None) -> str:
         """
         Create a new memory bank or get an existing one.
 
@@ -216,18 +216,18 @@ def create_mcp_server(memory: MemoryEngine) -> FastMCP:
         Args:
             bank_id: Unique identifier for the bank (e.g., 'user-123', 'agent-alpha')
             name: Optional human-friendly name for the bank
-            mission: Optional mission describing who the agent is and what they're trying to accomplish
+            background: Optional background context about the bank's owner/purpose
         """
         try:
             # get_bank_profile auto-creates bank if it doesn't exist
             profile = await memory.get_bank_profile(bank_id, request_context=RequestContext())
 
-            # Update name/mission if provided
-            if name is not None or mission is not None:
+            # Update name/background if provided
+            if name is not None or background is not None:
                 await memory.update_bank(
                     bank_id,
                     name=name,
-                    mission=mission,
+                    background=background,
                     request_context=RequestContext(),
                 )
                 # Fetch updated profile

hindsight-api/hindsight_api/config.py

@@ -4,12 +4,9 @@ Centralized configuration for Hindsight API.
 All environment variables and their defaults are defined here.
 """
 
-import json
 import logging
 import os
-import sys
 from dataclasses import dataclass
-from datetime import datetime, timezone
 
 from dotenv import find_dotenv, load_dotenv
 
@@ -44,19 +41,10 @@ ENV_EMBEDDINGS_LOCAL_MODEL = "HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL"
 ENV_EMBEDDINGS_TEI_URL = "HINDSIGHT_API_EMBEDDINGS_TEI_URL"
 ENV_EMBEDDINGS_OPENAI_API_KEY = "HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY"
 ENV_EMBEDDINGS_OPENAI_MODEL = "HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL"
-ENV_EMBEDDINGS_OPENAI_BASE_URL = "HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL"
 
 ENV_COHERE_API_KEY = "HINDSIGHT_API_COHERE_API_KEY"
 ENV_EMBEDDINGS_COHERE_MODEL = "HINDSIGHT_API_EMBEDDINGS_COHERE_MODEL"
-ENV_EMBEDDINGS_COHERE_BASE_URL = "HINDSIGHT_API_EMBEDDINGS_COHERE_BASE_URL"
 ENV_RERANKER_COHERE_MODEL = "HINDSIGHT_API_RERANKER_COHERE_MODEL"
-ENV_RERANKER_COHERE_BASE_URL = "HINDSIGHT_API_RERANKER_COHERE_BASE_URL"
-
-# LiteLLM gateway configuration (for embeddings and reranker via LiteLLM proxy)
-ENV_LITELLM_API_BASE = "HINDSIGHT_API_LITELLM_API_BASE"
-ENV_LITELLM_API_KEY = "HINDSIGHT_API_LITELLM_API_KEY"
-ENV_EMBEDDINGS_LITELLM_MODEL = "HINDSIGHT_API_EMBEDDINGS_LITELLM_MODEL"
-ENV_RERANKER_LITELLM_MODEL = "HINDSIGHT_API_RERANKER_LITELLM_MODEL"
 
 ENV_RERANKER_PROVIDER = "HINDSIGHT_API_RERANKER_PROVIDER"
 ENV_RERANKER_LOCAL_MODEL = "HINDSIGHT_API_RERANKER_LOCAL_MODEL"
@@ -64,23 +52,14 @@ ENV_RERANKER_LOCAL_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_LOCAL_MAX_CONCURRENT
 ENV_RERANKER_TEI_URL = "HINDSIGHT_API_RERANKER_TEI_URL"
 ENV_RERANKER_TEI_BATCH_SIZE = "HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE"
 ENV_RERANKER_TEI_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT"
-ENV_RERANKER_MAX_CANDIDATES = "HINDSIGHT_API_RERANKER_MAX_CANDIDATES"
-ENV_RERANKER_FLASHRANK_MODEL = "HINDSIGHT_API_RERANKER_FLASHRANK_MODEL"
-ENV_RERANKER_FLASHRANK_CACHE_DIR = "HINDSIGHT_API_RERANKER_FLASHRANK_CACHE_DIR"
 
 ENV_HOST = "HINDSIGHT_API_HOST"
 ENV_PORT = "HINDSIGHT_API_PORT"
 ENV_LOG_LEVEL = "HINDSIGHT_API_LOG_LEVEL"
-ENV_LOG_FORMAT = "HINDSIGHT_API_LOG_FORMAT"
-ENV_WORKERS = "HINDSIGHT_API_WORKERS"
 ENV_MCP_ENABLED = "HINDSIGHT_API_MCP_ENABLED"
 ENV_GRAPH_RETRIEVER = "HINDSIGHT_API_GRAPH_RETRIEVER"
-ENV_MPFP_TOP_K_NEIGHBORS = "HINDSIGHT_API_MPFP_TOP_K_NEIGHBORS"
-ENV_RECALL_MAX_CONCURRENT = "HINDSIGHT_API_RECALL_MAX_CONCURRENT"
-ENV_RECALL_CONNECTION_BUDGET = "HINDSIGHT_API_RECALL_CONNECTION_BUDGET"
 ENV_MCP_LOCAL_BANK_ID = "HINDSIGHT_API_MCP_LOCAL_BANK_ID"
 ENV_MCP_INSTRUCTIONS = "HINDSIGHT_API_MCP_INSTRUCTIONS"
-ENV_MENTAL_MODEL_REFRESH_CONCURRENCY = "HINDSIGHT_API_MENTAL_MODEL_REFRESH_CONCURRENCY"
 
 # Observation thresholds
 ENV_OBSERVATION_MIN_FACTS = "HINDSIGHT_API_OBSERVATION_MIN_FACTS"
@@ -106,16 +85,10 @@ ENV_DB_POOL_MAX_SIZE = "HINDSIGHT_API_DB_POOL_MAX_SIZE"
 ENV_DB_COMMAND_TIMEOUT = "HINDSIGHT_API_DB_COMMAND_TIMEOUT"
 ENV_DB_ACQUIRE_TIMEOUT = "HINDSIGHT_API_DB_ACQUIRE_TIMEOUT"
 
-# Worker configuration (distributed task processing)
-ENV_WORKER_ENABLED = "HINDSIGHT_API_WORKER_ENABLED"
-ENV_WORKER_ID = "HINDSIGHT_API_WORKER_ID"
-ENV_WORKER_POLL_INTERVAL_MS = "HINDSIGHT_API_WORKER_POLL_INTERVAL_MS"
-ENV_WORKER_MAX_RETRIES = "HINDSIGHT_API_WORKER_MAX_RETRIES"
-ENV_WORKER_BATCH_SIZE = "HINDSIGHT_API_WORKER_BATCH_SIZE"
-ENV_WORKER_HTTP_PORT = "HINDSIGHT_API_WORKER_HTTP_PORT"
-
-# Reflect agent settings
-ENV_REFLECT_MAX_ITERATIONS = "HINDSIGHT_API_REFLECT_MAX_ITERATIONS"
+# Background task processing
+ENV_TASK_BACKEND = "HINDSIGHT_API_TASK_BACKEND"
+ENV_TASK_BACKEND_MEMORY_BATCH_SIZE = "HINDSIGHT_API_TASK_BACKEND_MEMORY_BATCH_SIZE"
+ENV_TASK_BACKEND_MEMORY_BATCH_INTERVAL = "HINDSIGHT_API_TASK_BACKEND_MEMORY_BATCH_INTERVAL"
 
 # Default values
 DEFAULT_DATABASE_URL = "pg0"
@@ -134,30 +107,16 @@ DEFAULT_RERANKER_LOCAL_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"
 DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT = 4  # Limit concurrent CPU-bound reranking to prevent thrashing
 DEFAULT_RERANKER_TEI_BATCH_SIZE = 128
 DEFAULT_RERANKER_TEI_MAX_CONCURRENT = 8
-DEFAULT_RERANKER_MAX_CANDIDATES = 300
-DEFAULT_RERANKER_FLASHRANK_MODEL = "ms-marco-MiniLM-L-12-v2"  # Best balance of speed and quality
-DEFAULT_RERANKER_FLASHRANK_CACHE_DIR = None  # Use default cache directory
 
 DEFAULT_EMBEDDINGS_COHERE_MODEL = "embed-english-v3.0"
 DEFAULT_RERANKER_COHERE_MODEL = "rerank-english-v3.0"
 
-# LiteLLM defaults
-DEFAULT_LITELLM_API_BASE = "http://localhost:4000"
-DEFAULT_EMBEDDINGS_LITELLM_MODEL = "text-embedding-3-small"
-DEFAULT_RERANKER_LITELLM_MODEL = "cohere/rerank-english-v3.0"
-
 DEFAULT_HOST = "0.0.0.0"
 DEFAULT_PORT = 8888
 DEFAULT_LOG_LEVEL = "info"
-DEFAULT_LOG_FORMAT = "text"  # Options: "text", "json"
-DEFAULT_WORKERS = 1
 DEFAULT_MCP_ENABLED = True
-DEFAULT_GRAPH_RETRIEVER = "link_expansion"  # Options: "link_expansion", "mpfp", "bfs"
-DEFAULT_MPFP_TOP_K_NEIGHBORS = 20  # Fan-out limit per node in MPFP graph traversal
-DEFAULT_RECALL_MAX_CONCURRENT = 32  # Max concurrent recall operations per worker
-DEFAULT_RECALL_CONNECTION_BUDGET = 4  # Max concurrent DB connections per recall operation
+DEFAULT_GRAPH_RETRIEVER = "mpfp"  # Options: "mpfp", "bfs"
 DEFAULT_MCP_LOCAL_BANK_ID = "mcp"
-DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY = 8  # Max concurrent mental model refreshes
 
 # Observation thresholds
 DEFAULT_OBSERVATION_MIN_FACTS = 5  # Min facts required to generate entity observations
@@ -180,16 +139,10 @@ DEFAULT_DB_POOL_MAX_SIZE = 100
 DEFAULT_DB_COMMAND_TIMEOUT = 60  # seconds
 DEFAULT_DB_ACQUIRE_TIMEOUT = 30  # seconds
 
-# Worker configuration (distributed task processing)
-DEFAULT_WORKER_ENABLED = True  # API runs worker by default (standalone mode)
-DEFAULT_WORKER_ID = None  # Will use hostname if not specified
-DEFAULT_WORKER_POLL_INTERVAL_MS = 500  # Poll database every 500ms
-DEFAULT_WORKER_MAX_RETRIES = 3  # Max retries before marking task failed
-DEFAULT_WORKER_BATCH_SIZE = 10  # Tasks to claim per poll cycle
-DEFAULT_WORKER_HTTP_PORT = 8889  # HTTP port for worker metrics/health
-
-# Reflect agent settings
-DEFAULT_REFLECT_MAX_ITERATIONS = 10  # Max tool call iterations before forcing response
+# Background task processing
+DEFAULT_TASK_BACKEND = "memory"  # Options: "memory", "noop"
+DEFAULT_TASK_BACKEND_MEMORY_BATCH_SIZE = 10
+DEFAULT_TASK_BACKEND_MEMORY_BATCH_INTERVAL = 1.0  # seconds
 
 # Default MCP tool descriptions (can be customized via env vars)
 DEFAULT_MCP_RETAIN_DESCRIPTION = """Store important information to long-term memory.
@@ -215,36 +168,6 @@ Use this tool PROACTIVELY to:
 EMBEDDING_DIMENSION = DEFAULT_EMBEDDING_DIMENSION
 
 
-class JsonFormatter(logging.Formatter):
-    """JSON formatter for structured logging.
-
-    Outputs logs in JSON format with a 'severity' field that cloud logging
-    systems (GCP, AWS CloudWatch, etc.) can parse to correctly categorize log levels.
-    """
-
-    SEVERITY_MAP = {
-        logging.DEBUG: "DEBUG",
-        logging.INFO: "INFO",
-        logging.WARNING: "WARNING",
-        logging.ERROR: "ERROR",
-        logging.CRITICAL: "CRITICAL",
-    }
-
-    def format(self, record: logging.LogRecord) -> str:
-        log_entry = {
-            "severity": self.SEVERITY_MAP.get(record.levelno, "DEFAULT"),
-            "message": record.getMessage(),
-            "timestamp": datetime.now(timezone.utc).isoformat(),
-            "logger": record.name,
-        }
-
-        # Add exception info if present
-        if record.exc_info:
-            log_entry["exception"] = self.formatException(record.exc_info)
-
-        return json.dumps(log_entry)
-
-
 def _validate_extraction_mode(mode: str) -> str:
     """Validate and normalize extraction mode."""
     mode_lower = mode.lower()
@@ -287,8 +210,6 @@ class HindsightConfig:
     embeddings_provider: str
     embeddings_local_model: str
     embeddings_tei_url: str | None
-    embeddings_openai_base_url: str | None
-    embeddings_cohere_base_url: str | None
 
     # Reranker
     reranker_provider: str
@@ -296,22 +217,15 @@ class HindsightConfig:
     reranker_tei_url: str | None
     reranker_tei_batch_size: int
     reranker_tei_max_concurrent: int
-    reranker_max_candidates: int
-    reranker_cohere_base_url: str | None
 
     # Server
     host: str
     port: int
     log_level: str
-    log_format: str
     mcp_enabled: bool
 
     # Recall
     graph_retriever: str
-    mpfp_top_k_neighbors: int
-    recall_max_concurrent: int
-    recall_connection_budget: int
-    mental_model_refresh_concurrency: int
 
     # Observation thresholds
     observation_min_facts: int
@@ -337,16 +251,10 @@ class HindsightConfig:
     db_command_timeout: int
     db_acquire_timeout: int
 
-    # Worker configuration (distributed task processing)
-    worker_enabled: bool
-    worker_id: str | None
-    worker_poll_interval_ms: int
-    worker_max_retries: int
-    worker_batch_size: int
-    worker_http_port: int
-
-    # Reflect agent settings
-    reflect_max_iterations: int
+    # Background task processing
+    task_backend: str
+    task_backend_memory_batch_size: int
+    task_backend_memory_batch_interval: float
 
     @classmethod
     def from_env(cls) -> "HindsightConfig":
@@ -374,8 +282,6 @@ class HindsightConfig:
             embeddings_provider=os.getenv(ENV_EMBEDDINGS_PROVIDER, DEFAULT_EMBEDDINGS_PROVIDER),
             embeddings_local_model=os.getenv(ENV_EMBEDDINGS_LOCAL_MODEL, DEFAULT_EMBEDDINGS_LOCAL_MODEL),
             embeddings_tei_url=os.getenv(ENV_EMBEDDINGS_TEI_URL),
-            embeddings_openai_base_url=os.getenv(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None,
-            embeddings_cohere_base_url=os.getenv(ENV_EMBEDDINGS_COHERE_BASE_URL) or None,
             # Reranker
             reranker_provider=os.getenv(ENV_RERANKER_PROVIDER, DEFAULT_RERANKER_PROVIDER),
             reranker_local_model=os.getenv(ENV_RERANKER_LOCAL_MODEL, DEFAULT_RERANKER_LOCAL_MODEL),
@@ -384,24 +290,13 @@ class HindsightConfig:
             reranker_tei_max_concurrent=int(
                 os.getenv(ENV_RERANKER_TEI_MAX_CONCURRENT, str(DEFAULT_RERANKER_TEI_MAX_CONCURRENT))
             ),
-            reranker_max_candidates=int(os.getenv(ENV_RERANKER_MAX_CANDIDATES, str(DEFAULT_RERANKER_MAX_CANDIDATES))),
-            reranker_cohere_base_url=os.getenv(ENV_RERANKER_COHERE_BASE_URL) or None,
             # Server
             host=os.getenv(ENV_HOST, DEFAULT_HOST),
             port=int(os.getenv(ENV_PORT, DEFAULT_PORT)),
             log_level=os.getenv(ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL),
-            log_format=os.getenv(ENV_LOG_FORMAT, DEFAULT_LOG_FORMAT).lower(),
             mcp_enabled=os.getenv(ENV_MCP_ENABLED, str(DEFAULT_MCP_ENABLED)).lower() == "true",
             # Recall
             graph_retriever=os.getenv(ENV_GRAPH_RETRIEVER, DEFAULT_GRAPH_RETRIEVER),
-            mpfp_top_k_neighbors=int(os.getenv(ENV_MPFP_TOP_K_NEIGHBORS, str(DEFAULT_MPFP_TOP_K_NEIGHBORS))),
-            recall_max_concurrent=int(os.getenv(ENV_RECALL_MAX_CONCURRENT, str(DEFAULT_RECALL_MAX_CONCURRENT))),
-            recall_connection_budget=int(
-                os.getenv(ENV_RECALL_CONNECTION_BUDGET, str(DEFAULT_RECALL_CONNECTION_BUDGET))
-            ),
-            mental_model_refresh_concurrency=int(
-                os.getenv(ENV_MENTAL_MODEL_REFRESH_CONCURRENCY, str(DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY))
-            ),
             # Optimization flags
             skip_llm_verification=os.getenv(ENV_SKIP_LLM_VERIFICATION, "false").lower() == "true",
             lazy_reranker=os.getenv(ENV_LAZY_RERANKER, "false").lower() == "true",
@@ -433,15 +328,14 @@ class HindsightConfig:
             db_pool_max_size=int(os.getenv(ENV_DB_POOL_MAX_SIZE, str(DEFAULT_DB_POOL_MAX_SIZE))),
             db_command_timeout=int(os.getenv(ENV_DB_COMMAND_TIMEOUT, str(DEFAULT_DB_COMMAND_TIMEOUT))),
             db_acquire_timeout=int(os.getenv(ENV_DB_ACQUIRE_TIMEOUT, str(DEFAULT_DB_ACQUIRE_TIMEOUT))),
-            # Worker configuration
-            worker_enabled=os.getenv(ENV_WORKER_ENABLED, str(DEFAULT_WORKER_ENABLED)).lower() == "true",
-            worker_id=os.getenv(ENV_WORKER_ID) or DEFAULT_WORKER_ID,
-            worker_poll_interval_ms=int(os.getenv(ENV_WORKER_POLL_INTERVAL_MS, str(DEFAULT_WORKER_POLL_INTERVAL_MS))),
-            worker_max_retries=int(os.getenv(ENV_WORKER_MAX_RETRIES, str(DEFAULT_WORKER_MAX_RETRIES))),
-            worker_batch_size=int(os.getenv(ENV_WORKER_BATCH_SIZE, str(DEFAULT_WORKER_BATCH_SIZE))),
-            worker_http_port=int(os.getenv(ENV_WORKER_HTTP_PORT, str(DEFAULT_WORKER_HTTP_PORT))),
-            # Reflect agent settings
-            reflect_max_iterations=int(os.getenv(ENV_REFLECT_MAX_ITERATIONS, str(DEFAULT_REFLECT_MAX_ITERATIONS))),
+            # Background task processing
+            task_backend=os.getenv(ENV_TASK_BACKEND, DEFAULT_TASK_BACKEND),
+            task_backend_memory_batch_size=int(
+                os.getenv(ENV_TASK_BACKEND_MEMORY_BATCH_SIZE, str(DEFAULT_TASK_BACKEND_MEMORY_BATCH_SIZE))
+            ),
+            task_backend_memory_batch_interval=float(
+                os.getenv(ENV_TASK_BACKEND_MEMORY_BATCH_INTERVAL, str(DEFAULT_TASK_BACKEND_MEMORY_BATCH_INTERVAL))
+            ),
         )
 
     def get_llm_base_url(self) -> str:
@@ -472,28 +366,12 @@ class HindsightConfig:
         return log_level_map.get(self.log_level.lower(), logging.INFO)
 
     def configure_logging(self) -> None:
-        """Configure Python logging based on the log level and format.
-
-        When log_format is "json", outputs structured JSON logs with a severity
-        field that GCP Cloud Logging can parse for proper log level categorization.
-        """
-        root_logger = logging.getLogger()
-        root_logger.setLevel(self.get_python_log_level())
-
-        # Remove existing handlers
-        for handler in root_logger.handlers[:]:
-            root_logger.removeHandler(handler)
-
-        # Create handler writing to stdout (GCP treats stderr as ERROR)
-        handler = logging.StreamHandler(sys.stdout)
-        handler.setLevel(self.get_python_log_level())
-
-        if self.log_format == "json":
-            handler.setFormatter(JsonFormatter())
-        else:
-            handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s"))
-
-        root_logger.addHandler(handler)
+        """Configure Python logging based on the log level."""
+        logging.basicConfig(
+            level=self.get_python_log_level(),
+            format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+            force=True,  # Override any existing configuration
+        )
 
     def log_config(self) -> None:
         """Log the current configuration (without sensitive values)."""

hindsight-api/hindsight_api/engine/cross_encoder.py

@@ -15,24 +15,14 @@ from concurrent.futures import ThreadPoolExecutor
 import httpx
 
 from ..config import (
-    DEFAULT_LITELLM_API_BASE,
     DEFAULT_RERANKER_COHERE_MODEL,
-    DEFAULT_RERANKER_FLASHRANK_CACHE_DIR,
-    DEFAULT_RERANKER_FLASHRANK_MODEL,
-    DEFAULT_RERANKER_LITELLM_MODEL,
     DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT,
     DEFAULT_RERANKER_LOCAL_MODEL,
     DEFAULT_RERANKER_PROVIDER,
     DEFAULT_RERANKER_TEI_BATCH_SIZE,
     DEFAULT_RERANKER_TEI_MAX_CONCURRENT,
     ENV_COHERE_API_KEY,
-    ENV_LITELLM_API_BASE,
-    ENV_LITELLM_API_KEY,
-    ENV_RERANKER_COHERE_BASE_URL,
     ENV_RERANKER_COHERE_MODEL,
-    ENV_RERANKER_FLASHRANK_CACHE_DIR,
-    ENV_RERANKER_FLASHRANK_MODEL,
-    ENV_RERANKER_LITELLM_MODEL,
     ENV_RERANKER_LOCAL_MAX_CONCURRENT,
     ENV_RERANKER_LOCAL_MODEL,
     ENV_RERANKER_PROVIDER,
@@ -182,13 +172,8 @@ class RemoteTEICrossEncoder(CrossEncoderModel):
     Note: The TEI server must be running a cross-encoder/reranker model.
 
     Requests are made in parallel with configurable batch size and max concurrency (backpressure).
-    Uses a GLOBAL semaphore to limit concurrent requests across ALL recall operations.
     """
 
-    # Global semaphore shared across all instances and calls to prevent thundering herd
-    _global_semaphore: asyncio.Semaphore | None = None
-    _global_max_concurrent: int = DEFAULT_RERANKER_TEI_MAX_CONCURRENT
-
     def __init__(
         self,
         base_url: str,
@@ -205,8 +190,7 @@ class RemoteTEICrossEncoder(CrossEncoderModel):
             base_url: Base URL of the TEI server (e.g., "http://localhost:8080")
             timeout: Request timeout in seconds (default: 30.0)
             batch_size: Maximum batch size for rerank requests (default: 128)
-            max_concurrent: Maximum concurrent requests for backpressure (default: 8).
-                           This is a GLOBAL limit across all parallel recall operations.
+            max_concurrent: Maximum concurrent requests for backpressure (default: 8)
             max_retries: Maximum number of retries for failed requests (default: 3)
             retry_delay: Initial delay between retries in seconds, doubles each retry (default: 0.5)
         """
@@ -219,14 +203,6 @@ class RemoteTEICrossEncoder(CrossEncoderModel):
         self._async_client: httpx.AsyncClient | None = None
         self._model_id: str | None = None
 
-        # Update global semaphore if max_concurrent changed
-        if (
-            RemoteTEICrossEncoder._global_semaphore is None
-            or RemoteTEICrossEncoder._global_max_concurrent != max_concurrent
-        ):
-            RemoteTEICrossEncoder._global_max_concurrent = max_concurrent
-            RemoteTEICrossEncoder._global_semaphore = asyncio.Semaphore(max_concurrent)
-
     @property
     def provider_name(self) -> str:
         return "tei"
@@ -351,10 +327,9 @@ class RemoteTEICrossEncoder(CrossEncoderModel):
                 batch_texts = texts[i : i + self.batch_size]
                 tasks_info.append((query, batch_indices, batch_texts))
 
-        # Run all requests in parallel with GLOBAL semaphore for backpressure
-        # This ensures max_concurrent is respected across ALL parallel recall operations
+        # Run all requests in parallel with semaphore for backpressure
         all_scores = [0.0] * len(pairs)
-        semaphore = RemoteTEICrossEncoder._global_semaphore
+        semaphore = asyncio.Semaphore(self.max_concurrent)
 
         tasks = [
             self._rerank_query_group(self._async_client, semaphore, query, texts) for query, _, texts in tasks_info
@@ -398,7 +373,6 @@ class CohereCrossEncoder(CrossEncoderModel):
         self,
         api_key: str,
         model: str = DEFAULT_RERANKER_COHERE_MODEL,
-        base_url: str | None = None,
         timeout: float = 60.0,
     ):
         """
@@ -407,12 +381,10 @@ class CohereCrossEncoder(CrossEncoderModel):
         Args:
             api_key: Cohere API key
             model: Cohere rerank model name (default: rerank-english-v3.0)
-            base_url: Custom base URL for Cohere-compatible API (e.g., Azure-hosted endpoint)
             timeout: Request timeout in seconds (default: 60.0)
         """
         self.api_key = api_key
         self.model = model
-        self.base_url = base_url
         self.timeout = timeout
         self._client = None
 
@@ -430,14 +402,8 @@ class CohereCrossEncoder(CrossEncoderModel):
         except ImportError:
             raise ImportError("cohere is required for CohereCrossEncoder. Install it with: pip install cohere")
 
-        base_url_msg = f" at {self.base_url}" if self.base_url else ""
-        logger.info(f"Reranker: initializing Cohere provider with model {self.model}{base_url_msg}")
-
-        # Build client kwargs, only including base_url if set (for Azure or custom endpoints)
-        client_kwargs = {"api_key": self.api_key, "timeout": self.timeout}
-        if self.base_url:
-            client_kwargs["base_url"] = self.base_url
-        self._client = cohere.Client(**client_kwargs)
+        logger.info(f"Reranker: initializing Cohere provider with model {self.model}")
+        self._client = cohere.Client(api_key=self.api_key, timeout=self.timeout)
         logger.info("Reranker: Cohere provider initialized")
 
     async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
@@ -492,280 +458,6 @@ class CohereCrossEncoder(CrossEncoderModel):
         return all_scores
 
 
-class RRFPassthroughCrossEncoder(CrossEncoderModel):
-    """
-    Passthrough cross-encoder that preserves RRF scores without neural reranking.
-
-    This is useful for:
-    - Testing retrieval quality without reranking overhead
-    - Deployments where reranking latency is unacceptable
-    - Debugging to isolate retrieval vs reranking issues
-    """
-
-    def __init__(self):
-        """Initialize RRF passthrough cross-encoder."""
-        pass
-
-    @property
-    def provider_name(self) -> str:
-        return "rrf"
-
-    async def initialize(self) -> None:
-        """No initialization needed."""
-        logger.info("Reranker: RRF passthrough provider initialized (neural reranking disabled)")
-
-    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
-        """
-        Return neutral scores - actual ranking uses RRF scores from retrieval.
-
-        Args:
-            pairs: List of (query, document) tuples (ignored)
-
-        Returns:
-            List of 0.5 scores (neutral, lets RRF scores dominate)
-        """
-        # Return neutral scores so RRF ranking is preserved
-        return [0.5] * len(pairs)
-
-
-class FlashRankCrossEncoder(CrossEncoderModel):
-    """
-    FlashRank cross-encoder implementation.
-
-    FlashRank is an ultra-lite reranking library that runs on CPU without
-    requiring PyTorch or Transformers. It's ideal for serverless deployments
-    with minimal cold-start overhead.
-
-    Available models:
-    - ms-marco-TinyBERT-L-2-v2: Fastest, ~4MB
-    - ms-marco-MiniLM-L-12-v2: Best quality, ~34MB (default)
-    - rank-T5-flan: Best zero-shot, ~110MB
-    - ms-marco-MultiBERT-L-12: Multi-lingual, ~150MB
-    """
-
-    # Shared executor for CPU-bound reranking
-    _executor: ThreadPoolExecutor | None = None
-    _max_concurrent: int = 4
-
-    def __init__(
-        self,
-        model_name: str | None = None,
-        cache_dir: str | None = None,
-        max_length: int = 512,
-        max_concurrent: int = 4,
-    ):
-        """
-        Initialize FlashRank cross-encoder.
-
-        Args:
-            model_name: FlashRank model name. Default: ms-marco-MiniLM-L-12-v2
-            cache_dir: Directory to cache downloaded models. Default: system cache
-            max_length: Maximum sequence length for reranking. Default: 512
-            max_concurrent: Maximum concurrent reranking calls. Default: 4
-        """
-        self.model_name = model_name or DEFAULT_RERANKER_FLASHRANK_MODEL
-        self.cache_dir = cache_dir or DEFAULT_RERANKER_FLASHRANK_CACHE_DIR
-        self.max_length = max_length
-        self._ranker = None
-        FlashRankCrossEncoder._max_concurrent = max_concurrent
-
-    @property
-    def provider_name(self) -> str:
-        return "flashrank"
-
-    async def initialize(self) -> None:
-        """Load the FlashRank model."""
-        if self._ranker is not None:
-            return
-
-        try:
-            from flashrank import Ranker  # type: ignore[import-untyped]
-        except ImportError:
-            raise ImportError("flashrank is required for FlashRankCrossEncoder. Install it with: pip install flashrank")
-
-        logger.info(f"Reranker: initializing FlashRank provider with model {self.model_name}")
-
-        # Initialize ranker with optional cache directory
-        ranker_kwargs = {"model_name": self.model_name, "max_length": self.max_length}
-        if self.cache_dir:
-            ranker_kwargs["cache_dir"] = self.cache_dir
-
-        self._ranker = Ranker(**ranker_kwargs)
-
-        # Initialize shared executor
-        if FlashRankCrossEncoder._executor is None:
-            FlashRankCrossEncoder._executor = ThreadPoolExecutor(
-                max_workers=FlashRankCrossEncoder._max_concurrent,
-                thread_name_prefix="flashrank",
-            )
-            logger.info(
-                f"Reranker: FlashRank provider initialized (max_concurrent={FlashRankCrossEncoder._max_concurrent})"
-            )
-        else:
-            logger.info("Reranker: FlashRank provider initialized (using existing executor)")
-
-    def _predict_sync(self, pairs: list[tuple[str, str]]) -> list[float]:
-        """Synchronous predict - processes each query group."""
-        from flashrank import RerankRequest  # type: ignore[import-untyped]
-
-        if not pairs:
-            return []
-
-        # Group pairs by query
-        query_groups: dict[str, list[tuple[int, str]]] = {}
-        for idx, (query, text) in enumerate(pairs):
-            if query not in query_groups:
-                query_groups[query] = []
-            query_groups[query].append((idx, text))
-
-        all_scores = [0.0] * len(pairs)
-
-        for query, indexed_texts in query_groups.items():
-            # Build passages list for FlashRank
-            passages = [{"id": i, "text": text} for i, (_, text) in enumerate(indexed_texts)]
-            global_indices = [idx for idx, _ in indexed_texts]
-
-            # Create rerank request
-            request = RerankRequest(query=query, passages=passages)
-            results = self._ranker.rerank(request)
-
-            # Map scores back to original positions
-            for result in results:
-                local_idx = result["id"]
-                score = result["score"]
-                global_idx = global_indices[local_idx]
-                all_scores[global_idx] = score
-
-        return all_scores
-
-    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
-        """
-        Score query-document pairs using FlashRank.
-
-        Args:
-            pairs: List of (query, document) tuples to score
-
-        Returns:
-            List of relevance scores (higher = more relevant)
-        """
-        if self._ranker is None:
-            raise RuntimeError("Reranker not initialized. Call initialize() first.")
-
-        # Run in thread pool to avoid blocking event loop
-        loop = asyncio.get_event_loop()
-        return await loop.run_in_executor(FlashRankCrossEncoder._executor, self._predict_sync, pairs)
-
-
-class LiteLLMCrossEncoder(CrossEncoderModel):
-    """
-    LiteLLM cross-encoder implementation using LiteLLM proxy's /rerank endpoint.
-
-    LiteLLM provides a unified interface for multiple reranking providers via
-    the Cohere-compatible /rerank endpoint.
-    See: https://docs.litellm.ai/docs/rerank
-
-    Supported providers via LiteLLM:
-    - Cohere (rerank-english-v3.0, etc.) - prefix with cohere/
-    - Together AI - prefix with together_ai/
-    - Azure AI - prefix with azure_ai/
-    - Jina AI - prefix with jina_ai/
-    - AWS Bedrock - prefix with bedrock/
-    - Voyage AI - prefix with voyage/
-    """
-
-    def __init__(
-        self,
-        api_base: str = DEFAULT_LITELLM_API_BASE,
-        api_key: str | None = None,
-        model: str = DEFAULT_RERANKER_LITELLM_MODEL,
-        timeout: float = 60.0,
-    ):
-        """
-        Initialize LiteLLM cross-encoder client.
-
-        Args:
-            api_base: Base URL of the LiteLLM proxy (default: http://localhost:4000)
-            api_key: API key for the LiteLLM proxy (optional, depends on proxy config)
-            model: Reranking model name (default: cohere/rerank-english-v3.0)
-                   Use provider prefix (e.g., cohere/, together_ai/, voyage/)
-            timeout: Request timeout in seconds (default: 60.0)
-        """
-        self.api_base = api_base.rstrip("/")
-        self.api_key = api_key
-        self.model = model
-        self.timeout = timeout
-        self._async_client: httpx.AsyncClient | None = None
-
-    @property
-    def provider_name(self) -> str:
-        return "litellm"
-
-    async def initialize(self) -> None:
-        """Initialize the async HTTP client."""
-        if self._async_client is not None:
-            return
-
-        logger.info(f"Reranker: initializing LiteLLM provider at {self.api_base} with model {self.model}")
-
-        headers = {"Content-Type": "application/json"}
-        if self.api_key:
-            headers["Authorization"] = f"Bearer {self.api_key}"
-
-        self._async_client = httpx.AsyncClient(timeout=self.timeout, headers=headers)
-        logger.info("Reranker: LiteLLM provider initialized")
-
-    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
-        """
-        Score query-document pairs using the LiteLLM proxy's /rerank endpoint.
-
-        Args:
-            pairs: List of (query, document) tuples to score
-
-        Returns:
-            List of relevance scores
-        """
-        if self._async_client is None:
-            raise RuntimeError("Reranker not initialized. Call initialize() first.")
-
-        if not pairs:
-            return []
-
-        # Group pairs by query (LiteLLM rerank expects one query with multiple documents)
-        query_groups: dict[str, list[tuple[int, str]]] = {}
-        for idx, (query, text) in enumerate(pairs):
-            if query not in query_groups:
-                query_groups[query] = []
-            query_groups[query].append((idx, text))
-
-        all_scores = [0.0] * len(pairs)
-
-        for query, indexed_texts in query_groups.items():
-            texts = [text for _, text in indexed_texts]
-            indices = [idx for idx, _ in indexed_texts]
-
-            # LiteLLM /rerank follows Cohere API format
-            response = await self._async_client.post(
-                f"{self.api_base}/rerank",
-                json={
-                    "model": self.model,
-                    "query": query,
-                    "documents": texts,
-                    "top_n": len(texts),  # Return all scores
-                },
-            )
-            response.raise_for_status()
-            result = response.json()
-
-            # Map scores back to original positions
-            # Response format: {"results": [{"index": 0, "relevance_score": 0.9}, ...]}
-            for item in result.get("results", []):
-                original_idx = item["index"]
-                score = item.get("relevance_score", item.get("score", 0.0))
-                all_scores[indices[original_idx]] = score
-
-        return all_scores
-
-
 def create_cross_encoder_from_env() -> CrossEncoderModel:
     """
     Create a CrossEncoderModel instance based on environment variables.
@@ -796,20 +488,6 @@ def create_cross_encoder_from_env() -> CrossEncoderModel:
         if not api_key:
             raise ValueError(f"{ENV_COHERE_API_KEY} is required when {ENV_RERANKER_PROVIDER} is 'cohere'")
         model = os.environ.get(ENV_RERANKER_COHERE_MODEL, DEFAULT_RERANKER_COHERE_MODEL)
-        base_url = os.environ.get(ENV_RERANKER_COHERE_BASE_URL) or None
-        return CohereCrossEncoder(api_key=api_key, model=model, base_url=base_url)
-    elif provider == "flashrank":
-        model = os.environ.get(ENV_RERANKER_FLASHRANK_MODEL, DEFAULT_RERANKER_FLASHRANK_MODEL)
-        cache_dir = os.environ.get(ENV_RERANKER_FLASHRANK_CACHE_DIR, DEFAULT_RERANKER_FLASHRANK_CACHE_DIR)
-        return FlashRankCrossEncoder(model_name=model, cache_dir=cache_dir)
-    elif provider == "litellm":
-        api_base = os.environ.get(ENV_LITELLM_API_BASE, DEFAULT_LITELLM_API_BASE)
-        api_key = os.environ.get(ENV_LITELLM_API_KEY)
-        model = os.environ.get(ENV_RERANKER_LITELLM_MODEL, DEFAULT_RERANKER_LITELLM_MODEL)
-        return LiteLLMCrossEncoder(api_base=api_base, api_key=api_key, model=model)
-    elif provider == "rrf":
-        return RRFPassthroughCrossEncoder()
+        return CohereCrossEncoder(api_key=api_key, model=model)
     else:
-        raise ValueError(
-            f"Unknown reranker provider: {provider}. Supported: 'local', 'tei', 'cohere', 'flashrank', 'litellm', 'rrf'"
-        )
+        raise ValueError(f"Unknown reranker provider: {provider}. Supported: 'local', 'tei', 'cohere'")

hindsight-api/hindsight_api/engine/db_budget.py

@@ -1,284 +0,0 @@
-"""
-Database connection budget management.
-
-Limits concurrent database connections per operation to prevent
-a single operation (e.g., recall with parallel queries) from
-exhausting the connection pool.
-"""
-
-import asyncio
-import logging
-import uuid
-from contextlib import asynccontextmanager
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, AsyncIterator
-
-if TYPE_CHECKING:
-    import asyncpg
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class OperationBudget:
-    """
-    Tracks connection budget for a single operation.
-
-    Each operation gets a semaphore limiting its concurrent connections.
-    """
-
-    operation_id: str
-    max_connections: int
-    semaphore: asyncio.Semaphore = field(init=False)
-    active_count: int = field(default=0, init=False)
-
-    def __post_init__(self):
-        self.semaphore = asyncio.Semaphore(self.max_connections)
-
-
-class ConnectionBudgetManager:
-    """
-    Manages per-operation connection budgets.
-
-    Usage:
-        manager = ConnectionBudgetManager(default_budget=4)
-
-        # Start an operation
-        async with manager.operation(max_connections=2) as op:
-            # Acquire connections within the budget
-            async with op.acquire(pool) as conn:
-                await conn.fetch(...)
-
-            # Multiple connections respect the budget
-            async with op.acquire(pool) as conn1, op.acquire(pool) as conn2:
-                # At most 2 concurrent connections for this operation
-                ...
-    """
-
-    def __init__(self, default_budget: int = 4):
-        """
-        Initialize the budget manager.
-
-        Args:
-            default_budget: Default max connections per operation
-        """
-        self.default_budget = default_budget
-        self._operations: dict[str, OperationBudget] = {}
-        self._lock = asyncio.Lock()
-
-    @asynccontextmanager
-    async def operation(
-        self,
-        max_connections: int | None = None,
-        operation_id: str | None = None,
-    ) -> AsyncIterator["BudgetedOperation"]:
-        """
-        Create a budgeted operation context.
-
-        Args:
-            max_connections: Max concurrent connections for this operation.
-                           Defaults to manager's default_budget.
-            operation_id: Optional custom operation ID. Auto-generated if not provided.
-
-        Yields:
-            BudgetedOperation context for acquiring connections
-        """
-        op_id = operation_id or f"op-{uuid.uuid4().hex[:12]}"
-        budget = max_connections or self.default_budget
-
-        async with self._lock:
-            if op_id in self._operations:
-                raise ValueError(f"Operation {op_id} already exists")
-            self._operations[op_id] = OperationBudget(op_id, budget)
-
-        try:
-            yield BudgetedOperation(self, op_id)
-        finally:
-            async with self._lock:
-                self._operations.pop(op_id, None)
-
-    def _get_budget(self, operation_id: str) -> OperationBudget:
-        """Get budget for an operation (internal use)."""
-        budget = self._operations.get(operation_id)
-        if not budget:
-            raise ValueError(f"Operation {operation_id} not found")
-        return budget
-
-
-class BudgetedOperation:
-    """
-    A single operation with connection budget.
-
-    Provides methods to acquire connections within the budget.
-    """
-
-    def __init__(self, manager: ConnectionBudgetManager, operation_id: str):
-        self._manager = manager
-        self.operation_id = operation_id
-
-    @property
-    def budget(self) -> OperationBudget:
-        """Get the budget for this operation."""
-        return self._manager._get_budget(self.operation_id)
-
-    @asynccontextmanager
-    async def acquire(self, pool: "asyncpg.Pool") -> AsyncIterator["asyncpg.Connection"]:
-        """
-        Acquire a connection within the operation's budget.
-
-        Blocks if the operation has reached its connection limit.
-
-        Args:
-            pool: asyncpg connection pool
-
-        Yields:
-            Database connection
-        """
-        budget = self.budget
-        async with budget.semaphore:
-            budget.active_count += 1
-            conn = await pool.acquire()
-            try:
-                yield conn
-            finally:
-                budget.active_count -= 1
-                await pool.release(conn)
-
-    def wrap_pool(self, pool: "asyncpg.Pool") -> "BudgetedPool":
-        """
-        Wrap a pool with this operation's budget.
-
-        The returned BudgetedPool can be passed to functions expecting a pool,
-        and all acquire() calls will be limited by this operation's budget.
-
-        Args:
-            pool: asyncpg connection pool to wrap
-
-        Returns:
-            BudgetedPool that limits connections to this operation's budget
-        """
-        return BudgetedPool(pool, self)
-
-    async def acquire_many(
-        self,
-        pool: "asyncpg.Pool",
-        count: int,
-    ) -> AsyncIterator[list["asyncpg.Connection"]]:
-        """
-        Acquire multiple connections within the budget.
-
-        Note: This acquires connections sequentially to respect the budget.
-        For parallel acquisition, use multiple acquire() calls with asyncio.gather().
-
-        Args:
-            pool: asyncpg connection pool
-            count: Number of connections to acquire
-
-        Yields:
-            List of database connections
-        """
-        connections = []
-        try:
-            for _ in range(count):
-                conn = await pool.acquire()
-                connections.append(conn)
-            yield connections
-        finally:
-            for conn in connections:
-                await pool.release(conn)
-
-
-# Global default manager instance
-_default_manager: ConnectionBudgetManager | None = None
-
-
-def get_budget_manager(default_budget: int = 4) -> ConnectionBudgetManager:
-    """
-    Get or create the global budget manager.
-
-    Args:
-        default_budget: Default max connections per operation
-
-    Returns:
-        Global ConnectionBudgetManager instance
-    """
-    global _default_manager
-    if _default_manager is None:
-        _default_manager = ConnectionBudgetManager(default_budget=default_budget)
-    return _default_manager
-
-
-@asynccontextmanager
-async def budgeted_operation(
-    max_connections: int | None = None,
-    operation_id: str | None = None,
-    default_budget: int = 4,
-) -> AsyncIterator[BudgetedOperation]:
-    """
-    Convenience function to create a budgeted operation.
-
-    Args:
-        max_connections: Max concurrent connections for this operation
-        operation_id: Optional custom operation ID
-        default_budget: Default budget if manager not yet created
-
-    Yields:
-        BudgetedOperation context
-
-    Example:
-        async with budgeted_operation(max_connections=2) as op:
-            async with op.acquire(pool) as conn:
-                await conn.fetch(...)
-    """
-    manager = get_budget_manager(default_budget)
-    async with manager.operation(max_connections, operation_id) as op:
-        yield op
-
-
-class BudgetedPool:
-    """
-    A pool wrapper that limits concurrent connection acquisitions.
-
-    This can be passed to functions expecting a pool, and acquire()
-    calls will be limited by the budget semaphore.
-
-    Usage:
-        async with budgeted_operation(max_connections=4) as op:
-            budgeted_pool = op.wrap_pool(pool)
-            # Pass budgeted_pool to functions that expect a pool
-            await some_function(budgeted_pool, ...)
-    """
-
-    def __init__(self, pool: "asyncpg.Pool", operation: BudgetedOperation):
-        self._pool = pool
-        self._operation = operation
-
-    async def acquire(self) -> "asyncpg.Connection":
-        """
-        Acquire a connection within the budget.
-
-        Note: Caller must release the connection when done.
-        Prefer using as context manager via acquire_with_retry or op.acquire().
-        """
-        budget = self._operation.budget
-        await budget.semaphore.acquire()
-        budget.active_count += 1
-        try:
-            return await self._pool.acquire()
-        except Exception:
-            budget.active_count -= 1
-            budget.semaphore.release()
-            raise
-
-    async def release(self, conn: "asyncpg.Connection") -> None:
-        """Release a connection back to the pool."""
-        budget = self._operation.budget
-        try:
-            await self._pool.release(conn)
-        finally:
-            budget.active_count -= 1
-            budget.semaphore.release()
-
-    def __getattr__(self, name):
-        """Proxy other attributes to the underlying pool."""
-        return getattr(self._pool, name)

hindsight-api/hindsight_api/engine/embeddings.py

@@ -17,23 +17,16 @@ import httpx
 
 from ..config import (
     DEFAULT_EMBEDDINGS_COHERE_MODEL,
-    DEFAULT_EMBEDDINGS_LITELLM_MODEL,
     DEFAULT_EMBEDDINGS_LOCAL_MODEL,
     DEFAULT_EMBEDDINGS_OPENAI_MODEL,
     DEFAULT_EMBEDDINGS_PROVIDER,
-    DEFAULT_LITELLM_API_BASE,
     ENV_COHERE_API_KEY,
-    ENV_EMBEDDINGS_COHERE_BASE_URL,
     ENV_EMBEDDINGS_COHERE_MODEL,
-    ENV_EMBEDDINGS_LITELLM_MODEL,
     ENV_EMBEDDINGS_LOCAL_MODEL,
     ENV_EMBEDDINGS_OPENAI_API_KEY,
-    ENV_EMBEDDINGS_OPENAI_BASE_URL,
     ENV_EMBEDDINGS_OPENAI_MODEL,
     ENV_EMBEDDINGS_PROVIDER,
     ENV_EMBEDDINGS_TEI_URL,
-    ENV_LITELLM_API_BASE,
-    ENV_LITELLM_API_KEY,
     ENV_LLM_API_KEY,
 )
 
@@ -329,7 +322,6 @@ class OpenAIEmbeddings(Embeddings):
         self,
         api_key: str,
         model: str = DEFAULT_EMBEDDINGS_OPENAI_MODEL,
-        base_url: str | None = None,
         batch_size: int = 100,
         max_retries: int = 3,
     ):
@@ -339,13 +331,11 @@ class OpenAIEmbeddings(Embeddings):
         Args:
             api_key: OpenAI API key
             model: OpenAI embedding model name (default: text-embedding-3-small)
-            base_url: Custom base URL for OpenAI-compatible API (e.g., Azure OpenAI endpoint)
             batch_size: Maximum batch size for embedding requests (default: 100)
             max_retries: Maximum number of retries for failed requests (default: 3)
         """
         self.api_key = api_key
         self.model = model
-        self.base_url = base_url
         self.batch_size = batch_size
         self.max_retries = max_retries
         self._client = None
@@ -371,14 +361,8 @@ class OpenAIEmbeddings(Embeddings):
         except ImportError:
             raise ImportError("openai is required for OpenAIEmbeddings. Install it with: pip install openai")
 
-        base_url_msg = f" at {self.base_url}" if self.base_url else ""
-        logger.info(f"Embeddings: initializing OpenAI provider with model {self.model}{base_url_msg}")
-
-        # Build client kwargs, only including base_url if set (for Azure or custom endpoints)
-        client_kwargs = {"api_key": self.api_key, "max_retries": self.max_retries}
-        if self.base_url:
-            client_kwargs["base_url"] = self.base_url
-        self._client = OpenAI(**client_kwargs)
+        logger.info(f"Embeddings: initializing OpenAI provider with model {self.model}")
+        self._client = OpenAI(api_key=self.api_key, max_retries=self.max_retries)
 
         # Try to get dimension from known models, otherwise do a test embedding
         if self.model in self.MODEL_DIMENSIONS:
@@ -451,7 +435,6 @@ class CohereEmbeddings(Embeddings):
         self,
         api_key: str,
         model: str = DEFAULT_EMBEDDINGS_COHERE_MODEL,
-        base_url: str | None = None,
         batch_size: int = 96,
         timeout: float = 60.0,
         input_type: str = "search_document",
@@ -462,7 +445,6 @@ class CohereEmbeddings(Embeddings):
         Args:
             api_key: Cohere API key
             model: Cohere embedding model name (default: embed-english-v3.0)
-            base_url: Custom base URL for Cohere-compatible API (e.g., Azure-hosted endpoint)
             batch_size: Maximum batch size for embedding requests (default: 96, Cohere's limit)
             timeout: Request timeout in seconds (default: 60.0)
             input_type: Input type for embeddings (default: search_document).
@@ -470,7 +452,6 @@ class CohereEmbeddings(Embeddings):
         """
         self.api_key = api_key
         self.model = model
-        self.base_url = base_url
         self.batch_size = batch_size
         self.timeout = timeout
         self.input_type = input_type
@@ -497,14 +478,8 @@ class CohereEmbeddings(Embeddings):
         except ImportError:
             raise ImportError("cohere is required for CohereEmbeddings. Install it with: pip install cohere")
 
-        base_url_msg = f" at {self.base_url}" if self.base_url else ""
-        logger.info(f"Embeddings: initializing Cohere provider with model {self.model}{base_url_msg}")
-
-        # Build client kwargs, only including base_url if set (for Azure or custom endpoints)
-        client_kwargs = {"api_key": self.api_key, "timeout": self.timeout}
-        if self.base_url:
-            client_kwargs["base_url"] = self.base_url
-        self._client = cohere.Client(**client_kwargs)
+        logger.info(f"Embeddings: initializing Cohere provider with model {self.model}")
+        self._client = cohere.Client(api_key=self.api_key, timeout=self.timeout)
 
         # Try to get dimension from known models, otherwise do a test embedding
         if self.model in self.MODEL_DIMENSIONS:
@@ -554,123 +529,6 @@ class CohereEmbeddings(Embeddings):
         return all_embeddings
 
 
-class LiteLLMEmbeddings(Embeddings):
-    """
-    LiteLLM embeddings implementation using LiteLLM proxy's /embeddings endpoint.
-
-    LiteLLM provides a unified interface for multiple embedding providers.
-    The proxy exposes an OpenAI-compatible /embeddings endpoint.
-    See: https://docs.litellm.ai/docs/embedding/supported_embedding
-
-    Supported providers via LiteLLM:
-    - OpenAI (text-embedding-3-small, text-embedding-ada-002, etc.)
-    - Cohere (embed-english-v3.0, etc.) - prefix with cohere/
-    - Vertex AI (textembedding-gecko, etc.) - prefix with vertex_ai/
-    - HuggingFace, Mistral, Voyage AI, etc.
-
-    The embedding dimension is auto-detected from the model at initialization.
-    """
-
-    def __init__(
-        self,
-        api_base: str = DEFAULT_LITELLM_API_BASE,
-        api_key: str | None = None,
-        model: str = DEFAULT_EMBEDDINGS_LITELLM_MODEL,
-        batch_size: int = 100,
-        timeout: float = 60.0,
-    ):
-        """
-        Initialize LiteLLM embeddings client.
-
-        Args:
-            api_base: Base URL of the LiteLLM proxy (default: http://localhost:4000)
-            api_key: API key for the LiteLLM proxy (optional, depends on proxy config)
-            model: Embedding model name (default: text-embedding-3-small)
-                   Use provider prefix for non-OpenAI models (e.g., cohere/embed-english-v3.0)
-            batch_size: Maximum batch size for embedding requests (default: 100)
-            timeout: Request timeout in seconds (default: 60.0)
-        """
-        self.api_base = api_base.rstrip("/")
-        self.api_key = api_key
-        self.model = model
-        self.batch_size = batch_size
-        self.timeout = timeout
-        self._client: httpx.Client | None = None
-        self._dimension: int | None = None
-
-    @property
-    def provider_name(self) -> str:
-        return "litellm"
-
-    @property
-    def dimension(self) -> int:
-        if self._dimension is None:
-            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
-        return self._dimension
-
-    async def initialize(self) -> None:
-        """Initialize the HTTP client and detect embedding dimension."""
-        if self._client is not None:
-            return
-
-        logger.info(f"Embeddings: initializing LiteLLM provider at {self.api_base} with model {self.model}")
-
-        headers = {"Content-Type": "application/json"}
-        if self.api_key:
-            headers["Authorization"] = f"Bearer {self.api_key}"
-
-        self._client = httpx.Client(timeout=self.timeout, headers=headers)
-
-        # Do a test embedding to detect dimension
-        try:
-            response = self._client.post(
-                f"{self.api_base}/embeddings",
-                json={"model": self.model, "input": ["test"]},
-            )
-            response.raise_for_status()
-            result = response.json()
-            if result.get("data") and len(result["data"]) > 0:
-                self._dimension = len(result["data"][0]["embedding"])
-            logger.info(f"Embeddings: LiteLLM provider initialized (model: {self.model}, dim: {self._dimension})")
-        except httpx.HTTPError as e:
-            raise RuntimeError(f"Failed to connect to LiteLLM proxy at {self.api_base}: {e}")
-
-    def encode(self, texts: list[str]) -> list[list[float]]:
-        """
-        Generate embeddings using the LiteLLM proxy.
-
-        Args:
-            texts: List of text strings to encode
-
-        Returns:
-            List of embedding vectors
-        """
-        if self._client is None:
-            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
-
-        if not texts:
-            return []
-
-        all_embeddings = []
-
-        # Process in batches
-        for i in range(0, len(texts), self.batch_size):
-            batch = texts[i : i + self.batch_size]
-
-            response = self._client.post(
-                f"{self.api_base}/embeddings",
-                json={"model": self.model, "input": batch},
-            )
-            response.raise_for_status()
-            result = response.json()
-
-            # Sort by index to ensure correct order
-            batch_embeddings = sorted(result["data"], key=lambda x: x["index"])
-            all_embeddings.extend([e["embedding"] for e in batch_embeddings])
-
-        return all_embeddings
-
-
 def create_embeddings_from_env() -> Embeddings:
     """
     Create an Embeddings instance based on environment variables.
@@ -700,21 +558,12 @@ def create_embeddings_from_env() -> Embeddings:
                 f"when {ENV_EMBEDDINGS_PROVIDER} is 'openai'"
             )
         model = os.environ.get(ENV_EMBEDDINGS_OPENAI_MODEL, DEFAULT_EMBEDDINGS_OPENAI_MODEL)
-        base_url = os.environ.get(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None
-        return OpenAIEmbeddings(api_key=api_key, model=model, base_url=base_url)
+        return OpenAIEmbeddings(api_key=api_key, model=model)
     elif provider == "cohere":
         api_key = os.environ.get(ENV_COHERE_API_KEY)
         if not api_key:
             raise ValueError(f"{ENV_COHERE_API_KEY} is required when {ENV_EMBEDDINGS_PROVIDER} is 'cohere'")
         model = os.environ.get(ENV_EMBEDDINGS_COHERE_MODEL, DEFAULT_EMBEDDINGS_COHERE_MODEL)
-        base_url = os.environ.get(ENV_EMBEDDINGS_COHERE_BASE_URL) or None
-        return CohereEmbeddings(api_key=api_key, model=model, base_url=base_url)
-    elif provider == "litellm":
-        api_base = os.environ.get(ENV_LITELLM_API_BASE, DEFAULT_LITELLM_API_BASE)
-        api_key = os.environ.get(ENV_LITELLM_API_KEY)
-        model = os.environ.get(ENV_EMBEDDINGS_LITELLM_MODEL, DEFAULT_EMBEDDINGS_LITELLM_MODEL)
-        return LiteLLMEmbeddings(api_base=api_base, api_key=api_key, model=model)
+        return CohereEmbeddings(api_key=api_key, model=model)
     else:
-        raise ValueError(
-            f"Unknown embeddings provider: {provider}. Supported: 'local', 'tei', 'openai', 'cohere', 'litellm'"
-        )
+        raise ValueError(f"Unknown embeddings provider: {provider}. Supported: 'local', 'tei', 'openai', 'cohere'")

hindsight-api/hindsight_api/engine/interface.py

@@ -160,14 +160,14 @@ class MemoryEngineInterface(ABC):
         request_context: "RequestContext",
     ) -> dict[str, Any]:
         """
-        Get bank profile including disposition and mission.
+        Get bank profile including disposition and background.
 
         Args:
             bank_id: The memory bank ID.
             request_context: Request context for authentication.
 
         Returns:
-            Bank profile dict with bank_id, name, disposition, and mission.
+            Bank profile dict.
         """
         ...
 
@@ -190,44 +190,25 @@ class MemoryEngineInterface(ABC):
         ...
 
     @abstractmethod
-    async def merge_bank_mission(
+    async def merge_bank_background(
         self,
         bank_id: str,
         new_info: str,
         *,
+        update_disposition: bool = True,
         request_context: "RequestContext",
     ) -> dict[str, Any]:
         """
-        Merge new mission information into bank profile.
+        Merge new background information into bank profile.
 
         Args:
             bank_id: The memory bank ID.
-            new_info: New mission information to merge.
+            new_info: New background information to merge.
+            update_disposition: Whether to infer disposition from background.
             request_context: Request context for authentication.
 
         Returns:
-            Updated mission info.
-        """
-        ...
-
-    @abstractmethod
-    async def set_bank_mission(
-        self,
-        bank_id: str,
-        mission: str,
-        *,
-        request_context: "RequestContext",
-    ) -> dict[str, Any]:
-        """
-        Set the bank's mission (replaces existing).
-
-        Args:
-            bank_id: The memory bank ID.
-            mission: The mission text.
-            request_context: Request context for authentication.
-
-        Returns:
-            Dict with bank_id and mission.
+            Updated background info.
         """
         ...
 
@@ -425,20 +406,18 @@ class MemoryEngineInterface(ABC):
         bank_id: str,
         *,
         limit: int = 100,
-        offset: int = 0,
         request_context: "RequestContext",
-    ) -> dict[str, Any]:
+    ) -> list[dict[str, Any]]:
         """
-        List entities for a bank with pagination.
+        List entities for a bank.
 
         Args:
             bank_id: The memory bank ID.
             limit: Maximum results.
-            offset: Offset for pagination.
             request_context: Request context for authentication.
 
         Returns:
-            Dict with items, total, limit, offset.
+            List of entity dicts.
         """
         ...
 
@@ -537,7 +516,7 @@ class MemoryEngineInterface(ABC):
         bank_id: str,
         *,
         request_context: "RequestContext",
-    ) -> dict[str, Any]:
+    ) -> list[dict[str, Any]]:
         """
         List async operations for a bank.
 
@@ -546,7 +525,7 @@ class MemoryEngineInterface(ABC):
             request_context: Request context for authentication.
 
         Returns:
-            Dict with 'total' (int) and 'operations' (list of operation dicts).
+            List of operation dicts with id, task_type, status, etc.
         """
         ...
 
@@ -580,16 +559,16 @@ class MemoryEngineInterface(ABC):
         bank_id: str,
         *,
         name: str | None = None,
-        mission: str | None = None,
+        background: str | None = None,
         request_context: "RequestContext",
     ) -> dict[str, Any]:
         """
-        Update bank name and/or mission.
+        Update bank name and/or background.
 
         Args:
             bank_id: The memory bank ID.
             name: New bank name (optional).
-            mission: New mission text (optional, replaces existing).
+            background: New background text (optional, replaces existing).
             request_context: Request context for authentication.
 
         Returns:

hindsight-api/hindsight_api/engine/llm_wrapper.py

@@ -209,10 +209,10 @@ class LLMProvider:
             OutputTooLongError: If output exceeds token limits.
             Exception: Re-raises API errors after retries exhausted.
         """
-        semaphore_start = time.time()
+        queue_start_time = time.time()
         async with _global_llm_semaphore:
-            semaphore_wait_time = time.time() - semaphore_start
             start_time = time.time()
+            semaphore_wait_time = start_time - queue_start_time
 
             # Handle Mock provider (for testing)
             if self.provider == "mock":
@@ -318,44 +318,43 @@ class LLMProvider:
 
             last_exception = None
 
-            # Prepare response format ONCE before the retry loop
-            # (to avoid appending schema to messages on every retry)
-            if response_format is not None:
-                schema = None
-                if hasattr(response_format, "model_json_schema"):
-                    schema = response_format.model_json_schema()
-
-                if strict_schema and schema is not None:
-                    # Use OpenAI's strict JSON schema enforcement
-                    # This guarantees all required fields are returned
-                    call_params["response_format"] = {
-                        "type": "json_schema",
-                        "json_schema": {
-                            "name": "response",
-                            "strict": True,
-                            "schema": schema,
-                        },
-                    }
-                else:
-                    # Soft enforcement: add schema to prompt and use json_object mode
-                    if schema is not None:
-                        schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}"
-
-                        if call_params["messages"] and call_params["messages"][0].get("role") == "system":
-                            call_params["messages"][0]["content"] += schema_msg
-                        elif call_params["messages"]:
-                            call_params["messages"][0]["content"] = (
-                                schema_msg + "\n\n" + call_params["messages"][0]["content"]
-                            )
-                    if self.provider not in ("lmstudio", "ollama"):
-                        # LM Studio and Ollama don't support json_object response format reliably
-                        # We rely on the schema in the system message instead
-                        call_params["response_format"] = {"type": "json_object"}
-
             for attempt in range(max_retries + 1):
                 try:
                     if response_format is not None:
+                        schema = None
+                        if hasattr(response_format, "model_json_schema"):
+                            schema = response_format.model_json_schema()
+
+                        if strict_schema and schema is not None:
+                            # Use OpenAI's strict JSON schema enforcement
+                            # This guarantees all required fields are returned
+                            call_params["response_format"] = {
+                                "type": "json_schema",
+                                "json_schema": {
+                                    "name": "response",
+                                    "strict": True,
+                                    "schema": schema,
+                                },
+                            }
+                        else:
+                            # Soft enforcement: add schema to prompt and use json_object mode
+                            if schema is not None:
+                                schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}"
+
+                                if call_params["messages"] and call_params["messages"][0].get("role") == "system":
+                                    call_params["messages"][0]["content"] += schema_msg
+                                elif call_params["messages"]:
+                                    call_params["messages"][0]["content"] = (
+                                        schema_msg + "\n\n" + call_params["messages"][0]["content"]
+                                    )
+                            if self.provider not in ("lmstudio", "ollama"):
+                                # LM Studio and Ollama don't support json_object response format reliably
+                                # We rely on the schema in the system message instead
+                                call_params["response_format"] = {"type": "json_object"}
+
+                        logger.debug(f"Sending request to {self.provider}/{self.model} (timeout={self.timeout})")
                         response = await self._client.chat.completions.create(**call_params)
+                        logger.debug(f"Received response from {self.provider}/{self.model}")
 
                         content = response.choices[0].message.content
 
@@ -468,11 +467,13 @@ class LLMProvider:
 
                 except APIConnectionError as e:
                     last_exception = e
-                    status_code = getattr(e, "status_code", None) or getattr(
-                        getattr(e, "response", None), "status_code", None
-                    )
-                    logger.warning(f"APIConnectionError (HTTP {status_code}), attempt {attempt + 1}: {str(e)[:200]}")
                     if attempt < max_retries:
+                        status_code = getattr(e, "status_code", None) or getattr(
+                            getattr(e, "response", None), "status_code", None
+                        )
+                        logger.warning(
+                            f"Connection error, retrying... (attempt {attempt + 1}/{max_retries + 1}) - status_code={status_code}, message={e}"
+                        )
                         backoff = min(initial_backoff * (2**attempt), max_backoff)
                         await asyncio.sleep(backoff)
                         continue
@@ -486,45 +487,6 @@ class LLMProvider:
                         logger.error(f"Auth error (HTTP {e.status_code}), not retrying: {str(e)}")
                         raise
 
-                    # Handle tool_use_failed error - model outputted in tool call format
-                    # Convert to expected JSON format and continue
-                    if e.status_code == 400 and response_format is not None:
-                        try:
-                            error_body = e.body if hasattr(e, "body") else {}
-                            if isinstance(error_body, dict):
-                                error_info: dict[str, Any] = error_body.get("error") or {}
-                                if error_info.get("code") == "tool_use_failed":
-                                    failed_gen = error_info.get("failed_generation", "")
-                                    if failed_gen:
-                                        # Parse the tool call format and convert to actions format
-                                        tool_call = json.loads(failed_gen)
-                                        tool_name = tool_call.get("name", "")
-                                        tool_args = tool_call.get("arguments", {})
-                                        # Convert to actions format: {"actions": [{"tool": "name", ...args}]}
-                                        converted = {"actions": [{"tool": tool_name, **tool_args}]}
-                                        if skip_validation:
-                                            result = converted
-                                        else:
-                                            result = response_format.model_validate(converted)
-
-                                        # Record metrics for this successful recovery
-                                        duration = time.time() - start_time
-                                        metrics = get_metrics_collector()
-                                        metrics.record_llm_call(
-                                            provider=self.provider,
-                                            model=self.model,
-                                            scope=scope,
-                                            duration=duration,
-                                            input_tokens=0,
-                                            output_tokens=0,
-                                            success=True,
-                                        )
-                                        if return_usage:
-                                            return result, TokenUsage(input_tokens=0, output_tokens=0, total_tokens=0)
-                                        return result
-                        except (json.JSONDecodeError, KeyError, TypeError):
-                            pass  # Failed to parse tool_use_failed, continue with normal retry
-
                     last_exception = e
                     if attempt < max_retries:
                         backoff = min(initial_backoff * (2**attempt), max_backoff)
@@ -535,416 +497,14 @@ class LLMProvider:
                         logger.error(f"API error after {max_retries + 1} attempts: {str(e)}")
                         raise
 
-                except Exception:
+                except Exception as e:
+                    logger.error(f"Unexpected error during LLM call: {type(e).__name__}: {str(e)}")
                     raise
 
             if last_exception:
                 raise last_exception
             raise RuntimeError("LLM call failed after all retries with no exception captured")
 
-    async def call_with_tools(
-        self,
-        messages: list[dict[str, Any]],
-        tools: list[dict[str, Any]],
-        max_completion_tokens: int | None = None,
-        temperature: float | None = None,
-        scope: str = "tools",
-        max_retries: int = 5,
-        initial_backoff: float = 1.0,
-        max_backoff: float = 30.0,
-        tool_choice: str | dict[str, Any] = "auto",
-    ) -> "LLMToolCallResult":
-        """
-        Make an LLM API call with tool/function calling support.
-
-        Args:
-            messages: List of message dicts. Can include tool results with role='tool'.
-            tools: List of tool definitions in OpenAI format.
-            max_completion_tokens: Maximum tokens in response.
-            temperature: Sampling temperature (0.0-2.0).
-            scope: Scope identifier for tracking.
-            max_retries: Maximum retry attempts.
-            initial_backoff: Initial backoff time in seconds.
-            max_backoff: Maximum backoff time in seconds.
-            tool_choice: How to choose tools - "auto", "none", "required", or {"type": "function", "function": {"name": "..."}}
-
-        Returns:
-            LLMToolCallResult with content and/or tool_calls.
-        """
-        from .response_models import LLMToolCall, LLMToolCallResult
-
-        async with _global_llm_semaphore:
-            start_time = time.time()
-
-            # Handle Mock provider
-            if self.provider == "mock":
-                return await self._call_with_tools_mock(messages, tools, scope)
-
-            # Handle Anthropic separately (uses different tool format)
-            if self.provider == "anthropic":
-                return await self._call_with_tools_anthropic(
-                    messages, tools, max_completion_tokens, max_retries, initial_backoff, max_backoff, start_time, scope
-                )
-
-            # Handle Gemini (convert to Gemini tool format)
-            if self.provider == "gemini":
-                return await self._call_with_tools_gemini(
-                    messages, tools, max_retries, initial_backoff, max_backoff, start_time, scope
-                )
-
-            # OpenAI-compatible providers (OpenAI, Groq, Ollama, LMStudio)
-            call_params: dict[str, Any] = {
-                "model": self.model,
-                "messages": messages,
-                "tools": tools,
-                "tool_choice": tool_choice,
-            }
-
-            if max_completion_tokens is not None:
-                call_params["max_completion_tokens"] = max_completion_tokens
-            if temperature is not None:
-                call_params["temperature"] = temperature
-
-            # Provider-specific parameters
-            if self.provider == "groq":
-                call_params["seed"] = DEFAULT_LLM_SEED
-
-            last_exception = None
-
-            for attempt in range(max_retries + 1):
-                try:
-                    response = await self._client.chat.completions.create(**call_params)
-
-                    message = response.choices[0].message
-                    finish_reason = response.choices[0].finish_reason
-
-                    # Extract tool calls if present
-                    tool_calls: list[LLMToolCall] = []
-                    if message.tool_calls:
-                        for tc in message.tool_calls:
-                            try:
-                                args = json.loads(tc.function.arguments) if tc.function.arguments else {}
-                            except json.JSONDecodeError:
-                                args = {"_raw": tc.function.arguments}
-                            tool_calls.append(LLMToolCall(id=tc.id, name=tc.function.name, arguments=args))
-
-                    content = message.content
-
-                    # Record metrics
-                    duration = time.time() - start_time
-                    usage = response.usage
-                    input_tokens = usage.prompt_tokens or 0 if usage else 0
-                    output_tokens = usage.completion_tokens or 0 if usage else 0
-
-                    metrics = get_metrics_collector()
-                    metrics.record_llm_call(
-                        provider=self.provider,
-                        model=self.model,
-                        scope=scope,
-                        duration=duration,
-                        input_tokens=input_tokens,
-                        output_tokens=output_tokens,
-                        success=True,
-                    )
-
-                    return LLMToolCallResult(content=content, tool_calls=tool_calls, finish_reason=finish_reason)
-
-                except APIConnectionError as e:
-                    last_exception = e
-                    if attempt < max_retries:
-                        await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff))
-                        continue
-                    raise
-
-                except APIStatusError as e:
-                    if e.status_code in (401, 403):
-                        raise
-                    last_exception = e
-                    if attempt < max_retries:
-                        await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff))
-                        continue
-                    raise
-
-                except Exception:
-                    raise
-
-            if last_exception:
-                raise last_exception
-            raise RuntimeError("Tool call failed after all retries")
-
-    async def _call_with_tools_mock(
-        self,
-        messages: list[dict[str, Any]],
-        tools: list[dict[str, Any]],
-        scope: str,
-    ) -> "LLMToolCallResult":
-        """Handle mock tool calls for testing."""
-        from .response_models import LLMToolCallResult
-
-        call_record = {
-            "provider": self.provider,
-            "model": self.model,
-            "messages": messages,
-            "tools": [t.get("function", {}).get("name") for t in tools],
-            "scope": scope,
-        }
-        self._mock_calls.append(call_record)
-
-        if self._mock_response is not None:
-            if isinstance(self._mock_response, LLMToolCallResult):
-                return self._mock_response
-            # Allow setting just tool calls as a list
-            if isinstance(self._mock_response, list):
-                from .response_models import LLMToolCall
-
-                return LLMToolCallResult(
-                    tool_calls=[
-                        LLMToolCall(id=f"mock_{i}", name=tc["name"], arguments=tc.get("arguments", {}))
-                        for i, tc in enumerate(self._mock_response)
-                    ],
-                    finish_reason="tool_calls",
-                )
-
-        return LLMToolCallResult(content="mock response", finish_reason="stop")
-
-    async def _call_with_tools_anthropic(
-        self,
-        messages: list[dict[str, Any]],
-        tools: list[dict[str, Any]],
-        max_completion_tokens: int | None,
-        max_retries: int,
-        initial_backoff: float,
-        max_backoff: float,
-        start_time: float,
-        scope: str,
-    ) -> "LLMToolCallResult":
-        """Handle Anthropic tool calling."""
-        from anthropic import APIConnectionError, APIStatusError
-
-        from .response_models import LLMToolCall, LLMToolCallResult
-
-        # Convert OpenAI tool format to Anthropic format
-        anthropic_tools = []
-        for tool in tools:
-            func = tool.get("function", {})
-            anthropic_tools.append(
-                {
-                    "name": func.get("name", ""),
-                    "description": func.get("description", ""),
-                    "input_schema": func.get("parameters", {"type": "object", "properties": {}}),
-                }
-            )
-
-        # Convert messages - handle tool results
-        system_prompt = None
-        anthropic_messages = []
-        for msg in messages:
-            role = msg.get("role", "user")
-            content = msg.get("content", "")
-
-            if role == "system":
-                system_prompt = (system_prompt + "\n\n" + content) if system_prompt else content
-            elif role == "tool":
-                # Anthropic uses tool_result blocks
-                anthropic_messages.append(
-                    {
-                        "role": "user",
-                        "content": [
-                            {"type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": content}
-                        ],
-                    }
-                )
-            elif role == "assistant" and msg.get("tool_calls"):
-                # Convert assistant tool calls
-                tool_use_blocks = []
-                for tc in msg["tool_calls"]:
-                    tool_use_blocks.append(
-                        {
-                            "type": "tool_use",
-                            "id": tc.get("id", ""),
-                            "name": tc.get("function", {}).get("name", ""),
-                            "input": json.loads(tc.get("function", {}).get("arguments", "{}")),
-                        }
-                    )
-                anthropic_messages.append({"role": "assistant", "content": tool_use_blocks})
-            else:
-                anthropic_messages.append({"role": role, "content": content})
-
-        call_params: dict[str, Any] = {
-            "model": self.model,
-            "messages": anthropic_messages,
-            "tools": anthropic_tools,
-            "max_tokens": max_completion_tokens or 4096,
-        }
-        if system_prompt:
-            call_params["system"] = system_prompt
-
-        last_exception = None
-        for attempt in range(max_retries + 1):
-            try:
-                response = await self._anthropic_client.messages.create(**call_params)
-
-                # Extract content and tool calls
-                content_parts = []
-                tool_calls: list[LLMToolCall] = []
-
-                for block in response.content:
-                    if block.type == "text":
-                        content_parts.append(block.text)
-                    elif block.type == "tool_use":
-                        tool_calls.append(LLMToolCall(id=block.id, name=block.name, arguments=block.input or {}))
-
-                content = "".join(content_parts) if content_parts else None
-                finish_reason = "tool_calls" if tool_calls else "stop"
-
-                # Record metrics
-                metrics = get_metrics_collector()
-                metrics.record_llm_call(
-                    provider=self.provider,
-                    model=self.model,
-                    scope=scope,
-                    duration=time.time() - start_time,
-                    input_tokens=response.usage.input_tokens or 0,
-                    output_tokens=response.usage.output_tokens or 0,
-                    success=True,
-                )
-
-                return LLMToolCallResult(content=content, tool_calls=tool_calls, finish_reason=finish_reason)
-
-            except (APIConnectionError, APIStatusError) as e:
-                if isinstance(e, APIStatusError) and e.status_code in (401, 403):
-                    raise
-                last_exception = e
-                if attempt < max_retries:
-                    await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff))
-                    continue
-                raise
-
-        if last_exception:
-            raise last_exception
-        raise RuntimeError("Anthropic tool call failed")
-
-    async def _call_with_tools_gemini(
-        self,
-        messages: list[dict[str, Any]],
-        tools: list[dict[str, Any]],
-        max_retries: int,
-        initial_backoff: float,
-        max_backoff: float,
-        start_time: float,
-        scope: str,
-    ) -> "LLMToolCallResult":
-        """Handle Gemini tool calling."""
-        from .response_models import LLMToolCall, LLMToolCallResult
-
-        # Convert tools to Gemini format
-        gemini_tools = []
-        for tool in tools:
-            func = tool.get("function", {})
-            gemini_tools.append(
-                genai_types.Tool(
-                    function_declarations=[
-                        genai_types.FunctionDeclaration(
-                            name=func.get("name", ""),
-                            description=func.get("description", ""),
-                            parameters=func.get("parameters"),
-                        )
-                    ]
-                )
-            )
-
-        # Convert messages
-        system_instruction = None
-        gemini_contents = []
-        for msg in messages:
-            role = msg.get("role", "user")
-            content = msg.get("content", "")
-
-            if role == "system":
-                system_instruction = (system_instruction + "\n\n" + content) if system_instruction else content
-            elif role == "tool":
-                # Gemini uses function_response
-                gemini_contents.append(
-                    genai_types.Content(
-                        role="user",
-                        parts=[
-                            genai_types.Part(
-                                function_response=genai_types.FunctionResponse(
-                                    name=msg.get("name", ""),
-                                    response={"result": content},
-                                )
-                            )
-                        ],
-                    )
-                )
-            elif role == "assistant":
-                gemini_contents.append(genai_types.Content(role="model", parts=[genai_types.Part(text=content)]))
-            else:
-                gemini_contents.append(genai_types.Content(role="user", parts=[genai_types.Part(text=content)]))
-
-        config = genai_types.GenerateContentConfig(
-            system_instruction=system_instruction,
-            tools=gemini_tools,
-        )
-
-        last_exception = None
-        for attempt in range(max_retries + 1):
-            try:
-                response = await self._gemini_client.aio.models.generate_content(
-                    model=self.model,
-                    contents=gemini_contents,
-                    config=config,
-                )
-
-                # Extract content and tool calls
-                content = None
-                tool_calls: list[LLMToolCall] = []
-
-                if response.candidates and response.candidates[0].content:
-                    for part in response.candidates[0].content.parts:
-                        if hasattr(part, "text") and part.text:
-                            content = part.text
-                        if hasattr(part, "function_call") and part.function_call:
-                            fc = part.function_call
-                            tool_calls.append(
-                                LLMToolCall(
-                                    id=f"gemini_{len(tool_calls)}",
-                                    name=fc.name,
-                                    arguments=dict(fc.args) if fc.args else {},
-                                )
-                            )
-
-                finish_reason = "tool_calls" if tool_calls else "stop"
-
-                # Record metrics
-                metrics = get_metrics_collector()
-                input_tokens = response.usage_metadata.prompt_token_count if response.usage_metadata else 0
-                output_tokens = response.usage_metadata.candidates_token_count if response.usage_metadata else 0
-                metrics.record_llm_call(
-                    provider=self.provider,
-                    model=self.model,
-                    scope=scope,
-                    duration=time.time() - start_time,
-                    input_tokens=input_tokens,
-                    output_tokens=output_tokens,
-                    success=True,
-                )
-
-                return LLMToolCallResult(content=content, tool_calls=tool_calls, finish_reason=finish_reason)
-
-            except genai_errors.APIError as e:
-                if e.code in (401, 403):
-                    raise
-                last_exception = e
-                if attempt < max_retries:
-                    await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff))
-                    continue
-                raise
-
-        if last_exception:
-            raise last_exception
-        raise RuntimeError("Gemini tool call failed")
-
     async def _call_anthropic(
         self,
         messages: list[dict[str, str]],

hindsight-api/hindsight_api/engine/mental_models/__init__.py

@@ -1,18 +0,0 @@
-"""
-Mental models module for Hindsight.
-
-Mental models are synthesized summaries that represent understanding. They come
-in different subtypes based on how they were created:
-
-- Structural: Derived from the bank's mission (e.g., "Be a PM for engineering team")
-  These are created upfront based on what any agent with this role would need.
-
-- Emergent: Discovered from data patterns (named entities, temporal clusters, etc.)
-  These surface organically as facts are retained.
-
-- Pinned: User-defined models that persist across refreshes.
-"""
-
-from .models import MentalModel, MentalModelSubtype
-
-__all__ = ["MentalModel", "MentalModelSubtype"]

hindsight-api/hindsight_api/engine/mental_models/emergent.py

@@ -1,311 +0,0 @@
-"""
-Emergent mental model detection and promotion.
-
-Emergent models are discovered from data patterns:
-- Named entity extraction (people, projects, systems)
-- Temporal clustering (events with multiple references)
-- Causal patterns ("Because X, we do Y")
-- Behavioral anchors ("After X, we started Y")
-- Reference frequency (anything mentioned repeatedly)
-
-When a pattern is detected, it goes through a mission filter to check relevance,
-and if relevant, is promoted to a mental model.
-"""
-
-import logging
-from typing import TYPE_CHECKING
-
-from pydantic import BaseModel, Field
-
-from .models import EmergentCandidate
-
-if TYPE_CHECKING:
-    from ..llm_wrapper import LLMConfig
-
-logger = logging.getLogger(__name__)
-
-
-class MissionFilterCandidate(BaseModel):
-    """Result of mission filtering for a single candidate."""
-
-    name: str
-    promote: bool = Field(description="True if this is a specific named entity worth tracking")
-    reason: str = Field(description="Brief explanation for the decision")
-
-
-class MissionFilterResponse(BaseModel):
-    """Response from LLM for mission filtering."""
-
-    candidates: list[MissionFilterCandidate] = Field(description="Filtering decision for each candidate")
-
-
-def build_mission_filter_prompt(mission: str, candidates: list[EmergentCandidate]) -> str:
-    """Build the prompt for filtering candidates by mission relevance."""
-    candidate_list = "\n".join(
-        [f"- {c.name} (mentions: {c.mention_count}, method: {c.detection_method})" for c in candidates]
-    )
-
-    return f"""Filter these detected entities. For each one, decide: promote=true or promote=false.
-
-MISSION: {mission}
-
-DETECTED ENTITIES:
-{candidate_list}
-
-=== DECISION RULES ===
-
-Set promote=true ONLY for specific, named entities:
-- Person names: "John", "Maria", "Alice Chen", "Dr. Smith"
-- Named organizations: "Google", "Acme Corp", "Frontend Team"
-- Named places: "Central Park Zoo", "NYC Office", "Building A"
-- Named projects: "Project Phoenix", "Auth Service v2"
-
-Set promote=false for EVERYTHING ELSE, including:
-- Common English words: user, support, help, family, kids, parents, friends, people, team, photo, nature, park, office, home, work, school, joy, love, hope, fear, anger, gratitude, kindness, passion, motivation, inspiration, encouragement, positivity, energy, community, connection, commitment, collaboration, growth, impact, difference, success, progress, change, education, volunteering, veterans, homeless, shelter, meeting, project, system, process, event
-- Generic categories (even capitalized): Users, Customers, Team, Family, Kids, Veterans, Community
-- Abstract concepts: motivation, inspiration, gratitude, commitment, resilience
-
-THE TEST: Is this a specific name you'd find in a contact list or org chart?
-- "John" → YES (promote=true)
-- "kids" → NO (promote=false)
-- "community" → NO (promote=false)
-- "Maria" → YES (promote=true)
-- "park" → NO (promote=false)
-
-When in doubt, set promote=false."""
-
-
-def get_mission_filter_system_message() -> str:
-    """System message for mission filtering."""
-    return """You filter entities for promotion. Output JSON with 'candidates' array.
-
-Rules:
-- promote=true ONLY for specific names (people, organizations, named places/projects)
-- promote=false for common words, generic categories, abstract concepts
-
-Examples:
-- "John" → promote=true (person name)
-- "kids" → promote=false (generic category)
-- "community" → promote=false (abstract concept)
-- "Google" → promote=true (organization name)
-- "motivation" → promote=false (abstract concept)
-
-When in doubt, promote=false. Most entities should be rejected."""
-
-
-async def filter_candidates_by_mission(
-    llm_config: "LLMConfig",
-    mission: str,
-    candidates: list[EmergentCandidate],
-) -> list[EmergentCandidate]:
-    """
-    Filter emergent candidates to keep only specific, named entities.
-
-    Args:
-        llm_config: LLM configuration
-        mission: The bank's mission (used for context)
-        candidates: List of detected candidates
-
-    Returns:
-        Filtered list of candidates that are specific named entities
-    """
-    if not candidates:
-        return []
-
-    if not mission:
-        # No mission = no filtering, keep all candidates
-        logger.debug("[EMERGENT] No mission set, skipping filter")
-        return candidates
-
-    prompt = build_mission_filter_prompt(mission, candidates)
-
-    try:
-        result = await llm_config.call(
-            messages=[
-                {"role": "system", "content": get_mission_filter_system_message()},
-                {"role": "user", "content": prompt},
-            ],
-            response_format=MissionFilterResponse,
-            scope="mental_model_mission_filter",
-        )
-
-        # Build name -> promote map
-        promote_map = {c.name: c.promote for c in result.candidates}
-
-        # Filter candidates
-        filtered = []
-        for candidate in candidates:
-            if candidate.name in promote_map:
-                if promote_map[candidate.name]:
-                    filtered.append(candidate)
-                    logger.debug(f"[EMERGENT] Promoting '{candidate.name}'")
-                else:
-                    logger.debug(f"[EMERGENT] Rejecting '{candidate.name}'")
-            else:
-                # Candidate not in response - reject by default
-                logger.debug(f"[EMERGENT] '{candidate.name}' not in response, rejecting")
-
-        logger.info(f"[EMERGENT] Mission filter: {len(filtered)}/{len(candidates)} candidates promoted")
-        return filtered
-
-    except Exception as e:
-        logger.warning(f"[EMERGENT] Mission filter failed, rejecting all candidates: {e}")
-        return []
-
-
-async def evaluate_emergent_models(
-    llm_config: "LLMConfig",
-    models: list[dict],
-) -> list[str]:
-    """
-    Evaluate existing emergent models to check if they should be kept.
-
-    This re-evaluates emergent models using the same filtering criteria
-    as new candidates. Models that are generic/abstract will be removed.
-
-    Args:
-        llm_config: LLM configuration
-        models: List of existing emergent model dicts with 'name', 'id'
-
-    Returns:
-        List of model IDs that should be REMOVED (no longer valid)
-    """
-    if not models:
-        return []
-
-    # Convert existing models to candidates for evaluation
-    candidates = [
-        EmergentCandidate(
-            name=m["name"],
-            detection_method="existing_emergent_model",
-            mention_count=0,
-        )
-        for m in models
-    ]
-
-    # Build a simple prompt for re-evaluation
-    names_list = "\n".join([f"- {m['name']}" for m in models])
-    prompt = f"""Re-evaluate these existing mental models. For each one, decide: promote=true (keep) or promote=false (remove).
-
-EXISTING MODELS:
-{names_list}
-
-=== DECISION RULES ===
-
-Set promote=true ONLY for specific, named entities:
-- Person names: "John", "Maria", "Alice Chen", "Dr. Smith"
-- Named organizations: "Google", "Acme Corp", "Frontend Team"
-- Named places: "Central Park Zoo", "NYC Office", "Building A"
-- Named projects: "Project Phoenix", "Auth Service v2"
-
-Set promote=false for EVERYTHING ELSE, including:
-- Common English words: user, support, help, family, kids, parents, friends, people, team, photo, nature, park, office, home, work, school, joy, love, hope, fear, anger, gratitude, kindness, passion, motivation, inspiration, encouragement, positivity, energy, community, connection, commitment, collaboration, growth, impact, difference, success, progress, change, education, volunteering, veterans, homeless, shelter, meeting, project, system, process, event
-- Generic categories (even capitalized): Users, Customers, Team, Family, Kids, Veterans, Community
-- Abstract concepts: motivation, inspiration, gratitude, commitment, resilience
-
-THE TEST: Is this a specific name you'd find in a contact list or org chart?
-- "John" → YES (promote=true)
-- "kids" → NO (promote=false)
-- "community" → NO (promote=false)
-
-When in doubt, set promote=false."""
-
-    try:
-        result = await llm_config.call(
-            messages=[
-                {"role": "system", "content": get_mission_filter_system_message()},
-                {"role": "user", "content": prompt},
-            ],
-            response_format=MissionFilterResponse,
-            scope="mental_model_emergent_evaluation",
-        )
-
-        # Build name -> promote map
-        promote_map = {c.name: c.promote for c in result.candidates}
-
-        # Find models to remove
-        models_to_remove = []
-        for model in models:
-            name = model["name"]
-            if name in promote_map:
-                if not promote_map[name]:
-                    models_to_remove.append(model["id"])
-                else:
-                    logger.debug(f"[EMERGENT] Keeping '{name}'")
-            else:
-                # Model not in response - remove to be safe
-                logger.info(f"[EMERGENT] '{name}' not in evaluation response, marking for removal")
-                models_to_remove.append(model["id"])
-
-        logger.info(f"[EMERGENT] Evaluation: {len(models_to_remove)}/{len(models)} emergent models marked for removal")
-        return models_to_remove
-
-    except Exception as e:
-        logger.warning(f"[EMERGENT] Evaluation failed, keeping all models: {e}")
-        return []
-
-
-async def detect_entity_candidates(
-    pool,
-    bank_id: str,
-    min_mentions: int = 5,
-    top_percent: int = 20,
-) -> list[EmergentCandidate]:
-    """
-    Detect entities that are candidates for promotion to mental models.
-
-    Args:
-        pool: Database connection pool
-        bank_id: Bank identifier
-        min_mentions: Minimum mention count to consider
-        top_percent: Only consider top X% by mention count
-
-    Returns:
-        List of entity candidates
-    """
-    from ..db_utils import acquire_with_retry
-    from ..memory_engine import fq_table
-
-    candidates = []
-
-    async with acquire_with_retry(pool) as conn:
-        # Get entities that meet criteria and don't already have mental models
-        rows = await conn.fetch(
-            f"""
-            WITH ranked AS (
-                SELECT
-                    e.id,
-                    e.canonical_name,
-                    e.mention_count,
-                    PERCENT_RANK() OVER (ORDER BY e.mention_count DESC) as rank_pct
-                FROM {fq_table("entities")} e
-                LEFT JOIN {fq_table("mental_models")} mm
-                    ON mm.entity_id = e.id AND mm.bank_id = e.bank_id
-                WHERE e.bank_id = $1
-                  AND e.mention_count >= $2
-                  AND mm.id IS NULL  -- Not already a mental model
-            )
-            SELECT id, canonical_name, mention_count
-            FROM ranked
-            WHERE rank_pct <= $3
-            ORDER BY mention_count DESC
-            LIMIT 50
-            """,
-            bank_id,
-            min_mentions,
-            top_percent / 100.0,
-        )
-
-        for row in rows:
-            candidates.append(
-                EmergentCandidate(
-                    name=row["canonical_name"],
-                    detection_method="named_entity_extraction",
-                    mention_count=row["mention_count"],
-                    entity_id=str(row["id"]),
-                    relevance_score=0.0,
-                )
-            )
-
-    logger.debug(f"[EMERGENT] Detected {len(candidates)} entity candidates")
-    return candidates

hindsight-api/hindsight_api/engine/mental_models/models.py

@@ -1,98 +0,0 @@
-"""
-Pydantic models for mental models.
-"""
-
-from datetime import datetime, timezone
-from enum import Enum
-
-from pydantic import BaseModel, Field
-
-
-class MentalModelSubtype(str, Enum):
-    """Subtype of mental model - how it was created."""
-
-    STRUCTURAL = "structural"  # Derived from mission, created upfront
-    EMERGENT = "emergent"  # Discovered from data patterns
-    LEARNED = "learned"  # Formed through reflection
-    PINNED = "pinned"  # User-defined topic, observations LLM-generated
-    DIRECTIVE = "directive"  # User-defined hard rules, observations user-provided
-
-
-class MentalModel(BaseModel):
-    """
-    A mental model representing synthesized understanding.
-
-    Mental models are the agent's consolidated knowledge. Unlike raw facts,
-    mental models provide:
-    - A one-liner description for quick scanning/retrieval
-    - A full summary for deep understanding
-    - Links to related mental models
-    """
-
-    id: str = Field(description="Unique identifier within the bank")
-    bank_id: str = Field(description="Bank this mental model belongs to")
-    subtype: MentalModelSubtype = Field(description="How this model was created")
-    name: str = Field(description="Human-readable name")
-    description: str = Field(description="One-liner for quick scanning and retrieval matching")
-    summary: str | None = Field(default=None, description="Full synthesized understanding")
-
-    # References
-    entity_id: str | None = Field(default=None, description="Reference to entities table when type=entity")
-    source_facts: list[str] = Field(default_factory=list, description="Fact IDs used to generate summary")
-    links: list[str] = Field(default_factory=list, description="Related mental model IDs")
-
-    # Tags for scoped visibility (similar to document tags)
-    tags: list[str] = Field(default_factory=list, description="Tags for scoped visibility filtering")
-
-    # Timestamps
-    last_updated: datetime | None = Field(default=None, description="When summary was last regenerated")
-    created_at: datetime = Field(
-        default_factory=lambda: datetime.now(timezone.utc), description="When this model was created"
-    )
-
-
-class StructuralModelTemplate(BaseModel):
-    """
-    A template for a structural mental model.
-
-    Generated by LLM based on the bank's mission. Represents what any agent
-    with this role would need to track.
-    """
-
-    id: str = Field(default="", description="Existing model ID to keep, or empty for new models")
-    name: str = Field(description="Human-readable name")
-    description: str = Field(description="What this model should track")
-    initial_probes: list[str] = Field(default_factory=list, description="Initial search queries to populate this model")
-
-
-class StructuralModelDerivationResponse(BaseModel):
-    """Response from LLM for structural model derivation."""
-
-    templates: list[StructuralModelTemplate] = Field(description="Structural model templates derived from the mission")
-
-
-class EmergentCandidate(BaseModel):
-    """
-    A candidate for promotion to emergent mental model.
-
-    Detected through pattern analysis of facts.
-    """
-
-    name: str = Field(description="Name of the detected pattern/entity")
-    detection_method: str = Field(description="How this candidate was detected")
-    mention_count: int = Field(default=0, description="How many times referenced")
-    entity_id: str | None = Field(default=None, description="Entity ID if detected as entity")
-    relevance_score: float = Field(default=0.0, description="Score from mission filter (0-1)")
-
-
-class ResearchResult(BaseModel):
-    """
-    Result from the research endpoint.
-
-    Contains the answer along with the mental models and facts used.
-    """
-
-    answer: str = Field(description="The synthesized answer")
-    mental_models_used: list[str] = Field(default_factory=list, description="IDs of mental models that contributed")
-    facts_used: list[str] = Field(default_factory=list, description="Fact IDs that contributed")
-    question_type: str | None = Field(default=None, description="Detected question type (WHO, WHAT, HOW, etc.)")

hindsight-api/hindsight_api/engine/mental_models/structural.py

@@ -1,228 +0,0 @@
-"""
-Structural mental model derivation from bank mission.
-
-Structural models are derived from the bank's mission - they represent what
-any agent with this role would need to track. For example:
-
-Mission: "Be a PM for engineering team"
-Structural models:
-  - Team Structure (who's on the team, roles)
-  - Project Overview (current projects, status)
-  - Processes (how releases work, how decisions are made)
-  - Key Systems (what we own, dependencies)
-"""
-
-import logging
-from typing import TYPE_CHECKING
-
-from pydantic import BaseModel, Field
-
-from .models import StructuralModelTemplate
-
-if TYPE_CHECKING:
-    from ..llm_wrapper import LLMConfig
-
-logger = logging.getLogger(__name__)
-
-
-class StructuralDerivationResponse(BaseModel):
-    """Response from LLM for structural model derivation."""
-
-    templates: list[StructuralModelTemplate] = Field(description="Structural model templates derived from the mission")
-
-
-class StructuralRelevanceResult(BaseModel):
-    """Result of evaluating a structural model's relevance to the mission."""
-
-    name: str
-    relevant: bool
-    reason: str
-
-
-class StructuralRelevanceResponse(BaseModel):
-    """Response from LLM for structural model relevance evaluation."""
-
-    models: list[StructuralRelevanceResult] = Field(description="Relevance evaluation for each model")
-
-
-def build_structural_derivation_prompt(mission: str, existing_models: list[dict] | None = None) -> str:
-    """Build the prompt for deriving structural models from a mission."""
-    existing_section = ""
-    if existing_models:
-        model_list = "\n".join([f"- id='{m['id']}' name='{m['name']}': {m['description']}" for m in existing_models])
-        existing_section = f"""
-EXISTING STRUCTURAL MODELS:
-{model_list}
-
-IMPORTANT: If keeping an existing model, you MUST return its EXACT 'id' value.
-Models not included in your output will be REMOVED.
-"""
-
-    return f"""Given this agent mission, identify the KEY THINGS to track to achieve it.
-
-MISSION: {mission}
-{existing_section}
-IMPORTANT CONSTRAINTS:
-- Return 0-3 structural models MAXIMUM (less is better!)
-- Only include models for SPECIFIC, CONCRETE things the agent needs to track
-- Each model must be DIRECTLY tied to achieving the mission
-- If the mission is simple, return 0 models (empty array is fine)
-- If existing models are provided and you want to keep one, use its EXACT id
-- Do NOT create near-duplicates (e.g., don't create "topic-map" if "topic-connections" exists)
-
-GOOD examples (specific, actionable):
-- Mission: "Be a PM for engineering team" → "Team Members" (track who's on the team)
-- Mission: "Track customer feedback" → "Customer Issues" (track specific complaints/requests)
-- Mission: "Manage project X" → "Project X Milestones" (track progress)
-
-BAD examples (too generic, don't create these):
-- "Processes", "Workflows", "Key Systems", "Important Events"
-- "Communication", "Collaboration", "Progress", "Status"
-- Generic role-based models not tied to the specific mission
-
-For each model:
-1. id: Use EXACT existing id if keeping a model, or leave empty for new models
-2. name: Short, specific name (e.g., "Team Members", "Sprint Goals")
-3. description: One line describing what to track
-4. initial_probes: 2-3 search queries to find relevant information
-
-Return ONLY the models that should exist. Existing models not in your output will be deleted."""
-
-
-def get_structural_derivation_system_message() -> str:
-    """System message for structural model derivation."""
-    return """You identify the key things to track for a mission. Be VERY selective.
-
-Rules:
-- Maximum 3 models (prefer fewer)
-- Only SPECIFIC, CONCRETE things - not generic categories
-- Each must DIRECTLY help achieve the mission
-- Empty array is valid if no models are truly needed
-- If existing models are shown and you want to keep one, return its EXACT id
-- Never create duplicates - if a similar model exists, keep the existing one
-
-Output JSON with 'templates' array (can be empty)."""
-
-
-def _normalize_id(text: str) -> str:
-    """Normalize a string to a canonical form for comparison.
-
-    Removes common suffixes, pluralization, and normalizes separators.
-    """
-    # Lowercase and normalize separators
-    normalized = text.lower().replace(" ", "-").replace("_", "-")
-
-    # Remove common suffixes that indicate the same concept
-    suffixes_to_remove = ["-map", "-list", "-overview", "-tracker", "-s"]
-    for suffix in suffixes_to_remove:
-        if normalized.endswith(suffix) and len(normalized) > len(suffix):
-            normalized = normalized[: -len(suffix)]
-
-    return normalized
-
-
-def _find_similar_existing_id(new_id: str, existing_models: list[dict]) -> str | None:
-    """Find an existing model ID that is similar to the new ID.
-
-    Returns the existing ID if a similar one is found, None otherwise.
-    """
-    if not existing_models:
-        return None
-
-    new_normalized = _normalize_id(new_id)
-
-    for model in existing_models:
-        existing_id = model.get("id", "")
-        existing_normalized = _normalize_id(existing_id)
-
-        # Check if one is a prefix of the other (normalized)
-        if new_normalized.startswith(existing_normalized) or existing_normalized.startswith(new_normalized):
-            return existing_id
-
-        # Check if they're the same when normalized
-        if new_normalized == existing_normalized:
-            return existing_id
-
-    return None
-
-
-async def derive_structural_models(
-    llm_config: "LLMConfig",
-    mission: str,
-    existing_models: list[dict] | None = None,
-) -> tuple[list[StructuralModelTemplate], list[str]]:
-    """
-    Derive structural model templates from a bank's mission.
-
-    This combines derivation and evaluation in one call. The LLM sees existing
-    models and decides which to keep. Any existing model not in the output
-    will be marked for removal.
-
-    Args:
-        llm_config: LLM configuration for calling the model
-        mission: The bank's mission (e.g., "Be a PM for engineering team")
-        existing_models: Optional list of existing model dicts with 'name', 'description', 'id'
-
-    Returns:
-        Tuple of (templates to create/keep, IDs of existing models to remove)
-
-    Raises:
-        Exception: If LLM call fails
-    """
-    prompt = build_structural_derivation_prompt(mission, existing_models)
-
-    result = await llm_config.call(
-        messages=[
-            {"role": "system", "content": get_structural_derivation_system_message()},
-            {"role": "user", "content": prompt},
-        ],
-        response_format=StructuralDerivationResponse,
-        scope="mental_model_structural_derivation",
-    )
-
-    templates = result.templates
-    logger.info(f"[STRUCTURAL] LLM returned {len(templates)} structural models")
-
-    # Build set of existing IDs for quick lookup
-    existing_ids = {m["id"] for m in existing_models} if existing_models else set()
-
-    # Process templates: validate IDs, deduplicate, assign stable IDs
-    processed_templates: list[StructuralModelTemplate] = []
-    kept_existing_ids: set[str] = set()
-
-    for template in templates:
-        # If LLM returned an ID, check if it's a valid existing ID
-        if template.id and template.id in existing_ids:
-            # LLM is keeping an existing model
-            kept_existing_ids.add(template.id)
-            processed_templates.append(template)
-            logger.info(f"[STRUCTURAL] Keeping existing model: {template.id}")
-        else:
-            # New model or LLM didn't return a valid ID
-            # Generate ID from name
-            generated_id = template.name.lower().replace(" ", "-").replace("_", "-")
-
-            # Check for similar existing models to prevent near-duplicates
-            similar_id = _find_similar_existing_id(generated_id, existing_models)
-            if similar_id and similar_id not in kept_existing_ids:
-                # Use the existing similar model instead of creating a new one
-                logger.info(f"[STRUCTURAL] Detected near-duplicate: '{generated_id}' matches existing '{similar_id}'")
-                template.id = similar_id
-                kept_existing_ids.add(similar_id)
-            else:
-                template.id = generated_id
-
-            processed_templates.append(template)
-
-    # Find existing models to remove (not kept in LLM output)
-    models_to_remove = []
-    if existing_models:
-        for model in existing_models:
-            if model["id"] not in kept_existing_ids:
-                logger.info(f"[STRUCTURAL] Marking '{model['name']}' (id={model['id']}) for removal")
-                models_to_remove.append(model["id"])
-
-    if models_to_remove:
-        logger.info(f"[STRUCTURAL] {len(models_to_remove)} existing models will be removed")
-
-    return processed_templates, models_to_remove

hindsight-api/hindsight_api/engine/query_analyzer.py

@@ -84,7 +84,7 @@ class DateparserQueryAnalyzer(QueryAnalyzer):
 
     Performance:
     - ~10-50ms per query
-    - No model loading required (lazy import on first use)
+    - No model loading required
     """
 
     def __init__(self):
@@ -112,6 +112,8 @@ class DateparserQueryAnalyzer(QueryAnalyzer):
         Returns:
             QueryAnalysis with temporal_constraint if found
         """
+        self.load()
+
         if reference_date is None:
             reference_date = datetime.now()
 
@@ -121,9 +123,6 @@ class DateparserQueryAnalyzer(QueryAnalyzer):
         if period_result is not None:
             return QueryAnalysis(temporal_constraint=period_result)
 
-        # Lazy load dateparser (only imports on first call, then cached)
-        self.load()
-
         # Use dateparser's search_dates to find temporal expressions
         settings = {
             "RELATIVE_BASE": reference_date,

hindsight-api/hindsight_api/engine/reflect/__init__.py

@@ -1,20 +0,0 @@
-"""
-Reflect agent module for agentic reflection with tools.
-
-The reflect agent uses an iterative loop with tools to:
-1. Lookup mental models (existing knowledge)
-2. Recall facts (semantic + temporal search)
-3. Learn new insights (create/update mental models)
-4. Expand memories (get chunk/document context)
-"""
-
-from .agent import ReflectAgentResult, run_reflect_agent
-from .models import MentalModelInput, ReflectAction, ReflectActionBatch
-
-__all__ = [
-    "run_reflect_agent",
-    "ReflectAgentResult",
-    "ReflectAction",
-    "ReflectActionBatch",
-    "MentalModelInput",
-]

hindsight-api/hindsight_api/engine/reflect/agent.py

@@ -1,723 +0,0 @@
-"""
-Reflect agent - agentic loop for reflection with native tool calling.
-"""
-
-import asyncio
-import json
-import logging
-import time
-from typing import TYPE_CHECKING, Any, Awaitable, Callable
-
-from .models import DirectiveInfo, LLMCall, MentalModelInput, ReflectAgentResult, ToolCall
-from .prompts import FINAL_SYSTEM_PROMPT, _extract_directive_rules, build_final_prompt, build_system_prompt_for_tools
-from .tools_schema import get_reflect_tools
-
-
-def _build_directives_applied(directives: list[dict[str, Any]] | None) -> list[DirectiveInfo]:
-    """Build list of DirectiveInfo from directive mental models."""
-    if not directives:
-        return []
-
-    result = []
-    for directive in directives:
-        directive_id = directive.get("id", "")
-        directive_name = directive.get("name", "")
-        observations = directive.get("observations", [])
-
-        rules = []
-        for obs in observations:
-            # Support both Pydantic Observation objects and dicts
-            if hasattr(obs, "content"):
-                rules.append(obs.content)
-            elif isinstance(obs, dict) and obs.get("content"):
-                rules.append(obs["content"])
-
-        result.append(DirectiveInfo(id=directive_id, name=directive_name, rules=rules))
-
-    return result
-
-
-if TYPE_CHECKING:
-    from ..llm_wrapper import LLMProvider
-    from ..response_models import LLMToolCall
-
-logger = logging.getLogger(__name__)
-
-DEFAULT_MAX_ITERATIONS = 10
-
-
-async def _generate_structured_output(
-    answer: str,
-    response_schema: dict,
-    llm_config: "LLMProvider",
-    reflect_id: str,
-) -> dict[str, Any] | None:
-    """Generate structured output from an answer using the provided JSON schema.
-
-    Args:
-        answer: The text answer to extract structured data from
-        response_schema: JSON Schema for the expected output structure
-        llm_config: LLM provider for making the extraction call
-        reflect_id: Reflect ID for logging
-
-    Returns:
-        Structured output dict if successful, None otherwise
-    """
-    try:
-        from typing import Any as TypingAny
-
-        from pydantic import create_model
-
-        def _json_schema_type_to_python(field_schema: dict) -> type:
-            """Map JSON schema type to Python type for better LLM guidance."""
-            json_type = field_schema.get("type", "string")
-            if json_type == "array":
-                return list
-            elif json_type == "object":
-                return dict
-            elif json_type == "integer":
-                return int
-            elif json_type == "number":
-                return float
-            elif json_type == "boolean":
-                return bool
-            else:
-                return str
-
-        # Build fields from JSON schema properties
-        schema_props = response_schema.get("properties", {})
-        required_fields = set(response_schema.get("required", []))
-        fields: dict[str, TypingAny] = {}
-        for field_name, field_schema in schema_props.items():
-            field_type = _json_schema_type_to_python(field_schema)
-            default = ... if field_name in required_fields else None
-            fields[field_name] = (field_type, default)
-
-        if not fields:
-            return None
-
-        DynamicModel = create_model("StructuredResponse", **fields)
-
-        # Include the full schema in the prompt for better LLM guidance
-        schema_str = json.dumps(response_schema, indent=2)
-
-        # Call LLM with the answer to extract structured data
-        structured_prompt = f"""Based on this answer, extract the information into the requested structured format.
-
-Answer: {answer}
-
-JSON Schema to follow:
-```json
-{schema_str}
-```
-
-Return ONLY a valid JSON object that matches this exact schema. Pay special attention to field types:
-- "type": "array" means the value must be a JSON array/list, NOT a string
-- "type": "string" means the value must be a string
-- "type": "object" means the value must be a JSON object
-
-Do not include any explanation, only the JSON object."""
-
-        structured_result = await llm_config.call(
-            messages=[
-                {
-                    "role": "system",
-                    "content": "Extract structured data from the given answer. Return only valid JSON matching the provided schema exactly.",
-                },
-                {"role": "user", "content": structured_prompt},
-            ],
-            response_format=DynamicModel,
-            scope="reflect_structured",
-            skip_validation=True,  # We'll handle the dict ourselves
-        )
-
-        # Convert to dict
-        if hasattr(structured_result, "model_dump"):
-            structured_output = structured_result.model_dump()
-        elif isinstance(structured_result, dict):
-            structured_output = structured_result
-        else:
-            # Try to parse as JSON
-            structured_output = json.loads(str(structured_result))
-
-        logger.info(f"[REFLECT {reflect_id}] Generated structured output with {len(structured_output)} fields")
-        return structured_output
-
-    except Exception as e:
-        logger.warning(f"[REFLECT {reflect_id}] Failed to generate structured output: {e}")
-        return None
-
-
-async def run_reflect_agent(
-    llm_config: "LLMProvider",
-    bank_id: str,
-    query: str,
-    bank_profile: dict[str, Any],
-    lookup_fn: Callable[[str | None], Awaitable[dict[str, Any]]],
-    recall_fn: Callable[[str, int], Awaitable[dict[str, Any]]],
-    expand_fn: Callable[[list[str], str], Awaitable[dict[str, Any]]],
-    learn_fn: Callable[[MentalModelInput], Awaitable[dict[str, Any]]] | None = None,
-    context: str | None = None,
-    max_iterations: int = DEFAULT_MAX_ITERATIONS,
-    max_tokens: int | None = None,
-    response_schema: dict | None = None,
-    directives: list[dict[str, Any]] | None = None,
-) -> ReflectAgentResult:
-    """
-    Execute the reflect agent loop using native tool calling.
-
-    The agent iteratively calls tools to gather information and learn,
-    then provides a final answer via the done() tool.
-
-    Args:
-        llm_config: LLM provider for agent calls
-        bank_id: Bank identifier
-        query: Question to answer
-        bank_profile: Bank profile with name and mission
-        lookup_fn: Tool callback for lookup (model_id) -> result
-        recall_fn: Tool callback for recall (query, max_tokens) -> result
-        expand_fn: Tool callback for expand (memory_id, depth) -> result
-        learn_fn: Optional tool callback for learn (MentalModelInput) -> result.
-                  If None, learn tool is disabled.
-        context: Optional additional context
-        max_iterations: Maximum number of iterations before forcing response
-        max_tokens: Maximum tokens for the final response
-        response_schema: Optional JSON Schema for structured output in final response
-        directives: Optional list of directive mental models to inject as hard rules
-
-    Returns:
-        ReflectAgentResult with final answer and metadata
-    """
-    enable_learn = learn_fn is not None
-    reflect_id = f"{bank_id[:8]}-{int(time.time() * 1000) % 100000}"
-    start_time = time.time()
-
-    # Build directives_applied for the trace
-    directives_applied = _build_directives_applied(directives)
-
-    # Extract directive rules for tool schema (if any)
-    directive_rules = _extract_directive_rules(directives) if directives else None
-
-    # Get tools for this agent (with directive compliance field if directives exist)
-    tools = get_reflect_tools(enable_learn=enable_learn, directive_rules=directive_rules)
-
-    # Build initial messages (directives are injected into system prompt at START and END)
-    system_prompt = build_system_prompt_for_tools(bank_profile, context, directives=directives)
-    messages: list[dict[str, Any]] = [
-        {"role": "system", "content": system_prompt},
-        {"role": "user", "content": query},
-    ]
-
-    # Tracking
-    mental_models_created: list[str] = []
-    total_tools_called = 0
-    tool_trace: list[ToolCall] = []
-    tool_trace_summary: list[dict[str, Any]] = []
-    llm_trace: list[dict[str, Any]] = []
-    context_history: list[dict[str, Any]] = []  # For final prompt fallback
-
-    # Track available IDs for validation (prevents hallucinated citations)
-    available_memory_ids: set[str] = set()
-    available_model_ids: set[str] = set()
-
-    # Pre-fetch mental models so the agent always starts with this knowledge
-    prefetch_start = time.time()
-    models_result = await lookup_fn(None)  # List all mental models
-    prefetch_duration = int((time.time() - prefetch_start) * 1000)
-
-    # Track available model IDs
-    if isinstance(models_result, dict) and "models" in models_result:
-        for model in models_result["models"]:
-            if "id" in model:
-                available_model_ids.add(model["id"])
-
-    # Add to context history for the agent
-    context_history.append({"tool": "list_mental_models", "output": models_result})
-
-    # Add to tool trace
-    tool_trace.append(
-        ToolCall(
-            tool="list_mental_models",
-            input={"tool": "list_mental_models"},
-            output=models_result,
-            duration_ms=prefetch_duration,
-            iteration=0,
-        )
-    )
-    tool_trace_summary.append(
-        {
-            "tool": "list_mental_models",
-            "input_summary": "(prefetch)",
-            "duration_ms": prefetch_duration,
-            "output_chars": len(json.dumps(models_result, default=str)),
-        }
-    )
-    total_tools_called += 1
-
-    # Include in the user message so the agent sees it
-    models_info = json.dumps(models_result, indent=2, default=str)
-    messages[1]["content"] = f"{query}\n\n## Available Mental Models (pre-fetched)\n```json\n{models_info}\n```"
-
-    def _get_llm_trace() -> list[LLMCall]:
-        return [LLMCall(scope=c["scope"], duration_ms=c["duration_ms"]) for c in llm_trace]
-
-    def _log_completion(answer: str, iterations: int, forced: bool = False):
-        elapsed_ms = int((time.time() - start_time) * 1000)
-        tools_summary = (
-            ", ".join(
-                f"{t['tool']}({t['input_summary']})={t['duration_ms']}ms/{t.get('output_chars', 0)}c"
-                for t in tool_trace_summary
-            )
-            or "none"
-        )
-        llm_summary = ", ".join(f"{c['scope']}={c['duration_ms']}ms" for c in llm_trace) or "none"
-        total_llm_ms = sum(c["duration_ms"] for c in llm_trace)
-        total_tools_ms = sum(t["duration_ms"] for t in tool_trace_summary)
-
-        answer_preview = answer[:100] + "..." if len(answer) > 100 else answer
-        mode = "forced" if forced else "done"
-        logger.info(
-            f"[REFLECT {reflect_id}] {mode} | "
-            f"query='{query[:50]}...' | "
-            f"iterations={iterations} | "
-            f"llm=[{llm_summary}] ({total_llm_ms}ms) | "
-            f"tools=[{tools_summary}] ({total_tools_ms}ms) | "
-            f"answer='{answer_preview}' | "
-            f"total={elapsed_ms}ms"
-        )
-
-    for iteration in range(max_iterations):
-        is_last = iteration == max_iterations - 1
-
-        if is_last:
-            # Force text response on last iteration - no tools
-            prompt = build_final_prompt(query, context_history, bank_profile, context)
-            llm_start = time.time()
-            response = await llm_config.call(
-                messages=[
-                    {"role": "system", "content": FINAL_SYSTEM_PROMPT},
-                    {"role": "user", "content": prompt},
-                ],
-                scope="reflect_agent_final",
-                max_completion_tokens=max_tokens,
-            )
-            llm_trace.append({"scope": "final", "duration_ms": int((time.time() - llm_start) * 1000)})
-            answer = response.strip()
-
-            # Generate structured output if schema provided
-            structured_output = None
-            if response_schema and answer:
-                structured_output = await _generate_structured_output(answer, response_schema, llm_config, reflect_id)
-
-            _log_completion(answer, iteration + 1, forced=True)
-            return ReflectAgentResult(
-                text=answer,
-                structured_output=structured_output,
-                iterations=iteration + 1,
-                tools_called=total_tools_called,
-                mental_models_created=mental_models_created,
-                tool_trace=tool_trace,
-                llm_trace=_get_llm_trace(),
-                directives_applied=directives_applied,
-            )
-
-        # Call LLM with tools
-        llm_start = time.time()
-
-        try:
-            result = await llm_config.call_with_tools(
-                messages=messages,
-                tools=tools,
-                scope="reflect_agent",
-                tool_choice="required" if iteration == 0 else "auto",  # Force tool use on first iteration
-            )
-            llm_duration = int((time.time() - llm_start) * 1000)
-            llm_trace.append({"scope": f"agent_{iteration + 1}", "duration_ms": llm_duration})
-
-        except Exception:
-            llm_trace.append(
-                {"scope": f"agent_{iteration + 1}_err", "duration_ms": int((time.time() - llm_start) * 1000)}
-            )
-            # Guardrail: If no evidence gathered yet, retry
-            has_gathered_evidence = bool(available_memory_ids) or bool(available_model_ids)
-            if not has_gathered_evidence and iteration < max_iterations - 1:
-                continue
-            prompt = build_final_prompt(query, context_history, bank_profile, context)
-            llm_start = time.time()
-            response = await llm_config.call(
-                messages=[
-                    {"role": "system", "content": FINAL_SYSTEM_PROMPT},
-                    {"role": "user", "content": prompt},
-                ],
-                scope="reflect_agent_final",
-                max_completion_tokens=max_tokens,
-            )
-            llm_trace.append({"scope": "final", "duration_ms": int((time.time() - llm_start) * 1000)})
-            answer = response.strip()
-
-            # Generate structured output if schema provided
-            structured_output = None
-            if response_schema and answer:
-                structured_output = await _generate_structured_output(answer, response_schema, llm_config, reflect_id)
-
-            _log_completion(answer, iteration + 1, forced=True)
-            return ReflectAgentResult(
-                text=answer,
-                structured_output=structured_output,
-                iterations=iteration + 1,
-                tools_called=total_tools_called,
-                mental_models_created=mental_models_created,
-                tool_trace=tool_trace,
-                llm_trace=_get_llm_trace(),
-                directives_applied=directives_applied,
-            )
-
-        # No tool calls - LLM wants to respond with text
-        if not result.tool_calls:
-            if result.content:
-                answer = result.content.strip()
-
-                # Generate structured output if schema provided
-                structured_output = None
-                if response_schema and answer:
-                    structured_output = await _generate_structured_output(
-                        answer, response_schema, llm_config, reflect_id
-                    )
-
-                _log_completion(answer, iteration + 1)
-                return ReflectAgentResult(
-                    text=answer,
-                    structured_output=structured_output,
-                    iterations=iteration + 1,
-                    tools_called=total_tools_called,
-                    mental_models_created=mental_models_created,
-                    tool_trace=tool_trace,
-                    llm_trace=_get_llm_trace(),
-                    directives_applied=directives_applied,
-                )
-            # Empty response, force final
-            prompt = build_final_prompt(query, context_history, bank_profile, context)
-            llm_start = time.time()
-            response = await llm_config.call(
-                messages=[
-                    {"role": "system", "content": FINAL_SYSTEM_PROMPT},
-                    {"role": "user", "content": prompt},
-                ],
-                scope="reflect_agent_final",
-                max_completion_tokens=max_tokens,
-            )
-            llm_trace.append({"scope": "final", "duration_ms": int((time.time() - llm_start) * 1000)})
-            answer = response.strip()
-
-            # Generate structured output if schema provided
-            structured_output = None
-            if response_schema and answer:
-                structured_output = await _generate_structured_output(answer, response_schema, llm_config, reflect_id)
-
-            _log_completion(answer, iteration + 1, forced=True)
-            return ReflectAgentResult(
-                text=answer,
-                structured_output=structured_output,
-                iterations=iteration + 1,
-                tools_called=total_tools_called,
-                mental_models_created=mental_models_created,
-                tool_trace=tool_trace,
-                llm_trace=_get_llm_trace(),
-                directives_applied=directives_applied,
-            )
-
-        # Check for done tool call (handle both 'done' and 'functions.done')
-        done_call = next((tc for tc in result.tool_calls if tc.name == "done" or tc.name == "functions.done"), None)
-        if done_call:
-            # Guardrail: Require evidence before done
-            has_gathered_evidence = bool(available_memory_ids) or bool(available_model_ids)
-            if not has_gathered_evidence and iteration < max_iterations - 1:
-                # Add assistant message and fake tool result asking for evidence
-                messages.append(
-                    {
-                        "role": "assistant",
-                        "tool_calls": [_tool_call_to_dict(done_call)],
-                    }
-                )
-                messages.append(
-                    {
-                        "role": "tool",
-                        "tool_call_id": done_call.id,
-                        "content": json.dumps(
-                            {
-                                "error": "You must call recall() or list_mental_models() to gather evidence before providing your final answer."
-                            }
-                        ),
-                    }
-                )
-                continue
-
-            # Process done tool
-            return await _process_done_tool(
-                done_call,
-                available_memory_ids,
-                available_model_ids,
-                iteration + 1,
-                total_tools_called,
-                mental_models_created,
-                tool_trace,
-                _get_llm_trace(),
-                _log_completion,
-                reflect_id,
-                directives_applied=directives_applied,
-                llm_config=llm_config,
-                response_schema=response_schema,
-            )
-
-        # Execute other tools in parallel (exclude done and functions.done)
-        other_tools = [tc for tc in result.tool_calls if tc.name not in ("done", "functions.done")]
-        if other_tools:
-            # Add assistant message with tool calls
-            messages.append(
-                {
-                    "role": "assistant",
-                    "tool_calls": [_tool_call_to_dict(tc) for tc in other_tools],
-                }
-            )
-
-            # Execute tools in parallel
-            tool_tasks = [
-                _execute_tool_with_timing(tc, lookup_fn, recall_fn, expand_fn, learn_fn) for tc in other_tools
-            ]
-            tool_results = await asyncio.gather(*tool_tasks, return_exceptions=True)
-            total_tools_called += len(other_tools)
-
-            # Process results and add to messages
-            for tc, result_data in zip(other_tools, tool_results):
-                if isinstance(result_data, Exception):
-                    # Tool execution failed - log and raise to fail the request
-                    logger.error(f"[REFLECT {reflect_id}] Tool {tc.name} failed with exception: {result_data}")
-                    raise RuntimeError(f"Reflect tool '{tc.name}' failed: {result_data}")
-
-                output, duration_ms = result_data
-
-                # Check if tool returned an error response
-                if isinstance(output, dict) and "error" in output:
-                    logger.error(f"[REFLECT {reflect_id}] Tool {tc.name} returned error: {output['error']}")
-                    raise RuntimeError(f"Reflect tool '{tc.name}' error: {output['error']}")
-
-                # Track created mental models
-                if tc.name == "learn" and isinstance(output, dict) and "model_id" in output:
-                    mental_models_created.append(output["model_id"])
-
-                # Track available memory IDs from recall
-                if tc.name == "recall" and isinstance(output, dict) and "memories" in output:
-                    for memory in output["memories"]:
-                        if "id" in memory:
-                            available_memory_ids.add(memory["id"])
-
-                # Track available model IDs
-                if tc.name in ("list_mental_models", "get_mental_model") and isinstance(output, dict):
-                    if output.get("found") and "model" in output:
-                        model_id = output["model"].get("id")
-                        if model_id:
-                            available_model_ids.add(model_id)
-                    elif "models" in output:
-                        for model in output["models"]:
-                            if "id" in model:
-                                available_model_ids.add(model["id"])
-
-                # Add tool result message
-                messages.append(
-                    {
-                        "role": "tool",
-                        "tool_call_id": tc.id,
-                        "content": json.dumps(output, default=str),
-                    }
-                )
-
-                # Track for logging and context history
-                input_dict = {"tool": tc.name, **tc.arguments}
-                input_summary = _summarize_input(tc.name, tc.arguments)
-
-                tool_trace.append(
-                    ToolCall(
-                        tool=tc.name, input=input_dict, output=output, duration_ms=duration_ms, iteration=iteration + 1
-                    )
-                )
-
-                try:
-                    output_chars = len(json.dumps(output))
-                except (TypeError, ValueError):
-                    output_chars = len(str(output))
-
-                tool_trace_summary.append(
-                    {
-                        "tool": tc.name,
-                        "input_summary": input_summary,
-                        "duration_ms": duration_ms,
-                        "output_chars": output_chars,
-                    }
-                )
-
-                # Keep context history for fallback final prompt
-                context_history.append({"tool": tc.name, "input": input_dict, "output": output})
-
-    # Should not reach here
-    answer = "I was unable to formulate a complete answer within the iteration limit."
-    _log_completion(answer, max_iterations, forced=True)
-    return ReflectAgentResult(
-        text=answer,
-        iterations=max_iterations,
-        tools_called=total_tools_called,
-        mental_models_created=mental_models_created,
-        tool_trace=tool_trace,
-        llm_trace=_get_llm_trace(),
-        directives_applied=directives_applied,
-    )
-
-
-def _tool_call_to_dict(tc: "LLMToolCall") -> dict[str, Any]:
-    """Convert LLMToolCall to OpenAI message format."""
-    return {
-        "id": tc.id,
-        "type": "function",
-        "function": {
-            "name": tc.name,
-            "arguments": json.dumps(tc.arguments),
-        },
-    }
-
-
-async def _process_done_tool(
-    done_call: "LLMToolCall",
-    available_memory_ids: set[str],
-    available_model_ids: set[str],
-    iterations: int,
-    total_tools_called: int,
-    mental_models_created: list[str],
-    tool_trace: list[ToolCall],
-    llm_trace: list[LLMCall],
-    log_completion: Callable,
-    reflect_id: str,
-    directives_applied: list[DirectiveInfo],
-    llm_config: "LLMProvider | None" = None,
-    response_schema: dict | None = None,
-) -> ReflectAgentResult:
-    """Process the done tool call and return the result."""
-    args = done_call.arguments
-
-    answer = args.get("answer", "").strip()
-    if not answer:
-        answer = "No answer provided."
-
-    # Validate IDs
-    used_memory_ids = [mid for mid in args.get("memory_ids", []) if mid in available_memory_ids]
-    used_model_ids = [mid for mid in args.get("model_ids", []) if mid in available_model_ids]
-
-    # Generate structured output if schema provided
-    structured_output = None
-    if response_schema and llm_config and answer:
-        structured_output = await _generate_structured_output(answer, response_schema, llm_config, reflect_id)
-
-    log_completion(answer, iterations)
-    return ReflectAgentResult(
-        text=answer,
-        structured_output=structured_output,
-        iterations=iterations,
-        tools_called=total_tools_called,
-        mental_models_created=mental_models_created,
-        tool_trace=tool_trace,
-        llm_trace=llm_trace,
-        used_memory_ids=used_memory_ids,
-        used_model_ids=used_model_ids,
-        directives_applied=directives_applied,
-    )
-
-
-async def _execute_tool_with_timing(
-    tc: "LLMToolCall",
-    lookup_fn: Callable[[str | None], Awaitable[dict[str, Any]]],
-    recall_fn: Callable[[str, int], Awaitable[dict[str, Any]]],
-    expand_fn: Callable[[list[str], str], Awaitable[dict[str, Any]]],
-    learn_fn: Callable[[MentalModelInput], Awaitable[dict[str, Any]]] | None = None,
-) -> tuple[dict[str, Any], int]:
-    """Execute a tool call and return result with timing."""
-    start = time.time()
-    result = await _execute_tool(tc.name, tc.arguments, lookup_fn, recall_fn, expand_fn, learn_fn)
-    duration_ms = int((time.time() - start) * 1000)
-    return result, duration_ms
-
-
-async def _execute_tool(
-    tool_name: str,
-    args: dict[str, Any],
-    lookup_fn: Callable[[str | None], Awaitable[dict[str, Any]]],
-    recall_fn: Callable[[str, int], Awaitable[dict[str, Any]]],
-    expand_fn: Callable[[list[str], str], Awaitable[dict[str, Any]]],
-    learn_fn: Callable[[MentalModelInput], Awaitable[dict[str, Any]]] | None = None,
-) -> dict[str, Any]:
-    """Execute a single tool by name."""
-    # Normalize tool name - some LLMs return 'functions.done' instead of 'done'
-    if tool_name.startswith("functions."):
-        tool_name = tool_name[len("functions.") :]
-
-    if tool_name == "list_mental_models":
-        return await lookup_fn(None)
-
-    elif tool_name == "get_mental_model":
-        model_id = args.get("model_id")
-        if not model_id:
-            return {"error": "get_mental_model requires model_id"}
-        return await lookup_fn(model_id)
-
-    elif tool_name == "recall":
-        query = args.get("query")
-        if not query:
-            return {"error": "recall requires a query parameter"}
-        max_tokens = max(args.get("max_tokens") or 2048, 1000)  # Default 2048, min 1000
-        return await recall_fn(query, max_tokens)
-
-    elif tool_name == "learn":
-        if learn_fn is None:
-            return {"error": "learn tool is not available"}
-        name = args.get("name")
-        description = args.get("description")
-        if not name or not description:
-            return {"error": "learn requires name and description"}
-        return await learn_fn(MentalModelInput(name=name, description=description))
-
-    elif tool_name == "expand":
-        memory_ids = args.get("memory_ids", [])
-        if not memory_ids:
-            return {"error": "expand requires memory_ids"}
-        depth = args.get("depth", "chunk")
-        return await expand_fn(memory_ids, depth)
-
-    else:
-        return {"error": f"Unknown tool: {tool_name}"}
-
-
-def _summarize_input(tool_name: str, args: dict[str, Any]) -> str:
-    """Create a summary of tool input for logging, showing all params."""
-    if tool_name == "list_mental_models":
-        return "()"
-    elif tool_name == "get_mental_model":
-        return f"(model_id={args.get('model_id', '?')})"
-    elif tool_name == "recall":
-        query = args.get("query", "")
-        query_preview = f"'{query[:30]}...'" if len(query) > 30 else f"'{query}'"
-        # Show actual value used (default 2048, min 1000)
-        max_tokens = max(args.get("max_tokens") or 2048, 1000)
-        return f"(query={query_preview}, max_tokens={max_tokens})"
-    elif tool_name == "learn":
-        name = args.get("name", "?")
-        desc = args.get("description", "")
-        desc_preview = f"'{desc[:20]}...'" if len(desc) > 20 else f"'{desc}'"
-        return f"(name='{name}', description={desc_preview})"
-    elif tool_name == "expand":
-        memory_ids = args.get("memory_ids", [])
-        depth = args.get("depth", "chunk")
-        return f"(memory_ids=[{len(memory_ids)} ids], depth={depth})"
-    elif tool_name == "done":
-        answer = args.get("answer", "")
-        answer_preview = f"'{answer[:30]}...'" if len(answer) > 30 else f"'{answer}'"
-        memory_ids = args.get("memory_ids", [])
-        model_ids = args.get("model_ids", [])
-        return f"(answer={answer_preview}, memory_ids={len(memory_ids)}, model_ids={len(model_ids)})"
-    return str(args)

hindsight-api/hindsight_api/engine/reflect/models.py

@@ -1,114 +0,0 @@
-"""
-Pydantic models for the reflect agent.
-"""
-
-from typing import Any, Literal
-
-from pydantic import BaseModel, Field
-
-
-class MentalModelObservation(BaseModel):
-    """An observation within a mental model with its supporting memories."""
-
-    title: str = Field(description="Observation header (can be empty for intro)")
-    text: str = Field(description="Observation content - no headers, use lists/tables/bold")
-    memory_ids: list[str] = Field(default_factory=list, description="Memory IDs supporting this observation")
-
-
-class MentalModelInput(BaseModel):
-    """Input for the learn tool to create a mental model placeholder.
-
-    The agent only specifies name and description - the actual content/observations
-    are generated during refresh, similar to pinned models.
-    """
-
-    name: str = Field(description="Human-readable name for the mental model")
-    description: str = Field(description="What to track - used as prompt for content generation during refresh")
-    entity_id: str | None = Field(default=None, description="Optional link to existing entity ID")
-
-
-class AnswerSection(BaseModel):
-    """A section of the answer with its supporting evidence (DEPRECATED)."""
-
-    title: str = Field(description="Section header/title")
-    text: str = Field(description="Section content")
-    memory_ids: list[str] = Field(default_factory=list, description="Memory IDs supporting this section")
-    model_ids: list[str] = Field(default_factory=list, description="Mental model IDs supporting this section")
-
-
-class ReflectAction(BaseModel):
-    """Single action the reflect agent can take."""
-
-    tool: Literal["list_mental_models", "get_mental_model", "recall", "learn", "expand", "done"] = Field(
-        description="Tool to invoke: list_mental_models, get_mental_model, recall, learn, expand, or done"
-    )
-    # Tool-specific parameters
-    model_id: str | None = Field(default=None, description="Mental model ID for get_mental_model")
-    query: str | None = Field(default=None, description="Search query for recall")
-    max_tokens: int | None = Field(default=None, description="Max tokens for recall results (default 2048)")
-    mental_model: MentalModelInput | None = Field(default=None, description="Mental model to create/update for learn")
-    memory_ids: list[str] | None = Field(default=None, description="Memory unit IDs for expand (batched)")
-    depth: Literal["chunk", "document"] | None = Field(default=None, description="Expansion depth for expand")
-    sections: list[AnswerSection] | None = Field(default=None, description="DEPRECATED: Use answer field instead")
-    observations: list[MentalModelObservation] | None = Field(
-        default=None, description="Observations for done action (when output_mode=observations)"
-    )
-    # Plain text answer fields (for output_mode=answer)
-    answer: str | None = Field(default=None, description="Plain text answer for done action (no markdown)")
-    answer_memory_ids: list[str] | None = Field(
-        default=None, description="Memory IDs supporting the answer", alias="memory_ids"
-    )
-    answer_model_ids: list[str] | None = Field(
-        default=None, description="Mental model IDs supporting the answer", alias="model_ids"
-    )
-    reasoning: str | None = Field(default=None, description="Brief reasoning for this action")
-
-
-class ReflectActionBatch(BaseModel):
-    """Batch of actions for parallel execution."""
-
-    actions: list[ReflectAction] = Field(description="List of actions to execute in parallel")
-
-
-class ToolCall(BaseModel):
-    """A single tool call made during reflect."""
-
-    tool: str = Field(description="Tool name: lookup, recall, learn, expand")
-    input: dict = Field(description="Tool input parameters")
-    output: dict = Field(description="Tool output/result")
-    duration_ms: int = Field(description="Execution time in milliseconds")
-    iteration: int = Field(default=0, description="Iteration number (1-based) when this tool was called")
-
-
-class LLMCall(BaseModel):
-    """A single LLM call made during reflect."""
-
-    scope: str = Field(description="Call scope: agent_1, agent_2, final, etc.")
-    duration_ms: int = Field(description="Execution time in milliseconds")
-
-
-class DirectiveInfo(BaseModel):
-    """Information about a directive that was applied during reflect."""
-
-    id: str = Field(description="Directive mental model ID")
-    name: str = Field(description="Directive name")
-    rules: list[str] = Field(default_factory=list, description="Directive rules/observations that were applied")
-
-
-class ReflectAgentResult(BaseModel):
-    """Result from the reflect agent."""
-
-    text: str = Field(description="Final answer text")
-    structured_output: dict[str, Any] | None = Field(
-        default=None, description="Structured output parsed according to provided response_schema"
-    )
-    iterations: int = Field(default=0, description="Number of iterations taken")
-    tools_called: int = Field(default=0, description="Total number of tool calls made")
-    mental_models_created: list[str] = Field(default_factory=list, description="IDs of mental models created/updated")
-    tool_trace: list[ToolCall] = Field(default_factory=list, description="Trace of all tool calls made")
-    llm_trace: list[LLMCall] = Field(default_factory=list, description="Trace of all LLM calls made")
-    used_memory_ids: list[str] = Field(default_factory=list, description="Validated memory IDs actually used in answer")
-    used_model_ids: list[str] = Field(default_factory=list, description="Validated model IDs actually used in answer")
-    directives_applied: list[DirectiveInfo] = Field(
-        default_factory=list, description="Directive mental models that affected this reflection"
-    )

hindsight-api/hindsight_api/engine/reflect/observations.py

@@ -1,248 +0,0 @@
-"""
-Models and utilities for evidence-grounded observations with computed trends.
-
-Observations are part of mental models and represent patterns/beliefs derived
-from memories. Each observation must be grounded in specific evidence (quotes)
-from memories, and trends are computed algorithmically from evidence timestamps.
-"""
-
-from datetime import datetime, timedelta, timezone
-from enum import Enum
-
-from pydantic import BaseModel, Field, computed_field, field_validator
-
-
-class Trend(str, Enum):
-    """Computed trend for an observation based on evidence timestamps.
-
-    Trends indicate how an observation's evidence is distributed over time:
-    - STABLE: Evidence spread across time, continues to present
-    - STRENGTHENING: More/denser evidence recently than before
-    - WEAKENING: Evidence mostly old, sparse recently
-    - NEW: All evidence within recent window
-    - STALE: No evidence in recent window (may no longer apply)
-    """
-
-    STABLE = "stable"
-    STRENGTHENING = "strengthening"
-    WEAKENING = "weakening"
-    NEW = "new"
-    STALE = "stale"
-
-
-class ObservationEvidence(BaseModel):
-    """A single piece of evidence supporting an observation.
-
-    Each evidence item must include an exact quote from the source memory
-    to ensure observations are grounded and verifiable.
-    """
-
-    memory_id: str = Field(description="ID of the memory unit this evidence comes from")
-    quote: str = Field(description="Exact quote from the memory supporting the observation")
-    relevance: str = Field(default="", description="Brief explanation of how this quote supports the observation")
-    timestamp: datetime = Field(description="When the source memory was created")
-
-    @field_validator("timestamp", mode="before")
-    @classmethod
-    def ensure_timezone_aware(cls, v: datetime | str | None) -> datetime:
-        """Ensure timestamp is always timezone-aware UTC."""
-        if v is None:
-            return datetime.now(timezone.utc)
-        if isinstance(v, str):
-            # Parse ISO format string, handling 'Z' suffix
-            v = datetime.fromisoformat(v.replace("Z", "+00:00"))
-        if isinstance(v, datetime):
-            if v.tzinfo is None:
-                return v.replace(tzinfo=timezone.utc)
-            return v
-        raise ValueError(f"Invalid timestamp type: {type(v)}")
-
-
-class Observation(BaseModel):
-    """A single observation within a mental model.
-
-    Observations represent patterns, preferences, beliefs, or other insights
-    derived from memories. Each observation must be grounded in evidence
-    with exact quotes from source memories.
-    """
-
-    title: str = Field(description="Short summary title for the observation (5-10 words)")
-    content: str = Field(description="The observation content - detailed explanation of what we believe to be true")
-    evidence: list[ObservationEvidence] = Field(default_factory=list, description="Supporting evidence with quotes")
-    created_at: datetime = Field(
-        default_factory=lambda: datetime.now(timezone.utc), description="When this observation was first created"
-    )
-
-    @field_validator("created_at", mode="before")
-    @classmethod
-    def ensure_created_at_timezone_aware(cls, v: datetime | str | None) -> datetime:
-        """Ensure created_at is always timezone-aware UTC."""
-        if v is None:
-            return datetime.now(timezone.utc)
-        if isinstance(v, str):
-            v = datetime.fromisoformat(v.replace("Z", "+00:00"))
-        if isinstance(v, datetime):
-            if v.tzinfo is None:
-                return v.replace(tzinfo=timezone.utc)
-            return v
-        raise ValueError(f"Invalid created_at type: {type(v)}")
-
-    @computed_field
-    @property
-    def trend(self) -> Trend:
-        """Compute trend from evidence timestamps."""
-        return compute_trend(self.evidence)
-
-    @computed_field
-    @property
-    def evidence_span(self) -> dict[str, str | None]:
-        """Get the time span covered by evidence."""
-        if not self.evidence:
-            return {"from": None, "to": None}
-        timestamps = [e.timestamp for e in self.evidence]
-        return {
-            "from": min(timestamps).isoformat(),
-            "to": max(timestamps).isoformat(),
-        }
-
-    @computed_field
-    @property
-    def evidence_count(self) -> int:
-        """Number of evidence items supporting this observation."""
-        return len(self.evidence)
-
-
-def compute_trend(
-    evidence: list[ObservationEvidence],
-    now: datetime | None = None,
-    recent_days: int = 30,
-    old_days: int = 90,
-) -> Trend:
-    """Compute the trend for an observation based on evidence timestamps.
-
-    The trend indicates how the evidence is distributed over time:
-    - STABLE: Evidence spread across time, continues to present
-    - STRENGTHENING: More evidence recently than historically
-    - WEAKENING: Evidence mostly old, sparse recently
-    - NEW: All evidence is recent (within recent_days)
-    - STALE: No evidence in recent window
-
-    Args:
-        evidence: List of evidence items with timestamps
-        now: Reference time for calculations (defaults to current UTC time)
-        recent_days: Number of days to consider "recent" (default 30)
-        old_days: Number of days to consider "old" (default 90)
-
-    Returns:
-        Computed Trend enum value
-    """
-    if now is None:
-        now = datetime.now(timezone.utc)
-
-    # Ensure now is timezone-aware
-    if now.tzinfo is None:
-        now = now.replace(tzinfo=timezone.utc)
-
-    if not evidence:
-        return Trend.STALE
-
-    recent_cutoff = now - timedelta(days=recent_days)
-    old_cutoff = now - timedelta(days=old_days)
-
-    # Normalize timestamps to UTC for comparison
-    def normalize_ts(ts: datetime) -> datetime:
-        if ts.tzinfo is None:
-            return ts.replace(tzinfo=timezone.utc)
-        return ts
-
-    recent = [e for e in evidence if normalize_ts(e.timestamp) > recent_cutoff]
-    old = [e for e in evidence if normalize_ts(e.timestamp) < old_cutoff]
-    middle = [e for e in evidence if old_cutoff <= normalize_ts(e.timestamp) <= recent_cutoff]
-
-    # No recent evidence = stale
-    if not recent:
-        return Trend.STALE
-
-    # All evidence is recent = new
-    if not old and not middle:
-        return Trend.NEW
-
-    # Compare density (evidence per day)
-    recent_density = len(recent) / recent_days if recent_days > 0 else 0
-    older_period = old_days - recent_days
-    older_density = (len(old) + len(middle)) / older_period if older_period > 0 else 0
-
-    # Avoid division by zero
-    if older_density == 0:
-        return Trend.NEW
-
-    ratio = recent_density / older_density
-
-    if ratio > 1.5:
-        return Trend.STRENGTHENING
-    elif ratio < 0.5:
-        return Trend.WEAKENING
-    else:
-        return Trend.STABLE
-
-
-class CandidateObservation(BaseModel):
-    """A candidate observation generated during the seed phase.
-
-    Candidates are preliminary observations that need evidence validation
-    before becoming full observations.
-    """
-
-    content: str = Field(description="The proposed observation content")
-    seed_memory_ids: list[str] = Field(default_factory=list, description="Memory IDs that inspired this candidate")
-
-
-class CandidateWithEvidence(BaseModel):
-    """A candidate observation with gathered supporting and contradicting evidence."""
-
-    candidate: CandidateObservation
-    supporting_memories: list[dict] = Field(default_factory=list, description="Memories that support this observation")
-    contradicting_memories: list[dict] = Field(
-        default_factory=list, description="Memories that contradict this observation"
-    )
-
-
-class MentalModelSnapshot(BaseModel):
-    """A versioned snapshot of a mental model's observations.
-
-    Used for tracking changes over time and enabling diff views.
-    """
-
-    version: int = Field(description="Version number (1-indexed)")
-    observations: list[Observation] = Field(default_factory=list, description="Observations at this version")
-    created_at: datetime = Field(
-        default_factory=lambda: datetime.now(timezone.utc), description="When this version was created"
-    )
-    reflect_summary: str | None = Field(default=None, description="Summary of changes in this version")
-
-
-def verify_evidence_quotes(
-    observation: Observation,
-    memories: dict[str, str],
-) -> tuple[bool, list[str]]:
-    """Verify that all evidence quotes exist in the referenced memories.
-
-    Args:
-        observation: The observation to verify
-        memories: Dict mapping memory_id to memory content
-
-    Returns:
-        Tuple of (is_valid, list of error messages)
-    """
-    errors = []
-
-    for evidence in observation.evidence:
-        memory_content = memories.get(evidence.memory_id)
-        if memory_content is None:
-            errors.append(f"Memory {evidence.memory_id} not found")
-            continue
-
-        if evidence.quote not in memory_content:
-            errors.append(f"Quote not found in memory {evidence.memory_id}: '{evidence.quote[:50]}...'")
-
-    return len(errors) == 0, errors

hindsight-api/hindsight_api/engine/reflect/prompts.py

@@ -1,762 +0,0 @@
-"""
-System prompts for the reflect agent.
-"""
-
-import json
-from typing import Any
-
-
-def _extract_directive_rules(directives: list[dict[str, Any]]) -> list[str]:
-    """
-    Extract directive rules as a list of strings.
-
-    Args:
-        directives: List of directive mental models with observations
-
-    Returns:
-        List of directive rule strings
-    """
-    rules = []
-    for directive in directives:
-        directive_name = directive.get("name", "")
-        observations = directive.get("observations", [])
-        if observations:
-            for obs in observations:
-                # Support both Pydantic Observation objects and dicts
-                if hasattr(obs, "title"):
-                    title = obs.title
-                    content = obs.content
-                else:
-                    title = obs.get("title", "")
-                    content = obs.get("content", "")
-                if title and content:
-                    rules.append(f"**{title}**: {content}")
-                elif content:
-                    rules.append(content)
-        elif directive_name:
-            # Fallback to description if no observations
-            desc = directive.get("description", "")
-            if desc:
-                rules.append(f"**{directive_name}**: {desc}")
-    return rules
-
-
-def build_directives_section(directives: list[dict[str, Any]]) -> str:
-    """
-    Build the directives section for the system prompt.
-
-    Directives are hard rules that MUST be followed in all responses.
-
-    Args:
-        directives: List of directive mental models with observations
-    """
-    if not directives:
-        return ""
-
-    rules = _extract_directive_rules(directives)
-    if not rules:
-        return ""
-
-    parts = [
-        "## DIRECTIVES (MANDATORY)",
-        "These are hard rules you MUST follow in ALL responses:",
-        "",
-    ]
-
-    for rule in rules:
-        parts.append(f"- {rule}")
-
-    parts.extend(
-        [
-            "",
-            "NEVER violate these directives, even if other context suggests otherwise.",
-            "IMPORTANT: Do NOT explain or justify how you handled directives in your answer. Just follow them silently.",
-            "",
-        ]
-    )
-    return "\n".join(parts)
-
-
-def build_directives_reminder(directives: list[dict[str, Any]]) -> str:
-    """
-    Build a reminder section for directives to place at the end of the prompt.
-
-    Args:
-        directives: List of directive mental models with observations
-    """
-    if not directives:
-        return ""
-
-    rules = _extract_directive_rules(directives)
-    if not rules:
-        return ""
-
-    parts = [
-        "",
-        "## REMINDER: MANDATORY DIRECTIVES",
-        "Before responding, ensure your answer complies with ALL of these directives:",
-        "",
-    ]
-
-    for i, rule in enumerate(rules, 1):
-        parts.append(f"{i}. {rule}")
-
-    parts.append("")
-    parts.append("Your response will be REJECTED if it violates any directive above.")
-    parts.append("Do NOT include any commentary about how you handled directives - just follow them.")
-    return "\n".join(parts)
-
-
-def build_system_prompt_for_tools(
-    bank_profile: dict[str, Any],
-    context: str | None = None,
-    directives: list[dict[str, Any]] | None = None,
-) -> str:
-    """
-    Build the system prompt for tool-calling reflect agent.
-
-    This is a simplified prompt since tools are defined separately via the tools parameter.
-
-    Args:
-        bank_profile: Bank profile with name and mission
-        context: Optional additional context
-        directives: Optional list of directive mental models to inject as hard rules
-    """
-    name = bank_profile.get("name", "Assistant")
-    mission = bank_profile.get("mission", "")
-
-    no_info_rule = (
-        "- Only say 'I don't have information' AFTER trying list_mental_models AND recall with no relevant results"
-    )
-
-    parts = []
-
-    # Inject directives at the VERY START for maximum prominence
-    if directives:
-        parts.append(build_directives_section(directives))
-
-    parts.extend(
-        [
-            "You are a reflection agent that answers questions by reasoning over retrieved memories.",
-            "",
-        ]
-    )
-
-    parts.extend(
-        [
-            "## CRITICAL RULES",
-            "- You must NEVER fabricate information that has no basis in retrieved data",
-            "- You SHOULD synthesize, infer, and reason from the retrieved memories",
-            "- You MUST call recall() before saying you don't have information",
-            no_info_rule,
-            "",
-            "## How to Reason",
-            "- If memories mention someone did an activity, you can infer they likely enjoyed it",
-            "- Synthesize a coherent narrative from related memories",
-            "- Be a thoughtful interpreter, not just a literal repeater",
-            "- When the exact answer isn't stated, use what IS stated to give the best answer",
-            "",
-            "## Query Strategy (IMPORTANT)",
-            "recall() uses semantic search. NEVER just echo the user's question - decompose it into targeted searches:",
-            "",
-            "BAD: User asks 'recurring lesson themes between students' → recall('recurring lesson themes between students')",
-            "GOOD: Break it down into component searches:",
-            "  1. recall('lessons') - find all lesson-related memories",
-            "  2. recall('teaching sessions') - alternative phrasing",
-            "  3. recall('student progress') - find student-related memories",
-            "  4. recall('topics taught') - find subject matter",
-            "",
-            "Think: What ENTITIES and CONCEPTS does this question involve? Search for each separately.",
-            "- Questions about patterns → search for the individual instances first",
-            "- Questions comparing things → search for each thing separately",
-            "- Questions about relationships → search for each party involved",
-            "",
-            "## Workflow",
-        ]
-    )
-
-    # Answer mode: include mental model lookup in workflow
-    parts.extend(
-        [
-            "1. Review the pre-fetched mental models for relevant synthesized knowledge",
-            "2. If relevant, call get_mental_model(model_id) for full observations",
-            "3. DECOMPOSE the question into component searches (see Query Strategy above)",
-            "   - Identify entities and concepts in the question",
-            "   - Search for each separately with targeted queries",
-            "4. Run multiple recall() calls - don't just echo the user's question",
-            "5. Use expand() if you need more context on specific memories",
-            "6. BEFORE answering: Check if any person/project/concept from the memories deserves a mental model - use learn() if so",
-            "7. When ready, call done() with your answer and supporting memory_ids",
-            "",
-            "## When to Use learn() - IMPORTANT",
-            "ACTIVELY look for opportunities to use learn() when you discover:",
-            "- A person mentioned in 2+ memories who has no mental model yet",
-            "- A project or concept the user asks about that has no mental model",
-            "- A pattern or topic worth tracking for future questions",
-            "",
-            "DO NOT wait to be asked - proactively create models when you see the need.",
-            "Example: learn(name='Project Alpha', description='Track goals, status, and key decisions for Project Alpha')",
-            "",
-            "## Output Format: Plain Text Answer",
-            "Call done() with a plain text 'answer' field.",
-            "- Do NOT use markdown formatting",
-            "- NEVER include memory IDs, UUIDs, or 'Memory references' in the answer text",
-            "- Put memory IDs ONLY in the memory_ids array parameter, not in the answer",
-        ]
-    )
-
-    parts.append("")
-    parts.append(f"## Memory Bank: {name}")
-
-    if mission:
-        parts.append(f"Mission: {mission}")
-
-    # Disposition traits
-    disposition = bank_profile.get("disposition", {})
-    if disposition:
-        traits = []
-        if "skepticism" in disposition:
-            traits.append(f"skepticism={disposition['skepticism']}")
-        if "literalism" in disposition:
-            traits.append(f"literalism={disposition['literalism']}")
-        if "empathy" in disposition:
-            traits.append(f"empathy={disposition['empathy']}")
-        if traits:
-            parts.append(f"Disposition: {', '.join(traits)}")
-
-    if context:
-        parts.append(f"\n## Additional Context\n{context}")
-
-    # Add directive reminder at the END for recency effect
-    if directives:
-        parts.append(build_directives_reminder(directives))
-
-    return "\n".join(parts)
-
-
-def build_agent_prompt(
-    query: str,
-    context_history: list[dict],
-    bank_profile: dict,
-    additional_context: str | None = None,
-) -> str:
-    """Build the user prompt for the reflect agent."""
-    parts = []
-
-    # Bank identity
-    name = bank_profile.get("name", "Assistant")
-    mission = bank_profile.get("mission", "")
-
-    parts.append(f"## Memory Bank Context\nName: {name}")
-    if mission:
-        parts.append(f"Mission: {mission}")
-
-    # Disposition traits if present
-    disposition = bank_profile.get("disposition", {})
-    if disposition:
-        traits = []
-        if "skepticism" in disposition:
-            traits.append(f"skepticism={disposition['skepticism']}")
-        if "literalism" in disposition:
-            traits.append(f"literalism={disposition['literalism']}")
-        if "empathy" in disposition:
-            traits.append(f"empathy={disposition['empathy']}")
-        if traits:
-            parts.append(f"Disposition: {', '.join(traits)}")
-
-    # Additional context from caller
-    if additional_context:
-        parts.append(f"\n## Additional Context\n{additional_context}")
-
-    # Tool call history
-    if context_history:
-        parts.append("\n## Tool Results (synthesize and reason from this data)")
-        for i, entry in enumerate(context_history, 1):
-            tool = entry["tool"]
-            output = entry["output"]
-            # Format as proper JSON for LLM readability
-            try:
-                output_str = json.dumps(output, indent=2, default=str)
-            except (TypeError, ValueError):
-                output_str = str(output)
-            parts.append(f"\n### Call {i}: {tool}\n```json\n{output_str}\n```")
-
-    # The question
-    parts.append(f"\n## Question\n{query}")
-
-    # Instructions
-    if context_history:
-        parts.append(
-            "\n## Instructions\n"
-            "Based on the tool results above, either call more tools or provide your final answer. "
-            "Synthesize and reason from the data - make reasonable inferences when helpful. "
-            "If you have related information, use it to give the best possible answer."
-        )
-    else:
-        parts.append(
-            "\n## Instructions\n"
-            "Start by calling list_mental_models() to see available mental models - they contain pre-synthesized knowledge. "
-            "If a relevant model exists, use get_mental_model(model_id) to get its observations. "
-            "Then use recall(query) for specific details not covered by mental models."
-        )
-
-    return "\n".join(parts)
-
-
-def build_final_prompt(
-    query: str,
-    context_history: list[dict],
-    bank_profile: dict,
-    additional_context: str | None = None,
-) -> str:
-    """Build the final prompt when forcing a text response (no tools)."""
-    parts = []
-
-    # Bank identity
-    name = bank_profile.get("name", "Assistant")
-    mission = bank_profile.get("mission", "")
-
-    parts.append(f"## Memory Bank Context\nName: {name}")
-    if mission:
-        parts.append(f"Mission: {mission}")
-
-    # Disposition traits if present
-    disposition = bank_profile.get("disposition", {})
-    if disposition:
-        traits = []
-        if "skepticism" in disposition:
-            traits.append(f"skepticism={disposition['skepticism']}")
-        if "literalism" in disposition:
-            traits.append(f"literalism={disposition['literalism']}")
-        if "empathy" in disposition:
-            traits.append(f"empathy={disposition['empathy']}")
-        if traits:
-            parts.append(f"Disposition: {', '.join(traits)}")
-
-    # Additional context from caller
-    if additional_context:
-        parts.append(f"\n## Additional Context\n{additional_context}")
-
-    # Tool call history
-    if context_history:
-        parts.append("\n## Retrieved Data (synthesize and reason from this data)")
-        for entry in context_history:
-            tool = entry["tool"]
-            output = entry["output"]
-            # Format as proper JSON for LLM readability
-            try:
-                output_str = json.dumps(output, indent=2, default=str)
-            except (TypeError, ValueError):
-                output_str = str(output)
-            parts.append(f"\n### From {tool}:\n```json\n{output_str}\n```")
-    else:
-        parts.append("\n## Retrieved Data\nNo data was retrieved.")
-
-    # The question
-    parts.append(f"\n## Question\n{query}")
-
-    # Final instructions
-    parts.append(
-        "\n## Instructions\n"
-        "Provide a thoughtful answer by synthesizing and reasoning from the retrieved data above. "
-        "You can make reasonable inferences from the memories, but don't completely fabricate information."
-        "If the exact answer isn't stated, use what IS stated to give the best possible answer. "
-        "Only say 'I don't have information' if the retrieved data is truly unrelated to the question."
-    )
-
-    return "\n".join(parts)
-
-
-FINAL_SYSTEM_PROMPT = """You are a thoughtful assistant that synthesizes answers from retrieved memories.
-
-Your approach:
-- Reason over the retrieved memories to answer the question
-- Make reasonable inferences when the exact answer isn't explicitly stated
-- Connect related memories to form a complete picture
-- Be helpful - if you have related information, use it to give the best possible answer
-
-Only say "I don't have information" if the retrieved data is truly unrelated to the question.
-Do NOT fabricate information that has no basis in the retrieved data."""
-
-
-# =============================================================================
-# 4-Phase Mental Model Reflect Prompts
-# =============================================================================
-
-SEED_PHASE_SYSTEM_PROMPT = """You are analyzing memories to discover NEW patterns and generate candidate observations.
-
-Your task is to identify potential observations (beliefs, preferences, patterns, behaviors) that could be part of a mental model about this person/topic.
-
-## Important: Avoid Redundancy
-If existing observations are provided, DO NOT generate candidates that are essentially the same.
-Focus on discovering NEW patterns not already covered by existing observations.
-
-## Rules
-- Generate 5-15 candidate observations for NEW patterns only
-- Each candidate should be specific and testable (can be supported or contradicted by evidence)
-- Note which memory IDs inspired each candidate (these are seeds, not final evidence)
-- Focus on patterns that appear MULTIPLE TIMES across many memories - the more the better
-- The best candidates are ones you can find 10, 20, or even 50+ supporting memories for
-- Skip patterns that are already covered by existing observations
-
-## Output Format
-Return a JSON array of candidate observations:
-```json
-{
-  "candidates": [
-    {
-      "content": "The specific observation/belief/pattern - be detailed and specific",
-      "seed_memory_ids": ["memory_id_1", "memory_id_2", "memory_id_3"]
-    }
-  ]
-}
-```
-
-Focus on patterns that appear multiple times or have strong signals. Don't generate obvious or trivial observations.
-Prefer candidates with MORE seed memories - they're more likely to be real patterns.
-Return an empty candidates array if no genuinely new patterns are found."""
-
-
-def build_seed_phase_prompt(
-    memories: list[dict],
-    topic: str | None = None,
-    existing_observations: list[dict] | None = None,
-) -> str:
-    """Build the user prompt for the seed phase.
-
-    Args:
-        memories: List of memories to analyze
-        topic: Optional topic focus for the mental model
-        existing_observations: Optional list of existing observations to avoid rediscovering
-    """
-    parts = []
-
-    if topic:
-        parts.append(f"## Topic Focus\n{topic}\n")
-
-    # Include existing observations so we don't rediscover them
-    if existing_observations:
-        parts.append("## Existing Observations (DO NOT regenerate these)")
-        parts.append("These patterns are already tracked. Focus on discovering NEW patterns:\n")
-        for i, obs in enumerate(existing_observations, 1):
-            title = obs.get("title", "")
-            content = obs.get("content", "")
-            parts.append(f"{i}. **{title}**: {content}\n")
-        parts.append("")
-
-    parts.append("## Memories to Analyze")
-    parts.append("Review these memories and identify patterns, preferences, beliefs, and behaviors:\n")
-
-    for mem in memories:
-        mem_id = mem.get("id", "unknown")
-        content = mem.get("content", mem.get("text", ""))
-        timestamp = mem.get("timestamp", mem.get("created_at", ""))
-        parts.append(f"[{mem_id}] ({timestamp}): {content}\n")
-
-    parts.append("\n## Instructions")
-    if existing_observations:
-        parts.append("Generate candidate observations for NEW patterns not already covered above.")
-        parts.append("If all patterns are already covered by existing observations, return an empty candidates array.")
-    else:
-        parts.append("Generate candidate observations based on patterns you see in these memories.")
-    parts.append("Look for: recurring themes, stated preferences, behavioral patterns, beliefs, values, goals.")
-
-    return "\n".join(parts)
-
-
-VALIDATE_PHASE_SYSTEM_PROMPT = """You are validating candidate observations against evidence.
-
-For each candidate, you have:
-- Supporting memories (evidence FOR the observation)
-- Contradicting memories (evidence AGAINST the observation)
-
-## Your Task
-1. Evaluate each candidate based on the evidence
-2. For valid candidates, extract EXACT QUOTES from supporting memories
-3. Discard candidates with insufficient or contradicting evidence
-4. Merge similar candidates into single, refined observations
-
-## Rules for Quotes
-- Quotes must be EXACT text from the memory, not paraphrased
-- Each quote should directly support the observation
-- The MORE evidence quotes, the BETTER - don't limit yourself, include ALL relevant quotes (10, 20, 50+)
-- Observations with only 1-2 quotes are weak and should be discarded unless the evidence is exceptionally strong
-- Stronger observations have more supporting evidence - aim for comprehensive coverage
-
-## Output Format
-Return validated observations with evidence:
-```json
-{
-  "observations": [
-    {
-      "title": "Short descriptive title (3-8 words) - like a headline",
-      "content": "The full observation content - detailed explanation of the pattern/belief",
-      "evidence": [
-        {
-          "memory_id": "exact_memory_id",
-          "quote": "Exact quote from the memory text",
-          "relevance": "Brief explanation of how this supports the observation",
-          "timestamp": "2024-01-15T10:00:00Z"
-        }
-      ]
-    }
-  ],
-  "discarded": [
-    {
-      "content": "The discarded candidate",
-      "reason": "Why it was discarded (insufficient evidence, contradicted, etc.)"
-    }
-  ],
-  "merged": [
-    {
-      "from": ["candidate 1 content", "candidate 2 content"],
-      "into": "The merged observation content"
-    }
-  ]
-}
-```
-
-## Title Guidelines
-- Title should be a SHORT label (like "Prefers morning meetings" or "Coffee enthusiast")
-- NOT a truncated version of the content
-- Think of it as a category/tag for the observation
-
-Be rigorous: only keep observations with clear, verifiable evidence from multiple memories."""
-
-
-def build_validate_phase_prompt(candidates_with_evidence: list[dict]) -> str:
-    """Build the user prompt for the validate phase."""
-    parts = ["## Candidates to Validate\n"]
-
-    for i, item in enumerate(candidates_with_evidence, 1):
-        candidate = item.get("candidate", {})
-        supporting = item.get("supporting_memories", [])
-        contradicting = item.get("contradicting_memories", [])
-
-        parts.append(f"### Candidate {i}: {candidate.get('content', '')}")
-
-        if supporting:
-            parts.append("\n**Supporting Evidence:**")
-            for mem in supporting:
-                mem_id = mem.get("id", "unknown")
-                content = mem.get("content", mem.get("text", ""))
-                timestamp = mem.get("timestamp", mem.get("created_at", ""))
-                parts.append(f"- [{mem_id}] ({timestamp}): {content}")
-
-        if contradicting:
-            parts.append("\n**Contradicting Evidence:**")
-            for mem in contradicting:
-                mem_id = mem.get("id", "unknown")
-                content = mem.get("content", mem.get("text", ""))
-                timestamp = mem.get("timestamp", mem.get("created_at", ""))
-                parts.append(f"- [{mem_id}] ({timestamp}): {content}")
-
-        if not supporting and not contradicting:
-            parts.append("\n*No additional evidence found*")
-
-        parts.append("")
-
-    parts.append("## Instructions")
-    parts.append("1. Evaluate each candidate based on its evidence")
-    parts.append("2. Keep candidates with strong supporting evidence")
-    parts.append("3. Discard candidates with no evidence or strong contradictions")
-    parts.append("4. Merge similar candidates")
-    parts.append("5. Extract EXACT quotes (copy-paste from memory text) for evidence")
-
-    return "\n".join(parts)
-
-
-COMPARE_PHASE_SYSTEM_PROMPT = """You are merging new observations with an existing mental model.
-
-You have:
-- EXISTING observations (from the current mental model)
-- NEW observations (from this reflect cycle)
-
-## Your Task
-Produce the final, complete mental model by:
-1. Keeping existing observations that are still valid
-2. Updating existing observations with new evidence (ADD new evidence to existing)
-3. Adding new observations that don't overlap with existing
-4. Removing existing observations that are contradicted by new evidence
-5. Merging overlapping observations
-
-## Rules
-- The final model should have no contradictions
-- Each observation must have evidence with exact quotes
-- COMBINE evidence from both existing and new observations
-- If an existing observation has new supporting evidence, ADD ALL the new evidence to it
-- Include ALL relevant evidence - the more quotes the better (10, 20, 50+ is great)
-- Observations with more evidence are more reliable - don't limit the number of quotes
-
-## Output Format
-Return the complete, final mental model:
-```json
-{
-  "observations": [
-    {
-      "title": "Short descriptive title (3-8 words)",
-      "content": "Full observation content - detailed explanation",
-      "evidence": [
-        {
-          "memory_id": "id",
-          "quote": "exact quote",
-          "relevance": "explanation",
-          "timestamp": "ISO timestamp"
-        }
-      ],
-      "created_at": "ISO timestamp of when observation was first created"
-    }
-  ],
-  "changes": {
-    "kept": ["Observation that was kept unchanged"],
-    "updated": [{"from": "old content", "to": "new content", "reason": "why"}],
-    "added": ["New observation that was added"],
-    "removed": [{"content": "removed observation", "reason": "why removed"}],
-    "merged": [{"from": ["obs1", "obs2"], "into": "merged observation"}]
-  }
-}
-```"""
-
-
-def build_compare_phase_prompt(
-    existing_observations: list[dict],
-    new_observations: list[dict],
-) -> str:
-    """Build the user prompt for the compare phase."""
-    parts = []
-
-    parts.append("## Existing Mental Model Observations")
-    if existing_observations:
-        for i, obs in enumerate(existing_observations, 1):
-            title = obs.get("title", "")
-            content = obs.get("content", obs.get("text", ""))
-            evidence = obs.get("evidence", [])
-            parts.append(f"\n### Existing {i}: {title}")
-            parts.append(f"Content: {content}")
-            if evidence:
-                parts.append(f"Evidence ({len(evidence)} items):")
-                for ev in evidence[:5]:  # Show max 5 evidence items
-                    parts.append(f'  - [{ev.get("memory_id", "?")}]: "{ev.get("quote", "")}"')
-                if len(evidence) > 5:
-                    parts.append(f"  ... and {len(evidence) - 5} more")
-    else:
-        parts.append("*No existing observations*")
-
-    parts.append("\n## New Observations from This Reflect")
-    if new_observations:
-        for i, obs in enumerate(new_observations, 1):
-            title = obs.get("title", "")
-            content = obs.get("content", "")
-            evidence = obs.get("evidence", [])
-            parts.append(f"\n### New {i}: {title}")
-            parts.append(f"Content: {content}")
-            if evidence:
-                parts.append(f"Evidence ({len(evidence)} items):")
-                for ev in evidence:
-                    parts.append(f'  - [{ev.get("memory_id", "?")}]: "{ev.get("quote", "")}"')
-    else:
-        parts.append("*No new observations*")
-
-    parts.append("\n## Instructions")
-    parts.append("Merge these into a coherent, non-contradictory mental model.")
-    parts.append("Preserve all valid evidence. Remove stale or contradicted observations.")
-
-    return "\n".join(parts)
-
-
-# =============================================================================
-# UPDATE EXISTING Phase Prompts (for diff-based refresh)
-# =============================================================================
-
-UPDATE_EXISTING_SYSTEM_PROMPT = """You are updating existing observations with newly found evidence.
-
-For each existing observation, you have been given:
-- The original observation (title, content, existing evidence)
-- Newly found supporting memories
-- Newly found contradicting memories
-
-## Your Task
-1. Extract EXACT QUOTES from new supporting memories to add to the observation
-2. Flag observations with strong contradicting evidence for potential removal
-3. Keep existing evidence intact - only ADD new evidence
-
-## Rules for Quotes
-- Quotes must be EXACT text from the memory, not paraphrased
-- Each quote should directly support the observation
-- Include ALL relevant quotes from the new memories
-
-## Output Format
-Return updated observations with new evidence:
-```json
-{
-  "updated_observations": [
-    {
-      "title": "Original title",
-      "content": "Original content",
-      "existing_evidence_count": 5,
-      "new_evidence": [
-        {
-          "memory_id": "exact_memory_id",
-          "quote": "Exact quote from the memory text",
-          "relevance": "Brief explanation of how this supports the observation",
-          "timestamp": "2024-01-15T10:00:00Z"
-        }
-      ],
-      "has_contradiction": false,
-      "contradiction_note": null
-    }
-  ]
-}
-```
-
-If an observation has strong contradicting evidence, set has_contradiction=true and explain in contradiction_note."""
-
-
-def build_update_existing_prompt(observations_with_evidence: list[dict]) -> str:
-    """Build the user prompt for the update existing phase.
-
-    Args:
-        observations_with_evidence: List of existing observations with new evidence found
-    """
-    parts = ["## Existing Observations to Update\n"]
-
-    for i, item in enumerate(observations_with_evidence, 1):
-        obs = item.get("observation", {})
-        supporting = item.get("supporting_memories", [])
-        contradicting = item.get("contradicting_memories", [])
-
-        title = obs.get("title", "")
-        content = obs.get("content", "")
-        existing_evidence = obs.get("evidence", [])
-
-        parts.append(f"### Observation {i}: {title}")
-        parts.append(f"Content: {content}")
-        parts.append(f"Existing evidence count: {len(existing_evidence)}")
-
-        if supporting:
-            parts.append("\n**New Supporting Memories:**")
-            for mem in supporting:
-                mem_id = mem.get("id", "unknown")
-                mem_content = mem.get("content", mem.get("text", ""))
-                timestamp = mem.get("timestamp", mem.get("created_at", ""))
-                parts.append(f"- [{mem_id}] ({timestamp}): {mem_content}")
-
-        if contradicting:
-            parts.append("\n**New Contradicting Memories:**")
-            for mem in contradicting:
-                mem_id = mem.get("id", "unknown")
-                mem_content = mem.get("content", mem.get("text", ""))
-                timestamp = mem.get("timestamp", mem.get("created_at", ""))
-                parts.append(f"- [{mem_id}] ({timestamp}): {mem_content}")
-
-        if not supporting and not contradicting:
-            parts.append("\n*No new evidence found*")
-
-        parts.append("")
-
-    parts.append("## Instructions")
-    parts.append("1. Extract EXACT quotes from new supporting memories")
-    parts.append("2. Flag observations with strong contradictions")
-    parts.append("3. Return the updated observations with new evidence added")
-
-    return "\n".join(parts)

hindsight-api/hindsight_api/engine/reflect/tools.py

@@ -1,450 +0,0 @@
-"""
-Tool implementations for the reflect agent.
-"""
-
-import logging
-import re
-import uuid
-from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Any
-
-from .models import MentalModelInput
-from .observations import Observation, ObservationEvidence, Trend
-
-if TYPE_CHECKING:
-    from asyncpg import Connection
-
-    from ...api.http import RequestContext
-    from ..memory_engine import MemoryEngine
-
-logger = logging.getLogger(__name__)
-
-
-def generate_model_id(name: str) -> str:
-    """Generate a stable ID from mental model name."""
-    # Normalize: lowercase, replace spaces/special chars with hyphens
-    normalized = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
-    # Truncate to reasonable length
-    return normalized[:50]
-
-
-def _parse_observations(observations_raw: list) -> list[Observation]:
-    """Parse raw observation dicts into typed Observation models."""
-    observations: list[Observation] = []
-    for obs in observations_raw:
-        if not isinstance(obs, dict):
-            continue
-
-        try:
-            parsed = Observation(
-                title=obs.get("title", ""),
-                content=obs.get("content", ""),
-                evidence=[
-                    ObservationEvidence(
-                        memory_id=ev.get("memory_id", ""),
-                        quote=ev.get("quote", ""),
-                        relevance=ev.get("relevance", ""),
-                        timestamp=ev.get("timestamp"),
-                    )
-                    for ev in obs.get("evidence", [])
-                    if isinstance(ev, dict)
-                ],
-                created_at=obs.get("created_at"),
-            )
-            observations.append(parsed)
-        except Exception as e:
-            logger.warning(f"Failed to parse observation: {e}")
-            continue
-
-    return observations
-
-
-async def tool_lookup(
-    conn: "Connection",
-    bank_id: str,
-    model_id: str | None = None,
-    tags: list[str] | None = None,
-    tags_match: str = "any",
-) -> dict[str, Any]:
-    """
-    List or get mental models.
-
-    Args:
-        conn: Database connection
-        bank_id: Bank identifier
-        model_id: Optional specific model ID to get (if None, lists all)
-        tags: Optional tags to filter models (when listing)
-        tags_match: How to match tags - "any" (OR), "all" (AND)
-
-    Returns:
-        Dict with either a list of models or a single model's details
-    """
-    if model_id:
-        # Get specific mental model with full details including observations
-        row = await conn.fetchrow(
-            """
-            SELECT id, subtype, name, description, observations, entity_id, last_updated
-            FROM mental_models
-            WHERE id = $1 AND bank_id = $2
-            """,
-            model_id,
-            bank_id,
-        )
-        if row:
-            # Parse observations JSON
-            obs_data = row["observations"] or {"observations": []}
-            if isinstance(obs_data, str):
-                import json
-
-                obs_data = json.loads(obs_data)
-            observations_raw = obs_data.get("observations", []) if isinstance(obs_data, dict) else obs_data
-
-            # Parse observations into typed models
-            observations = _parse_observations(observations_raw)
-
-            return {
-                "found": True,
-                "model": {
-                    "id": row["id"],
-                    "subtype": row["subtype"],
-                    "name": row["name"],
-                    "description": row["description"],
-                    "observations": observations,
-                    "entity_id": str(row["entity_id"]) if row["entity_id"] else None,
-                    "last_updated": row["last_updated"].isoformat() if row["last_updated"] else None,
-                },
-            }
-        return {"found": False, "model_id": model_id}
-    else:
-        # List mental models (compact: id, name, description only)
-        # Full observations are retrieved via get_mental_model(model_id)
-        # NOTE: Directives (subtype='directive') are excluded from listing -
-        # they are injected into the system prompt, not discoverable via tools
-        # Filter by tags if provided
-        if tags:
-            if tags_match == "all":
-                # All tags must match
-                rows = await conn.fetch(
-                    """
-                    SELECT id, subtype, name, description
-                    FROM mental_models
-                    WHERE bank_id = $1 AND tags @> $2::varchar[] AND subtype != 'directive'
-                    ORDER BY last_updated DESC NULLS LAST, created_at DESC
-                    """,
-                    bank_id,
-                    tags,
-                )
-            else:
-                # Any tag matches (OR) - default
-                rows = await conn.fetch(
-                    """
-                    SELECT id, subtype, name, description
-                    FROM mental_models
-                    WHERE bank_id = $1 AND tags && $2::varchar[] AND subtype != 'directive'
-                    ORDER BY last_updated DESC NULLS LAST, created_at DESC
-                    """,
-                    bank_id,
-                    tags,
-                )
-        else:
-            rows = await conn.fetch(
-                """
-                SELECT id, subtype, name, description
-                FROM mental_models
-                WHERE bank_id = $1 AND subtype != 'directive'
-                ORDER BY last_updated DESC NULLS LAST, created_at DESC
-                """,
-                bank_id,
-            )
-
-        return {
-            "count": len(rows),
-            "models": [
-                {
-                    "id": row["id"],
-                    "subtype": row["subtype"],
-                    "name": row["name"],
-                    "description": row["description"],
-                }
-                for row in rows
-            ],
-        }
-
-
-async def tool_recall(
-    memory_engine: "MemoryEngine",
-    bank_id: str,
-    query: str,
-    request_context: "RequestContext",
-    max_tokens: int = 2048,
-    max_results: int = 50,
-    tags: list[str] | None = None,
-    tags_match: str = "any",
-    connection_budget: int = 1,
-) -> dict[str, Any]:
-    """
-    Search memories using TEMPR retrieval.
-
-    Args:
-        memory_engine: Memory engine instance
-        bank_id: Bank identifier
-        query: Search query
-        request_context: Request context for authentication
-        max_tokens: Maximum tokens for results (default 2048)
-        max_results: Maximum number of results
-        tags: Filter by tags (includes untagged memories)
-        tags_match: How to match tags - "any" (OR), "all" (AND), or "exact"
-        connection_budget: Max DB connections for this recall (default 1 for internal ops)
-
-    Returns:
-        Dict with list of matching memories
-    """
-    result = await memory_engine.recall_async(
-        bank_id=bank_id,
-        query=query,
-        fact_type=["experience", "world"],  # Exclude opinions
-        max_tokens=max_tokens,
-        enable_trace=False,
-        request_context=request_context,
-        tags=tags,
-        tags_match=tags_match,
-        _connection_budget=connection_budget,
-    )
-
-    memories = []
-    for m in result.results[:max_results]:
-        memories.append(
-            {
-                "id": str(m.id),
-                "text": m.text,
-                "type": m.fact_type,
-                "entities": m.entities or [],
-                "occurred": m.occurred_start,  # Already ISO format string
-            }
-        )
-
-    return {
-        "query": query,
-        "count": len(memories),
-        "memories": memories,
-    }
-
-
-async def tool_learn(
-    conn: "Connection",
-    bank_id: str,
-    input: MentalModelInput,
-    tags: list[str] | None = None,
-) -> dict[str, Any]:
-    """
-    Create a mental model placeholder with subtype='learned'.
-
-    The agent only specifies name and description - actual observations are generated
-    in the background via refresh, similar to pinned models.
-
-    Args:
-        conn: Database connection
-        bank_id: Bank identifier
-        input: Mental model input data (name, description, optional entity_id)
-        tags: Tags to apply to new mental models (from reflect context)
-
-    Returns:
-        Dict with created model info including model_id for background generation
-    """
-    model_id = generate_model_id(input.name)
-
-    # Parse entity_id if provided
-    entity_uuid = None
-    if input.entity_id:
-        try:
-            entity_uuid = uuid.UUID(input.entity_id)
-        except ValueError:
-            logger.warning(f"Invalid entity_id format: {input.entity_id}")
-
-    # Check if model exists
-    existing = await conn.fetchrow(
-        "SELECT id FROM mental_models WHERE id = $1 AND bank_id = $2",
-        model_id,
-        bank_id,
-    )
-
-    if existing:
-        # Update description only - observations will be regenerated
-        await conn.execute(
-            """
-            UPDATE mental_models SET
-                description = $3,
-                entity_id = $4
-            WHERE id = $1 AND bank_id = $2
-            """,
-            model_id,
-            bank_id,
-            input.description,
-            entity_uuid,
-        )
-        status = "updated"
-    else:
-        # Insert new model placeholder - observations will be generated in background
-        await conn.execute(
-            """
-            INSERT INTO mental_models (id, bank_id, subtype, name, description, observations, entity_id, tags, created_at)
-            VALUES ($1, $2, 'learned', $3, $4, '{}'::jsonb, $5, $6, NOW())
-            """,
-            model_id,
-            bank_id,
-            input.name,
-            input.description,
-            entity_uuid,
-            tags or [],
-        )
-        status = "created"
-
-    logger.info(f"[REFLECT] Mental model '{model_id}' {status} in bank {bank_id} - pending background generation")
-
-    return {
-        "status": status,
-        "model_id": model_id,
-        "name": input.name,
-        "pending_generation": True,
-    }
-
-
-async def tool_expand(
-    conn: "Connection",
-    bank_id: str,
-    memory_ids: list[str],
-    depth: str,
-) -> dict[str, Any]:
-    """
-    Expand multiple memories to get chunk or document context.
-
-    Args:
-        conn: Database connection
-        bank_id: Bank identifier
-        memory_ids: List of memory unit IDs
-        depth: "chunk" or "document"
-
-    Returns:
-        Dict with results array, each containing memory, chunk, and optionally document data
-    """
-    if not memory_ids:
-        return {"error": "memory_ids is required and must not be empty"}
-
-    # Validate and convert UUIDs
-    valid_uuids: list[uuid.UUID] = []
-    errors: dict[str, str] = {}
-    for mid in memory_ids:
-        try:
-            valid_uuids.append(uuid.UUID(mid))
-        except ValueError:
-            errors[mid] = f"Invalid memory_id format: {mid}"
-
-    if not valid_uuids:
-        return {"error": "No valid memory IDs provided", "details": errors}
-
-    # Batch fetch all memory units
-    memories = await conn.fetch(
-        """
-        SELECT id, text, chunk_id, document_id, fact_type, context
-        FROM memory_units
-        WHERE id = ANY($1) AND bank_id = $2
-        """,
-        valid_uuids,
-        bank_id,
-    )
-    memory_map = {row["id"]: row for row in memories}
-
-    # Collect chunk_ids and document_ids for batch fetching
-    chunk_ids = [m["chunk_id"] for m in memories if m["chunk_id"]]
-    doc_ids_from_chunks: set[str] = set()
-    doc_ids_direct: set[str] = set()
-
-    # Batch fetch all chunks
-    chunk_map: dict[str, Any] = {}
-    if chunk_ids:
-        chunks = await conn.fetch(
-            """
-            SELECT chunk_id, chunk_text, chunk_index, document_id
-            FROM chunks
-            WHERE chunk_id = ANY($1)
-            """,
-            chunk_ids,
-        )
-        chunk_map = {row["chunk_id"]: row for row in chunks}
-        if depth == "document":
-            doc_ids_from_chunks = {c["document_id"] for c in chunks if c["document_id"]}
-
-    # Collect direct document IDs (memories without chunks)
-    if depth == "document":
-        for m in memories:
-            if not m["chunk_id"] and m["document_id"]:
-                doc_ids_direct.add(m["document_id"])
-
-    # Batch fetch all documents
-    doc_map: dict[str, Any] = {}
-    all_doc_ids = list(doc_ids_from_chunks | doc_ids_direct)
-    if all_doc_ids:
-        docs = await conn.fetch(
-            """
-            SELECT id, original_text, metadata, retain_params
-            FROM documents
-            WHERE id = ANY($1) AND bank_id = $2
-            """,
-            all_doc_ids,
-            bank_id,
-        )
-        doc_map = {row["id"]: row for row in docs}
-
-    # Build results
-    results: list[dict[str, Any]] = []
-    for mid, mem_uuid in zip(memory_ids, valid_uuids):
-        if mid in errors:
-            results.append({"memory_id": mid, "error": errors[mid]})
-            continue
-
-        memory = memory_map.get(mem_uuid)
-        if not memory:
-            results.append({"memory_id": mid, "error": f"Memory not found: {mid}"})
-            continue
-
-        item: dict[str, Any] = {
-            "memory_id": mid,
-            "memory": {
-                "id": str(memory["id"]),
-                "text": memory["text"],
-                "type": memory["fact_type"],
-                "context": memory["context"],
-            },
-        }
-
-        # Add chunk if available
-        if memory["chunk_id"] and memory["chunk_id"] in chunk_map:
-            chunk = chunk_map[memory["chunk_id"]]
-            item["chunk"] = {
-                "id": chunk["chunk_id"],
-                "text": chunk["chunk_text"],
-                "index": chunk["chunk_index"],
-                "document_id": chunk["document_id"],
-            }
-            # Add document if depth=document
-            if depth == "document" and chunk["document_id"] in doc_map:
-                doc = doc_map[chunk["document_id"]]
-                item["document"] = {
-                    "id": doc["id"],
-                    "full_text": doc["original_text"],
-                    "metadata": doc["metadata"],
-                    "retain_params": doc["retain_params"],
-                }
-        elif memory["document_id"] and depth == "document" and memory["document_id"] in doc_map:
-            # No chunk, but has document_id
-            doc = doc_map[memory["document_id"]]
-            item["document"] = {
-                "id": doc["id"],
-                "full_text": doc["original_text"],
-                "metadata": doc["metadata"],
-                "retain_params": doc["retain_params"],
-            }
-
-        results.append(item)
-
-    return {"results": results, "count": len(results)}

hindsight-api/hindsight_api/engine/reflect/tools_schema.py

@@ -1,218 +0,0 @@
-"""
-Tool schema definitions for the reflect agent.
-
-These are OpenAI-format tool definitions used with native tool calling.
-"""
-
-# Tool definitions in OpenAI format
-TOOL_LIST_MENTAL_MODELS = {
-    "type": "function",
-    "function": {
-        "name": "list_mental_models",
-        "description": "List all available mental models - your synthesized knowledge about entities, concepts, and events. Returns an array of models with id, name, and description.",
-        "parameters": {
-            "type": "object",
-            "properties": {},
-            "required": [],
-        },
-    },
-}
-
-TOOL_GET_MENTAL_MODEL = {
-    "type": "function",
-    "function": {
-        "name": "get_mental_model",
-        "description": "Get full details of a specific mental model including all observations and memory references.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "model_id": {
-                    "type": "string",
-                    "description": "ID of the mental model (from list_mental_models results)",
-                },
-            },
-            "required": ["model_id"],
-        },
-    },
-}
-
-TOOL_RECALL = {
-    "type": "function",
-    "function": {
-        "name": "recall",
-        "description": "Search memories using semantic + temporal retrieval. Returns relevant memories from experience and world knowledge, each with an 'id' you can reference.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "query": {
-                    "type": "string",
-                    "description": "Search query string",
-                },
-                "max_tokens": {
-                    "type": "integer",
-                    "description": "Optional limit on result size (default 2048). Use higher values for broader searches.",
-                },
-            },
-            "required": ["query"],
-        },
-    },
-}
-
-TOOL_LEARN = {
-    "type": "function",
-    "function": {
-        "name": "learn",
-        "description": "Create a new mental model to track an important recurring topic. Use when you discover a person, project, concept, or pattern that appears frequently and would benefit from synthesized knowledge. The model content will be generated automatically.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "name": {
-                    "type": "string",
-                    "description": "Human-readable name (e.g., 'Project Alpha', 'John Smith', 'Product Strategy')",
-                },
-                "description": {
-                    "type": "string",
-                    "description": "What to track and synthesize (e.g., 'Track goals, milestones, blockers, and key decisions for Project Alpha')",
-                },
-            },
-            "required": ["name", "description"],
-        },
-    },
-}
-
-TOOL_EXPAND = {
-    "type": "function",
-    "function": {
-        "name": "expand",
-        "description": "Get more context for one or more memories. Memory hierarchy: memory -> chunk -> document.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "memory_ids": {
-                    "type": "array",
-                    "items": {"type": "string"},
-                    "description": "Array of memory IDs from recall results (batch multiple for efficiency)",
-                },
-                "depth": {
-                    "type": "string",
-                    "enum": ["chunk", "document"],
-                    "description": "chunk: surrounding text chunk, document: full source document",
-                },
-            },
-            "required": ["memory_ids", "depth"],
-        },
-    },
-}
-
-TOOL_DONE_ANSWER = {
-    "type": "function",
-    "function": {
-        "name": "done",
-        "description": "Signal completion with your final answer. Use this when you have gathered enough information to answer the question.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "answer": {
-                    "type": "string",
-                    "description": "Your response as plain text. Do NOT use markdown formatting. NEVER include memory IDs, UUIDs, or 'Memory references' in this text - put IDs only in memory_ids array.",
-                },
-                "memory_ids": {
-                    "type": "array",
-                    "items": {"type": "string"},
-                    "description": "Array of memory IDs that support your answer (put IDs here, NOT in answer text)",
-                },
-                "model_ids": {
-                    "type": "array",
-                    "items": {"type": "string"},
-                    "description": "Array of mental model IDs that support your answer",
-                },
-            },
-            "required": ["answer"],
-        },
-    },
-}
-
-
-def _build_done_tool_with_directives(directive_rules: list[str]) -> dict:
-    """
-    Build the done tool schema with directive compliance field.
-
-    When directives are present, adds a required field that forces the agent
-    to confirm compliance with each directive before submitting.
-
-    Args:
-        directive_rules: List of directive rule strings
-    """
-    from typing import Any, cast
-
-    # Build rules list for description
-    rules_list = "\n".join(f"  {i + 1}. {rule}" for i, rule in enumerate(directive_rules))
-
-    # Build the tool with directive compliance field
-    return {
-        "type": "function",
-        "function": {
-            "name": "done",
-            "description": (
-                "Signal completion with your final answer. IMPORTANT: You must confirm directive compliance before submitting. "
-                "Your answer will be REJECTED if it violates any directive."
-            ),
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "answer": {
-                        "type": "string",
-                        "description": "Your response as plain text. Do NOT use markdown formatting. NEVER include memory IDs, UUIDs, or 'Memory references' in this text - put IDs only in memory_ids array.",
-                    },
-                    "memory_ids": {
-                        "type": "array",
-                        "items": {"type": "string"},
-                        "description": "Array of memory IDs that support your answer (put IDs here, NOT in answer text)",
-                    },
-                    "model_ids": {
-                        "type": "array",
-                        "items": {"type": "string"},
-                        "description": "Array of mental model IDs that support your answer",
-                    },
-                    "directive_compliance": {
-                        "type": "string",
-                        "description": f"REQUIRED: Confirm your answer complies with ALL directives. List each directive and how your answer follows it:\n{rules_list}\n\nFormat: 'Directive 1: [how answer complies]. Directive 2: [how answer complies]...'",
-                    },
-                },
-                "required": ["answer", "directive_compliance"],
-            },
-        },
-    }
-
-
-def get_reflect_tools(enable_learn: bool = True, directive_rules: list[str] | None = None) -> list[dict]:
-    """
-    Get the list of tools for the reflect agent.
-
-    Args:
-        enable_learn: Whether to include the learn tool
-        directive_rules: Optional list of directive rule strings. If provided,
-                        the done() tool will require directive compliance confirmation.
-
-    Returns:
-        List of tool definitions in OpenAI format
-    """
-    tools = []
-
-    # Include mental model tools for lookup
-    tools.append(TOOL_LIST_MENTAL_MODELS)
-    tools.append(TOOL_GET_MENTAL_MODEL)
-    tools.append(TOOL_RECALL)
-
-    if enable_learn:
-        tools.append(TOOL_LEARN)
-
-    tools.append(TOOL_EXPAND)
-
-    # Use directive-aware done tool if directives are present
-    if directive_rules:
-        tools.append(_build_done_tool_with_directives(directive_rules))
-    else:
-        tools.append(TOOL_DONE_ANSWER)
-
-    return tools

hindsight-api/hindsight_api/engine/response_models.py

@@ -10,60 +10,8 @@ from typing import Any
 
 from pydantic import BaseModel, ConfigDict, Field
 
-# Valid fact types for recall operations (excludes 'observation' which is internal, and 'opinion' which is deprecated)
-VALID_RECALL_FACT_TYPES = frozenset(["world", "experience"])
-
-
-class LLMToolCall(BaseModel):
-    """A tool call requested by the LLM."""
-
-    id: str = Field(description="Unique identifier for this tool call")
-    name: str = Field(description="Name of the tool to call")
-    arguments: dict[str, Any] = Field(description="Arguments to pass to the tool")
-
-
-class LLMToolCallResult(BaseModel):
-    """Result from an LLM call that may include tool calls."""
-
-    content: str | None = Field(default=None, description="Text content if any")
-    tool_calls: list[LLMToolCall] = Field(default_factory=list, description="Tool calls requested by the LLM")
-    finish_reason: str | None = Field(default=None, description="Reason the LLM stopped: 'stop', 'tool_calls', etc.")
-
-
-class ToolCallTrace(BaseModel):
-    """A single tool call made during reflect."""
-
-    tool: str = Field(description="Tool name: lookup, recall, learn, expand")
-    input: dict = Field(description="Tool input parameters")
-    output: dict = Field(description="Tool output/result")
-    duration_ms: int = Field(description="Execution time in milliseconds")
-    iteration: int = Field(default=0, description="Iteration number (1-based) when this tool was called")
-
-
-class LLMCallTrace(BaseModel):
-    """A single LLM call made during reflect."""
-
-    scope: str = Field(description="Call scope: agent_1, agent_2, final, etc.")
-    duration_ms: int = Field(description="Execution time in milliseconds")
-
-
-class MentalModelRef(BaseModel):
-    """Reference to a mental model accessed during reflect."""
-
-    id: str = Field(description="Mental model ID")
-    name: str = Field(description="Mental model name")
-    type: str = Field(description="Mental model type: entity, concept, event")
-    subtype: str = Field(description="Mental model subtype: structural, emergent, learned")
-    description: str = Field(description="Brief description")
-    summary: str | None = Field(default=None, description="Full summary (when looked up in detail)")
-
-
-class DirectiveRef(BaseModel):
-    """Reference to a directive that was applied during reflect."""
-
-    id: str = Field(description="Directive mental model ID")
-    name: str = Field(description="Directive name")
-    rules: list[str] = Field(default_factory=list, description="Directive rules/observations that were applied")
+# Valid fact types for recall operations (excludes 'observation' which is internal)
+VALID_RECALL_FACT_TYPES = frozenset(["world", "experience", "opinion"])
 
 
 class TokenUsage(BaseModel):
@@ -137,7 +85,6 @@ class MemoryFact(BaseModel):
                 "metadata": {"source": "slack"},
                 "chunk_id": "bank123_session_abc123_0",
                 "activation": 0.95,
-                "tags": ["user_a", "session_123"],
             }
         }
     )
@@ -155,7 +102,6 @@ class MemoryFact(BaseModel):
     chunk_id: str | None = Field(
         None, description="ID of the chunk this fact was extracted from (format: bank_id_document_id_chunk_index)"
     )
-    tags: list[str] | None = Field(None, description="Visibility scope tags associated with this fact")
 
 
 class ChunkInfo(BaseModel):
@@ -250,22 +196,6 @@ class ReflectResult(BaseModel):
         default=None,
         description="Token usage metrics for the LLM calls made during this reflect operation.",
     )
-    tool_trace: list[ToolCallTrace] = Field(
-        default_factory=list,
-        description="Trace of tool calls made during reflection. Only present when include.tool_calls is enabled.",
-    )
-    llm_trace: list[LLMCallTrace] = Field(
-        default_factory=list,
-        description="Trace of LLM calls made during reflection. Only present when include.tool_calls is enabled.",
-    )
-    mental_models: list[MentalModelRef] = Field(
-        default_factory=list,
-        description="Mental models accessed during reflection, including directives (subtype='directive').",
-    )
-    directives_applied: list[DirectiveRef] = Field(
-        default_factory=list,
-        description="Directive mental models that were applied during this reflection.",
-    )
 
 
 class Opinion(BaseModel):
@@ -329,32 +259,3 @@ class EntityState(BaseModel):
     observations: list[EntityObservation] = Field(
         default_factory=list, description="List of observations about this entity"
     )
-
-
-class MentalModel(BaseModel):
-    """
-    A manually configured mental model for tracking specific topics/areas.
-
-    Mental models are user-defined focus areas that the agent should track
-    and maintain summaries for, unlike auto-extracted entities.
-    """
-
-    model_config = ConfigDict(
-        json_schema_extra={
-            "example": {
-                "id": "team-dynamics",
-                "name": "Team Dynamics",
-                "description": "Track how the team collaborates, communication patterns, conflicts, and resolutions",
-                "summary": "The team has strong collaboration...",
-                "summary_updated_at": "2024-01-15T10:30:00Z",
-                "created_at": "2024-01-10T08:00:00Z",
-            }
-        }
-    )
-
-    id: str = Field(description="Unique identifier (alphanumeric lowercase)")
-    name: str = Field(description="Display name for the mental model")
-    description: str = Field(description="Prompt/directions for what to track and summarize")
-    summary: str | None = Field(None, description="Generated summary based on relevant facts")
-    summary_updated_at: str | None = Field(None, description="ISO format date when summary was last updated")
-    created_at: str = Field(description="ISO format date when the mental model was created")

hindsight-api/hindsight_api/engine/retain/bank_utils.py

@@ -1,5 +1,5 @@
 """
-bank profile utilities for disposition and mission management.
+bank profile utilities for disposition and background management.
 """
 
 import json
@@ -27,18 +27,19 @@ class BankProfile(TypedDict):
 
     name: str
     disposition: DispositionTraits
-    mission: str
+    background: str
 
 
-class MissionMergeResponse(BaseModel):
-    """LLM response for mission merge."""
+class BackgroundMergeResponse(BaseModel):
+    """LLM response for background merge with disposition inference."""
 
-    mission: str = Field(description="Merged mission in first person perspective")
+    background: str = Field(description="Merged background in first person perspective")
+    disposition: DispositionTraits = Field(description="Inferred disposition traits (skepticism, literalism, empathy)")
 
 
 async def get_bank_profile(pool, bank_id: str) -> BankProfile:
     """
-    Get bank profile (name, disposition + mission).
+    Get bank profile (name, disposition + background).
     Auto-creates bank with default values if not exists.
 
     Args:
@@ -46,13 +47,13 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
         bank_id: bank IDentifier
 
     Returns:
-        BankProfile with name, typed DispositionTraits, and mission
+        BankProfile with name, typed DispositionTraits, and background
     """
     async with acquire_with_retry(pool) as conn:
         # Try to get existing bank
         row = await conn.fetchrow(
             f"""
-            SELECT name, disposition, mission
+            SELECT name, disposition, background
             FROM {fq_table("banks")} WHERE bank_id = $1
             """,
             bank_id,
@@ -65,15 +66,13 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
                 disposition_data = json.loads(disposition_data)
 
             return BankProfile(
-                name=row["name"],
-                disposition=DispositionTraits(**disposition_data),
-                mission=row["mission"] or "",
+                name=row["name"], disposition=DispositionTraits(**disposition_data), background=row["background"]
             )
 
         # Bank doesn't exist, create with defaults
         await conn.execute(
             f"""
-            INSERT INTO {fq_table("banks")} (bank_id, name, disposition, mission)
+            INSERT INTO {fq_table("banks")} (bank_id, name, disposition, background)
             VALUES ($1, $2, $3::jsonb, $4)
             ON CONFLICT (bank_id) DO NOTHING
             """,
@@ -83,7 +82,7 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
             "",
         )
 
-        return BankProfile(name=bank_id, disposition=DispositionTraits(**DEFAULT_DISPOSITION), mission="")
+        return BankProfile(name=bank_id, disposition=DispositionTraits(**DEFAULT_DISPOSITION), background="")
 
 
 async def update_bank_disposition(pool, bank_id: str, disposition: dict[str, int]) -> None:
@@ -111,121 +110,244 @@ async def update_bank_disposition(pool, bank_id: str, disposition: dict[str, int
         )
 
 
-async def set_bank_mission(pool, bank_id: str, mission: str) -> None:
+async def merge_bank_background(pool, llm_config, bank_id: str, new_info: str, update_disposition: bool = True) -> dict:
     """
-    Set bank mission (replacing any existing mission).
-
-    Args:
-        pool: Database connection pool
-        bank_id: bank IDentifier
-        mission: The mission text
-    """
-    # Ensure bank exists first
-    await get_bank_profile(pool, bank_id)
-
-    async with acquire_with_retry(pool) as conn:
-        await conn.execute(
-            f"""
-            UPDATE {fq_table("banks")}
-            SET mission = $2,
-                updated_at = NOW()
-            WHERE bank_id = $1
-            """,
-            bank_id,
-            mission,
-        )
-
-
-async def merge_bank_mission(pool, llm_config, bank_id: str, new_info: str) -> dict:
-    """
-    Merge new mission information with existing mission using LLM.
+    Merge new background information with existing background using LLM.
     Normalizes to first person ("I") and resolves conflicts.
+    Optionally infers disposition traits from the merged background.
 
     Args:
         pool: Database connection pool
-        llm_config: LLM configuration for mission merging
+        llm_config: LLM configuration for background merging
         bank_id: bank IDentifier
-        new_info: New mission information to add/merge
+        new_info: New background information to add/merge
+        update_disposition: If True, infer Big Five traits from background (default: True)
 
     Returns:
-        Dict with 'mission' (str) key
+        Dict with 'background' (str) and optionally 'disposition' (dict) keys
     """
     # Get current profile
     profile = await get_bank_profile(pool, bank_id)
-    current_mission = profile["mission"]
+    current_background = profile["background"]
 
-    # Use LLM to merge missions
-    result = await _llm_merge_mission(llm_config, current_mission, new_info)
+    # Use LLM to merge backgrounds and optionally infer disposition
+    result = await _llm_merge_background(llm_config, current_background, new_info, infer_disposition=update_disposition)
 
-    merged_mission = result["mission"]
+    merged_background = result["background"]
+    inferred_disposition = result.get("disposition")
 
     # Update in database
     async with acquire_with_retry(pool) as conn:
-        await conn.execute(
-            f"""
-            UPDATE {fq_table("banks")}
-            SET mission = $2,
-                updated_at = NOW()
-            WHERE bank_id = $1
-            """,
-            bank_id,
-            merged_mission,
-        )
+        if inferred_disposition:
+            # Update both background and disposition
+            await conn.execute(
+                f"""
+                UPDATE {fq_table("banks")}
+                SET background = $2,
+                    disposition = $3::jsonb,
+                    updated_at = NOW()
+                WHERE bank_id = $1
+                """,
+                bank_id,
+                merged_background,
+                json.dumps(inferred_disposition),
+            )
+        else:
+            # Update only background
+            await conn.execute(
+                f"""
+                UPDATE {fq_table("banks")}
+                SET background = $2,
+                    updated_at = NOW()
+                WHERE bank_id = $1
+                """,
+                bank_id,
+                merged_background,
+            )
 
-    return {"mission": merged_mission}
+    response = {"background": merged_background}
+    if inferred_disposition:
+        response["disposition"] = inferred_disposition
+
+    return response
 
 
-async def _llm_merge_mission(llm_config, current: str, new_info: str) -> dict:
+async def _llm_merge_background(llm_config, current: str, new_info: str, infer_disposition: bool = False) -> dict:
     """
-    Use LLM to intelligently merge mission information.
+    Use LLM to intelligently merge background information.
+    Optionally infer Big Five disposition traits from the merged background.
 
     Args:
         llm_config: LLM configuration to use
-        current: Current mission text
+        current: Current background text
         new_info: New information to merge
+        infer_disposition: If True, also infer disposition traits
 
     Returns:
-        Dict with 'mission' (str) key
+        Dict with 'background' (str) and optionally 'disposition' (dict) keys
     """
-    prompt = f"""You are helping maintain an agent's mission statement.
+    if infer_disposition:
+        prompt = f"""You are helping maintain a memory bank's background/profile and infer their disposition. You MUST respond with ONLY valid JSON.
 
-Current mission: {current if current else "(empty)"}
+Current background: {current if current else "(empty)"}
 
 New information to add: {new_info}
 
 Instructions:
-1. Merge the new information with the current mission
-2. If there are conflicts, the NEW information overwrites the old
+1. Merge the new information with the current background
+2. If there are conflicts (e.g., different birthplaces), the NEW information overwrites the old
+3. Keep additions that don't conflict
+4. Output in FIRST PERSON ("I") perspective
+5. Be concise - keep merged background under 500 characters
+6. Infer disposition traits from the merged background (each 1-5 integer):
+   - Skepticism: 1-5 (1=trusting, takes things at face value; 5=skeptical, questions everything)
+   - Literalism: 1-5 (1=flexible interpretation, reads between lines; 5=literal, exact interpretation)
+   - Empathy: 1-5 (1=detached, focuses on facts; 5=empathetic, considers emotional context)
+
+CRITICAL: You MUST respond with ONLY a valid JSON object. No markdown, no code blocks, no explanations. Just the JSON.
+
+Format:
+{{
+  "background": "the merged background text in first person",
+  "disposition": {{
+    "skepticism": 3,
+    "literalism": 3,
+    "empathy": 3
+  }}
+}}
+
+Trait inference examples:
+- "I'm a lawyer" → skepticism: 4, literalism: 5, empathy: 2
+- "I'm a therapist" → skepticism: 2, literalism: 2, empathy: 5
+- "I'm an engineer" → skepticism: 3, literalism: 4, empathy: 3
+- "I've been burned before by trusting people" → skepticism: 5, literalism: 3, empathy: 3
+- "I try to understand what people really mean" → skepticism: 3, literalism: 2, empathy: 4
+- "I take contracts very seriously" → skepticism: 4, literalism: 5, empathy: 2"""
+    else:
+        prompt = f"""You are helping maintain a memory bank's background/profile.
+
+Current background: {current if current else "(empty)"}
+
+New information to add: {new_info}
+
+Instructions:
+1. Merge the new information with the current background
+2. If there are conflicts (e.g., different birthplaces), the NEW information overwrites the old
 3. Keep additions that don't conflict
 4. Output in FIRST PERSON ("I") perspective
 5. Be concise - keep it under 500 characters
-6. Return ONLY the merged mission text, no explanations
+6. Return ONLY the merged background text, no explanations
 
-Merged mission:"""
+Merged background:"""
 
     try:
+        # Prepare messages
         messages = [{"role": "user", "content": prompt}]
 
+        if infer_disposition:
+            # Use structured output with Pydantic model for disposition inference
+            try:
+                parsed = await llm_config.call(
+                    messages=messages,
+                    response_format=BackgroundMergeResponse,
+                    scope="bank_background",
+                    temperature=0.3,
+                    max_completion_tokens=8192,
+                )
+                logger.info(f"Successfully got structured response: background={parsed.background[:100]}")
+
+                # Convert Pydantic model to dict format
+                return {"background": parsed.background, "disposition": parsed.disposition.model_dump()}
+            except Exception as e:
+                logger.warning(f"Structured output failed, falling back to manual parsing: {e}")
+                # Fall through to manual parsing below
+
+        # Manual parsing fallback or non-disposition merge
         content = await llm_config.call(
-            messages=messages, scope="bank_mission", temperature=0.3, max_completion_tokens=8192
+            messages=messages, scope="bank_background", temperature=0.3, max_completion_tokens=8192
         )
 
-        logger.info(f"LLM response for mission merge (first 500 chars): {content[:500]}")
+        logger.info(f"LLM response for background merge (first 500 chars): {content[:500]}")
 
-        merged = content.strip()
-        if not merged or merged.lower() in ["(empty)", "none", "n/a"]:
-            merged = new_info if new_info else ""
-        return {"mission": merged}
+        if infer_disposition:
+            # Parse JSON response - try multiple extraction methods
+            result = None
+
+            # Method 1: Direct parse
+            try:
+                result = json.loads(content)
+                logger.info("Successfully parsed JSON directly")
+            except json.JSONDecodeError:
+                pass
+
+            # Method 2: Extract from markdown code blocks
+            if result is None:
+                # Remove markdown code blocks
+                code_block_match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", content, re.DOTALL)
+                if code_block_match:
+                    try:
+                        result = json.loads(code_block_match.group(1))
+                        logger.info("Successfully extracted JSON from markdown code block")
+                    except json.JSONDecodeError:
+                        pass
+
+            # Method 3: Find nested JSON structure
+            if result is None:
+                # Look for JSON object with nested structure
+                json_match = re.search(
+                    r'\{[^{}]*"background"[^{}]*"disposition"[^{}]*\{[^{}]*\}[^{}]*\}', content, re.DOTALL
+                )
+                if json_match:
+                    try:
+                        result = json.loads(json_match.group())
+                        logger.info("Successfully extracted JSON using nested pattern")
+                    except json.JSONDecodeError:
+                        pass
+
+            # All parsing methods failed - use fallback
+            if result is None:
+                logger.warning(f"Failed to extract JSON from LLM response. Raw content: {content[:200]}")
+                # Fallback: use new_info as background with default disposition
+                return {
+                    "background": new_info if new_info else current if current else "",
+                    "disposition": DEFAULT_DISPOSITION.copy(),
+                }
+
+            # Validate disposition values
+            disposition = result.get("disposition", {})
+            for key in ["skepticism", "literalism", "empathy"]:
+                if key not in disposition:
+                    disposition[key] = 3  # Default to neutral
+                else:
+                    # Clamp to [1, 5] and convert to int
+                    disposition[key] = max(1, min(5, int(disposition[key])))
+
+            result["disposition"] = disposition
+
+            # Ensure background exists
+            if "background" not in result or not result["background"]:
+                result["background"] = new_info if new_info else ""
+
+            return result
+        else:
+            # Just background merge
+            merged = content
+            if not merged or merged.lower() in ["(empty)", "none", "n/a"]:
+                merged = new_info if new_info else ""
+            return {"background": merged}
 
     except Exception as e:
-        logger.error(f"Error merging mission with LLM: {e}")
+        logger.error(f"Error merging background with LLM: {e}")
         # Fallback: just append new info
         if current:
             merged = f"{current} {new_info}".strip()
         else:
             merged = new_info
 
-        return {"mission": merged}
+        result = {"background": merged}
+        if infer_disposition:
+            result["disposition"] = DEFAULT_DISPOSITION.copy()
+        return result
 
 
 async def list_banks(pool) -> list:
@@ -236,12 +358,12 @@ async def list_banks(pool) -> list:
         pool: Database connection pool
 
     Returns:
-        List of dicts with bank_id, name, disposition, mission, created_at, updated_at
+        List of dicts with bank_id, name, disposition, background, created_at, updated_at
     """
     async with acquire_with_retry(pool) as conn:
         rows = await conn.fetch(
             f"""
-            SELECT bank_id, name, disposition, mission, created_at, updated_at
+            SELECT bank_id, name, disposition, background, created_at, updated_at
             FROM {fq_table("banks")}
             ORDER BY updated_at DESC
             """
@@ -259,7 +381,7 @@ async def list_banks(pool) -> list:
                     "bank_id": row["bank_id"],
                     "name": row["name"],
                     "disposition": disposition_data,
-                    "mission": row["mission"] or "",
+                    "background": row["background"],
                     "created_at": row["created_at"].isoformat() if row["created_at"] else None,
                     "updated_at": row["updated_at"].isoformat() if row["updated_at"] else None,
                 }

hindsight-api/hindsight_api/engine/retain/fact_extraction.py

@@ -1268,7 +1268,6 @@ async def extract_facts_from_contents(
                         # mentioned_at: always the event_date (when the conversation/document occurred)
                         mentioned_at=content.event_date,
                         metadata=content.metadata,
-                        tags=content.tags,
                     )
 
                     extracted_facts.append(extracted_fact)

hindsight-api/hindsight_api/engine/retain/fact_storage.py

@@ -45,7 +45,6 @@ async def insert_facts_batch(
     metadata_jsons = []
     chunk_ids = []
     document_ids = []
-    tags_list = []
 
     for fact in facts:
         fact_texts.append(fact.fact_text)
@@ -66,31 +65,16 @@ async def insert_facts_batch(
         chunk_ids.append(fact.chunk_id)
         # Use per-fact document_id if available, otherwise fallback to batch-level document_id
         document_ids.append(fact.document_id if fact.document_id else document_id)
-        # Convert tags to JSON string for proper batch insertion (PostgreSQL unnest doesn't handle 2D arrays well)
-        tags_list.append(json.dumps(fact.tags if fact.tags else []))
 
     # Batch insert all facts
-    # Note: tags are passed as JSON strings and converted back to varchar[] via jsonb_array_elements_text + array_agg
     results = await conn.fetch(
         f"""
-        WITH input_data AS (
-            SELECT * FROM unnest(
-                $2::text[], $3::vector[], $4::timestamptz[], $5::timestamptz[], $6::timestamptz[], $7::timestamptz[],
-                $8::text[], $9::text[], $10::float[], $11::int[], $12::jsonb[], $13::text[], $14::text[], $15::jsonb[]
-            ) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
-                   context, fact_type, confidence_score, access_count, metadata, chunk_id, document_id, tags_json)
-        )
         INSERT INTO {fq_table("memory_units")} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
-                                 context, fact_type, confidence_score, access_count, metadata, chunk_id, document_id, tags)
-        SELECT
-            $1,
-            text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
-            context, fact_type, confidence_score, access_count, metadata, chunk_id, document_id,
-            COALESCE(
-                (SELECT array_agg(elem) FROM jsonb_array_elements_text(tags_json) AS elem),
-                '{{}}'::varchar[]
-            )
-        FROM input_data
+                                 context, fact_type, confidence_score, access_count, metadata, chunk_id, document_id)
+        SELECT $1, * FROM unnest(
+            $2::text[], $3::vector[], $4::timestamptz[], $5::timestamptz[], $6::timestamptz[], $7::timestamptz[],
+            $8::text[], $9::text[], $10::float[], $11::int[], $12::jsonb[], $13::text[], $14::text[]
+        )
         RETURNING id
         """,
         bank_id,
@@ -107,7 +91,6 @@ async def insert_facts_batch(
         metadata_jsons,
         chunk_ids,
         document_ids,
-        tags_list,
     )
 
     unit_ids = [str(row["id"]) for row in results]
@@ -126,7 +109,7 @@ async def ensure_bank_exists(conn, bank_id: str) -> None:
     """
     await conn.execute(
         f"""
-        INSERT INTO {fq_table("banks")} (bank_id, disposition, mission)
+        INSERT INTO {fq_table("banks")} (bank_id, disposition, background)
         VALUES ($1, $2::jsonb, $3)
         ON CONFLICT (bank_id) DO UPDATE
         SET updated_at = NOW()
@@ -138,13 +121,7 @@ async def ensure_bank_exists(conn, bank_id: str) -> None:
 
 
 async def handle_document_tracking(
-    conn,
-    bank_id: str,
-    document_id: str,
-    combined_content: str,
-    is_first_batch: bool,
-    retain_params: dict | None = None,
-    document_tags: list[str] | None = None,
+    conn, bank_id: str, document_id: str, combined_content: str, is_first_batch: bool, retain_params: dict | None = None
 ) -> None:
     """
     Handle document tracking in the database.
@@ -156,7 +133,6 @@ async def handle_document_tracking(
         combined_content: Combined content text from all content items
         is_first_batch: Whether this is the first batch (for chunked operations)
         retain_params: Optional parameters passed during retain (context, event_date, etc.)
-        document_tags: Optional list of tags to associate with the document
     """
     import hashlib
 
@@ -173,14 +149,13 @@ async def handle_document_tracking(
     # Insert document (or update if exists from concurrent operations)
     await conn.execute(
         f"""
-        INSERT INTO {fq_table("documents")} (id, bank_id, original_text, content_hash, metadata, retain_params, tags)
-        VALUES ($1, $2, $3, $4, $5, $6, $7)
+        INSERT INTO {fq_table("documents")} (id, bank_id, original_text, content_hash, metadata, retain_params)
+        VALUES ($1, $2, $3, $4, $5, $6)
         ON CONFLICT (id, bank_id) DO UPDATE
         SET original_text = EXCLUDED.original_text,
             content_hash = EXCLUDED.content_hash,
             metadata = EXCLUDED.metadata,
             retain_params = EXCLUDED.retain_params,
-            tags = EXCLUDED.tags,
             updated_at = NOW()
         """,
         document_id,
@@ -189,5 +164,4 @@ async def handle_document_tracking(
         content_hash,
         json.dumps({}),  # Empty metadata dict
         json.dumps(retain_params) if retain_params else None,
-        document_tags or [],
     )

hindsight-api/hindsight_api/engine/retain/observation_regeneration.py

@@ -0,0 +1,254 @@
+"""
+Observation regeneration for retain pipeline.
+
+Regenerates entity observations as part of the retain transaction.
+"""
+
+import logging
+import time
+import uuid
+from datetime import UTC, datetime
+
+from ...config import get_config
+from ..memory_engine import fq_table
+from ..search import observation_utils
+from . import embedding_utils
+from .types import EntityLink
+
+logger = logging.getLogger(__name__)
+
+
+def utcnow():
+    """Get current UTC time."""
+    return datetime.now(UTC)
+
+
+# Simple dataclass-like container for facts (avoid importing from memory_engine)
+class MemoryFactForObservation:
+    def __init__(self, id: str, text: str, fact_type: str, context: str, occurred_start: str | None):
+        self.id = id
+        self.text = text
+        self.fact_type = fact_type
+        self.context = context
+        self.occurred_start = occurred_start
+
+
+async def regenerate_observations_batch(
+    conn, embeddings_model, llm_config, bank_id: str, entity_links: list[EntityLink], log_buffer: list[str] = None
+) -> None:
+    """
+    Regenerate observations for top entities in this batch.
+
+    Called INSIDE the retain transaction for atomicity - if observations
+    fail, the entire retain batch is rolled back.
+
+    Args:
+        conn: Database connection (from the retain transaction)
+        embeddings_model: Embeddings model for generating observation embeddings
+        llm_config: LLM configuration for observation extraction
+        bank_id: Bank identifier
+        entity_links: Entity links from this batch
+        log_buffer: Optional log buffer for timing
+    """
+    config = get_config()
+    TOP_N_ENTITIES = config.observation_top_entities
+    MIN_FACTS_THRESHOLD = config.observation_min_facts
+
+    if not entity_links:
+        return
+
+    # Count mentions per entity in this batch
+    entity_mention_counts: dict[str, int] = {}
+    for link in entity_links:
+        if link.entity_id:
+            entity_id = str(link.entity_id)
+            entity_mention_counts[entity_id] = entity_mention_counts.get(entity_id, 0) + 1
+
+    if not entity_mention_counts:
+        return
+
+    # Sort by mention count descending and take top N
+    sorted_entities = sorted(entity_mention_counts.items(), key=lambda x: x[1], reverse=True)
+    entities_to_process = [e[0] for e in sorted_entities[:TOP_N_ENTITIES]]
+
+    obs_start = time.time()
+
+    # Convert to UUIDs
+    entity_uuids = [uuid.UUID(eid) if isinstance(eid, str) else eid for eid in entities_to_process]
+
+    # Batch query for entity names
+    entity_rows = await conn.fetch(
+        f"""
+        SELECT id, canonical_name FROM {fq_table("entities")}
+        WHERE id = ANY($1) AND bank_id = $2
+        """,
+        entity_uuids,
+        bank_id,
+    )
+    entity_names = {row["id"]: row["canonical_name"] for row in entity_rows}
+
+    # Batch query for fact counts
+    fact_counts = await conn.fetch(
+        f"""
+        SELECT ue.entity_id, COUNT(*) as cnt
+        FROM {fq_table("unit_entities")} ue
+        JOIN {fq_table("memory_units")} mu ON ue.unit_id = mu.id
+        WHERE ue.entity_id = ANY($1) AND mu.bank_id = $2
+        GROUP BY ue.entity_id
+        """,
+        entity_uuids,
+        bank_id,
+    )
+    entity_fact_counts = {row["entity_id"]: row["cnt"] for row in fact_counts}
+
+    # Filter entities that meet the threshold
+    entities_with_names = []
+    for entity_id in entities_to_process:
+        entity_uuid = uuid.UUID(entity_id) if isinstance(entity_id, str) else entity_id
+        if entity_uuid not in entity_names:
+            continue
+        fact_count = entity_fact_counts.get(entity_uuid, 0)
+        if fact_count >= MIN_FACTS_THRESHOLD:
+            entities_with_names.append((entity_id, entity_names[entity_uuid]))
+
+    if not entities_with_names:
+        return
+
+    # Process entities SEQUENTIALLY (asyncpg doesn't allow concurrent queries on same connection)
+    # We must use the same connection to stay in the retain transaction
+    total_observations = 0
+
+    for entity_id, entity_name in entities_with_names:
+        try:
+            obs_ids = await _regenerate_entity_observations(
+                conn, embeddings_model, llm_config, bank_id, entity_id, entity_name
+            )
+            total_observations += len(obs_ids)
+        except Exception as e:
+            logger.error(f"[OBSERVATIONS] Error processing entity {entity_id}: {e}")
+
+    obs_time = time.time() - obs_start
+    if log_buffer is not None:
+        log_buffer.append(
+            f"[11] Observations: {total_observations} observations for {len(entities_with_names)} entities in {obs_time:.3f}s"
+        )
+
+
+async def _regenerate_entity_observations(
+    conn, embeddings_model, llm_config, bank_id: str, entity_id: str, entity_name: str
+) -> list[str]:
+    """
+    Regenerate observations for a single entity.
+
+    Uses the provided connection (part of retain transaction).
+
+    Args:
+        conn: Database connection (from the retain transaction)
+        embeddings_model: Embeddings model
+        llm_config: LLM configuration
+        bank_id: Bank identifier
+        entity_id: Entity UUID
+        entity_name: Canonical name of the entity
+
+    Returns:
+        List of created observation IDs
+    """
+    entity_uuid = uuid.UUID(entity_id) if isinstance(entity_id, str) else entity_id
+
+    # Get all facts mentioning this entity (exclude observations themselves)
+    rows = await conn.fetch(
+        f"""
+        SELECT mu.id, mu.text, mu.context, mu.occurred_start, mu.fact_type
+        FROM {fq_table("memory_units")} mu
+        JOIN {fq_table("unit_entities")} ue ON mu.id = ue.unit_id
+        WHERE mu.bank_id = $1
+          AND ue.entity_id = $2
+          AND mu.fact_type IN ('world', 'experience')
+        ORDER BY mu.occurred_start DESC
+        LIMIT 50
+        """,
+        bank_id,
+        entity_uuid,
+    )
+
+    if not rows:
+        return []
+
+    # Convert to fact objects for observation extraction
+    facts = []
+    for row in rows:
+        occurred_start = row["occurred_start"].isoformat() if row["occurred_start"] else None
+        facts.append(
+            MemoryFactForObservation(
+                id=str(row["id"]),
+                text=row["text"],
+                fact_type=row["fact_type"],
+                context=row["context"],
+                occurred_start=occurred_start,
+            )
+        )
+
+    # Extract observations using LLM
+    observations = await observation_utils.extract_observations_from_facts(llm_config, entity_name, facts)
+
+    if not observations:
+        return []
+
+    # Delete old observations for this entity
+    await conn.execute(
+        f"""
+        DELETE FROM {fq_table("memory_units")}
+        WHERE id IN (
+            SELECT mu.id
+            FROM {fq_table("memory_units")} mu
+            JOIN {fq_table("unit_entities")} ue ON mu.id = ue.unit_id
+            WHERE mu.bank_id = $1
+              AND mu.fact_type = 'observation'
+              AND ue.entity_id = $2
+        )
+        """,
+        bank_id,
+        entity_uuid,
+    )
+
+    # Generate embeddings for new observations
+    embeddings = await embedding_utils.generate_embeddings_batch(embeddings_model, observations)
+
+    # Insert new observations
+    current_time = utcnow()
+    created_ids = []
+
+    for obs_text, embedding in zip(observations, embeddings):
+        result = await conn.fetchrow(
+            f"""
+            INSERT INTO {fq_table("memory_units")} (
+                bank_id, text, embedding, context, event_date,
+                occurred_start, occurred_end, mentioned_at,
+                fact_type, access_count
+            )
+            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'observation', 0)
+            RETURNING id
+            """,
+            bank_id,
+            obs_text,
+            str(embedding),
+            f"observation about {entity_name}",
+            current_time,
+            current_time,
+            current_time,
+            current_time,
+        )
+        obs_id = str(result["id"])
+        created_ids.append(obs_id)
+
+        # Link observation to entity
+        await conn.execute(
+            f"""
+            INSERT INTO {fq_table("unit_entities")} (unit_id, entity_id)
+            VALUES ($1, $2)
+            """,
+            uuid.UUID(obs_id),
+            entity_uuid,
+        )
+
+    return created_ids

hindsight-api/hindsight_api/engine/retain/orchestrator.py

@@ -9,6 +9,7 @@ import time
 import uuid
 from datetime import UTC, datetime
 
+from ...config import get_config
 from ..db_utils import acquire_with_retry
 from . import bank_utils
 
@@ -27,8 +28,9 @@ from . import (
     fact_extraction,
     fact_storage,
     link_creation,
+    observation_regeneration,
 )
-from .types import EntityLink, ExtractedFact, ProcessedFact, RetainContent, RetainContentDict
+from .types import ExtractedFact, ProcessedFact, RetainContent, RetainContentDict
 
 logger = logging.getLogger(__name__)
 
@@ -38,6 +40,7 @@ async def retain_batch(
     embeddings_model,
     llm_config,
     entity_resolver,
+    task_backend,
     format_date_fn,
     duplicate_checker_fn,
     bank_id: str,
@@ -46,7 +49,6 @@ async def retain_batch(
     is_first_batch: bool = True,
     fact_type_override: str | None = None,
     confidence_score: float | None = None,
-    document_tags: list[str] | None = None,
 ) -> tuple[list[list[str]], TokenUsage]:
     """
     Process a batch of content through the retain pipeline.
@@ -56,6 +58,7 @@ async def retain_batch(
         embeddings_model: Embeddings model for generating embeddings
         llm_config: LLM configuration for fact extraction
         entity_resolver: Entity resolver for entity processing
+        task_backend: Task backend for background jobs
         format_date_fn: Function to format datetime to readable string
         duplicate_checker_fn: Function to check for duplicate facts
         bank_id: Bank identifier
@@ -64,7 +67,6 @@ async def retain_batch(
         is_first_batch: Whether this is the first batch
         fact_type_override: Override fact type for all facts
         confidence_score: Confidence score for opinions
-        document_tags: Tags applied to all items in this batch
 
     Returns:
         Tuple of (unit ID lists, token usage for fact extraction)
@@ -86,16 +88,12 @@ async def retain_batch(
     # Convert dicts to RetainContent objects
     contents = []
     for item in contents_dicts:
-        # Merge item-level tags with document-level tags
-        item_tags = item.get("tags", []) or []
-        merged_tags = list(set(item_tags + (document_tags or [])))
         content = RetainContent(
             content=item["content"],
             context=item.get("context", ""),
             event_date=item.get("event_date") or utcnow(),
             metadata=item.get("metadata", {}),
             entities=item.get("entities", []),
-            tags=merged_tags,
         )
         contents.append(content)
 
@@ -133,7 +131,7 @@ async def retain_batch(
                         if first_item.get("metadata"):
                             retain_params["metadata"] = first_item["metadata"]
                     await fact_storage.handle_document_tracking(
-                        conn, bank_id, document_id, combined_content, is_first_batch, retain_params, document_tags
+                        conn, bank_id, document_id, combined_content, is_first_batch, retain_params
                     )
                 else:
                     # Check for per-item document_ids
@@ -161,7 +159,7 @@ async def retain_batch(
                             if first_item.get("metadata"):
                                 retain_params["metadata"] = first_item["metadata"]
                         await fact_storage.handle_document_tracking(
-                            conn, bank_id, doc_id, combined_content, is_first_batch, retain_params, document_tags
+                            conn, bank_id, doc_id, combined_content, is_first_batch, retain_params
                         )
 
         total_time = time.time() - start_time
@@ -227,7 +225,7 @@ async def retain_batch(
                         retain_params["metadata"] = first_item["metadata"]
 
                 await fact_storage.handle_document_tracking(
-                    conn, bank_id, document_id, combined_content, is_first_batch, retain_params, document_tags
+                    conn, bank_id, document_id, combined_content, is_first_batch, retain_params
                 )
                 document_ids_added.append(document_id)
                 doc_id_mapping[None] = document_id  # For backwards compatibility
@@ -271,13 +269,7 @@ async def retain_batch(
                                     retain_params["metadata"] = first_item["metadata"]
 
                             await fact_storage.handle_document_tracking(
-                                conn,
-                                bank_id,
-                                actual_doc_id,
-                                combined_content,
-                                is_first_batch,
-                                retain_params,
-                                document_tags,
+                                conn, bank_id, actual_doc_id, combined_content, is_first_batch, retain_params
                             )
                             document_ids_added.append(actual_doc_id)
 
@@ -404,9 +396,27 @@ async def retain_batch(
             causal_link_count = await link_creation.create_causal_links_batch(conn, unit_ids, non_duplicate_facts)
             log_buffer.append(f"[10] Causal links: {causal_link_count} links in {time.time() - step_start:.3f}s")
 
+            # Regenerate observations - sync (in transaction) or async (background task)
+            config = get_config()
+            if config.retain_observations_async:
+                # Queue for async processing after transaction commits
+                entity_ids_for_async = list(set(link.entity_id for link in entity_links)) if entity_links else []
+                log_buffer.append(
+                    f"[11] Observations: queued {len(entity_ids_for_async)} entities for async processing"
+                )
+            else:
+                # Run synchronously inside transaction for atomicity
+                await observation_regeneration.regenerate_observations_batch(
+                    conn, embeddings_model, llm_config, bank_id, entity_links, log_buffer
+                )
+                entity_ids_for_async = []
+
             # Map results back to original content items
             result_unit_ids = _map_results_to_contents(contents, extracted_facts, is_duplicate_flags, unit_ids)
 
+        # Trigger background tasks AFTER transaction commits
+        await _trigger_background_tasks(task_backend, bank_id, unit_ids, non_duplicate_facts, entity_ids_for_async)
+
         # Log final summary
         total_time = time.time() - start_time
         log_buffer.append(f"{'=' * 60}")
@@ -448,3 +458,35 @@ def _map_results_to_contents(
         result_unit_ids.append(content_unit_ids)
 
     return result_unit_ids
+
+
+async def _trigger_background_tasks(
+    task_backend,
+    bank_id: str,
+    unit_ids: list[str],
+    facts: list[ProcessedFact],
+    entity_ids_for_observations: list[str] | None = None,
+) -> None:
+    """Trigger background tasks after transaction commits."""
+    # Trigger opinion reinforcement if there are entities
+    fact_entities = [[e.name for e in fact.entities] for fact in facts]
+    if any(fact_entities):
+        await task_backend.submit_task(
+            {
+                "type": "reinforce_opinion",
+                "bank_id": bank_id,
+                "created_unit_ids": unit_ids,
+                "unit_texts": [fact.fact_text for fact in facts],
+                "unit_entities": fact_entities,
+            }
+        )
+
+    # Trigger observation regeneration if async mode is enabled
+    if entity_ids_for_observations:
+        await task_backend.submit_task(
+            {
+                "type": "regenerate_observations",
+                "bank_id": bank_id,
+                "entity_ids": entity_ids_for_observations,
+            }
+        )

hindsight-api/hindsight_api/engine/retain/types.py

@@ -21,7 +21,6 @@ class RetainContentDict(TypedDict, total=False):
         metadata: Custom key-value metadata (optional)
         document_id: Document ID for this content item (optional)
         entities: User-provided entities to merge with extracted entities (optional)
-        tags: Visibility scope tags for this content item (optional)
     """
 
     content: str  # Required
@@ -30,7 +29,6 @@ class RetainContentDict(TypedDict, total=False):
     metadata: dict[str, str]
     document_id: str
     entities: list[dict[str, str]]  # [{"text": "...", "type": "..."}]
-    tags: list[str]  # Visibility scope tags
 
 
 def _now_utc() -> datetime:
@@ -51,7 +49,6 @@ class RetainContent:
     event_date: datetime = field(default_factory=_now_utc)
     metadata: dict[str, str] = field(default_factory=dict)
     entities: list[dict[str, str]] = field(default_factory=list)  # User-provided entities
-    tags: list[str] = field(default_factory=list)  # Visibility scope tags
 
 
 @dataclass
@@ -116,7 +113,6 @@ class ExtractedFact:
     context: str = ""
     mentioned_at: datetime | None = None
     metadata: dict[str, str] = field(default_factory=dict)
-    tags: list[str] = field(default_factory=list)  # Visibility scope tags
 
 
 @dataclass
@@ -162,9 +158,6 @@ class ProcessedFact:
     # Track which content this fact came from (for user entity merging)
     content_index: int = 0
 
-    # Visibility scope tags
-    tags: list[str] = field(default_factory=list)
-
     @property
     def is_duplicate(self) -> bool:
         """Check if this fact was marked as a duplicate."""
@@ -208,7 +201,6 @@ class ProcessedFact:
             causal_relations=extracted_fact.causal_relations,
             chunk_id=chunk_id,
             content_index=extracted_fact.content_index,
-            tags=extracted_fact.tags,
         )
 
 
@@ -240,7 +232,6 @@ class RetainBatch:
     document_id: str | None = None
     fact_type_override: str | None = None
     confidence_score: float | None = None
-    document_tags: list[str] = field(default_factory=list)  # Tags applied to all items
 
     # Extracted data (populated during processing)
     extracted_facts: list[ExtractedFact] = field(default_factory=list)

hindsight-api/hindsight_api/engine/search/graph_retrieval.py

@@ -11,7 +11,6 @@ from abc import ABC, abstractmethod
 
 from ..db_utils import acquire_with_retry
 from ..memory_engine import fq_table
-from .tags import TagsMatch, filter_results_by_tags
 from .types import MPFPTimings, RetrievalResult
 
 logger = logging.getLogger(__name__)
@@ -44,8 +43,6 @@ class GraphRetriever(ABC):
         semantic_seeds: list[RetrievalResult] | None = None,
         temporal_seeds: list[RetrievalResult] | None = None,
         adjacency=None,  # TypedAdjacency, optional pre-loaded graph
-        tags: list[str] | None = None,  # Visibility scope tags for filtering
-        tags_match: TagsMatch = "any",  # How to match tags: 'any' (OR) or 'all' (AND)
     ) -> tuple[list[RetrievalResult], MPFPTimings | None]:
         """
         Retrieve relevant facts via graph traversal.
@@ -60,7 +57,6 @@ class GraphRetriever(ABC):
             semantic_seeds: Pre-computed semantic entry points (from semantic retrieval)
             temporal_seeds: Pre-computed temporal entry points (from temporal retrieval)
             adjacency: Pre-loaded typed adjacency graph (optional, for MPFP)
-            tags: Optional list of tags for visibility filtering (OR matching)
 
         Returns:
             Tuple of (List of RetrievalResult with activation scores, optional timing info)
@@ -118,8 +114,6 @@ class BFSGraphRetriever(GraphRetriever):
         semantic_seeds: list[RetrievalResult] | None = None,
         temporal_seeds: list[RetrievalResult] | None = None,
         adjacency=None,  # Not used by BFS
-        tags: list[str] | None = None,
-        tags_match: TagsMatch = "any",
     ) -> tuple[list[RetrievalResult], MPFPTimings | None]:
         """
         Retrieve facts using BFS spreading activation.
@@ -135,9 +129,7 @@ class BFSGraphRetriever(GraphRetriever):
         for interface compatibility but not used.
         """
         async with acquire_with_retry(pool) as conn:
-            results = await self._retrieve_with_conn(
-                conn, query_embedding_str, bank_id, fact_type, budget, tags=tags, tags_match=tags_match
-            )
+            results = await self._retrieve_with_conn(conn, query_embedding_str, bank_id, fact_type, budget)
             return results, None
 
     async def _retrieve_with_conn(
@@ -147,46 +139,33 @@ class BFSGraphRetriever(GraphRetriever):
         bank_id: str,
         fact_type: str,
         budget: int,
-        tags: list[str] | None = None,
-        tags_match: TagsMatch = "any",
     ) -> list[RetrievalResult]:
         """Internal implementation with connection."""
-        from .tags import build_tags_where_clause_simple
-
-        tags_clause = build_tags_where_clause_simple(tags, 6, match=tags_match)
-        params = [query_embedding_str, bank_id, fact_type, self.entry_point_threshold, self.entry_point_limit]
-        if tags:
-            params.append(tags)
 
         # Step 1: Find entry points
         entry_points = await conn.fetch(
             f"""
             SELECT id, text, context, event_date, occurred_start, occurred_end,
-                   mentioned_at, access_count, embedding, fact_type, document_id, chunk_id, tags,
+                   mentioned_at, access_count, embedding, fact_type, document_id, chunk_id,
                    1 - (embedding <=> $1::vector) AS similarity
             FROM {fq_table("memory_units")}
             WHERE bank_id = $2
               AND embedding IS NOT NULL
               AND fact_type = $3
               AND (1 - (embedding <=> $1::vector)) >= $4
-              {tags_clause}
             ORDER BY embedding <=> $1::vector
             LIMIT $5
             """,
-            *params,
+            query_embedding_str,
+            bank_id,
+            fact_type,
+            self.entry_point_threshold,
+            self.entry_point_limit,
         )
 
         if not entry_points:
-            logger.debug(
-                f"[BFS] No entry points found for fact_type={fact_type} (tags={tags}, tags_match={tags_match})"
-            )
             return []
 
-        logger.debug(
-            f"[BFS] Found {len(entry_points)} entry points for fact_type={fact_type} "
-            f"(tags={tags}, tags_match={tags_match})"
-        )
-
         # Step 2: BFS spreading activation
         visited = set()
         results = []
@@ -217,7 +196,7 @@ class BFSGraphRetriever(GraphRetriever):
                     f"""
                     SELECT mu.id, mu.text, mu.context, mu.occurred_start, mu.occurred_end,
                            mu.mentioned_at, mu.access_count, mu.embedding, mu.fact_type,
-                           mu.document_id, mu.chunk_id, mu.tags,
+                           mu.document_id, mu.chunk_id,
                            ml.weight, ml.link_type, ml.from_unit_id
                     FROM {fq_table("memory_links")} ml
                     JOIN {fq_table("memory_units")} mu ON ml.to_unit_id = mu.id
@@ -257,8 +236,4 @@ class BFSGraphRetriever(GraphRetriever):
                             neighbor_result = RetrievalResult.from_db_row(dict(n))
                             queue.append((neighbor_result, new_activation))
 
-        # Apply tags filtering (BFS may traverse into memories that don't match tags criteria)
-        if tags:
-            results = filter_results_by_tags(results, tags, match=tags_match)
-
         return results

hindsight-api/hindsight_api/engine/search/link_expansion_retrieval.py

@@ -1,256 +0,0 @@
-"""
-Link Expansion graph retrieval.
-
-A simple, fast graph retrieval that expands from seeds via:
-1. Entity links: Find facts sharing entities with seeds (filtered by entity frequency)
-2. Causal links: Find facts causally linked to seeds (top-k by weight)
-
-Characteristics:
-- 2-3 DB queries (seed finding + parallel entity/causal expansion)
-- Sublinear: only touches connected facts via indexes
-- No iteration, no propagation, no normalization
-- Target: <100ms
-"""
-
-import logging
-import time
-
-from ..db_utils import acquire_with_retry
-from ..memory_engine import fq_table
-from .graph_retrieval import GraphRetriever
-from .tags import TagsMatch, filter_results_by_tags
-from .types import MPFPTimings, RetrievalResult
-
-logger = logging.getLogger(__name__)
-
-
-async def _find_semantic_seeds(
-    conn,
-    query_embedding_str: str,
-    bank_id: str,
-    fact_type: str,
-    limit: int = 20,
-    threshold: float = 0.3,
-    tags: list[str] | None = None,
-    tags_match: TagsMatch = "any",
-) -> list[RetrievalResult]:
-    """Find semantic seeds via embedding search."""
-    from .tags import build_tags_where_clause_simple
-
-    tags_clause = build_tags_where_clause_simple(tags, 6, match=tags_match)
-    params = [query_embedding_str, bank_id, fact_type, threshold, limit]
-    if tags:
-        params.append(tags)
-
-    rows = await conn.fetch(
-        f"""
-        SELECT id, text, context, event_date, occurred_start, occurred_end,
-               mentioned_at, access_count, embedding, fact_type, document_id, chunk_id, tags,
-               1 - (embedding <=> $1::vector) AS similarity
-        FROM {fq_table("memory_units")}
-        WHERE bank_id = $2
-          AND embedding IS NOT NULL
-          AND fact_type = $3
-          AND (1 - (embedding <=> $1::vector)) >= $4
-          {tags_clause}
-        ORDER BY embedding <=> $1::vector
-        LIMIT $5
-        """,
-        *params,
-    )
-    return [RetrievalResult.from_db_row(dict(r)) for r in rows]
-
-
-class LinkExpansionRetriever(GraphRetriever):
-    """
-    Graph retrieval via direct link expansion from seeds.
-
-    Expands through entity co-occurrence and causal links in a single query.
-    Fast and simple alternative to MPFP.
-    """
-
-    def __init__(
-        self,
-        max_entity_frequency: int = 500,
-        causal_weight_threshold: float = 0.3,
-        causal_limit_per_seed: int = 10,
-    ):
-        """
-        Initialize link expansion retriever.
-
-        Args:
-            max_entity_frequency: Skip entities appearing in more than this many facts
-            causal_weight_threshold: Minimum weight for causal links
-            causal_limit_per_seed: Max causal links to follow per seed
-        """
-        self.max_entity_frequency = max_entity_frequency
-        self.causal_weight_threshold = causal_weight_threshold
-        self.causal_limit_per_seed = causal_limit_per_seed
-
-    @property
-    def name(self) -> str:
-        return "link_expansion"
-
-    async def retrieve(
-        self,
-        pool,
-        query_embedding_str: str,
-        bank_id: str,
-        fact_type: str,
-        budget: int,
-        query_text: str | None = None,
-        semantic_seeds: list[RetrievalResult] | None = None,
-        temporal_seeds: list[RetrievalResult] | None = None,
-        adjacency=None,
-        tags: list[str] | None = None,
-        tags_match: TagsMatch = "any",
-    ) -> tuple[list[RetrievalResult], MPFPTimings | None]:
-        """
-        Retrieve facts by expanding links from seeds.
-
-        Args:
-            pool: Database connection pool
-            query_embedding_str: Query embedding (unused, kept for interface)
-            bank_id: Memory bank ID
-            fact_type: Fact type to filter
-            budget: Maximum results to return
-            query_text: Original query text (unused)
-            semantic_seeds: Pre-computed semantic entry points
-            temporal_seeds: Pre-computed temporal entry points
-            adjacency: Unused, kept for interface compatibility
-            tags: Optional list of tags for visibility filtering (OR matching)
-
-        Returns:
-            Tuple of (results, timings)
-        """
-        start_time = time.time()
-        timings = MPFPTimings(fact_type=fact_type)
-
-        # Use single connection for all queries to reduce pool pressure
-        # (queries are fast ~50ms each, connection acquisition is the bottleneck)
-        async with acquire_with_retry(pool) as conn:
-            # Find seeds if not provided
-            if semantic_seeds:
-                all_seeds = list(semantic_seeds)
-            else:
-                seeds_start = time.time()
-                all_seeds = await _find_semantic_seeds(
-                    conn,
-                    query_embedding_str,
-                    bank_id,
-                    fact_type,
-                    limit=20,
-                    threshold=0.3,
-                    tags=tags,
-                    tags_match=tags_match,
-                )
-                timings.seeds_time = time.time() - seeds_start
-                logger.debug(
-                    f"[LinkExpansion] Found {len(all_seeds)} semantic seeds for fact_type={fact_type} "
-                    f"(tags={tags}, tags_match={tags_match})"
-                )
-
-            # Add temporal seeds if provided
-            if temporal_seeds:
-                all_seeds.extend(temporal_seeds)
-
-            if not all_seeds:
-                logger.debug("[LinkExpansion] No seeds found, returning empty results")
-                return [], timings
-
-            seed_ids = list({s.id for s in all_seeds})
-            timings.pattern_count = len(seed_ids)
-
-            # Run entity and causal expansion sequentially on same connection
-            query_start = time.time()
-
-            entity_rows = await conn.fetch(
-                f"""
-                SELECT
-                    mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
-                    mu.occurred_end, mu.mentioned_at, mu.access_count, mu.embedding,
-                    mu.fact_type, mu.document_id, mu.chunk_id, mu.tags,
-                    COUNT(*)::float AS score
-                FROM {fq_table("unit_entities")} seed_ue
-                JOIN {fq_table("entities")} e ON seed_ue.entity_id = e.id
-                JOIN {fq_table("unit_entities")} other_ue ON seed_ue.entity_id = other_ue.entity_id
-                JOIN {fq_table("memory_units")} mu ON other_ue.unit_id = mu.id
-                WHERE seed_ue.unit_id = ANY($1::uuid[])
-                  AND e.mention_count < $2
-                  AND mu.id != ALL($1::uuid[])
-                  AND mu.fact_type = $3
-                GROUP BY mu.id
-                ORDER BY score DESC
-                LIMIT $4
-                """,
-                seed_ids,
-                self.max_entity_frequency,
-                fact_type,
-                budget,
-            )
-
-            causal_rows = await conn.fetch(
-                f"""
-                SELECT DISTINCT ON (mu.id)
-                    mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
-                    mu.occurred_end, mu.mentioned_at, mu.access_count, mu.embedding,
-                    mu.fact_type, mu.document_id, mu.chunk_id, mu.tags,
-                    ml.weight + 1.0 AS score
-                FROM {fq_table("memory_links")} ml
-                JOIN {fq_table("memory_units")} mu ON ml.to_unit_id = mu.id
-                WHERE ml.from_unit_id = ANY($1::uuid[])
-                  AND ml.link_type IN ('causes', 'caused_by', 'enables', 'prevents')
-                  AND ml.weight >= $2
-                  AND mu.fact_type = $3
-                ORDER BY mu.id, ml.weight DESC
-                LIMIT $4
-                """,
-                seed_ids,
-                self.causal_weight_threshold,
-                fact_type,
-                budget,
-            )
-
-            timings.edge_load_time = time.time() - query_start
-            timings.db_queries = 2
-            timings.edge_count = len(entity_rows) + len(causal_rows)
-
-        # Merge results, taking max score per fact
-        score_map: dict[str, float] = {}
-        row_map: dict[str, dict] = {}
-
-        for row in entity_rows:
-            fact_id = str(row["id"])
-            score_map[fact_id] = max(score_map.get(fact_id, 0), row["score"])
-            row_map[fact_id] = dict(row)
-
-        for row in causal_rows:
-            fact_id = str(row["id"])
-            score_map[fact_id] = max(score_map.get(fact_id, 0), row["score"])
-            if fact_id not in row_map:
-                row_map[fact_id] = dict(row)
-
-        # Sort by score and limit
-        sorted_ids = sorted(score_map.keys(), key=lambda x: score_map[x], reverse=True)[:budget]
-        rows = [row_map[fact_id] for fact_id in sorted_ids]
-
-        # Convert to results
-        results = []
-        for row in rows:
-            result = RetrievalResult.from_db_row(dict(row))
-            result.activation = row["score"]
-            results.append(result)
-
-        # Apply tags filtering (graph expansion may reach untagged memories)
-        if tags:
-            results = filter_results_by_tags(results, tags, match=tags_match)
-
-        timings.result_count = len(results)
-        timings.traverse = time.time() - start_time
-
-        logger.debug(
-            f"LinkExpansion: {len(results)} results from {len(seed_ids)} seeds "
-            f"in {timings.traverse * 1000:.1f}ms (query: {timings.edge_load_time * 1000:.1f}ms)"
-        )
-
-        return results, timings

hindsight-api/hindsight_api/engine/search/mpfp_retrieval.py

@@ -23,7 +23,6 @@ from dataclasses import dataclass, field
 from ..db_utils import acquire_with_retry
 from ..memory_engine import fq_table
 from .graph_retrieval import GraphRetriever
-from .tags import TagsMatch
 from .types import MPFPTimings, RetrievalResult
 
 logger = logging.getLogger(__name__)
@@ -49,21 +48,12 @@ class EdgeCache:
 
     Grows per-hop as edges are loaded for frontier nodes.
     Shared across patterns to avoid redundant loads.
-    Loads ALL edge types at once to minimize DB queries.
-    Thread-safe via asyncio lock to prevent redundant concurrent loads.
     """
 
     # edge_type -> from_node_id -> list of EdgeTarget
     graphs: dict[str, dict[str, list[EdgeTarget]]] = field(default_factory=dict)
-    # Track which nodes have been fully loaded (all edge types)
-    _fully_loaded: set[str] = field(default_factory=set)
-    # Timing stats
-    db_queries: int = 0
-    edge_load_time: float = 0.0
-    # Detailed hop timing for debugging
-    hop_details: list[dict] = field(default_factory=list)
-    # Lock to prevent redundant concurrent loads
-    _lock: asyncio.Lock = field(default_factory=asyncio.Lock)
+    # Track which (edge_type, node_id) have been loaded
+    _loaded: set[tuple[str, str]] = field(default_factory=set)
 
     def get_neighbors(self, edge_type: str, node_id: str) -> list[EdgeTarget]:
         """Get neighbors for a node via a specific edge type."""
@@ -81,30 +71,32 @@ class EdgeCache:
 
         return [EdgeTarget(node_id=n.node_id, weight=n.weight / total) for n in neighbors]
 
-    def is_fully_loaded(self, node_id: str) -> bool:
-        """Check if all edges for this node have been loaded."""
-        return node_id in self._fully_loaded
+    def is_loaded(self, edge_type: str, node_id: str) -> bool:
+        """Check if edges for this node+type have been loaded."""
+        return (edge_type, node_id) in self._loaded
 
-    def get_uncached(self, node_ids: list[str]) -> list[str]:
-        """Get node IDs that haven't been fully loaded yet."""
-        return [n for n in node_ids if not self.is_fully_loaded(n)]
+    def get_uncached(self, edge_type: str, node_ids: list[str]) -> list[str]:
+        """Get node IDs that haven't been loaded yet for this edge type."""
+        return [n for n in node_ids if not self.is_loaded(edge_type, n)]
 
-    def add_all_edges(self, edges_by_type: dict[str, dict[str, list[EdgeTarget]]], all_queried: list[str]):
+    def add_edges(self, edge_type: str, edges: dict[str, list[EdgeTarget]], all_queried: list[str]):
         """
-        Add loaded edges to the cache (all edge types at once).
+        Add loaded edges to the cache.
 
         Args:
-            edges_by_type: Dict mapping edge_type -> from_node_id -> list of EdgeTarget
-            all_queried: All node IDs that were queried (marks them as fully loaded)
+            edge_type: Type of edges
+            edges: Dict mapping from_node_id -> list of EdgeTarget
+            all_queried: All node IDs that were queried (marks them as loaded even if no edges)
         """
-        for edge_type, edges in edges_by_type.items():
-            if edge_type not in self.graphs:
-                self.graphs[edge_type] = {}
-            for node_id, neighbors in edges.items():
-                self.graphs[edge_type][node_id] = neighbors
+        if edge_type not in self.graphs:
+            self.graphs[edge_type] = {}
 
-        # Mark all queried nodes as fully loaded (even if they have no edges)
-        self._fully_loaded.update(all_queried)
+        for node_id, neighbors in edges.items():
+            self.graphs[edge_type][node_id] = neighbors
+
+        # Mark all queried nodes as loaded (even if they have no edges)
+        for node_id in all_queried:
+            self._loaded.add((edge_type, node_id))
 
 
 @dataclass
@@ -156,64 +148,49 @@ class SeedNode:
 # -----------------------------------------------------------------------------
 
 
-async def load_all_edges_for_frontier(
+async def load_edges_for_frontier(
     pool,
+    bank_id: str,
+    edge_type: str,
     node_ids: list[str],
-    top_k_per_type: int = 20,
-) -> dict[str, dict[str, list[EdgeTarget]]]:
+) -> dict[str, list[EdgeTarget]]:
     """
-    Load top-k edges per (node, edge_type) for frontier nodes.
-
-    Uses a LATERAL join to efficiently fetch only the top-k edges per type,
-    avoiding loading hundreds of entity edges when only 20 are needed.
-
-    Requires composite index: (from_unit_id, link_type, weight DESC)
+    Load edges for specific frontier nodes only.
 
     Args:
         pool: Database connection pool
+        bank_id: Memory bank ID
+        edge_type: Type of edges to load
         node_ids: Frontier node IDs to load edges for
-        top_k_per_type: Max edges to load per (node, link_type) pair
 
     Returns:
-        Dict mapping edge_type -> from_node_id -> list of EdgeTarget
+        Dict mapping from_node_id -> list of EdgeTarget
     """
     if not node_ids:
         return {}
 
     async with acquire_with_retry(pool) as conn:
-        # Use LATERAL join to get top-k per (from_node, link_type)
-        # This leverages the composite index for efficient early termination
         rows = await conn.fetch(
             f"""
-            WITH frontier(node_id) AS (SELECT unnest($1::uuid[]))
-            SELECT f.node_id as from_unit_id, lt.link_type, edges.to_unit_id, edges.weight
-            FROM frontier f
-            CROSS JOIN (VALUES ('semantic'), ('temporal'), ('entity'), ('causes'), ('caused_by')) AS lt(link_type)
-            CROSS JOIN LATERAL (
-                SELECT ml.to_unit_id, ml.weight
-                FROM {fq_table("memory_links")} ml
-                WHERE ml.from_unit_id = f.node_id
-                  AND ml.link_type = lt.link_type
-                  AND ml.weight >= 0.1
-                ORDER BY ml.weight DESC
-                LIMIT $2
-            ) edges
+            SELECT ml.from_unit_id, ml.to_unit_id, ml.weight
+            FROM {fq_table("memory_links")} ml
+            WHERE ml.from_unit_id = ANY($1::uuid[])
+              AND ml.link_type = $2
+              AND ml.weight >= 0.1
+            ORDER BY ml.from_unit_id, ml.weight DESC
             """,
             node_ids,
-            top_k_per_type,
+            edge_type,
         )
 
-    # Group by edge_type -> from_node -> neighbors
-    result: dict[str, dict[str, list[EdgeTarget]]] = defaultdict(lambda: defaultdict(list))
+    result: dict[str, list[EdgeTarget]] = defaultdict(list)
     for row in rows:
-        edge_type = row["link_type"]
         from_id = str(row["from_unit_id"])
         to_id = str(row["to_unit_id"])
         weight = row["weight"]
-        result[edge_type][from_id].append(EdgeTarget(node_id=to_id, weight=weight))
+        result[from_id].append(EdgeTarget(node_id=to_id, weight=weight))
 
-    # Convert nested defaultdicts to regular dicts
-    return {edge_type: dict(edges) for edge_type, edges in result.items()}
+    return dict(result)
 
 
 # -----------------------------------------------------------------------------
@@ -221,164 +198,9 @@ async def load_all_edges_for_frontier(
 # -----------------------------------------------------------------------------
 
 
-@dataclass
-class PatternState:
-    """State for a pattern traversal between hops."""
-
-    pattern: list[str]
-    hop_index: int
-    scores: dict[str, float]
-    frontier: dict[str, float]
-
-
-def _init_pattern_state(seeds: list[SeedNode], pattern: list[str]) -> PatternState:
-    """Initialize pattern state from seeds."""
-    if not seeds:
-        return PatternState(pattern=pattern, hop_index=0, scores={}, frontier={})
-
-    total_seed_score = sum(s.score for s in seeds)
-    if total_seed_score == 0:
-        total_seed_score = len(seeds)
-
-    frontier = {s.node_id: s.score / total_seed_score for s in seeds}
-    return PatternState(pattern=pattern, hop_index=0, scores={}, frontier=frontier)
-
-
-def _execute_hop(state: PatternState, cache: EdgeCache, config: MPFPConfig) -> set[str]:
-    """
-    Execute ONE hop of traversal, return frontier nodes for next hop.
-
-    This is a pure function that uses cached edges (no DB access).
-    Returns set of uncached nodes needed for next hop.
-    """
-    if state.hop_index >= len(state.pattern):
-        return set()
-
-    edge_type = state.pattern[state.hop_index]
-
-    # Collect active nodes above threshold
-    active_nodes = [node_id for node_id, mass in state.frontier.items() if mass >= config.threshold]
-    if not active_nodes:
-        state.frontier = {}
-        return set()
-
-    # Propagate mass using cached edges
-    next_frontier: dict[str, float] = {}
-    uncached_for_next: set[str] = set()
-
-    for node_id, mass in state.frontier.items():
-        if mass < config.threshold:
-            continue
-
-        # Keep α portion for this node
-        state.scores[node_id] = state.scores.get(node_id, 0) + config.alpha * mass
-
-        # Push (1-α) to neighbors
-        push_mass = (1 - config.alpha) * mass
-        neighbors = cache.get_normalized_neighbors(edge_type, node_id, config.top_k_neighbors)
-
-        for neighbor in neighbors:
-            next_frontier[neighbor.node_id] = next_frontier.get(neighbor.node_id, 0) + push_mass * neighbor.weight
-            # Track if we'll need edges for this node in the next hop
-            if not cache.is_fully_loaded(neighbor.node_id):
-                uncached_for_next.add(neighbor.node_id)
-
-    state.frontier = next_frontier
-    state.hop_index += 1
-
-    return uncached_for_next
-
-
-def _finalize_pattern(state: PatternState, config: MPFPConfig) -> PatternResult:
-    """Finalize pattern by adding remaining frontier mass to scores."""
-    for node_id, mass in state.frontier.items():
-        if mass >= config.threshold:
-            state.scores[node_id] = state.scores.get(node_id, 0) + mass
-
-    return PatternResult(pattern=state.pattern, scores=state.scores)
-
-
-async def mpfp_traverse_hop_synchronized(
-    pool,
-    pattern_jobs: list[tuple[list[SeedNode], list[str]]],
-    config: MPFPConfig,
-    cache: EdgeCache,
-) -> list[PatternResult]:
-    """
-    Execute ALL patterns with hop-synchronized edge loading.
-
-    Instead of running each pattern independently (causing multiple DB queries),
-    this function:
-    1. Runs hop 1 for ALL patterns (using pre-warmed seed edges)
-    2. Collects ALL unique hop-2 frontier nodes across patterns
-    3. Pre-warms hop-2 edges in ONE query
-    4. Runs hop 2 for ALL patterns
-
-    This reduces DB queries from O(patterns * hops) to O(hops).
-
-    Args:
-        pool: Database connection pool
-        pattern_jobs: List of (seeds, pattern) tuples
-        config: Algorithm parameters
-        cache: Shared edge cache (should be pre-warmed with seed edges)
-
-    Returns:
-        List of PatternResult for each pattern
-    """
-    import time
-
-    # Initialize all pattern states
-    states = [_init_pattern_state(seeds, pattern) for seeds, pattern in pattern_jobs]
-
-    # Determine max hops (all patterns should be same length, but be safe)
-    max_hops = max((len(p) for _, p in pattern_jobs), default=0)
-
-    # Detailed timing for debugging
-    hop_times: list[dict] = []
-
-    # Execute hop-by-hop across ALL patterns
-    for hop in range(max_hops):
-        hop_start = time.time()
-        hop_timing = {"hop": hop, "patterns_executed": 0, "uncached_count": 0, "load_time": 0.0}
-
-        # Execute this hop for all patterns, collect uncached nodes for next hop
-        all_uncached: set[str] = set()
-        exec_start = time.time()
-        for state in states:
-            if state.hop_index < len(state.pattern):
-                uncached = _execute_hop(state, cache, config)
-                all_uncached.update(uncached)
-                hop_timing["patterns_executed"] += 1
-        hop_timing["exec_time"] = time.time() - exec_start
-
-        # Pre-warm edges for ALL uncached nodes before next hop
-        hop_timing["uncached_count"] = len(all_uncached)
-        if all_uncached:
-            uncached_list = list(all_uncached - cache._fully_loaded)
-            hop_timing["uncached_after_filter"] = len(uncached_list)
-            if uncached_list:
-                load_start = time.time()
-                edges_by_type = await load_all_edges_for_frontier(pool, uncached_list, config.top_k_neighbors)
-                hop_timing["load_time"] = time.time() - load_start
-                cache.edge_load_time += hop_timing["load_time"]
-                cache.db_queries += 1
-                cache.add_all_edges(edges_by_type, uncached_list)
-                hop_timing["edges_loaded"] = sum(
-                    len(neighbors) for edges in edges_by_type.values() for neighbors in edges.values()
-                )
-
-        hop_timing["total_time"] = time.time() - hop_start
-        hop_times.append(hop_timing)
-
-    # Store hop timing details in cache for logging
-    cache.hop_details = hop_times
-
-    # Finalize all patterns
-    return [_finalize_pattern(state, config) for state in states]
-
-
 async def mpfp_traverse_async(
     pool,
+    bank_id: str,
     seeds: list[SeedNode],
     pattern: list[str],
     config: MPFPConfig,
@@ -387,14 +209,72 @@ async def mpfp_traverse_async(
     """
     Async Forward Push traversal with lazy edge loading.
 
-    NOTE: For better performance with multiple patterns, use mpfp_traverse_hop_synchronized().
-    This function is kept for single-pattern use cases.
+    Loads edges on-demand per hop, only for frontier nodes.
+
+    Args:
+        pool: Database connection pool
+        bank_id: Memory bank ID
+        seeds: Entry point nodes with initial scores
+        pattern: Sequence of edge types to follow
+        config: Algorithm parameters
+        cache: Shared edge cache (grows as edges are loaded)
+
+    Returns:
+        PatternResult with accumulated scores per node
     """
     if not seeds:
         return PatternResult(pattern=pattern, scores={})
 
-    results = await mpfp_traverse_hop_synchronized(pool, [(seeds, pattern)], config, cache)
-    return results[0] if results else PatternResult(pattern=pattern, scores={})
+    scores: dict[str, float] = {}
+
+    # Initialize frontier with seed masses (normalized)
+    total_seed_score = sum(s.score for s in seeds)
+    if total_seed_score == 0:
+        total_seed_score = len(seeds)  # fallback to uniform
+
+    frontier: dict[str, float] = {s.node_id: s.score / total_seed_score for s in seeds}
+
+    # Follow pattern hop by hop
+    for edge_type in pattern:
+        # Collect frontier nodes above threshold
+        active_nodes = [node_id for node_id, mass in frontier.items() if mass >= config.threshold]
+
+        if not active_nodes:
+            break
+
+        # Find nodes that need edge loading
+        uncached = cache.get_uncached(edge_type, active_nodes)
+
+        # Batch load edges for uncached nodes
+        if uncached:
+            edges = await load_edges_for_frontier(pool, bank_id, edge_type, uncached)
+            cache.add_edges(edge_type, edges, uncached)
+
+        # Propagate mass
+        next_frontier: dict[str, float] = {}
+
+        for node_id, mass in frontier.items():
+            if mass < config.threshold:
+                continue
+
+            # Keep α portion for this node
+            scores[node_id] = scores.get(node_id, 0) + config.alpha * mass
+
+            # Push (1-α) to neighbors
+            push_mass = (1 - config.alpha) * mass
+            neighbors = cache.get_normalized_neighbors(edge_type, node_id, config.top_k_neighbors)
+
+            for neighbor in neighbors:
+                next_frontier[neighbor.node_id] = next_frontier.get(neighbor.node_id, 0) + push_mass * neighbor.weight
+
+        frontier = next_frontier
+
+    # Final frontier nodes get their remaining mass
+    for node_id, mass in frontier.items():
+        if mass >= config.threshold:
+            scores[node_id] = scores.get(node_id, 0) + mass
+
+    return PatternResult(pattern=pattern, scores=scores)
 
 
 def rrf_fusion(
@@ -449,7 +329,7 @@ async def fetch_memory_units_by_ids(
         rows = await conn.fetch(
             f"""
             SELECT id, text, context, event_date, occurred_start, occurred_end,
-                   mentioned_at, access_count, embedding, fact_type, document_id, chunk_id, tags
+                   mentioned_at, access_count, embedding, fact_type, document_id, chunk_id
             FROM {fq_table("memory_units")}
             WHERE id = ANY($1::uuid[])
               AND fact_type = $2
@@ -481,13 +361,7 @@ class MPFPGraphRetriever(GraphRetriever):
         Args:
             config: Algorithm configuration (uses defaults if None)
         """
-        if config is None:
-            # Read top_k_neighbors from global config
-            from ...config import get_config
-
-            global_config = get_config()
-            config = MPFPConfig(top_k_neighbors=global_config.mpfp_top_k_neighbors)
-        self.config = config
+        self.config = config or MPFPConfig()
 
     @property
     def name(self) -> str:
@@ -504,8 +378,6 @@ class MPFPGraphRetriever(GraphRetriever):
         semantic_seeds: list[RetrievalResult] | None = None,
         temporal_seeds: list[RetrievalResult] | None = None,
         adjacency=None,  # Ignored - kept for interface compatibility
-        tags: list[str] | None = None,
-        tags_match: TagsMatch = "any",
     ) -> tuple[list[RetrievalResult], MPFPTimings | None]:
         """
         Retrieve facts using MPFP algorithm with lazy edge loading.
@@ -520,7 +392,6 @@ class MPFPGraphRetriever(GraphRetriever):
             semantic_seeds: Pre-computed semantic entry points
             temporal_seeds: Pre-computed temporal entry points
             adjacency: Ignored (kept for interface compatibility)
-            tags: Optional list of tags for visibility filtering (OR matching)
 
         Returns:
             Tuple of (List of RetrievalResult with activation scores, MPFPTimings)
@@ -535,14 +406,7 @@ class MPFPGraphRetriever(GraphRetriever):
 
         # If no semantic seeds provided, fall back to finding our own
         if not semantic_seed_nodes:
-            seeds_start = time.time()
-            semantic_seed_nodes = await self._find_semantic_seeds(
-                pool, query_embedding_str, bank_id, fact_type, tags=tags, tags_match=tags_match
-            )
-            timings.seeds_time = time.time() - seeds_start
-            logger.debug(
-                f"[MPFP] Found {len(semantic_seed_nodes)} semantic seeds for fact_type={fact_type} (tags={tags}, tags_match={tags_match})"
-            )
+            semantic_seed_nodes = await self._find_semantic_seeds(pool, query_embedding_str, bank_id, fact_type)
 
         # Collect all pattern jobs
         pattern_jobs = []
@@ -558,9 +422,6 @@ class MPFPGraphRetriever(GraphRetriever):
                 pattern_jobs.append((temporal_seed_nodes, pattern))
 
         if not pattern_jobs:
-            logger.debug(
-                f"[MPFP] No pattern jobs (semantic_seeds={len(semantic_seed_nodes)}, temporal_seeds={len(temporal_seed_nodes)})"
-            )
             return [], timings
 
         timings.pattern_count = len(pattern_jobs)
@@ -568,30 +429,16 @@ class MPFPGraphRetriever(GraphRetriever):
         # Shared edge cache across all patterns
         cache = EdgeCache()
 
-        # Pre-warm cache with ALL seed node edges BEFORE running patterns
-        # This prevents redundant DB queries at hop 1
-        all_seed_ids = list({s.node_id for seeds, _ in pattern_jobs for s in seeds})
-        if all_seed_ids:
-            import time as time_module
-
-            prewarm_start = time_module.time()
-            edges_by_type = await load_all_edges_for_frontier(pool, all_seed_ids, self.config.top_k_neighbors)
-            cache.edge_load_time += time_module.time() - prewarm_start
-            cache.db_queries += 1
-            cache.add_all_edges(edges_by_type, all_seed_ids)
-
-        # Run all patterns with HOP-SYNCHRONIZED edge loading
-        # This batches hop-2 edge loads across ALL patterns into ONE query
-        # Reduces DB queries from O(patterns * hops) to O(hops)
+        # Run all patterns in parallel (each does lazy edge loading)
         step_start = time.time()
-        pattern_results = await mpfp_traverse_hop_synchronized(pool, pattern_jobs, self.config, cache)
+        pattern_tasks = [
+            mpfp_traverse_async(pool, bank_id, seeds, pattern, self.config, cache) for seeds, pattern in pattern_jobs
+        ]
+        pattern_results = await asyncio.gather(*pattern_tasks)
         timings.traverse = time.time() - step_start
 
-        # Record edge loading stats from cache
+        # Count edges loaded
         timings.edge_count = sum(len(neighbors) for g in cache.graphs.values() for neighbors in g.values())
-        timings.db_queries = cache.db_queries
-        timings.edge_load_time = cache.edge_load_time
-        timings.hop_details = cache.hop_details
 
         # Fuse results
         step_start = time.time()
@@ -599,7 +446,6 @@ class MPFPGraphRetriever(GraphRetriever):
         timings.fusion = time.time() - step_start
 
         if not fused:
-            logger.debug(f"[MPFP] No fused results after RRF fusion (pattern_count={len(pattern_results)})")
             return [], timings
 
         # Get top result IDs
@@ -609,13 +455,6 @@ class MPFPGraphRetriever(GraphRetriever):
         step_start = time.time()
         results = await fetch_memory_units_by_ids(pool, result_ids, fact_type)
         timings.fetch = time.time() - step_start
-
-        # Filter results by tags (graph traversal may have picked up unfiltered memories)
-        if tags:
-            from .tags import filter_results_by_tags
-
-            results = filter_results_by_tags(results, tags, match=tags_match)
-
         timings.result_count = len(results)
 
         # Add activation scores from fusion
@@ -654,17 +493,8 @@ class MPFPGraphRetriever(GraphRetriever):
         fact_type: str,
         limit: int = 20,
         threshold: float = 0.3,
-        tags: list[str] | None = None,
-        tags_match: TagsMatch = "any",
     ) -> list[SeedNode]:
         """Fallback: find semantic seeds via embedding search."""
-        from .tags import build_tags_where_clause_simple
-
-        tags_clause = build_tags_where_clause_simple(tags, 6, match=tags_match)
-        params = [query_embedding_str, bank_id, fact_type, threshold, limit]
-        if tags:
-            params.append(tags)
-
         async with acquire_with_retry(pool) as conn:
             rows = await conn.fetch(
                 f"""
@@ -674,11 +504,14 @@ class MPFPGraphRetriever(GraphRetriever):
                   AND embedding IS NOT NULL
                   AND fact_type = $3
                   AND (1 - (embedding <=> $1::vector)) >= $4
-                  {tags_clause}
                 ORDER BY embedding <=> $1::vector
                 LIMIT $5
                 """,
-                *params,
+                query_embedding_str,
+                bank_id,
+                fact_type,
+                threshold,
+                limit,
             )
 
         return [SeedNode(node_id=str(r["id"]), score=r["similarity"]) for r in rows]

hindsight-api/hindsight_api/engine/search/observation_utils.py

@@ -0,0 +1,125 @@
+"""
+Observation utilities for generating entity observations from facts.
+
+Observations are objective facts synthesized from multiple memory facts
+about an entity, without personality influence.
+"""
+
+import logging
+
+from pydantic import BaseModel, Field
+
+from ..response_models import MemoryFact
+
+logger = logging.getLogger(__name__)
+
+
+class Observation(BaseModel):
+    """An observation about an entity."""
+
+    observation: str = Field(description="The observation text - a factual statement about the entity")
+
+
+class ObservationExtractionResponse(BaseModel):
+    """Response containing extracted observations."""
+
+    observations: list[Observation] = Field(default_factory=list, description="List of observations about the entity")
+
+
+def format_facts_for_observation_prompt(facts: list[MemoryFact]) -> str:
+    """Format facts as text for observation extraction prompt."""
+    import json
+
+    if not facts:
+        return "[]"
+    formatted = []
+    for fact in facts:
+        fact_obj = {"text": fact.text}
+
+        # Add context if available
+        if fact.context:
+            fact_obj["context"] = fact.context
+
+        # Add occurred_start if available
+        if fact.occurred_start:
+            fact_obj["occurred_at"] = fact.occurred_start
+
+        formatted.append(fact_obj)
+
+    return json.dumps(formatted, indent=2)
+
+
+def build_observation_prompt(
+    entity_name: str,
+    facts_text: str,
+) -> str:
+    """Build the observation extraction prompt for the LLM."""
+    return f"""Based on the following facts about "{entity_name}", generate a list of key observations.
+
+FACTS ABOUT {entity_name.upper()}:
+{facts_text}
+
+Your task: Synthesize the facts into clear, objective observations about {entity_name}.
+
+GUIDELINES:
+1. Each observation should be a factual statement about {entity_name}
+2. Combine related facts into single observations where appropriate
+3. Be objective - do not add opinions, judgments, or interpretations
+4. Focus on what we KNOW about {entity_name}, not what we assume
+5. Include observations about: identity, characteristics, roles, relationships, activities
+6. Write in third person (e.g., "John is..." not "I think John is...")
+7. If there are conflicting facts, note the most recent or most supported one
+
+EXAMPLES of good observations:
+- "John works at Google as a software engineer"
+- "John is detail-oriented and methodical in his approach"
+- "John collaborates frequently with Sarah on the AI project"
+- "John joined the company in 2023"
+
+EXAMPLES of bad observations (avoid these):
+- "John seems like a good person" (opinion/judgment)
+- "John probably likes his job" (assumption)
+- "I believe John is reliable" (first-person opinion)
+
+Generate 3-7 observations based on the available facts. If there are very few facts, generate fewer observations."""
+
+
+def get_observation_system_message() -> str:
+    """Get the system message for observation extraction."""
+    return "You are an objective observer synthesizing facts about an entity. Generate clear, factual observations without opinions or personality influence. Be concise and accurate."
+
+
+async def extract_observations_from_facts(llm_config, entity_name: str, facts: list[MemoryFact]) -> list[str]:
+    """
+    Extract observations from facts about an entity using LLM.
+
+    Args:
+        llm_config: LLM configuration to use
+        entity_name: Name of the entity to generate observations about
+        facts: List of facts mentioning the entity
+
+    Returns:
+        List of observation strings
+    """
+    if not facts:
+        return []
+
+    facts_text = format_facts_for_observation_prompt(facts)
+    prompt = build_observation_prompt(entity_name, facts_text)
+
+    try:
+        result = await llm_config.call(
+            messages=[
+                {"role": "system", "content": get_observation_system_message()},
+                {"role": "user", "content": prompt},
+            ],
+            response_format=ObservationExtractionResponse,
+            scope="memory_extract_observation",
+        )
+
+        observations = [op.observation for op in result.observations]
+        return observations
+
+    except Exception as e:
+        logger.warning(f"Failed to extract observations for {entity_name}: {str(e)}")
+        return []

hindsight-api/hindsight_api/engine/search/tags.py

@@ -1,172 +0,0 @@
-"""
-Tags filtering utilities for retrieval.
-
-Provides SQL building functions for filtering memories by tags.
-Supports four matching modes via TagsMatch enum:
-- "any": OR matching, includes untagged memories (default, backward compatible)
-- "all": AND matching, includes untagged memories
-- "any_strict": OR matching, excludes untagged memories
-- "all_strict": AND matching, excludes untagged memories
-
-OR matching (any/any_strict): Memory matches if ANY of its tags overlap with request tags
-AND matching (all/all_strict): Memory matches if ALL request tags are present in its tags
-"""
-
-from typing import Literal
-
-TagsMatch = Literal["any", "all", "any_strict", "all_strict"]
-
-
-def _parse_tags_match(match: TagsMatch) -> tuple[str, bool]:
-    """
-    Parse TagsMatch into operator and include_untagged flag.
-
-    Returns:
-        Tuple of (operator, include_untagged)
-        - operator: "&&" for any/any_strict, "@>" for all/all_strict
-        - include_untagged: True for any/all, False for any_strict/all_strict
-    """
-    if match == "any":
-        return "&&", True
-    elif match == "all":
-        return "@>", True
-    elif match == "any_strict":
-        return "&&", False
-    elif match == "all_strict":
-        return "@>", False
-    else:
-        # Default to "any" behavior
-        return "&&", True
-
-
-def build_tags_where_clause(
-    tags: list[str] | None,
-    param_offset: int = 1,
-    table_alias: str = "",
-    match: TagsMatch = "any",
-) -> tuple[str, list, int]:
-    """
-    Build a SQL WHERE clause for filtering by tags.
-
-    Supports four matching modes:
-    - "any" (default): OR matching, includes untagged memories
-    - "all": AND matching, includes untagged memories
-    - "any_strict": OR matching, excludes untagged memories
-    - "all_strict": AND matching, excludes untagged memories
-
-    Args:
-        tags: List of tags to filter by. If None or empty, returns empty clause (no filtering).
-        param_offset: Starting parameter number for SQL placeholders (default 1).
-        table_alias: Optional table alias prefix (e.g., "mu." for "memory_units mu").
-        match: Matching mode. Defaults to "any".
-
-    Returns:
-        Tuple of (sql_clause, params, next_param_offset):
-        - sql_clause: SQL WHERE clause string
-        - params: List of parameter values to bind
-        - next_param_offset: Next available parameter number
-
-    Example:
-        >>> clause, params, next_offset = build_tags_where_clause(['user_a'], 3, 'mu.', 'any_strict')
-        >>> print(clause)  # "AND mu.tags IS NOT NULL AND mu.tags != '{}' AND mu.tags && $3"
-    """
-    if not tags:
-        return "", [], param_offset
-
-    column = f"{table_alias}tags" if table_alias else "tags"
-    operator, include_untagged = _parse_tags_match(match)
-
-    if include_untagged:
-        # Include untagged memories (NULL or empty array) OR matching tags
-        clause = f"AND ({column} IS NULL OR {column} = '{{}}' OR {column} {operator} ${param_offset})"
-    else:
-        # Strict: only memories with matching tags (exclude NULL and empty)
-        clause = f"AND {column} IS NOT NULL AND {column} != '{{}}' AND {column} {operator} ${param_offset}"
-
-    return clause, [tags], param_offset + 1
-
-
-def build_tags_where_clause_simple(
-    tags: list[str] | None,
-    param_num: int,
-    table_alias: str = "",
-    match: TagsMatch = "any",
-) -> str:
-    """
-    Build a simple SQL WHERE clause for tags filtering.
-
-    This is a convenience version that returns just the clause string,
-    assuming the caller will add the tags array to their params list.
-
-    Args:
-        tags: List of tags to filter by. If None or empty, returns empty string.
-        param_num: Parameter number to use in the clause.
-        table_alias: Optional table alias prefix.
-        match: Matching mode. Defaults to "any".
-
-    Returns:
-        SQL clause string or empty string.
-    """
-    if not tags:
-        return ""
-
-    column = f"{table_alias}tags" if table_alias else "tags"
-    operator, include_untagged = _parse_tags_match(match)
-
-    if include_untagged:
-        # Include untagged memories (NULL or empty array) OR matching tags
-        return f"AND ({column} IS NULL OR {column} = '{{}}' OR {column} {operator} ${param_num})"
-    else:
-        # Strict: only memories with matching tags (exclude NULL and empty)
-        return f"AND {column} IS NOT NULL AND {column} != '{{}}' AND {column} {operator} ${param_num}"
-
-
-def filter_results_by_tags(
-    results: list,
-    tags: list[str] | None,
-    match: TagsMatch = "any",
-) -> list:
-    """
-    Filter retrieval results by tags in Python (for post-processing).
-
-    Used when SQL filtering isn't possible (e.g., graph traversal results).
-
-    Args:
-        results: List of RetrievalResult objects with a 'tags' attribute.
-        tags: List of tags to filter by. If None or empty, returns all results.
-        match: Matching mode. Defaults to "any".
-
-    Returns:
-        Filtered list of results.
-    """
-    if not tags:
-        return results
-
-    _, include_untagged = _parse_tags_match(match)
-    is_any_match = match in ("any", "any_strict")
-
-    tags_set = set(tags)
-    filtered = []
-
-    for result in results:
-        result_tags = getattr(result, "tags", None)
-
-        # Check if untagged
-        is_untagged = result_tags is None or len(result_tags) == 0
-
-        if is_untagged:
-            if include_untagged:
-                filtered.append(result)
-            # else: skip untagged
-        else:
-            result_tags_set = set(result_tags)
-            if is_any_match:
-                # Any overlap
-                if result_tags_set & tags_set:
-                    filtered.append(result)
-            else:
-                # All tags must be present
-                if tags_set <= result_tags_set:
-                    filtered.append(result)
-
-    return filtered

hindsight-api/hindsight_api/engine/search/think_utils.py

@@ -3,13 +3,31 @@ Think operation utilities for formulating answers based on agent and world facts
 """
 
 import logging
+import re
 from datetime import datetime
 
+from pydantic import BaseModel, Field
+
 from ..response_models import DispositionTraits, MemoryFact
 
 logger = logging.getLogger(__name__)
 
 
+class Opinion(BaseModel):
+    """An opinion formed by the bank."""
+
+    opinion: str = Field(description="The opinion or perspective with reasoning included")
+    confidence: float = Field(description="Confidence score for this opinion (0.0 to 1.0, where 1.0 is very confident)")
+
+
+class OpinionExtractionResponse(BaseModel):
+    """Response containing extracted opinions."""
+
+    opinions: list[Opinion] = Field(
+        default_factory=list, description="List of opinions formed with their supporting reasons and confidence scores"
+    )
+
+
 def describe_trait_level(value: int) -> str:
     """Convert trait value (1-5) to descriptive text."""
     levels = {1: "very low", 2: "low", 3: "moderate", 4: "high", 5: "very high"}
@@ -75,46 +93,17 @@ def format_facts_for_prompt(facts: list[MemoryFact]) -> str:
     return json.dumps(formatted, indent=2)
 
 
-def format_entity_summaries_for_prompt(entities: dict) -> str:
-    """Format entity summaries for inclusion in the reflect prompt.
-
-    Args:
-        entities: Dict mapping entity name to EntityState objects
-
-    Returns:
-        Formatted string with entity summaries, or empty string if no summaries
-    """
-    if not entities:
-        return ""
-
-    summaries = []
-    for name, state in entities.items():
-        # Get summary from observations (summary is stored as single observation)
-        if state.observations:
-            summary_text = state.observations[0].text
-            summaries.append(f"## {name}\n{summary_text}")
-
-    if not summaries:
-        return ""
-
-    return "\n\n".join(summaries)
-
-
 def build_think_prompt(
     agent_facts_text: str,
     world_facts_text: str,
+    opinion_facts_text: str,
     query: str,
     name: str,
     disposition: DispositionTraits,
     background: str,
     context: str | None = None,
-    entity_summaries_text: str | None = None,
 ) -> str:
-    """Build the think prompt for the LLM.
-
-    Note: opinion_facts_text parameter removed - opinions are now stored as mental models
-    and included via entity_summaries_text.
-    """
+    """Build the think prompt for the LLM."""
     disposition_desc = build_disposition_description(disposition)
 
     name_section = f"""
@@ -136,14 +125,6 @@ Your background:
 ADDITIONAL CONTEXT:
 {context}
 
-"""
-
-    entity_section = ""
-    if entity_summaries_text:
-        entity_section = f"""
-KEY PEOPLE, PLACES & THINGS I KNOW ABOUT:
-{entity_summaries_text}
-
 """
 
     return f"""Here's what I know and have experienced:
@@ -154,11 +135,14 @@ MY IDENTITY & EXPERIENCES:
 WHAT I KNOW ABOUT THE WORLD:
 {world_facts_text}
 
-{entity_section}{context_section}{name_section}{disposition_desc}{background_section}
+MY EXISTING OPINIONS & BELIEFS:
+{opinion_facts_text}
+
+{context_section}{name_section}{disposition_desc}{background_section}
 
 QUESTION: {query}
 
-Based on everything I know, believe, and who I am (including my name, disposition and background), here's what I genuinely think about this question. I'll draw on my experiences, knowledge, and personal traits to give you my honest perspective."""
+Based on everything I know, believe, and who I am (including my name, disposition and background), here's what I genuinely think about this question. I'll draw on my experiences, knowledge, opinions, and personal traits to give you my honest perspective."""
 
 
 def get_system_message(disposition: DispositionTraits) -> str:
@@ -191,11 +175,122 @@ def get_system_message(disposition: DispositionTraits) -> str:
     return f"You are a person with your own thoughts, experiences, opinions, and disposition. When asked a question, respond naturally like a human would - share what you genuinely think based on what you know and have experienced. {disposition_instruction} Be direct, express your views confidently, and use 'I think', 'I believe', 'in my view', etc. Respond in plain text without markdown formatting. IMPORTANT: Detect the language of the question and respond in the SAME language. Do not translate to English if the question is in another language."
 
 
+async def extract_opinions_from_text(llm_config, text: str, query: str) -> list[Opinion]:
+    """
+    Extract opinions with reasons and confidence from text using LLM.
+
+    Args:
+        llm_config: LLM configuration to use
+        text: Text to extract opinions from
+        query: The original query that prompted this response
+
+    Returns:
+        List of Opinion objects with text and confidence
+    """
+    extraction_prompt = f"""Extract any NEW opinions or perspectives from the answer below and rewrite them in FIRST-PERSON as if YOU are stating the opinion directly.
+
+ORIGINAL QUESTION:
+{query}
+
+ANSWER PROVIDED:
+{text}
+
+Your task: Find opinions in the answer and rewrite them AS IF YOU ARE THE ONE SAYING THEM.
+
+An opinion is a judgment, viewpoint, or conclusion that goes beyond just stating facts.
+
+IMPORTANT: Do NOT extract statements like:
+- "I don't have enough information"
+- "The facts don't contain information about X"
+- "I cannot answer because..."
+
+ONLY extract actual opinions about substantive topics.
+
+CRITICAL FORMAT REQUIREMENTS:
+1. **ALWAYS start with first-person phrases**: "I think...", "I believe...", "In my view...", "I've come to believe...", "Previously I thought... but now..."
+2. **NEVER use third-person**: Do NOT say "The speaker thinks..." or "They believe..." - always use "I"
+3. Include the reasoning naturally within the statement
+4. Provide a confidence score (0.0 to 1.0)
+
+CORRECT Examples (✓ FIRST-PERSON):
+- "I think Alice is more reliable because she consistently delivers on time and writes clean code"
+- "Previously I thought all engineers were equal, but now I feel that experience and track record really matter"
+- "I believe reliability is best measured by consistent output over time"
+- "I've come to believe that track records are more important than potential"
+
+WRONG Examples (✗ THIRD-PERSON - DO NOT USE):
+- "The speaker thinks Alice is more reliable"
+- "They believe reliability matters"
+- "It is believed that Alice is better"
+
+If no genuine opinions are expressed (e.g., the response just says "I don't know"), return an empty list."""
+
+    try:
+        result = await llm_config.call(
+            messages=[
+                {
+                    "role": "system",
+                    "content": "You are converting opinions from text into first-person statements. Always use 'I think', 'I believe', 'I feel', etc. NEVER use third-person like 'The speaker' or 'They'.",
+                },
+                {"role": "user", "content": extraction_prompt},
+            ],
+            response_format=OpinionExtractionResponse,
+            scope="memory_extract_opinion",
+        )
+
+        # Format opinions with confidence score and convert to first-person
+        formatted_opinions = []
+        for op in result.opinions:
+            # Convert third-person to first-person if needed
+            opinion_text = op.opinion
+
+            # Replace common third-person patterns with first-person
+            def singularize_verb(verb):
+                if verb.endswith("es"):
+                    return verb[:-1]  # believes -> believe
+                elif verb.endswith("s"):
+                    return verb[:-1]  # thinks -> think
+                return verb
+
+            # Pattern: "The speaker/user [verb]..." -> "I [verb]..."
+            match = re.match(
+                r"^(The speaker|The user|They|It is believed) (believes?|thinks?|feels?|says|asserts?|considers?)(\s+that)?(.*)$",
+                opinion_text,
+                re.IGNORECASE,
+            )
+            if match:
+                verb = singularize_verb(match.group(2))
+                that_part = match.group(3) or ""  # Keep " that" if present
+                rest = match.group(4)
+                opinion_text = f"I {verb}{that_part}{rest}"
+
+            # If still doesn't start with first-person, prepend "I believe that "
+            first_person_starters = [
+                "I think",
+                "I believe",
+                "I feel",
+                "In my view",
+                "I've come to believe",
+                "Previously I",
+            ]
+            if not any(opinion_text.startswith(starter) for starter in first_person_starters):
+                opinion_text = "I believe that " + opinion_text[0].lower() + opinion_text[1:]
+
+            formatted_opinions.append(Opinion(opinion=opinion_text, confidence=op.confidence))
+
+        return formatted_opinions
+
+    except Exception as e:
+        logger.warning(f"Failed to extract opinions: {str(e)}")
+        return []
+
+
 async def reflect(
     llm_config,
     query: str,
     experience_facts: list[str] = None,
     world_facts: list[str] = None,
+    opinion_facts: list[str] = None,
     name: str = "Assistant",
     disposition: DispositionTraits = None,
     background: str = "",
@@ -212,6 +307,7 @@ async def reflect(
         query: Question to answer
         experience_facts: List of experience/agent fact strings
         world_facts: List of world fact strings
+        opinion_facts: List of opinion fact strings
         name: Name of the agent/persona
         disposition: Disposition traits (defaults to neutral)
         background: Background information
@@ -232,15 +328,18 @@ async def reflect(
 
     agent_results = to_memory_facts(experience_facts or [], "experience")
     world_results = to_memory_facts(world_facts or [], "world")
+    opinion_results = to_memory_facts(opinion_facts or [], "opinion")
 
     # Format facts for prompt
     agent_facts_text = format_facts_for_prompt(agent_results)
     world_facts_text = format_facts_for_prompt(world_results)
+    opinion_facts_text = format_facts_for_prompt(opinion_results)
 
     # Build prompt
     prompt = build_think_prompt(
         agent_facts_text=agent_facts_text,
         world_facts_text=world_facts_text,
+        opinion_facts_text=opinion_facts_text,
         query=query,
         name=name,
         disposition=disposition,

hindsight-api/hindsight_api/engine/search/trace.py

@@ -11,13 +11,6 @@ from typing import Any, Literal
 from pydantic import BaseModel, Field
 
 
-class TemporalConstraint(BaseModel):
-    """Detected temporal constraint from query analysis."""
-
-    start: datetime | None = Field(default=None, description="Start of temporal range")
-    end: datetime | None = Field(default=None, description="End of temporal range")
-
-
 class QueryInfo(BaseModel):
     """Information about the search query."""
 
@@ -26,11 +19,6 @@ class QueryInfo(BaseModel):
     timestamp: datetime = Field(description="When the query was executed")
     budget: int = Field(description="Maximum nodes to explore")
     max_tokens: int = Field(description="Maximum tokens to return in results")
-    tags: list[str] | None = Field(default=None, description="Tags filter applied to recall")
-    tags_match: str | None = Field(default=None, description="Tags matching mode: any, all, any_strict, all_strict")
-    temporal_constraint: TemporalConstraint | None = Field(
-        default=None, description="Detected temporal range from query"
-    )
 
 
 class EntryPoint(BaseModel):

hindsight-api/hindsight_api/engine/search/tracer.py

@@ -22,7 +22,6 @@ from .trace import (
     SearchPhaseMetrics,
     SearchSummary,
     SearchTrace,
-    TemporalConstraint,
     WeightComponents,
 )
 
@@ -46,14 +45,7 @@ class SearchTracer:
         json_output = trace.to_json()
     """
 
-    def __init__(
-        self,
-        query: str,
-        budget: int,
-        max_tokens: int,
-        tags: list[str] | None = None,
-        tags_match: str | None = None,
-    ):
+    def __init__(self, query: str, budget: int, max_tokens: int):
         """
         Initialize tracer.
 
@@ -61,14 +53,10 @@ class SearchTracer:
             query: Search query text
             budget: Maximum nodes to explore
             max_tokens: Maximum tokens to return in results
-            tags: Tags filter applied to recall
-            tags_match: Tags matching mode (any, all, any_strict, all_strict)
         """
         self.query_text = query
         self.budget = budget
         self.max_tokens = max_tokens
-        self.tags = tags
-        self.tags_match = tags_match
 
         # Trace data
         self.query_embedding: list[float] | None = None
@@ -78,9 +66,6 @@ class SearchTracer:
         self.pruned: list[PruningDecision] = []
         self.phase_metrics: list[SearchPhaseMetrics] = []
 
-        # Temporal constraint detected from query
-        self.temporal_constraint: TemporalConstraint | None = None
-
         # New 4-way retrieval tracking
         self.retrieval_results: list[RetrievalMethodResults] = []
         self.rrf_merged: list[RRFMergeResult] = []
@@ -103,11 +88,6 @@ class SearchTracer:
         """Record the query embedding."""
         self.query_embedding = embedding
 
-    def record_temporal_constraint(self, start: datetime | None, end: datetime | None):
-        """Record the detected temporal constraint from query analysis."""
-        if start is not None or end is not None:
-            self.temporal_constraint = TemporalConstraint(start=start, end=end)
-
     def add_entry_point(self, node_id: str, text: str, similarity: float, rank: int):
         """
         Record an entry point.
@@ -448,9 +428,6 @@ class SearchTracer:
             timestamp=datetime.now(UTC),
             budget=self.budget,
             max_tokens=self.max_tokens,
-            tags=self.tags,
-            tags_match=self.tags_match,
-            temporal_constraint=self.temporal_constraint,
         )
 
         # Create summary

hindsight-api/hindsight_api/engine/search/types.py

@@ -15,17 +15,14 @@ class MPFPTimings:
     """Timing breakdown for a single MPFP retrieval call."""
 
     fact_type: str
-    edge_count: int = 0  # Total edges loaded
-    db_queries: int = 0  # Number of DB queries for edge loading
-    edge_load_time: float = 0.0  # Time spent loading edges from DB
-    traverse: float = 0.0  # Total traversal time (includes edge loading)
-    pattern_count: int = 0  # Number of patterns executed
-    fusion: float = 0.0  # Time for RRF fusion
-    fetch: float = 0.0  # Time to fetch memory unit details
-    seeds_time: float = 0.0  # Time to find semantic seeds (if fallback used)
-    result_count: int = 0  # Number of results returned
-    # Detailed per-hop timing: list of {hop, exec_time, uncached, load_time, edges_loaded, total_time}
-    hop_details: list[dict] = field(default_factory=list)
+    adjacency_query: float = 0.0
+    adjacency_process: float = 0.0
+    edge_count: int = 0
+    traverse: float = 0.0
+    pattern_count: int = 0
+    fusion: float = 0.0
+    fetch: float = 0.0
+    result_count: int = 0
 
 
 @dataclass
@@ -48,7 +45,6 @@ class RetrievalResult:
     chunk_id: str | None = None
     access_count: int = 0
     embedding: list[float] | None = None
-    tags: list[str] | None = None  # Visibility scope tags
 
     # Retrieval-specific scores (only one will be set depending on retrieval method)
     similarity: float | None = None  # Semantic retrieval
@@ -73,7 +69,6 @@ class RetrievalResult:
             chunk_id=row.get("chunk_id"),
             access_count=row.get("access_count", 0),
             embedding=row.get("embedding"),
-            tags=row.get("tags"),
             similarity=row.get("similarity"),
             bm25_score=row.get("bm25_score"),
             activation=row.get("activation"),
@@ -158,7 +153,6 @@ class ScoredResult:
             "chunk_id": self.retrieval.chunk_id,
             "access_count": self.retrieval.access_count,
             "embedding": self.retrieval.embedding,
-            "tags": self.retrieval.tags,
             "semantic_similarity": self.retrieval.similarity,
             "bm25_score": self.retrieval.bm25_score,
         }

hindsight-api/hindsight_api/engine/task_backend.py

@@ -1,40 +1,31 @@
 """
-Task backend for distributed task processing.
+Abstract task backend for running async tasks.
 
-This provides an abstraction for task storage and execution:
-- BrokerTaskBackend: Uses PostgreSQL as broker (production)
-- SyncTaskBackend: Executes tasks immediately (testing/embedded)
+This provides an abstraction that can be adapted to different execution models:
+- AsyncIO queue (default implementation)
+- Pub/Sub architectures (future)
+- Message brokers (future)
 """
 
-import json
+import asyncio
 import logging
 from abc import ABC, abstractmethod
 from collections.abc import Awaitable, Callable
-from typing import TYPE_CHECKING, Any
-
-if TYPE_CHECKING:
-    import asyncpg
+from typing import Any
 
 logger = logging.getLogger(__name__)
 
 
-def fq_table(table: str, schema: str | None = None) -> str:
-    """Get fully-qualified table name with optional schema prefix."""
-    if schema:
-        return f'"{schema}".{table}'
-    return table
-
-
 class TaskBackend(ABC):
     """
     Abstract base class for task execution backends.
 
     Implementations must:
     1. Store/publish task events (as serializable dicts)
-    2. Execute tasks through a provided executor callback (optional)
+    2. Execute tasks through a provided executor callback
 
     The backend treats tasks as pure dictionaries that can be serialized
-    and stored in the database. The executor (typically MemoryEngine.execute_task)
+    and sent over the network. The executor (typically MemoryEngine.execute_task)
     receives the dict and routes it to the appropriate handler.
     """
 
@@ -55,7 +46,7 @@ class TaskBackend(ABC):
     @abstractmethod
     async def initialize(self):
         """
-        Initialize the backend (e.g., connect to database).
+        Initialize the backend (e.g., start workers, connect to broker).
         """
         pass
 
@@ -72,7 +63,7 @@ class TaskBackend(ABC):
     @abstractmethod
     async def shutdown(self):
         """
-        Shutdown the backend gracefully.
+        Shutdown the backend gracefully (e.g., stop workers, close connections).
         """
         pass
 
@@ -102,8 +93,9 @@ class SyncTaskBackend(TaskBackend):
     """
     Synchronous task backend that executes tasks immediately.
 
-    This is useful for tests and embedded/CLI usage where we don't want
-    background workers. Tasks are executed inline rather than being queued.
+    This is useful for embedded/CLI usage where we don't want background
+    workers that prevent clean exit. Tasks are executed inline rather than
+    being queued.
     """
 
     async def initialize(self):
@@ -129,123 +121,221 @@ class SyncTaskBackend(TaskBackend):
         logger.debug("SyncTaskBackend shutdown")
 
 
-class BrokerTaskBackend(TaskBackend):
+class NoopTaskBackend(TaskBackend):
     """
-    Task backend using PostgreSQL as broker.
+    No-op task backend that discards all tasks.
 
-    submit_task() stores task_payload in async_operations table.
-    Actual polling and execution is handled separately by WorkerPoller.
-
-    This backend is used by the API to store tasks. Workers poll
-    the database separately to claim and execute tasks.
+    This is useful for tests where background task execution is not needed
+    and would only slow down the test suite.
     """
 
-    def __init__(
-        self,
-        pool_getter: Callable[[], "asyncpg.Pool"],
-        schema: str | None = None,
-    ):
-        """
-        Initialize the broker task backend.
-
-        Args:
-            pool_getter: Callable that returns the asyncpg connection pool
-            schema: Database schema for multi-tenant support (optional)
-        """
-        super().__init__()
-        self._pool_getter = pool_getter
-        self._schema = schema
-
     async def initialize(self):
-        """Initialize the backend."""
+        """No-op."""
         self._initialized = True
-        logger.info("BrokerTaskBackend initialized")
+        logger.debug("NoopTaskBackend initialized")
+
+    async def submit_task(self, task_dict: dict[str, Any]):
+        """Discard the task (do nothing)."""
+        pass
+
+    async def shutdown(self):
+        """No-op."""
+        self._initialized = False
+        logger.debug("NoopTaskBackend shutdown")
+
+
+class AsyncIOQueueBackend(TaskBackend):
+    """
+    Task backend implementation using asyncio queues.
+
+    This is the default implementation that uses in-process asyncio queues
+    and a periodic consumer worker.
+    """
+
+    def __init__(self, batch_size: int = 10, batch_interval: float = 1.0):
+        """
+        Initialize AsyncIO queue backend.
+
+        Args:
+            batch_size: Maximum number of tasks to process in one batch
+            batch_interval: Maximum time (seconds) to wait before processing batch
+        """
+        super().__init__()
+        self._queue: asyncio.Queue | None = None
+        self._worker_task: asyncio.Task | None = None
+        self._shutdown_event: asyncio.Event | None = None
+        self._batch_size = batch_size
+        self._batch_interval = batch_interval
+        self._in_flight_count = 0
+        self._in_flight_lock = asyncio.Lock()
+
+    async def initialize(self):
+        """Initialize the queue and start the worker."""
+        if self._initialized:
+            return
+
+        self._queue = asyncio.Queue()
+        self._shutdown_event = asyncio.Event()
+        self._worker_task = asyncio.create_task(self._worker())
+        self._initialized = True
+        logger.info("AsyncIOQueueBackend initialized")
 
     async def submit_task(self, task_dict: dict[str, Any]):
         """
-        Store task payload in async_operations table.
-
-        The task_dict should contain an 'operation_id' if updating an existing
-        operation record, otherwise a new operation will be created.
+        Submit a task by putting it in the queue.
 
         Args:
-            task_dict: Task dictionary to store (must be JSON serializable)
+            task_dict: Task dictionary to execute
         """
         if not self._initialized:
             await self.initialize()
 
-        pool = self._pool_getter()
-        operation_id = task_dict.get("operation_id")
-        task_type = task_dict.get("type", "unknown")
-        bank_id = task_dict.get("bank_id")
-        payload_json = json.dumps(task_dict)
-
-        table = fq_table("async_operations", self._schema)
-
-        if operation_id:
-            # Update existing operation with task payload
-            await pool.execute(
-                f"""
-                UPDATE {table}
-                SET task_payload = $1::jsonb, updated_at = now()
-                WHERE operation_id = $2
-                """,
-                payload_json,
-                operation_id,
-            )
-            logger.debug(f"Updated task payload for operation {operation_id}")
-        else:
-            # Insert new operation (for tasks without pre-created records)
-            # e.g., access_count_update tasks
-            import uuid
-
-            new_id = uuid.uuid4()
-            await pool.execute(
-                f"""
-                INSERT INTO {table} (operation_id, bank_id, operation_type, status, task_payload)
-                VALUES ($1, $2, $3, 'pending', $4::jsonb)
-                """,
-                new_id,
-                bank_id,
-                task_type,
-                payload_json,
-            )
-            logger.debug(f"Created new operation {new_id} for task type {task_type}")
-
-    async def shutdown(self):
-        """Shutdown the backend."""
-        self._initialized = False
-        logger.info("BrokerTaskBackend shutdown")
+        await self._queue.put(task_dict)
 
     async def wait_for_pending_tasks(self, timeout: float = 120.0):
         """
-        Wait for pending tasks to be processed.
+        Wait for all pending tasks in the queue and in-flight tasks to complete.
 
-        In the broker model, this polls the database to check if tasks
-        for this process have been completed. This is useful in tests
-        when worker_enabled=True (API processes its own tasks).
+        This is useful in tests to ensure background tasks complete before assertions.
 
         Args:
-            timeout: Maximum time to wait in seconds
+            timeout: Maximum time to wait in seconds (default 120s for long-running tasks)
         """
-        import asyncio
-
-        pool = self._pool_getter()
-        table = fq_table("async_operations", self._schema)
+        if not self._initialized or self._queue is None:
+            return
 
+        # Wait for queue to be empty AND no in-flight tasks
         start_time = asyncio.get_event_loop().time()
         while asyncio.get_event_loop().time() - start_time < timeout:
-            # Check if there are any pending tasks with payloads
-            count = await pool.fetchval(
-                f"""
-                SELECT COUNT(*) FROM {table}
-                WHERE status = 'pending' AND task_payload IS NOT NULL
-                """
-            )
+            async with self._in_flight_lock:
+                in_flight = self._in_flight_count
 
-            if count == 0:
+            if self._queue.empty() and in_flight == 0:
+                # Queue is empty and no tasks in flight, we're done
                 return
 
+            # Wait a bit before checking again
             await asyncio.sleep(0.5)
 
-        logger.warning(f"Timeout waiting for pending tasks after {timeout}s")
+    async def shutdown(self):
+        """Shutdown the worker and drain the queue."""
+        if not self._initialized:
+            return
+
+        logger.info("Shutting down AsyncIOQueueBackend...")
+
+        # Signal shutdown
+        self._shutdown_event.set()
+
+        # Cancel worker
+        if self._worker_task is not None:
+            self._worker_task.cancel()
+            try:
+                await self._worker_task
+            except asyncio.CancelledError:
+                pass  # Worker cancelled successfully
+
+        self._initialized = False
+        logger.info("AsyncIOQueueBackend shutdown complete")
+
+    async def _execute_task_with_tracking(self, task_dict: dict[str, Any]):
+        """Execute a task and track its in-flight status."""
+        async with self._in_flight_lock:
+            self._in_flight_count += 1
+        try:
+            await self._execute_task(task_dict)
+        finally:
+            async with self._in_flight_lock:
+                self._in_flight_count -= 1
+
+    async def _execute_task_no_tracking(self, task_dict: dict[str, Any]):
+        """Execute a task without in-flight tracking (tracking done at batch level)."""
+        await self._execute_task(task_dict)
+
+    def _get_queue_stats(self) -> tuple[int, dict[str, int]]:
+        """Get current queue size and bank_id distribution."""
+        queue_size = self._queue.qsize() if self._queue else 0
+        bank_distribution: dict[str, int] = {}
+
+        if queue_size > 0 and self._queue:
+            # Peek at queue items without removing them
+            # Note: This is a snapshot and may not be perfectly accurate due to concurrency
+            try:
+                # Access internal deque for logging purposes only
+                items = list(self._queue._queue)  # type: ignore[attr-defined]
+                for item in items:
+                    bank_id = item.get("bank_id", "unknown")
+                    bank_distribution[bank_id] = bank_distribution.get(bank_id, 0) + 1
+            except Exception:
+                pass  # Queue access failed, return empty distribution
+
+        return queue_size, bank_distribution
+
+    async def _worker(self):
+        """
+        Background worker that processes tasks in batches.
+
+        Collects tasks for up to batch_interval seconds or batch_size items,
+        then processes them.
+        """
+        while not self._shutdown_event.is_set():
+            try:
+                # Collect tasks for batching
+                tasks = []
+                deadline = asyncio.get_event_loop().time() + self._batch_interval
+
+                while len(tasks) < self._batch_size and asyncio.get_event_loop().time() < deadline:
+                    try:
+                        remaining_time = max(0.1, deadline - asyncio.get_event_loop().time())
+                        task_dict = await asyncio.wait_for(self._queue.get(), timeout=remaining_time)
+                        # Track task as in-flight immediately when picked up from queue
+                        # This prevents wait_for_pending_tasks from returning too early
+                        async with self._in_flight_lock:
+                            self._in_flight_count += 1
+                        tasks.append(task_dict)
+                    except TimeoutError:
+                        break
+
+                # Process batch
+                if tasks:
+                    # Log batch start with queue stats
+                    queue_size, bank_distribution = self._get_queue_stats()
+
+                    # Summarize batch by task type and bank
+                    batch_summary: dict[str, dict[str, int]] = {}
+                    for task_dict in tasks:
+                        task_type = task_dict.get("type", "unknown")
+                        bank_id = task_dict.get("bank_id", "unknown")
+                        if task_type not in batch_summary:
+                            batch_summary[task_type] = {}
+                        batch_summary[task_type][bank_id] = batch_summary[task_type].get(bank_id, 0) + 1
+
+                    # Build log message
+                    batch_parts = []
+                    for task_type, banks in sorted(batch_summary.items()):
+                        bank_str = ", ".join(f"{b}:{c}" for b, c in sorted(banks.items()))
+                        batch_parts.append(f"{task_type}[{bank_str}]")
+                    batch_str = ", ".join(batch_parts)
+
+                    if queue_size > 0:
+                        pending_str = ", ".join(f"{k}:{v}" for k, v in sorted(bank_distribution.items()))
+                        logger.info(
+                            f"Processing {len(tasks)} tasks: {batch_str} (pending={queue_size} [{pending_str}])"
+                        )
+                    else:
+                        logger.info(f"Processing {len(tasks)} tasks: {batch_str}")
+
+                    # Execute tasks concurrently (in_flight already tracked when picked up)
+                    await asyncio.gather(
+                        *[self._execute_task_no_tracking(task_dict) for task_dict in tasks], return_exceptions=True
+                    )
+
+                    # Decrement in_flight count after all tasks complete
+                    async with self._in_flight_lock:
+                        self._in_flight_count -= len(tasks)
+
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Worker error: {e}")
+                await asyncio.sleep(1)  # Backoff on error

hindsight-api/hindsight_api/extensions/__init__.py

@@ -27,8 +27,6 @@ from hindsight_api.extensions.operation_validator import (
     RecallResult,
     ReflectContext,
     ReflectResultContext,
-    RefreshMentalModelContext,
-    RefreshMentalModelResult,
     RetainContext,
     RetainResult,
     ValidationResult,
@@ -56,8 +54,6 @@ __all__ = [
     "RecallResult",
     "ReflectContext",
     "ReflectResultContext",
-    "RefreshMentalModelContext",
-    "RefreshMentalModelResult",
     "RetainContext",
     "RetainResult",
     "ValidationResult",

hindsight-api/hindsight_api/extensions/operation_validator.py

@@ -97,18 +97,6 @@ class ReflectContext:
     context: str | None = None
 
 
-@dataclass
-class RefreshMentalModelContext:
-    """Context for a refresh mental model operation validation (pre-operation).
-
-    Contains ALL user-provided parameters for the refresh mental model operation.
-    """
-
-    bank_id: str
-    model_id: str
-    request_context: "RequestContext"
-
-
 # =============================================================================
 # Post-operation Contexts (includes results)
 # =============================================================================
@@ -176,27 +164,6 @@ class ReflectResultContext:
     error: str | None = None
 
 
-@dataclass
-class RefreshMentalModelResult:
-    """Result context for post-refresh-mental-model hook.
-
-    Contains the operation parameters and the result including token usage.
-    """
-
-    bank_id: str
-    model_id: str
-    request_context: "RequestContext"
-    # Result
-    model_name: str | None = None
-    observations_count: int = 0
-    input_tokens: int = 0
-    output_tokens: int = 0
-    total_tokens: int = 0
-    duration_ms: int = 0
-    success: bool = True
-    error: str | None = None
-
-
 class OperationValidatorExtension(Extension, ABC):
     """
     Validates and hooks into retain/recall/reflect operations.
@@ -298,25 +265,6 @@ class OperationValidatorExtension(Extension, ABC):
         """
         ...
 
-    @abstractmethod
-    async def validate_refresh_mental_model(self, ctx: RefreshMentalModelContext) -> ValidationResult:
-        """
-        Validate a refresh mental model operation before execution.
-
-        Called before the refresh mental model operation is processed.
-        Return ValidationResult.reject() to prevent the operation from executing.
-
-        Args:
-            ctx: Context containing all user-provided parameters:
-                - bank_id: Bank identifier
-                - model_id: Mental model ID to refresh
-                - request_context: Request context with auth info
-
-        Returns:
-            ValidationResult indicating whether the operation is allowed.
-        """
-        ...
-
     # =========================================================================
     # Post-operation hooks (optional - override to implement)
     # =========================================================================
@@ -377,28 +325,3 @@ class OperationValidatorExtension(Extension, ABC):
                 - error: Error message (if failed)
         """
         pass
-
-    async def on_refresh_mental_model_complete(self, result: RefreshMentalModelResult) -> None:
-        """
-        Called after a refresh mental model operation completes (success or failure).
-
-        Override this method to implement post-operation logic such as:
-        - Token usage tracking and billing
-        - Audit logging
-        - Metrics collection
-
-        Args:
-            result: Result context containing:
-                - bank_id: Bank identifier
-                - model_id: Mental model ID
-                - request_context: Request context with auth info
-                - model_name: Name of the mental model (if success)
-                - observations_count: Number of observations generated
-                - input_tokens: Number of input tokens used
-                - output_tokens: Number of output tokens used
-                - total_tokens: Total tokens used (input + output)
-                - duration_ms: Total operation duration in milliseconds
-                - success: Whether the operation succeeded
-                - error: Error message (if failed)
-        """
-        pass

hindsight-api/hindsight_api/main.py

@@ -23,7 +23,7 @@ import uvicorn
 from . import MemoryEngine
 from .api import create_app
 from .banner import print_banner
-from .config import DEFAULT_WORKERS, ENV_WORKERS, HindsightConfig, get_config
+from .config import HindsightConfig, get_config
 from .daemon import (
     DEFAULT_DAEMON_PORT,
     DEFAULT_IDLE_TIMEOUT,
@@ -95,12 +95,7 @@ def main():
 
     # Development options
     parser.add_argument("--reload", action="store_true", help="Enable auto-reload on code changes (development only)")
-    parser.add_argument(
-        "--workers",
-        type=int,
-        default=int(os.getenv(ENV_WORKERS, str(DEFAULT_WORKERS))),
-        help=f"Number of worker processes (env: {ENV_WORKERS}, default: {DEFAULT_WORKERS})",
-    )
+    parser.add_argument("--workers", type=int, default=1, help="Number of worker processes (default: 1)")
 
     # Access log options
     parser.add_argument("--access-log", action="store_true", help="Enable access log")
@@ -187,24 +182,16 @@ def main():
             embeddings_provider=config.embeddings_provider,
             embeddings_local_model=config.embeddings_local_model,
             embeddings_tei_url=config.embeddings_tei_url,
-            embeddings_openai_base_url=config.embeddings_openai_base_url,
-            embeddings_cohere_base_url=config.embeddings_cohere_base_url,
             reranker_provider=config.reranker_provider,
             reranker_local_model=config.reranker_local_model,
             reranker_tei_url=config.reranker_tei_url,
             reranker_tei_batch_size=config.reranker_tei_batch_size,
             reranker_tei_max_concurrent=config.reranker_tei_max_concurrent,
-            reranker_max_candidates=config.reranker_max_candidates,
-            reranker_cohere_base_url=config.reranker_cohere_base_url,
             host=args.host,
             port=args.port,
             log_level=args.log_level,
-            log_format=config.log_format,
             mcp_enabled=config.mcp_enabled,
             graph_retriever=config.graph_retriever,
-            mpfp_top_k_neighbors=config.mpfp_top_k_neighbors,
-            recall_max_concurrent=config.recall_max_concurrent,
-            recall_connection_budget=config.recall_connection_budget,
             observation_min_facts=config.observation_min_facts,
             observation_top_entities=config.observation_top_entities,
             retain_max_completion_tokens=config.retain_max_completion_tokens,
@@ -219,14 +206,9 @@ def main():
             db_pool_max_size=config.db_pool_max_size,
             db_command_timeout=config.db_command_timeout,
             db_acquire_timeout=config.db_acquire_timeout,
-            worker_enabled=config.worker_enabled,
-            worker_id=config.worker_id,
-            worker_poll_interval_ms=config.worker_poll_interval_ms,
-            worker_max_retries=config.worker_max_retries,
-            worker_batch_size=config.worker_batch_size,
-            worker_http_port=config.worker_http_port,
-            reflect_max_iterations=config.reflect_max_iterations,
-            mental_model_refresh_concurrency=config.mental_model_refresh_concurrency,
+            task_backend=config.task_backend,
+            task_backend_memory_batch_size=config.task_backend_memory_batch_size,
+            task_backend_memory_batch_interval=config.task_backend_memory_batch_interval,
         )
     config.configure_logging()
     if not args.daemon:
@@ -283,27 +265,14 @@ def main():
         app = idle_middleware
 
     # Prepare uvicorn config
-    # When using workers or reload, we must use import string so each worker can import the app
-    use_import_string = args.workers > 1 or args.reload
-    # Check for uvloop availability
-    try:
-        import uvloop  # noqa: F401
-
-        loop_impl = "uvloop"
-        print("uvloop available, will use for event loop")
-    except ImportError:
-        loop_impl = "asyncio"
-        print("uvloop not installed, using default asyncio event loop")
-
     uvicorn_config = {
-        "app": "hindsight_api.server:app" if use_import_string else app,
+        "app": app,
         "host": args.host,
         "port": args.port,
         "log_level": args.log_level,
         "access_log": args.access_log,
         "proxy_headers": args.proxy_headers,
         "ws": "wsproto",  # Use wsproto instead of websockets to avoid deprecation warnings
-        "loop": loop_impl,  # Explicitly set event loop implementation
     }
 
     # Add optional parameters if provided

hindsight-api/hindsight_api/metrics.py

@@ -28,15 +28,6 @@ from opentelemetry.sdk.resources import Resource
 if TYPE_CHECKING:
     import asyncpg
 
-
-def _get_tenant() -> str:
-    """Get current tenant (schema) from context for metrics labeling."""
-    # Import here to avoid circular imports
-    from hindsight_api.engine.memory_engine import get_current_schema
-
-    return get_current_schema()
-
-
 # Custom bucket boundaries for operation duration (in seconds)
 # Fine granularity in 0-30s range where most operations complete
 DURATION_BUCKETS = (0.1, 0.25, 0.5, 0.75, 1.0, 2.0, 3.0, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, 120.0)
@@ -332,7 +323,6 @@ class MetricsCollector(MetricsCollectorBase):
             "operation": operation,
             "bank_id": bank_id,
             "source": source,
-            "tenant": _get_tenant(),
         }
         if budget:
             attributes["budget"] = budget
@@ -383,7 +373,6 @@ class MetricsCollector(MetricsCollectorBase):
             "model": model,
             "scope": scope,
             "success": str(success).lower(),
-            "tenant": _get_tenant(),
         }
 
         # Record duration
@@ -436,14 +425,10 @@ class MetricsCollector(MetricsCollectorBase):
             status_code = status_code_getter()
             status_class = f"{status_code // 100}xx"
 
-            # Get tenant from context (may be set during request processing)
-            tenant = _get_tenant()
-
             attributes = {
                 **base_attributes,
                 "status_code": str(status_code),
                 "status_class": status_class,
-                "tenant": tenant,
             }
 
             # Record duration and count

hindsight-api/hindsight_api/migrations.py

@@ -22,7 +22,6 @@ from pathlib import Path
 
 from alembic import command
 from alembic.config import Config
-from alembic.script.revision import ResolutionError
 from sqlalchemy import create_engine, text
 
 logger = logging.getLogger(__name__)
@@ -79,18 +78,7 @@ def _run_migrations_internal(database_url: str, script_location: str, schema: st
         alembic_cfg.set_main_option("target_schema", schema)
 
     # Run migrations
-    try:
-        command.upgrade(alembic_cfg, "head")
-    except ResolutionError as e:
-        # This happens during rolling deployments when a newer version of the code
-        # has already run migrations, and this older replica doesn't have the new
-        # migration files. The database is already at a newer revision than we know.
-        # This is safe to ignore - the newer code has already applied its migrations.
-        logger.warning(
-            f"Database is at a newer migration revision than this code version knows about. "
-            f"This is expected during rolling deployments. Skipping migrations. Error: {e}"
-        )
-        return
+    command.upgrade(alembic_cfg, "head")
 
     logger.info(f"Database migrations completed successfully for schema '{schema_name}'")
 

hindsight-api/hindsight_api/server.py

@@ -7,7 +7,6 @@ This module provides the ASGI app for uvicorn import string usage:
 For CLI usage, use the hindsight-api command instead.
 """
 
-import logging
 import os
 import warnings
 
@@ -18,12 +17,6 @@ warnings.filterwarnings("ignore", message="websockets.server.WebSocketServerProt
 from hindsight_api import MemoryEngine
 from hindsight_api.api import create_app
 from hindsight_api.config import get_config
-from hindsight_api.extensions import (
-    DefaultExtensionContext,
-    OperationValidatorExtension,
-    TenantExtension,
-    load_extension,
-)
 
 # Disable tokenizers parallelism to avoid warnings
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
@@ -32,42 +25,12 @@ os.environ["TOKENIZERS_PARALLELISM"] = "false"
 config = get_config()
 config.configure_logging()
 
-# Load operation validator extension if configured
-operation_validator = load_extension("OPERATION_VALIDATOR", OperationValidatorExtension)
-if operation_validator:
-    logging.info(f"Loaded operation validator: {operation_validator.__class__.__name__}")
-
-# Load tenant extension if configured
-tenant_extension = load_extension("TENANT", TenantExtension)
-if tenant_extension:
-    logging.info(f"Loaded tenant extension: {tenant_extension.__class__.__name__}")
-
 # Create app at module level (required for uvicorn import string)
 # MemoryEngine reads configuration from environment variables automatically
-# Note: run_migrations=True by default, but migrations are idempotent so safe with workers
-_memory = MemoryEngine(
-    operation_validator=operation_validator,
-    tenant_extension=tenant_extension,
-    run_migrations=config.run_migrations_on_startup,
-)
-
-# Set extension context on tenant extension (needed for schema provisioning)
-if tenant_extension:
-    extension_context = DefaultExtensionContext(
-        database_url=config.database_url,
-        memory_engine=_memory,
-    )
-    tenant_extension.set_context(extension_context)
-    logging.info("Extension context set on tenant extension")
+_memory = MemoryEngine()
 
 # Create unified app with both HTTP and optionally MCP
-app = create_app(
-    memory=_memory,
-    http_api_enabled=True,
-    mcp_api_enabled=config.mcp_enabled,
-    mcp_mount_path="/mcp",
-    initialize_memory=True,
-)
+app = create_app(memory=_memory, http_api_enabled=True, mcp_api_enabled=config.mcp_enabled, mcp_mount_path="/mcp")
 
 
 if __name__ == "__main__":

hindsight-api/hindsight_api/worker/__init__.py

@@ -1,11 +0,0 @@
-"""
-Worker package for distributed task processing.
-
-This package provides:
-- WorkerPoller: Polls PostgreSQL for pending tasks and executes them
-- main: CLI entry point for hindsight-worker
-"""
-
-from .poller import WorkerPoller
-
-__all__ = ["WorkerPoller"]

hindsight-api/hindsight_api/worker/main.py

@@ -1,285 +0,0 @@
-"""
-Command-line interface for Hindsight Worker.
-
-Run the worker with:
-    hindsight-worker
-
-Stop with Ctrl+C (graceful shutdown).
-"""
-
-import argparse
-import asyncio
-import atexit
-import logging
-import os
-import signal
-import socket
-import sys
-import warnings
-
-from ..config import get_config
-from ..engine.task_backend import SyncTaskBackend
-from .poller import WorkerPoller
-
-# Filter deprecation warnings from third-party libraries
-warnings.filterwarnings("ignore", message="websockets.legacy is deprecated")
-warnings.filterwarnings("ignore", message="websockets.server.WebSocketServerProtocol is deprecated")
-
-# Disable tokenizers parallelism to avoid warnings
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
-
-logger = logging.getLogger(__name__)
-
-
-def create_worker_app(poller: WorkerPoller, memory):
-    """Create a minimal FastAPI app for worker metrics and health."""
-    from fastapi import FastAPI
-    from fastapi.responses import JSONResponse, Response
-    from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
-
-    from ..metrics import create_metrics_collector, get_metrics_collector, initialize_metrics
-
-    app = FastAPI(
-        title="Hindsight Worker",
-        description="Worker process for distributed task execution",
-    )
-
-    # Initialize OpenTelemetry metrics
-    try:
-        prometheus_reader = initialize_metrics(service_name="hindsight-worker", service_version="1.0.0")
-        create_metrics_collector()
-        app.state.prometheus_reader = prometheus_reader
-        logger.info("Metrics initialized - available at /metrics endpoint")
-    except Exception as e:
-        logger.warning(f"Failed to initialize metrics: {e}. Metrics will be disabled.")
-        app.state.prometheus_reader = None
-
-    # Set up DB pool metrics if available
-    metrics_collector = get_metrics_collector()
-    if memory._pool is not None and hasattr(metrics_collector, "set_db_pool"):
-        metrics_collector.set_db_pool(memory._pool)
-        logger.info("DB pool metrics configured")
-
-    @app.get(
-        "/health",
-        summary="Health check endpoint",
-        description="Returns worker health status including database connectivity",
-        tags=["Monitoring"],
-    )
-    async def health_endpoint():
-        """Health check endpoint."""
-        health = await memory.health_check()
-        health["worker_id"] = poller.worker_id
-        health["is_shutdown"] = poller.is_shutdown
-        status_code = 200 if health.get("status") == "healthy" else 503
-        return JSONResponse(content=health, status_code=status_code)
-
-    @app.get(
-        "/metrics",
-        summary="Prometheus metrics endpoint",
-        description="Exports metrics in Prometheus format for scraping",
-        tags=["Monitoring"],
-    )
-    async def metrics_endpoint():
-        """Return Prometheus metrics."""
-        metrics_data = generate_latest()
-        return Response(content=metrics_data, media_type=CONTENT_TYPE_LATEST)
-
-    @app.get(
-        "/",
-        summary="Worker info",
-        description="Basic worker information",
-        tags=["Info"],
-    )
-    async def root():
-        """Return basic worker info."""
-        return {
-            "service": "hindsight-worker",
-            "worker_id": poller.worker_id,
-            "is_shutdown": poller.is_shutdown,
-        }
-
-    return app
-
-
-def main():
-    """Main entry point for the hindsight-worker CLI."""
-    # Load configuration from environment
-    config = get_config()
-
-    parser = argparse.ArgumentParser(
-        prog="hindsight-worker",
-        description="Hindsight Worker - distributed task processor",
-    )
-
-    # Worker options
-    parser.add_argument(
-        "--worker-id",
-        default=config.worker_id or socket.gethostname(),
-        help="Worker identifier (default: hostname, env: HINDSIGHT_API_WORKER_ID)",
-    )
-    parser.add_argument(
-        "--poll-interval",
-        type=int,
-        default=config.worker_poll_interval_ms,
-        help=f"Poll interval in milliseconds (default: {config.worker_poll_interval_ms}, env: HINDSIGHT_API_WORKER_POLL_INTERVAL_MS)",
-    )
-    parser.add_argument(
-        "--batch-size",
-        type=int,
-        default=config.worker_batch_size,
-        help=f"Tasks to claim per poll (default: {config.worker_batch_size}, env: HINDSIGHT_API_WORKER_BATCH_SIZE)",
-    )
-    parser.add_argument(
-        "--max-retries",
-        type=int,
-        default=config.worker_max_retries,
-        help=f"Max retries before marking failed (default: {config.worker_max_retries}, env: HINDSIGHT_API_WORKER_MAX_RETRIES)",
-    )
-
-    # HTTP server options
-    parser.add_argument(
-        "--http-port",
-        type=int,
-        default=config.worker_http_port,
-        help=f"HTTP port for metrics/health endpoints (default: {config.worker_http_port}, env: HINDSIGHT_API_WORKER_HTTP_PORT)",
-    )
-    parser.add_argument(
-        "--http-host",
-        default="0.0.0.0",
-        help="HTTP host to bind (default: 0.0.0.0)",
-    )
-
-    # Logging options
-    parser.add_argument(
-        "--log-level",
-        default=config.log_level,
-        choices=["critical", "error", "warning", "info", "debug", "trace"],
-        help=f"Log level (default: {config.log_level}, env: HINDSIGHT_API_LOG_LEVEL)",
-    )
-
-    args = parser.parse_args()
-
-    # Configure logging
-    config.configure_logging()
-
-    # Import MemoryEngine here to avoid circular imports
-    from .. import MemoryEngine
-
-    print(f"Starting Hindsight Worker: {args.worker_id}")
-    print(f"  Poll interval: {args.poll_interval}ms")
-    print(f"  Batch size: {args.batch_size}")
-    print(f"  Max retries: {args.max_retries}")
-    print(f"  HTTP server: {args.http_host}:{args.http_port}")
-    print()
-
-    # Global references for cleanup
-    memory = None
-    poller = None
-
-    async def run():
-        nonlocal memory, poller
-        import uvicorn
-
-        # Initialize MemoryEngine
-        # Workers use SyncTaskBackend because they execute tasks directly,
-        # they don't need to store tasks (they poll from DB)
-        memory = MemoryEngine(
-            run_migrations=False,  # Workers don't run migrations
-            task_backend=SyncTaskBackend(),
-        )
-
-        await memory.initialize()
-
-        print(f"Database connected: {config.database_url}")
-
-        # Create and start the poller
-        poller = WorkerPoller(
-            pool=memory._pool,
-            worker_id=args.worker_id,
-            executor=memory.execute_task,
-            poll_interval_ms=args.poll_interval,
-            batch_size=args.batch_size,
-            max_retries=args.max_retries,
-        )
-
-        # Create the HTTP app for metrics/health
-        app = create_worker_app(poller, memory)
-
-        # Setup signal handlers for graceful shutdown
-        shutdown_requested = asyncio.Event()
-
-        def signal_handler(signum, frame):
-            print(f"\nReceived signal {signum}, initiating graceful shutdown...")
-            shutdown_requested.set()
-
-        signal.signal(signal.SIGINT, signal_handler)
-        signal.signal(signal.SIGTERM, signal_handler)
-
-        # Create uvicorn config and server
-        uvicorn_config = uvicorn.Config(
-            app,
-            host=args.http_host,
-            port=args.http_port,
-            log_level="info",  # Reduce uvicorn noise
-            access_log=False,
-        )
-        server = uvicorn.Server(uvicorn_config)
-
-        # Run the poller and HTTP server concurrently
-        poller_task = asyncio.create_task(poller.run())
-        http_task = asyncio.create_task(server.serve())
-
-        print(f"Worker started. Metrics available at http://{args.http_host}:{args.http_port}/metrics")
-
-        # Wait for shutdown signal
-        await shutdown_requested.wait()
-
-        # Graceful shutdown
-        print("Shutting down HTTP server...")
-        server.should_exit = True
-
-        print("Waiting for poller to finish...")
-        await poller.shutdown_graceful(timeout=30.0)
-        poller_task.cancel()
-        try:
-            await poller_task
-        except asyncio.CancelledError:
-            pass
-
-        # Wait for HTTP server to finish
-        try:
-            await asyncio.wait_for(http_task, timeout=5.0)
-        except asyncio.TimeoutError:
-            http_task.cancel()
-            try:
-                await http_task
-            except asyncio.CancelledError:
-                pass
-
-        # Close memory engine
-        await memory.close()
-        print("Worker shutdown complete")
-
-    def cleanup():
-        """Synchronous cleanup for atexit."""
-        if memory is not None and memory._pg0 is not None:
-            try:
-                loop = asyncio.new_event_loop()
-                loop.run_until_complete(memory._pg0.stop())
-                loop.close()
-                print("\npg0 stopped.")
-            except Exception as e:
-                print(f"\nError stopping pg0: {e}")
-
-    atexit.register(cleanup)
-
-    try:
-        asyncio.run(run())
-    except KeyboardInterrupt:
-        print("\nWorker interrupted")
-        sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()

hindsight-api/hindsight_api/worker/poller.py

@@ -1,281 +0,0 @@
-"""
-Worker poller for distributed task execution.
-
-Polls PostgreSQL for pending tasks and executes them using
-FOR UPDATE SKIP LOCKED for safe concurrent claiming.
-"""
-
-import asyncio
-import json
-import logging
-import traceback
-from collections.abc import Awaitable, Callable
-from typing import TYPE_CHECKING, Any
-
-if TYPE_CHECKING:
-    import asyncpg
-
-logger = logging.getLogger(__name__)
-
-
-def fq_table(table: str, schema: str | None = None) -> str:
-    """Get fully-qualified table name with optional schema prefix."""
-    if schema:
-        return f'"{schema}".{table}'
-    return table
-
-
-class WorkerPoller:
-    """
-    Polls PostgreSQL for pending tasks and executes them.
-
-    Uses FOR UPDATE SKIP LOCKED for safe distributed claiming,
-    allowing multiple workers to process tasks without conflicts.
-    """
-
-    def __init__(
-        self,
-        pool: "asyncpg.Pool",
-        worker_id: str,
-        executor: Callable[[dict[str, Any]], Awaitable[None]],
-        poll_interval_ms: int = 500,
-        batch_size: int = 10,
-        max_retries: int = 3,
-        schema: str | None = None,
-    ):
-        """
-        Initialize the worker poller.
-
-        Args:
-            pool: asyncpg connection pool
-            worker_id: Unique identifier for this worker
-            executor: Async function to execute tasks (typically MemoryEngine.execute_task)
-            poll_interval_ms: Interval between polls when no tasks found (milliseconds)
-            batch_size: Maximum number of tasks to claim per poll cycle
-            max_retries: Maximum retry attempts before marking task as failed
-            schema: Database schema for multi-tenant support (optional)
-        """
-        self._pool = pool
-        self._worker_id = worker_id
-        self._executor = executor
-        self._poll_interval_ms = poll_interval_ms
-        self._batch_size = batch_size
-        self._max_retries = max_retries
-        self._schema = schema
-        self._shutdown = asyncio.Event()
-        self._current_tasks: set[asyncio.Task] = set()
-        self._in_flight_count = 0
-        self._in_flight_lock = asyncio.Lock()
-
-    async def claim_batch(self) -> list[tuple[str, dict[str, Any]]]:
-        """
-        Claim up to batch_size pending tasks atomically.
-
-        Uses FOR UPDATE SKIP LOCKED to ensure no conflicts with other workers.
-
-        Returns:
-            List of tuples (operation_id, task_dict)
-        """
-        table = fq_table("async_operations", self._schema)
-
-        async with self._pool.acquire() as conn:
-            async with conn.transaction():
-                # Select and lock pending tasks
-                rows = await conn.fetch(
-                    f"""
-                    SELECT operation_id, task_payload
-                    FROM {table}
-                    WHERE status = 'pending' AND task_payload IS NOT NULL
-                    ORDER BY created_at
-                    LIMIT $1
-                    FOR UPDATE SKIP LOCKED
-                    """,
-                    self._batch_size,
-                )
-
-                if not rows:
-                    return []
-
-                # Claim the tasks by updating status and worker_id
-                operation_ids = [row["operation_id"] for row in rows]
-                await conn.execute(
-                    f"""
-                    UPDATE {table}
-                    SET status = 'processing', worker_id = $1, claimed_at = now(), updated_at = now()
-                    WHERE operation_id = ANY($2)
-                    """,
-                    self._worker_id,
-                    operation_ids,
-                )
-
-                # Parse and return task payloads
-                return [(str(row["operation_id"]), json.loads(row["task_payload"])) for row in rows]
-
-    async def _mark_completed(self, operation_id: str):
-        """Mark a task as completed."""
-        table = fq_table("async_operations", self._schema)
-        await self._pool.execute(
-            f"""
-            UPDATE {table}
-            SET status = 'completed', completed_at = now(), updated_at = now()
-            WHERE operation_id = $1
-            """,
-            operation_id,
-        )
-
-    async def _mark_failed(self, operation_id: str, error_message: str):
-        """Mark a task as failed with error message."""
-        table = fq_table("async_operations", self._schema)
-        # Truncate error message if too long (max 5000 chars in schema)
-        error_message = error_message[:5000] if len(error_message) > 5000 else error_message
-        await self._pool.execute(
-            f"""
-            UPDATE {table}
-            SET status = 'failed', error_message = $2, completed_at = now(), updated_at = now()
-            WHERE operation_id = $1
-            """,
-            operation_id,
-            error_message,
-        )
-
-    async def _retry_or_fail(self, operation_id: str, error_message: str):
-        """Increment retry count or mark as failed if max retries exceeded."""
-        table = fq_table("async_operations", self._schema)
-
-        # Get current retry count
-        row = await self._pool.fetchrow(
-            f"SELECT retry_count FROM {table} WHERE operation_id = $1",
-            operation_id,
-        )
-
-        if row is None:
-            logger.warning(f"Operation {operation_id} not found, cannot retry")
-            return
-
-        retry_count = row["retry_count"]
-
-        if retry_count >= self._max_retries:
-            # Max retries exceeded, mark as failed
-            await self._mark_failed(
-                operation_id, f"Max retries ({self._max_retries}) exceeded. Last error: {error_message}"
-            )
-            logger.error(f"Task {operation_id} failed after {retry_count} retries")
-        else:
-            # Increment retry and reset to pending
-            await self._pool.execute(
-                f"""
-                UPDATE {table}
-                SET status = 'pending', worker_id = NULL, claimed_at = NULL,
-                    retry_count = retry_count + 1, updated_at = now()
-                WHERE operation_id = $1
-                """,
-                operation_id,
-            )
-            logger.warning(f"Task {operation_id} failed, will retry (attempt {retry_count + 1}/{self._max_retries})")
-
-    async def execute_task(self, operation_id: str, task_dict: dict[str, Any]):
-        """Execute a single task and update its status."""
-        task_type = task_dict.get("type", "unknown")
-        bank_id = task_dict.get("bank_id", "unknown")
-
-        try:
-            logger.debug(f"Executing task {operation_id} (type={task_type}, bank={bank_id})")
-            await self._executor(task_dict)
-            await self._mark_completed(operation_id)
-            logger.debug(f"Task {operation_id} completed successfully")
-        except Exception as e:
-            error_msg = f"{type(e).__name__}: {e}\n{traceback.format_exc()}"
-            logger.error(f"Task {operation_id} failed: {e}")
-            await self._retry_or_fail(operation_id, error_msg)
-
-    async def run(self):
-        """
-        Main polling loop.
-
-        Continuously polls for pending tasks, claims them, and executes them
-        until shutdown is signaled.
-        """
-        logger.info(f"Worker {self._worker_id} starting polling loop")
-
-        while not self._shutdown.is_set():
-            try:
-                # Claim a batch of tasks
-                tasks = await self.claim_batch()
-
-                if tasks:
-                    # Log batch info
-                    task_types = {}
-                    for _, task_dict in tasks:
-                        t = task_dict.get("type", "unknown")
-                        task_types[t] = task_types.get(t, 0) + 1
-                    types_str = ", ".join(f"{k}:{v}" for k, v in task_types.items())
-                    logger.info(f"Worker {self._worker_id} claimed {len(tasks)} tasks: {types_str}")
-
-                    # Track in-flight tasks
-                    async with self._in_flight_lock:
-                        self._in_flight_count += len(tasks)
-
-                    # Execute tasks concurrently
-                    try:
-                        await asyncio.gather(
-                            *[self.execute_task(op_id, task_dict) for op_id, task_dict in tasks],
-                            return_exceptions=True,
-                        )
-                    finally:
-                        async with self._in_flight_lock:
-                            self._in_flight_count -= len(tasks)
-                else:
-                    # No tasks found, wait before polling again
-                    try:
-                        await asyncio.wait_for(
-                            self._shutdown.wait(),
-                            timeout=self._poll_interval_ms / 1000,
-                        )
-                    except asyncio.TimeoutError:
-                        pass  # Normal timeout, continue polling
-
-            except asyncio.CancelledError:
-                logger.info(f"Worker {self._worker_id} polling loop cancelled")
-                break
-            except Exception as e:
-                logger.error(f"Worker {self._worker_id} error in polling loop: {e}")
-                traceback.print_exc()
-                # Backoff on error
-                await asyncio.sleep(1)
-
-        logger.info(f"Worker {self._worker_id} polling loop stopped")
-
-    async def shutdown_graceful(self, timeout: float = 30.0):
-        """
-        Signal shutdown and wait for current tasks to complete.
-
-        Args:
-            timeout: Maximum time to wait for in-flight tasks (seconds)
-        """
-        logger.info(f"Worker {self._worker_id} initiating graceful shutdown")
-        self._shutdown.set()
-
-        # Wait for in-flight tasks to complete
-        start_time = asyncio.get_event_loop().time()
-        while asyncio.get_event_loop().time() - start_time < timeout:
-            async with self._in_flight_lock:
-                in_flight = self._in_flight_count
-
-            if in_flight == 0:
-                logger.info(f"Worker {self._worker_id} graceful shutdown complete")
-                return
-
-            logger.info(f"Worker {self._worker_id} waiting for {in_flight} in-flight tasks")
-            await asyncio.sleep(0.5)
-
-        logger.warning(f"Worker {self._worker_id} shutdown timeout after {timeout}s")
-
-    @property
-    def worker_id(self) -> str:
-        """Get the worker ID."""
-        return self._worker_id
-
-    @property
-    def is_shutdown(self) -> bool:
-        """Check if shutdown has been signaled."""
-        return self._shutdown.is_set()

hindsight-api/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-api"
-version = "0.3.0"
+version = "0.2.1"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -25,7 +25,7 @@ dependencies = [
     "psycopg2-binary>=2.9.11",
     "tiktoken>=0.12.0",
     "httpx>=0.27.0",
-    "fastmcp>=2.14.0",  # CVE-2025-66416
+    "fastmcp>=2.3.0",
     "pg0-embedded>=0.11.0",
     "python-dateutil>=2.8.0",
     "opentelemetry-api>=1.20.0",
@@ -37,19 +37,10 @@ dependencies = [
     "anthropic>=0.40.0",
     "typer>=0.9.0",
     "cohere>=5.0.0",
-    "flashrank>=0.2.0",
     # Local ML models for embeddings/reranking - can be excluded in Docker with INCLUDE_LOCAL_MODELS=false
-    "sentence-transformers>=3.3.0",
-    "transformers>=4.53.0",  # Security fixes for ReDoS vulnerabilities
-    "torch>=2.6.0",  # CVE fix for remote code execution
-    "uvloop>=0.22.1",
-    # Transitive dependency security fixes
-    "pyasn1>=0.6.2",  # DoS vulnerability fix
-    "urllib3>=2.6.3",  # Decompression-bomb safeguards bypass fix
-    "langchain-core>=1.2.5",  # Serialization injection vulnerability fix
-    "filelock>=3.20.1",  # TOCTOU race condition fix
-    "authlib>=1.6.6",  # Account takeover vulnerability fix
-    "aiohttp>=3.13.3",  # Multiple DoS vulnerabilities
+    "sentence-transformers>=3.0.0,<3.3.0",
+    "transformers>=4.30.0,<4.46.0",
+    "torch>=2.0.0",
 ]
 
 [project.optional-dependencies]
@@ -58,12 +49,11 @@ test = [
     "pytest-asyncio>=0.21.0",
     "pytest-timeout>=2.4.0",
     "pytest-xdist>=3.0.0",
-    "filelock>=3.20.1",  # TOCTOU race condition fix
+    "filelock>=3.0.0",
 ]
 
 [project.scripts]
 hindsight-api = "hindsight_api.main:main"
-hindsight-worker = "hindsight_api.worker.main:main"
 hindsight-local-mcp = "hindsight_api.mcp_local:main"
 hindsight-admin = "hindsight_api.admin.cli:main"
 
@@ -105,7 +95,7 @@ dev = [
     "pytest-timeout>=2.4.0",
     "pytest-xdist>=3.8.0",
     "python-dotenv>=1.2.1",
-    "filelock>=3.20.1",  # TOCTOU race condition fix
+    "filelock>=3.0.0",
     "ruff>=0.8.0",
     "ty>=0.0.1",
 ]

hindsight-api/tests/conftest.py

@@ -12,7 +12,6 @@ from hindsight_api import MemoryEngine, LLMConfig, LocalSTEmbeddings, RequestCon
 
 from hindsight_api.engine.cross_encoder import LocalSTCrossEncoder
 from hindsight_api.engine.query_analyzer import DateparserQueryAnalyzer
-from hindsight_api.engine.task_backend import SyncTaskBackend
 from hindsight_api.pg0 import EmbeddedPostgres
 
 # Default pg0 instance configuration for tests
@@ -148,7 +147,6 @@ async def memory(pg0_db_url, embeddings, cross_encoder, query_analyzer):
     Uses pg0_db_url (a postgresql:// URL) directly, so MemoryEngine won't try to
     manage pg0 lifecycle - that's handled by the session-scoped pg0_db_url fixture.
     Migrations are disabled here since they're run once at session scope in pg0_db_url.
-    Uses SyncTaskBackend so async tasks execute immediately (no worker needed).
     """
     mem = MemoryEngine(
         db_url=pg0_db_url,  # Direct postgresql:// URL, not pg0://
@@ -162,7 +160,6 @@ async def memory(pg0_db_url, embeddings, cross_encoder, query_analyzer):
         pool_min_size=1,
         pool_max_size=5,
         run_migrations=False,  # Migrations already run at session scope
-        task_backend=SyncTaskBackend(),  # Execute tasks immediately in tests
     )
     await mem.initialize()
     yield mem

hindsight-api/tests/test_agents_api.py

@@ -1,5 +1,5 @@
 """
-Tests for agent management API (profile, disposition).
+Tests for agent management API (profile, disposition, background).
 """
 import pytest
 import uuid
@@ -25,12 +25,15 @@ class TestAgentProfile:
 
         assert profile is not None
         assert "disposition" in profile
+        assert "background" in profile
 
         disposition = profile["disposition"]
         assert disposition.skepticism == 3
         assert disposition.literalism == 3
         assert disposition.empathy == 3
 
+        assert profile["background"] == ""
+
     @pytest.mark.asyncio
     async def test_update_agent_disposition(self, memory: MemoryEngine, request_context):
         """Test updating agent disposition traits."""
@@ -73,10 +76,63 @@ class TestAgentProfile:
         for agent in agents:
             assert "bank_id" in agent
             assert "disposition" in agent
+            assert "background" in agent
             assert "created_at" in agent
             assert "updated_at" in agent
 
 
+class TestAgentBackground:
+    """Tests for agent background management."""
+
+    @pytest.mark.asyncio
+    async def test_merge_agent_background(self, memory: MemoryEngine, request_context):
+        """Test merging agent background information."""
+        bank_id = unique_agent_id("test_profile_merge")
+
+        profile = await memory.get_bank_profile(bank_id, request_context=request_context)
+        assert profile["background"] == ""
+
+        result1 = await memory.merge_bank_background(
+            bank_id,
+            "I was born in Texas",
+            update_disposition=False,
+            request_context=request_context,
+        )
+        assert "Texas" in result1["background"]
+
+        result2 = await memory.merge_bank_background(
+            bank_id,
+            "I have 10 years of startup experience",
+            update_disposition=False,
+            request_context=request_context,
+        )
+        assert "Texas" in result2["background"] or "startup" in result2["background"]
+
+        final_profile = await memory.get_bank_profile(bank_id, request_context=request_context)
+        assert final_profile["background"] != ""
+
+    @pytest.mark.asyncio
+    async def test_merge_background_handles_conflicts(self, memory: MemoryEngine, request_context):
+        """Test that merging background handles conflicts (new overwrites old)."""
+        bank_id = unique_agent_id("test_profile_conflict")
+
+        result1 = await memory.merge_bank_background(
+            bank_id,
+            "I was born in Colorado",
+            update_disposition=False,
+            request_context=request_context,
+        )
+        assert "Colorado" in result1["background"]
+
+        result2 = await memory.merge_bank_background(
+            bank_id,
+            "You were born in Texas",
+            update_disposition=False,
+            request_context=request_context,
+        )
+        assert "Texas" in result2["background"]
+
+
 class TestAgentEndpoint:
     """Tests for agent PUT endpoint logic."""
 
@@ -91,6 +147,7 @@ class TestAgentEndpoint:
                 literalism=5,
                 empathy=2
             ),
+            background="I am a creative software engineer"
         )
 
         profile = await memory.get_bank_profile(bank_id, request_context=request_context)
@@ -102,10 +159,55 @@ class TestAgentEndpoint:
                 request_context=request_context,
             )
 
+        if request.background is not None:
+            pool = await memory._get_pool()
+            async with pool.acquire() as conn:
+                await conn.execute(
+                    """
+                    UPDATE banks
+                    SET background = $2,
+                        updated_at = NOW()
+                    WHERE bank_id = $1
+                    """,
+                    bank_id,
+                    request.background
+                )
+
         final_profile = await memory.get_bank_profile(bank_id, request_context=request_context)
 
         assert final_profile["disposition"].skepticism == 4
         assert final_profile["disposition"].literalism == 5
+        assert final_profile["background"] == "I am a creative software engineer"
+
+    @pytest.mark.asyncio
+    async def test_put_agent_partial_update(self, memory: MemoryEngine, request_context):
+        """Test updating only background."""
+        bank_id = unique_agent_id("test_put_partial")
+
+        request = CreateBankRequest(
+            background="I am a data scientist"
+        )
+
+        profile = await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        if request.background is not None:
+            pool = await memory._get_pool()
+            async with pool.acquire() as conn:
+                await conn.execute(
+                    """
+                    UPDATE banks
+                    SET background = $2,
+                        updated_at = NOW()
+                    WHERE bank_id = $1
+                    """,
+                    bank_id,
+                    request.background
+                )
+
+        final_profile = await memory.get_bank_profile(bank_id, request_context=request_context)
+
+        assert final_profile["disposition"].skepticism == 3  # Default
+        assert final_profile["background"] == "I am a data scientist"
 
 
 class TestAgentDispositionIntegration:
@@ -123,6 +225,13 @@ class TestAgentDispositionIntegration:
         }
         await memory.update_bank_disposition(bank_id, disposition, request_context=request_context)
 
+        await memory.merge_bank_background(
+            bank_id,
+            "I am a creative artist who values innovation over tradition",
+            update_disposition=False,
+            request_context=request_context,
+        )
+
         await memory.retain_batch_async(
             bank_id=bank_id,
             contents=[

hindsight-api/tests/test_custom_embedding_dimension.py

@@ -17,7 +17,6 @@ from hindsight_api import MemoryEngine, RequestContext
 from hindsight_api.engine.embeddings import LocalSTEmbeddings, OpenAIEmbeddings, CohereEmbeddings
 from hindsight_api.engine.cross_encoder import LocalSTCrossEncoder, CohereCrossEncoder
 from hindsight_api.engine.query_analyzer import DateparserQueryAnalyzer
-from hindsight_api.engine.task_backend import SyncTaskBackend
 from hindsight_api.extensions import TenantExtension, TenantContext
 from hindsight_api.migrations import run_migrations, ensure_embedding_dimension
 
@@ -324,7 +323,6 @@ class TestOpenAIEmbeddings:
             pool_max_size=3,
             run_migrations=False,
             tenant_extension=SchemaTenantExtension(schema_name),
-            task_backend=SyncTaskBackend(),
         )
 
         try:
@@ -394,7 +392,6 @@ class TestOpenAIEmbeddings:
             pool_max_size=3,
             run_migrations=False,
             tenant_extension=SchemaTenantExtension(schema_name),
-            task_backend=SyncTaskBackend(),
         )
 
         try:
@@ -562,7 +559,6 @@ class TestCohereIntegration:
             pool_max_size=3,
             run_migrations=False,
             tenant_extension=SchemaTenantExtension(schema_name),
-            task_backend=SyncTaskBackend(),
         )
 
         try:

hindsight-api/tests/test_emergent_filtering.py

@@ -1,516 +0,0 @@
-"""Tests for emergent entity filtering."""
-
-import pytest
-from unittest.mock import AsyncMock, MagicMock
-
-from hindsight_api.engine.mental_models.emergent import (
-    build_mission_filter_prompt,
-    evaluate_emergent_models,
-    filter_candidates_by_mission,
-    MissionFilterResponse,
-    MissionFilterCandidate,
-)
-from hindsight_api.engine.mental_models.models import EmergentCandidate
-
-
-class TestBuildMissionFilterPrompt:
-    """Test prompt building for mission filtering."""
-
-    def test_prompt_contains_mission(self):
-        """Test that prompt includes the mission."""
-        candidates = [
-            EmergentCandidate(
-                name="Alice",
-                detection_method="named_entity_extraction",
-                mention_count=10,
-            )
-        ]
-        prompt = build_mission_filter_prompt("Be a PM for engineering team", candidates)
-        assert "Be a PM for engineering team" in prompt
-
-    def test_prompt_contains_candidates(self):
-        """Test that prompt includes all candidates."""
-        candidates = [
-            EmergentCandidate(
-                name="Alice Chen",
-                detection_method="named_entity_extraction",
-                mention_count=10,
-            ),
-            EmergentCandidate(
-                name="Project Phoenix",
-                detection_method="named_entity_extraction",
-                mention_count=5,
-            ),
-        ]
-        prompt = build_mission_filter_prompt("Track projects", candidates)
-        assert "Alice Chen" in prompt
-        assert "Project Phoenix" in prompt
-
-    def test_prompt_contains_rejection_guidance(self):
-        """Test that prompt contains guidance to reject generic entities."""
-        candidates = [
-            EmergentCandidate(
-                name="test",
-                detection_method="named_entity_extraction",
-                mention_count=1,
-            )
-        ]
-        prompt = build_mission_filter_prompt("Test mission", candidates)
-
-        # Should contain rejection guidance for generic terms
-        assert "promote=false" in prompt
-        assert "kids" in prompt  # Example of generic term to reject
-        assert "community" in prompt  # Example of abstract concept to reject
-        assert "motivation" in prompt  # Example of abstract concept to reject
-
-
-class TestFilterCandidatesByMission:
-    """Test the filter_candidates_by_mission function."""
-
-    @pytest.fixture
-    def mock_llm_config(self):
-        """Create a mock LLM config."""
-        config = MagicMock()
-        config.call = AsyncMock()
-        return config
-
-    async def test_empty_candidates(self, mock_llm_config):
-        """Test with empty candidate list."""
-        result = await filter_candidates_by_mission(
-            llm_config=mock_llm_config,
-            mission="Test mission",
-            candidates=[],
-        )
-        assert result == []
-        mock_llm_config.call.assert_not_called()
-
-    async def test_no_mission_keeps_all(self, mock_llm_config):
-        """Test that no mission keeps all candidates (skips filtering)."""
-        candidates = [
-            EmergentCandidate(
-                name="Alice",
-                detection_method="named_entity_extraction",
-                mention_count=10,
-            )
-        ]
-        result = await filter_candidates_by_mission(
-            llm_config=mock_llm_config,
-            mission="",  # Empty mission
-            candidates=candidates,
-        )
-        assert len(result) == 1
-        assert result[0].name == "Alice"
-        mock_llm_config.call.assert_not_called()
-
-    async def test_filters_by_promote_flag(self, mock_llm_config):
-        """Test that candidates are filtered by promote flag."""
-        candidates = [
-            EmergentCandidate(
-                name="Alice Chen",
-                detection_method="named_entity_extraction",
-                mention_count=10,
-            ),
-            EmergentCandidate(
-                name="community",
-                detection_method="named_entity_extraction",
-                mention_count=5,
-            ),
-        ]
-
-        # Mock LLM response - Alice is promoted, community is not
-        mock_llm_config.call.return_value = MissionFilterResponse(
-            candidates=[
-                MissionFilterCandidate(name="Alice Chen", promote=True, reason="Specific person"),
-                MissionFilterCandidate(name="community", promote=False, reason="Generic abstract concept"),
-            ]
-        )
-
-        result = await filter_candidates_by_mission(
-            llm_config=mock_llm_config,
-            mission="Be a PM for engineering team",
-            candidates=candidates,
-        )
-
-        assert len(result) == 1
-        assert result[0].name == "Alice Chen"
-
-    async def test_rejects_generic_entities(self, mock_llm_config):
-        """Test that generic entities are rejected."""
-        # These are all generic/abstract terms that should be rejected
-        generic_names = [
-            "user", "support", "community", "family", "motivation",
-            "photo", "gratitude", "difference", "volunteering",
-            "kids", "veterans", "impact", "kindness", "encouragement",
-            "education", "nature", "joy", "positivity", "inspiration",
-            "help", "commitment", "passion", "energy", "connection",
-        ]
-        candidates = [
-            EmergentCandidate(
-                name=name,
-                detection_method="named_entity_extraction",
-                mention_count=10,
-            )
-            for name in generic_names
-        ]
-
-        # Add some valid candidates
-        valid_candidates = [
-            EmergentCandidate(
-                name="John",
-                detection_method="named_entity_extraction",
-                mention_count=10,
-            ),
-            EmergentCandidate(
-                name="Maria",
-                detection_method="named_entity_extraction",
-                mention_count=8,
-            ),
-            EmergentCandidate(
-                name="Max",
-                detection_method="named_entity_extraction",
-                mention_count=6,
-            ),
-        ]
-        candidates.extend(valid_candidates)
-
-        # Mock LLM response - reject all generic, promote only specific names
-        response_candidates = [
-            MissionFilterCandidate(name=name, promote=False, reason="Generic/abstract term")
-            for name in generic_names
-        ]
-        response_candidates.extend([
-            MissionFilterCandidate(name=c.name, promote=True, reason="Specific person name")
-            for c in valid_candidates
-        ])
-
-        mock_llm_config.call.return_value = MissionFilterResponse(candidates=response_candidates)
-
-        result = await filter_candidates_by_mission(
-            llm_config=mock_llm_config,
-            mission="Be a health coach",
-            candidates=candidates,
-        )
-
-        # Should only have John, Maria, and Max
-        result_names = {c.name for c in result}
-        assert result_names == {"John", "Maria", "Max"}
-
-    async def test_accepts_specific_named_entities(self, mock_llm_config):
-        """Test that specific named entities are accepted."""
-        # These should all be accepted
-        valid_names = [
-            "Alice Chen",       # Full name
-            "Dr. Smith",        # Title + name
-            "John",             # First name (when it's clearly a person)
-            "Google",           # Organization
-            "Frontend Team",    # Named team
-            "Project Phoenix",  # Named project
-            "NYC Office",       # Named place
-            "Q4 Planning",      # Named event
-            "Sprint 23 Review", # Named meeting
-        ]
-        candidates = [
-            EmergentCandidate(
-                name=name,
-                detection_method="named_entity_extraction",
-                mention_count=10,
-            )
-            for name in valid_names
-        ]
-
-        # Mock LLM response - promote all
-        response_candidates = [
-            MissionFilterCandidate(name=name, promote=True, reason="Specific named entity")
-            for name in valid_names
-        ]
-        mock_llm_config.call.return_value = MissionFilterResponse(candidates=response_candidates)
-
-        result = await filter_candidates_by_mission(
-            llm_config=mock_llm_config,
-            mission="Be a PM for engineering team",
-            candidates=candidates,
-        )
-
-        # Should have all valid names
-        result_names = {c.name for c in result}
-        assert result_names == set(valid_names)
-
-    async def test_llm_error_rejects_all_candidates(self, mock_llm_config):
-        """Test that LLM errors result in rejecting all candidates (fail-safe)."""
-        candidates = [
-            EmergentCandidate(
-                name="Alice",
-                detection_method="named_entity_extraction",
-                mention_count=10,
-            )
-        ]
-
-        mock_llm_config.call.side_effect = Exception("LLM error")
-
-        result = await filter_candidates_by_mission(
-            llm_config=mock_llm_config,
-            mission="Test mission",
-            candidates=candidates,
-        )
-
-        # Should reject all candidates on error (fail-safe)
-        assert len(result) == 0
-
-    async def test_missing_candidate_in_response_is_rejected(self, mock_llm_config):
-        """Test that candidates not in LLM response are rejected by default."""
-        candidates = [
-            EmergentCandidate(
-                name="Alice",
-                detection_method="named_entity_extraction",
-                mention_count=10,
-            ),
-            EmergentCandidate(
-                name="Bob",
-                detection_method="named_entity_extraction",
-                mention_count=5,
-            ),
-        ]
-
-        # Mock LLM response - only includes Alice, not Bob
-        mock_llm_config.call.return_value = MissionFilterResponse(
-            candidates=[
-                MissionFilterCandidate(name="Alice", promote=True, reason="Specific person"),
-            ]
-        )
-
-        result = await filter_candidates_by_mission(
-            llm_config=mock_llm_config,
-            mission="Test mission",
-            candidates=candidates,
-        )
-
-        # Only Alice should be in result (Bob was missing from response, so rejected)
-        assert len(result) == 1
-        assert result[0].name == "Alice"
-
-
-class TestEvaluateEmergentModels:
-    """Test the evaluate_emergent_models function for cleanup of existing models."""
-
-    @pytest.fixture
-    def mock_llm_config(self):
-        """Create a mock LLM config."""
-        config = MagicMock()
-        config.call = AsyncMock()
-        return config
-
-    async def test_empty_models(self, mock_llm_config):
-        """Test with empty model list."""
-        result = await evaluate_emergent_models(
-            llm_config=mock_llm_config,
-            models=[],
-        )
-        assert result == []
-        mock_llm_config.call.assert_not_called()
-
-    async def test_removes_generic_models(self, mock_llm_config):
-        """Test that generic/abstract models are marked for removal."""
-        models = [
-            {"id": "id-kids", "name": "kids"},
-            {"id": "id-community", "name": "community"},
-            {"id": "id-motivation", "name": "motivation"},
-            {"id": "id-john", "name": "John"},
-            {"id": "id-maria", "name": "Maria"},
-        ]
-
-        # Mock LLM response - reject generic, keep specific names
-        mock_llm_config.call.return_value = MissionFilterResponse(
-            candidates=[
-                MissionFilterCandidate(name="kids", promote=False, reason="Generic category"),
-                MissionFilterCandidate(name="community", promote=False, reason="Abstract concept"),
-                MissionFilterCandidate(name="motivation", promote=False, reason="Abstract concept"),
-                MissionFilterCandidate(name="John", promote=True, reason="Person name"),
-                MissionFilterCandidate(name="Maria", promote=True, reason="Person name"),
-            ]
-        )
-
-        result = await evaluate_emergent_models(
-            llm_config=mock_llm_config,
-            models=models,
-        )
-
-        # Should return IDs of generic models to remove
-        assert set(result) == {"id-kids", "id-community", "id-motivation"}
-
-    async def test_keeps_specific_named_models(self, mock_llm_config):
-        """Test that specific named models are kept."""
-        models = [
-            {"id": "id-john", "name": "John"},
-            {"id": "id-google", "name": "Google"},
-            {"id": "id-project", "name": "Project Phoenix"},
-        ]
-
-        # Mock LLM response - keep all
-        mock_llm_config.call.return_value = MissionFilterResponse(
-            candidates=[
-                MissionFilterCandidate(name="John", promote=True, reason="Person name"),
-                MissionFilterCandidate(name="Google", promote=True, reason="Organization"),
-                MissionFilterCandidate(name="Project Phoenix", promote=True, reason="Named project"),
-            ]
-        )
-
-        result = await evaluate_emergent_models(
-            llm_config=mock_llm_config,
-            models=models,
-        )
-
-        # No models should be removed
-        assert result == []
-
-    async def test_llm_error_keeps_all_models(self, mock_llm_config):
-        """Test that LLM errors result in keeping all models (safe default)."""
-        models = [
-            {"id": "id-kids", "name": "kids"},
-            {"id": "id-john", "name": "John"},
-        ]
-
-        mock_llm_config.call.side_effect = Exception("LLM error")
-
-        result = await evaluate_emergent_models(
-            llm_config=mock_llm_config,
-            models=models,
-        )
-
-        # Should keep all models on error (return empty removal list)
-        assert result == []
-
-    async def test_missing_model_in_response_is_removed(self, mock_llm_config):
-        """Test that models not in LLM response are marked for removal."""
-        models = [
-            {"id": "id-alice", "name": "Alice"},
-            {"id": "id-bob", "name": "Bob"},
-        ]
-
-        # Mock LLM response - only includes Alice
-        mock_llm_config.call.return_value = MissionFilterResponse(
-            candidates=[
-                MissionFilterCandidate(name="Alice", promote=True, reason="Person name"),
-            ]
-        )
-
-        result = await evaluate_emergent_models(
-            llm_config=mock_llm_config,
-            models=models,
-        )
-
-        # Bob should be marked for removal (missing from response)
-        assert result == ["id-bob"]
-
-
-class TestRemovedEntitiesNotRepromoted:
-    """Test that entities removed by evaluation are not re-promoted.
-
-    This tests the fix for a bug where:
-    1. evaluate_emergent_models returns model IDs to remove (e.g., 'entity-maya')
-    2. We delete those models
-    3. detect_entity_candidates finds the same entities (now eligible since model was deleted)
-    4. filter_candidates_by_goal approves them (different LLM call)
-    5. BUG: We were re-promoting the same entities we just removed
-
-    The fix tracks removed entity_ids and excludes them from promotion.
-    """
-
-    async def test_removed_entity_ids_excluded_from_promotion(self):
-        """Test that entities whose models were removed are not re-promoted."""
-        from hindsight_api.engine.mental_models.models import EmergentCandidate
-
-        # Simulate the scenario from the bug:
-        # - existing_emergent has model 'entity-maya' with entity_id='uuid-maya'
-        # - evaluate_emergent_models says to remove 'entity-maya'
-        # - detect_entity_candidates returns 'Maya' with entity_id='uuid-maya' (now eligible)
-        # - filter_candidates_by_goal says to promote 'Maya'
-        # - But we should NOT promote because we just removed it
-
-        existing_emergent = [
-            {"id": "entity-maya", "name": "Maya", "entity_id": "uuid-maya"},
-            {"id": "entity-alex", "name": "Alex", "entity_id": "uuid-alex"},
-            {"id": "entity-john", "name": "John", "entity_id": "uuid-john"},  # This one will be kept
-        ]
-
-        # Models to remove (evaluate_emergent_models would return these)
-        models_to_remove = ["entity-maya", "entity-alex"]
-
-        # Build model_id -> entity_id mapping (this is what the fix does)
-        model_to_entity = {m["id"]: m.get("entity_id") for m in existing_emergent}
-
-        # Track removed entity_ids
-        removed_entity_ids: set[str] = set()
-        for model_id in models_to_remove:
-            entity_id = model_to_entity.get(model_id)
-            if entity_id:
-                removed_entity_ids.add(str(entity_id))
-
-        # Verify we tracked the right entity_ids
-        assert removed_entity_ids == {"uuid-maya", "uuid-alex"}
-
-        # Now simulate candidates that were detected (includes removed entities)
-        candidates = [
-            EmergentCandidate(
-                name="Maya", entity_id="uuid-maya", detection_method="named_entity", mention_count=10
-            ),
-            EmergentCandidate(
-                name="Alex", entity_id="uuid-alex", detection_method="named_entity", mention_count=8
-            ),
-            EmergentCandidate(
-                name="NewPerson", entity_id="uuid-new", detection_method="named_entity", mention_count=5
-            ),
-        ]
-
-        # Filter out candidates whose entity was just removed (the fix)
-        filtered_candidates = [c for c in candidates if c.entity_id not in removed_entity_ids]
-
-        # Only NewPerson should remain - Maya and Alex were removed and should not be re-promoted
-        assert len(filtered_candidates) == 1
-        assert filtered_candidates[0].name == "NewPerson"
-        assert filtered_candidates[0].entity_id == "uuid-new"
-
-    async def test_candidates_without_matching_removal_are_kept(self):
-        """Test that candidates not in the removed set are still promoted."""
-        from hindsight_api.engine.mental_models.models import EmergentCandidate
-
-        # No models removed
-        removed_entity_ids: set[str] = set()
-
-        candidates = [
-            EmergentCandidate(
-                name="Alice", entity_id="uuid-alice", detection_method="named_entity", mention_count=10
-            ),
-            EmergentCandidate(
-                name="Bob", entity_id="uuid-bob", detection_method="named_entity", mention_count=8
-            ),
-        ]
-
-        # Filter (should keep all since nothing was removed)
-        filtered_candidates = [c for c in candidates if c.entity_id not in removed_entity_ids]
-
-        assert len(filtered_candidates) == 2
-        assert {c.name for c in filtered_candidates} == {"Alice", "Bob"}
-
-    async def test_partial_removal_keeps_other_candidates(self):
-        """Test that only removed entities are excluded, others pass through."""
-        from hindsight_api.engine.mental_models.models import EmergentCandidate
-
-        # Only one entity removed
-        removed_entity_ids = {"uuid-removed"}
-
-        candidates = [
-            EmergentCandidate(
-                name="Removed", entity_id="uuid-removed", detection_method="named_entity", mention_count=10
-            ),
-            EmergentCandidate(
-                name="Kept1", entity_id="uuid-kept1", detection_method="named_entity", mention_count=8
-            ),
-            EmergentCandidate(
-                name="Kept2", entity_id="uuid-kept2", detection_method="named_entity", mention_count=5
-            ),
-        ]
-
-        filtered_candidates = [c for c in candidates if c.entity_id not in removed_entity_ids]
-
-        assert len(filtered_candidates) == 2
-        assert {c.name for c in filtered_candidates} == {"Kept1", "Kept2"}

hindsight-api/tests/test_extensions.py

@@ -17,8 +17,6 @@ from hindsight_api.extensions import (
     RecallResult,
     ReflectContext,
     ReflectResultContext,
-    RefreshMentalModelContext,
-    RefreshMentalModelResult,
     RequestContext,
     RetainContext,
     RetainResult,
@@ -95,7 +93,6 @@ class RateLimitingValidator(OperationValidatorExtension):
         self.retain_counts: dict[str, int] = defaultdict(int)
         self.recall_counts: dict[str, int] = defaultdict(int)
         self.reflect_counts: dict[str, int] = defaultdict(int)
-        self.refresh_mental_model_counts: dict[str, int] = defaultdict(int)
 
     async def validate_retain(self, ctx: RetainContext) -> ValidationResult:
         self.retain_counts[ctx.bank_id] += 1
@@ -121,16 +118,6 @@ class RateLimitingValidator(OperationValidatorExtension):
             )
         return ValidationResult.accept()
 
-    async def validate_refresh_mental_model(
-        self, ctx: RefreshMentalModelContext
-    ) -> ValidationResult:
-        self.refresh_mental_model_counts[ctx.bank_id] += 1
-        if self.refresh_mental_model_counts[ctx.bank_id] > self.max_attempts:
-            return ValidationResult.reject(
-                f"Refresh mental model limit exceeded for bank {ctx.bank_id}"
-            )
-        return ValidationResult.accept()
-
 
 class TrackingValidator(OperationValidatorExtension):
     """
@@ -145,12 +132,10 @@ class TrackingValidator(OperationValidatorExtension):
         self.pre_retain_calls: list[RetainContext] = []
         self.pre_recall_calls: list[RecallContext] = []
         self.pre_reflect_calls: list[ReflectContext] = []
-        self.pre_refresh_mental_model_calls: list[RefreshMentalModelContext] = []
         # Post-hook tracking
         self.post_retain_calls: list[RetainResult] = []
         self.post_recall_calls: list[RecallResult] = []
         self.post_reflect_calls: list[ReflectResultContext] = []
-        self.post_refresh_mental_model_calls: list[RefreshMentalModelResult] = []
 
     async def validate_retain(self, ctx: RetainContext) -> ValidationResult:
         self.pre_retain_calls.append(ctx)
@@ -164,12 +149,6 @@ class TrackingValidator(OperationValidatorExtension):
         self.pre_reflect_calls.append(ctx)
         return ValidationResult.accept()
 
-    async def validate_refresh_mental_model(
-        self, ctx: RefreshMentalModelContext
-    ) -> ValidationResult:
-        self.pre_refresh_mental_model_calls.append(ctx)
-        return ValidationResult.accept()
-
     async def on_retain_complete(self, result: RetainResult) -> None:
         self.post_retain_calls.append(result)
 
@@ -179,11 +158,6 @@ class TrackingValidator(OperationValidatorExtension):
     async def on_reflect_complete(self, result: ReflectResultContext) -> None:
         self.post_reflect_calls.append(result)
 
-    async def on_refresh_mental_model_complete(
-        self, result: RefreshMentalModelResult
-    ) -> None:
-        self.post_refresh_mental_model_calls.append(result)
-
 
 class TestMemoryEngineValidation:
     """Tests for validation integration with MemoryEngine.
@@ -541,105 +515,6 @@ class TestOperationHooksParameters:
         assert len(validator.pre_recall_calls) == 1
         assert len(validator.post_recall_calls) == 1
 
-    @pytest.mark.asyncio
-    async def test_refresh_mental_model_pre_hook_receives_all_parameters(
-        self, memory_with_tracking_validator
-    ):
-        """Pre-refresh-mental-model hook receives all user-provided parameters."""
-        import uuid
-
-        memory, validator = memory_with_tracking_validator
-        bank_id = f"test-refresh-mm-params-{uuid.uuid4().hex[:8]}"
-        ctx = RequestContext(api_key="test-key")
-
-        # Create bank first (get_bank_profile auto-creates if needed)
-        await memory.get_bank_profile(bank_id, request_context=ctx)
-
-        # Create a pinned mental model
-        model = await memory.create_mental_model(
-            bank_id=bank_id,
-            name="Test Model",
-            description="Test description",
-            subtype="pinned",
-            request_context=ctx,
-        )
-
-        assert model is not None
-        model_id = model["id"]
-
-        # Attempt to refresh (may not actually refresh if no data, but hook should be called)
-        try:
-            await memory.refresh_mental_model(
-                bank_id=bank_id,
-                model_id=model_id,
-                request_context=ctx,
-            )
-        except Exception:
-            pass  # May fail if no data
-
-        # Check pre-hook was called
-        assert len(validator.pre_refresh_mental_model_calls) == 1
-        pre_ctx = validator.pre_refresh_mental_model_calls[0]
-        assert pre_ctx.bank_id == bank_id
-        assert pre_ctx.model_id == model_id
-        assert pre_ctx.request_context == ctx
-
-    @pytest.mark.asyncio
-    async def test_refresh_mental_model_post_hook_receives_token_usage(
-        self, memory_with_tracking_validator
-    ):
-        """Post-refresh-mental-model hook receives token usage information."""
-        import uuid
-
-        memory, validator = memory_with_tracking_validator
-        bank_id = f"test-refresh-mm-tokens-{uuid.uuid4().hex[:8]}"
-        ctx = RequestContext(api_key="test-key")
-
-        # Store some content first
-        await memory.retain_batch_async(
-            bank_id=bank_id,
-            contents=[
-                {"content": "Alice is a software engineer who works on machine learning."},
-                {"content": "Alice enjoys hiking and outdoor activities on weekends."},
-                {"content": "Alice has been working at the company for 5 years."},
-            ],
-            request_context=ctx,
-        )
-
-        # Create a pinned mental model
-        model = await memory.create_mental_model(
-            bank_id=bank_id,
-            name="Alice Profile",
-            description="Profile of Alice including work and hobbies",
-            subtype="pinned",
-            request_context=ctx,
-        )
-
-        if model:
-            model_id = model["id"]
-
-            # Refresh the mental model
-            result = await memory.refresh_mental_model(
-                bank_id=bank_id,
-                model_id=model_id,
-                request_context=ctx,
-            )
-
-            # Check post-hook was called with token usage
-            if validator.post_refresh_mental_model_calls:
-                post_result = validator.post_refresh_mental_model_calls[0]
-                assert post_result.bank_id == bank_id
-                assert post_result.model_id == model_id
-                assert post_result.request_context == ctx
-                assert post_result.success is True
-                assert post_result.error is None
-
-                # Token usage should be populated (may be 0 if refresh was skipped)
-                assert post_result.total_tokens >= 0
-                assert post_result.input_tokens >= 0
-                assert post_result.output_tokens >= 0
-                assert post_result.duration_ms >= 0
-
 
 class TestTenantExtension:
     """Tests for TenantExtension and ApiKeyTenantExtension."""

hindsight-api/tests/test_fact_extraction_quality.py

@@ -947,3 +947,172 @@ so the algorithm learns to box out. See you next week!
                     raise e
 
 
+# =============================================================================
+# DISPOSITION INFERENCE TESTS
+# =============================================================================
+
+class TestDispositionInference:
+    """Tests for LLM-based disposition trait inference from background."""
+
+    @pytest.mark.asyncio
+    async def test_background_merge_with_disposition_inference(self, memory, request_context):
+        """Test that background merge infers disposition traits by default."""
+        import uuid
+        bank_id = f"test_infer_{uuid.uuid4().hex[:8]}"
+
+        result = await memory.merge_bank_background(
+            bank_id,
+            "I am a creative software engineer who loves innovation and trying new technologies",
+            update_disposition=True,
+            request_context=request_context,
+        )
+
+        assert "background" in result
+        assert "disposition" in result
+
+        background = result["background"]
+        disposition = result["disposition"]
+
+        assert "creative" in background.lower() or "innovation" in background.lower()
+
+        # Check that new traits are present with valid values (1-5)
+        required_traits = ["skepticism", "literalism", "empathy"]
+        for trait in required_traits:
+            assert trait in disposition
+            assert 1 <= disposition[trait] <= 5
+
+    @pytest.mark.asyncio
+    async def test_background_merge_without_disposition_inference(self, memory, request_context):
+        """Test that background merge skips disposition inference when disabled."""
+        import uuid
+        bank_id = f"test_no_infer_{uuid.uuid4().hex[:8]}"
+
+        initial_profile = await memory.get_bank_profile(bank_id, request_context=request_context)
+        initial_disposition = initial_profile["disposition"]
+
+        result = await memory.merge_bank_background(
+            bank_id,
+            "I am a data scientist",
+            update_disposition=False,
+            request_context=request_context,
+        )
+
+        assert "background" in result
+        assert "disposition" not in result
+
+        final_profile = await memory.get_bank_profile(bank_id, request_context=request_context)
+        final_disposition = final_profile["disposition"]
+
+        assert initial_disposition == final_disposition
+
+    @pytest.mark.asyncio
+    async def test_disposition_inference_for_lawyer(self, memory, request_context):
+        """Test disposition inference for lawyer profile (high skepticism, high literalism)."""
+        import uuid
+        bank_id = f"test_lawyer_{uuid.uuid4().hex[:8]}"
+
+        result = await memory.merge_bank_background(
+            bank_id,
+            "I am a lawyer who focuses on contract details and never takes claims at face value",
+            update_disposition=True,
+            request_context=request_context,
+        )
+
+        disposition = result["disposition"]
+
+        # Lawyers should have higher skepticism and literalism
+        assert disposition["skepticism"] >= 3
+        assert disposition["literalism"] >= 3
+
+    @pytest.mark.asyncio
+    async def test_disposition_inference_for_therapist(self, memory, request_context):
+        """Test disposition inference for therapist profile (high empathy)."""
+        import uuid
+        bank_id = f"test_therapist_{uuid.uuid4().hex[:8]}"
+
+        result = await memory.merge_bank_background(
+            bank_id,
+            "I am a therapist who deeply understands and connects with people's emotional struggles",
+            update_disposition=True,
+            request_context=request_context,
+        )
+
+        disposition = result["disposition"]
+
+        # Therapists should have higher empathy
+        assert disposition["empathy"] >= 3
+
+    @pytest.mark.asyncio
+    async def test_disposition_updates_in_database(self, memory, request_context):
+        """Test that inferred disposition is actually stored in database."""
+        import uuid
+        bank_id = f"test_db_update_{uuid.uuid4().hex[:8]}"
+
+        result = await memory.merge_bank_background(
+            bank_id,
+            "I am an innovative designer",
+            update_disposition=True,
+            request_context=request_context,
+        )
+
+        inferred_disposition = result["disposition"]
+
+        profile = await memory.get_bank_profile(bank_id, request_context=request_context)
+        db_disposition = profile["disposition"]
+
+        # Compare values (db_disposition is a Pydantic model)
+        assert db_disposition.skepticism == inferred_disposition["skepticism"]
+        assert db_disposition.literalism == inferred_disposition["literalism"]
+        assert db_disposition.empathy == inferred_disposition["empathy"]
+
+    @pytest.mark.asyncio
+    async def test_multiple_background_merges_update_disposition(self, memory, request_context):
+        """Test that each background merge can update disposition."""
+        import uuid
+        bank_id = f"test_multi_merge_{uuid.uuid4().hex[:8]}"
+
+        result1 = await memory.merge_bank_background(
+            bank_id,
+            "I am a software engineer",
+            update_disposition=True,
+            request_context=request_context,
+        )
+        disposition1 = result1["disposition"]
+
+        result2 = await memory.merge_bank_background(
+            bank_id,
+            "I love creative problem solving and innovation",
+            update_disposition=True,
+            request_context=request_context,
+        )
+        disposition2 = result2["disposition"]
+
+        assert "engineer" in result2["background"].lower() or "software" in result2["background"].lower()
+        assert "creative" in result2["background"].lower() or "innovation" in result2["background"].lower()
+
+    @pytest.mark.asyncio
+    async def test_background_merge_conflict_resolution_with_disposition(self, memory, request_context):
+        """Test that conflicts are resolved and disposition reflects final background."""
+        import uuid
+        bank_id = f"test_conflict_{uuid.uuid4().hex[:8]}"
+
+        await memory.merge_bank_background(
+            bank_id,
+            "I was born in Colorado and prefer stability",
+            update_disposition=True,
+            request_context=request_context,
+        )
+
+        result = await memory.merge_bank_background(
+            bank_id,
+            "You were born in Texas and are very skeptical of people",
+            update_disposition=True,
+            request_context=request_context,
+        )
+
+        background = result["background"]
+        disposition = result["disposition"]
+
+        assert "texas" in background.lower()
+        # Higher skepticism expected from "very skeptical of people"
+        assert disposition["skepticism"] >= 3

hindsight-api/tests/test_fact_ordering.py

@@ -51,7 +51,7 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
     results = await memory.recall_async(
         bank_id=bank_id,
         query="Marcus prediction Rams",
-        fact_type=['experience', 'world'],
+        fact_type=['opinion', 'experience', 'world'],
         budget=Budget.LOW,
         max_tokens=8192,
         request_context=request_context,
@@ -61,8 +61,8 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
     for i, result in enumerate(results.results):
         print(f"{i+1}. [{result.mentioned_at}] {result.text[:100]}")
 
-    # Get all facts (Marcus's predictions/statements)
-    agent_facts = results.results
+    # Get all opinion facts (Marcus's predictions/statements)
+    agent_facts = [r for r in results.results if r.fact_type == 'opinion']
 
     print(f"\n=== Agent facts (Marcus's statements) ===")
     for i, fact in enumerate(agent_facts):
@@ -70,7 +70,6 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
 
     # Check that agent facts have different timestamps
     if len(agent_facts) >= 2:
-        # Parse timestamps
         timestamps = [datetime.fromisoformat(f.mentioned_at.replace('Z', '+00:00')) for f in agent_facts]
 
         # Verify timestamps are different (have time offsets)
@@ -78,40 +77,42 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
         assert len(unique_timestamps) == len(timestamps), \
             f"Expected unique timestamps for each fact, but got duplicates: {timestamps}"
 
-        # Sort facts by timestamp for ordering check
-        # Note: recall returns by relevance, not time order
-        sorted_facts = sorted(agent_facts, key=lambda f: datetime.fromisoformat(f.mentioned_at.replace('Z', '+00:00')))
-        sorted_timestamps = [datetime.fromisoformat(f.mentioned_at.replace('Z', '+00:00')) for f in sorted_facts]
-
-        # Verify sorted timestamps are in ascending order
-        for i in range(len(sorted_timestamps) - 1):
-            assert sorted_timestamps[i] < sorted_timestamps[i + 1], \
-                f"Facts should have sequential timestamps. Fact {i} ({sorted_timestamps[i]}) >= Fact {i+1} ({sorted_timestamps[i+1]})"
+        # Verify timestamps are in order (ascending)
+        for i in range(len(timestamps) - 1):
+            assert timestamps[i] < timestamps[i + 1], \
+                f"Facts should be ordered by time. Fact {i} ({timestamps[i]}) >= Fact {i+1} ({timestamps[i+1]})"
 
         # Verify reasonable time spacing (should be ~10 seconds apart)
-        time_diffs = [(sorted_timestamps[i+1] - sorted_timestamps[i]).total_seconds() for i in range(len(sorted_timestamps) - 1)]
+        time_diffs = [(timestamps[i+1] - timestamps[i]).total_seconds() for i in range(len(timestamps) - 1)]
         print(f"\n=== Time differences between facts: {time_diffs} seconds ===")
 
         # Each fact should be 10+ seconds apart (allowing for some flexibility)
         for diff in time_diffs:
             assert diff >= 5, f"Expected at least 5 seconds between facts, got {diff}"
 
-        # Update agent_facts to be sorted for subsequent checks
-        agent_facts = sorted_facts
-        timestamps = sorted_timestamps
-
         print(f"\n✅ All {len(agent_facts)} agent facts have properly ordered timestamps")
 
-    # Verify that facts capture the key information
-    # Note: LLM may merge related predictions into single facts
+    # Verify that retrieval returns facts in chronological order
+    # The first prediction should come before the changed prediction
     agent_texts = [f.text.lower() for f in agent_facts]
-    all_text = " ".join(agent_texts)
 
-    # Look for evidence of the predictions being captured (may be merged or separate)
-    has_prediction_info = '27' in all_text or 'rams' in all_text or 'prediction' in all_text
+    # Look for evidence of the sequence
+    has_first_prediction = any('27' in text and '24' in text for text in agent_texts)
+    has_changed_prediction = any('chang' in text or 'by 3' in text or 'realized' in text for text in agent_texts)
 
-    assert has_prediction_info, "Facts should contain information about Marcus's predictions"
-    print(f"\n✅ Facts capture prediction information")
+    if has_first_prediction and has_changed_prediction:
+        # Find indices
+        first_idx = next(i for i, text in enumerate(agent_texts) if '27' in text and '24' in text)
+        changed_idx = next(i for i, text in enumerate(agent_texts) if 'chang' in text or 'by 3' in text or 'realized' in text)
+
+        print(f"\nFirst prediction at index {first_idx}: {agent_facts[first_idx].text[:100]}")
+        print(f"Changed prediction at index {changed_idx}: {agent_facts[changed_idx].text[:100]}")
+
+        # The original prediction should come before the changed one
+        assert timestamps[first_idx] < timestamps[changed_idx], \
+            "Original prediction should have earlier timestamp than changed prediction"
+
+        print(f"\n✅ Temporal ordering preserved: First prediction came before changed prediction")
 
     # Cleanup
     await memory.delete_bank(bank_id, request_context=request_context)
@@ -155,14 +156,14 @@ Alice: I reconsidered the team's experience level.
     results = await memory.recall_async(
         bank_id=bank_id,
         query="Alice preference React Vue",
-        fact_type=['experience', 'world'],
+        fact_type=['opinion', 'experience'],
         budget=Budget.LOW,
         max_tokens=8192,
         request_context=request_context,
     )
 
     print(f"\n=== Retrieved {len(results.results)} agent facts ===")
-    agent_facts = results.results
+    agent_facts = [r for r in results.results if r.fact_type in ('opinion', 'experience')]
 
     for i, fact in enumerate(agent_facts):
         print(f"{i+1}. [{fact.mentioned_at}] {fact.text[:80]}")

hindsight-api/tests/test_http_api_integration.py

@@ -60,6 +60,17 @@ async def test_full_api_workflow(api_client, test_bank_id):
     assert response.status_code == 200
     profile = response.json()
     assert "disposition" in profile
+    assert "background" in profile
+
+    # Add background
+    response = await api_client.post(
+        f"/v1/default/banks/{test_bank_id}/background",
+        json={
+            "content": "A software engineer passionate about AI and memory systems."
+        }
+    )
+    assert response.status_code == 200
+    assert "software engineer" in response.json()["background"].lower()
 
     # ================================================================
     # 2. Memory Storage
@@ -233,42 +244,17 @@ async def test_full_api_workflow(api_client, test_bank_id):
     response = await api_client.get(f"/v1/default/banks/{test_bank_id}/profile")
     assert response.status_code == 200
     updated_profile = response.json()
-    assert updated_profile["disposition"]["skepticism"] == 4
-    assert updated_profile["disposition"]["literalism"] == 3
-    assert updated_profile["disposition"]["empathy"] == 4
+    assert "software engineer" in updated_profile["background"].lower()
 
     # ================================================================
     # 8. Test Entity Endpoints
     # ================================================================
 
-    # List entities with pagination
+    # List entities
     response = await api_client.get(f"/v1/default/banks/{test_bank_id}/entities")
     assert response.status_code == 200
     entities_data = response.json()
     assert "items" in entities_data
-    assert "total" in entities_data
-    assert "limit" in entities_data
-    assert "offset" in entities_data
-    assert entities_data["offset"] == 0
-    assert entities_data["limit"] == 100  # default limit
-
-    # Test pagination with custom limit and offset
-    response = await api_client.get(f"/v1/default/banks/{test_bank_id}/entities?limit=5&offset=0")
-    assert response.status_code == 200
-    paginated_data = response.json()
-    assert paginated_data["limit"] == 5
-    assert paginated_data["offset"] == 0
-    assert len(paginated_data["items"]) <= 5
-
-    # Test offset
-    if entities_data["total"] > 1:
-        response = await api_client.get(f"/v1/default/banks/{test_bank_id}/entities?limit=1&offset=1")
-        assert response.status_code == 200
-        offset_data = response.json()
-        assert offset_data["offset"] == 1
-        # With offset=1, we should get different entity than first one (if there are multiple)
-        if len(offset_data["items"]) > 0 and len(entities_data["items"]) > 1:
-            assert offset_data["items"][0]["id"] != entities_data["items"][0]["id"]
 
     # Get specific entity if any exist
     if len(entities_data['items']) > 0:
@@ -280,11 +266,11 @@ async def test_full_api_workflow(api_client, test_bank_id):
         entity_detail = response.json()
         assert "id" in entity_detail
 
-        # Test regenerate observations (deprecated - returns 410 Gone)
+        # Test regenerate observations
         response = await api_client.post(
             f"/v1/default/banks/{test_bank_id}/entities/{entity_id}/regenerate"
         )
-        assert response.status_code == 410  # Deprecated endpoint
+        assert response.status_code == 200
 
     # ================================================================
     # 9. List All Banks (should include our test bank)
@@ -836,8 +822,9 @@ async def test_reflect_structured_output(api_client):
     assert response.status_code == 200
     result = response.json()
 
-    # Verify text field exists (may contain text even with structured output)
+    # Verify text field exists (empty when using structured output)
     assert "text" in result
+    assert result["text"] == ""
 
     # Verify structured output exists and has expected structure
     assert "structured_output" in result
@@ -969,24 +956,20 @@ async def test_reflect_returns_token_usage(api_client):
     assert "text" in result
     assert len(result["text"]) > 0
 
-    # Verify usage field exists (may be None for agentic reflect which makes multiple LLM calls)
+    # Verify usage field exists and has expected structure
     assert "usage" in result, "Response should include 'usage' field"
     usage = result["usage"]
+    assert usage is not None, "Usage should not be None for reflect"
+    assert "input_tokens" in usage, "Usage should have 'input_tokens'"
+    assert "output_tokens" in usage, "Usage should have 'output_tokens'"
+    assert "total_tokens" in usage, "Usage should have 'total_tokens'"
 
-    # Usage is optional - agentic reflect doesn't aggregate multiple LLM call usages
-    if usage is not None:
-        assert "input_tokens" in usage, "Usage should have 'input_tokens'"
-        assert "output_tokens" in usage, "Usage should have 'output_tokens'"
-        assert "total_tokens" in usage, "Usage should have 'total_tokens'"
+    # Verify token counts are valid
+    assert usage["input_tokens"] > 0, f"Expected input_tokens > 0, got {usage['input_tokens']}"
+    assert usage["output_tokens"] >= 0, f"Expected output_tokens >= 0, got {usage['output_tokens']}"
+    assert usage["total_tokens"] == usage["input_tokens"] + usage["output_tokens"]
 
-        # Verify token counts are valid
-        assert usage["input_tokens"] > 0, f"Expected input_tokens > 0, got {usage['input_tokens']}"
-        assert usage["output_tokens"] >= 0, f"Expected output_tokens >= 0, got {usage['output_tokens']}"
-        assert usage["total_tokens"] == usage["input_tokens"] + usage["output_tokens"]
-
-        print(f"Reflect token usage: input={usage['input_tokens']}, output={usage['output_tokens']}, total={usage['total_tokens']}")
-    else:
-        print("Reflect usage is None (expected for agentic reflect)")
+    print(f"Reflect token usage: input={usage['input_tokens']}, output={usage['output_tokens']}, total={usage['total_tokens']}")
 
 
 @pytest.mark.asyncio

hindsight-api/tests/test_llm_tools.py

@@ -1,325 +0,0 @@
-"""
-Tests for LLM tool calling functionality.
-"""
-
-import pytest
-
-from hindsight_api.engine.llm_wrapper import LLMProvider
-from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult
-
-
-# Sample tools for testing
-SAMPLE_TOOLS = [
-    {
-        "type": "function",
-        "function": {
-            "name": "get_weather",
-            "description": "Get weather for a location",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "location": {"type": "string", "description": "City name"},
-                    "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
-                },
-                "required": ["location"],
-            },
-        },
-    },
-    {
-        "type": "function",
-        "function": {
-            "name": "search",
-            "description": "Search for information",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "query": {"type": "string", "description": "Search query"},
-                },
-                "required": ["query"],
-            },
-        },
-    },
-]
-
-
-class TestMockToolCalling:
-    """Test tool calling with mock provider."""
-
-    @pytest.mark.asyncio
-    async def test_call_with_tools_returns_tool_calls(self):
-        """Test that mock provider can return tool calls."""
-        llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
-
-        # Set mock response to return tool calls
-        llm.set_mock_response([
-            {"name": "get_weather", "arguments": {"location": "Paris", "unit": "celsius"}},
-        ])
-
-        result = await llm.call_with_tools(
-            messages=[{"role": "user", "content": "What's the weather in Paris?"}],
-            tools=SAMPLE_TOOLS,
-        )
-
-        assert isinstance(result, LLMToolCallResult)
-        assert len(result.tool_calls) == 1
-        assert result.tool_calls[0].name == "get_weather"
-        assert result.tool_calls[0].arguments == {"location": "Paris", "unit": "celsius"}
-        assert result.finish_reason == "tool_calls"
-
-    @pytest.mark.asyncio
-    async def test_call_with_tools_returns_content(self):
-        """Test that mock provider can return plain content."""
-        llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
-
-        # Default mock response is plain content
-        result = await llm.call_with_tools(
-            messages=[{"role": "user", "content": "Hello"}],
-            tools=SAMPLE_TOOLS,
-        )
-
-        assert isinstance(result, LLMToolCallResult)
-        assert result.content == "mock response"
-        assert len(result.tool_calls) == 0
-        assert result.finish_reason == "stop"
-
-    @pytest.mark.asyncio
-    async def test_call_with_tools_records_calls(self):
-        """Test that mock calls are recorded."""
-        llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
-        llm.clear_mock_calls()
-
-        await llm.call_with_tools(
-            messages=[{"role": "user", "content": "Test message"}],
-            tools=SAMPLE_TOOLS,
-            scope="test_scope",
-        )
-
-        calls = llm.get_mock_calls()
-        assert len(calls) == 1
-        assert calls[0]["scope"] == "test_scope"
-        assert "get_weather" in calls[0]["tools"]
-        assert "search" in calls[0]["tools"]
-
-    @pytest.mark.asyncio
-    async def test_call_with_tools_multiple_tool_calls(self):
-        """Test handling multiple tool calls in one response."""
-        llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
-
-        llm.set_mock_response([
-            {"name": "get_weather", "arguments": {"location": "Paris"}},
-            {"name": "search", "arguments": {"query": "weather forecast"}},
-        ])
-
-        result = await llm.call_with_tools(
-            messages=[{"role": "user", "content": "Weather in Paris and search for forecasts"}],
-            tools=SAMPLE_TOOLS,
-        )
-
-        assert len(result.tool_calls) == 2
-        assert result.tool_calls[0].name == "get_weather"
-        assert result.tool_calls[1].name == "search"
-
-    @pytest.mark.asyncio
-    async def test_call_with_tools_accepts_llm_tool_call_result(self):
-        """Test that mock can accept LLMToolCallResult directly."""
-        llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
-
-        expected_result = LLMToolCallResult(
-            content="Here's the info",
-            tool_calls=[LLMToolCall(id="call_123", name="search", arguments={"query": "test"})],
-            finish_reason="tool_calls",
-        )
-        llm.set_mock_response(expected_result)
-
-        result = await llm.call_with_tools(
-            messages=[{"role": "user", "content": "Search for test"}],
-            tools=SAMPLE_TOOLS,
-        )
-
-        assert result == expected_result
-
-
-class TestToolCallConversation:
-    """Test tool call conversation flow."""
-
-    @pytest.mark.asyncio
-    async def test_tool_result_message_format(self):
-        """Test that tool result messages can be passed in subsequent calls."""
-        llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
-
-        # First call returns tool call
-        llm.set_mock_response([{"name": "get_weather", "arguments": {"location": "Paris"}}])
-
-        result1 = await llm.call_with_tools(
-            messages=[{"role": "user", "content": "What's the weather?"}],
-            tools=SAMPLE_TOOLS,
-        )
-
-        # Build conversation with tool result
-        messages = [
-            {"role": "user", "content": "What's the weather?"},
-            {
-                "role": "assistant",
-                "tool_calls": [
-                    {
-                        "id": result1.tool_calls[0].id,
-                        "type": "function",
-                        "function": {
-                            "name": result1.tool_calls[0].name,
-                            "arguments": '{"location": "Paris"}',
-                        },
-                    }
-                ],
-            },
-            {
-                "role": "tool",
-                "tool_call_id": result1.tool_calls[0].id,
-                "content": '{"temperature": 20, "conditions": "sunny"}',
-            },
-        ]
-
-        # Second call should work with tool result in history
-        llm.set_mock_response(None)  # Reset to default
-        result2 = await llm.call_with_tools(
-            messages=messages,
-            tools=SAMPLE_TOOLS,
-        )
-
-        assert result2.content == "mock response"
-
-
-class TestToolSchemas:
-    """Test tool schema handling."""
-
-    @pytest.mark.asyncio
-    async def test_empty_tools_list(self):
-        """Test calling with empty tools list."""
-        llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
-
-        result = await llm.call_with_tools(
-            messages=[{"role": "user", "content": "Hello"}],
-            tools=[],
-        )
-
-        assert result.content == "mock response"
-
-    @pytest.mark.asyncio
-    async def test_tool_with_no_required_params(self):
-        """Test tool with no required parameters."""
-        llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
-
-        tools = [
-            {
-                "type": "function",
-                "function": {
-                    "name": "list_items",
-                    "description": "List all items",
-                    "parameters": {"type": "object", "properties": {}, "required": []},
-                },
-            }
-        ]
-
-        llm.set_mock_response([{"name": "list_items", "arguments": {}}])
-
-        result = await llm.call_with_tools(
-            messages=[{"role": "user", "content": "List items"}],
-            tools=tools,
-        )
-
-        assert len(result.tool_calls) == 1
-        assert result.tool_calls[0].name == "list_items"
-        assert result.tool_calls[0].arguments == {}
-
-
-class TestReflectToolSchemas:
-    """Test reflect agent tool schemas."""
-
-    def test_get_reflect_tools_default(self):
-        """Test getting default reflect tools."""
-        from hindsight_api.engine.reflect.tools_schema import get_reflect_tools
-
-        tools = get_reflect_tools()
-
-        tool_names = [t["function"]["name"] for t in tools]
-        assert "list_mental_models" in tool_names
-        assert "get_mental_model" in tool_names
-        assert "recall" in tool_names
-        assert "learn" in tool_names
-        assert "expand" in tool_names
-        assert "done" in tool_names
-
-    def test_get_reflect_tools_without_learn(self):
-        """Test getting reflect tools without learn."""
-        from hindsight_api.engine.reflect.tools_schema import get_reflect_tools
-
-        tools = get_reflect_tools(enable_learn=False)
-
-        tool_names = [t["function"]["name"] for t in tools]
-        assert "learn" not in tool_names
-        assert "recall" in tool_names
-        assert "done" in tool_names
-
-    def test_get_reflect_tools_answer_mode(self):
-        """Test getting reflect tools with answer output mode."""
-        from hindsight_api.engine.reflect.tools_schema import get_reflect_tools
-
-        tools = get_reflect_tools()
-
-        done_tool = next(t for t in tools if t["function"]["name"] == "done")
-        params = done_tool["function"]["parameters"]["properties"]
-
-        assert "answer" in params
-        assert "memory_ids" in params
-        assert "model_ids" in params
-
-
-class TestLLMToolCallResult:
-    """Test LLMToolCallResult model."""
-
-    def test_tool_call_result_defaults(self):
-        """Test default values for LLMToolCallResult."""
-        result = LLMToolCallResult()
-
-        assert result.content is None
-        assert result.tool_calls == []
-        assert result.finish_reason is None
-
-    def test_tool_call_result_with_content(self):
-        """Test LLMToolCallResult with content."""
-        result = LLMToolCallResult(content="Hello", finish_reason="stop")
-
-        assert result.content == "Hello"
-        assert result.tool_calls == []
-        assert result.finish_reason == "stop"
-
-    def test_tool_call_result_with_tool_calls(self):
-        """Test LLMToolCallResult with tool calls."""
-        result = LLMToolCallResult(
-            tool_calls=[
-                LLMToolCall(id="call_1", name="test_tool", arguments={"arg": "value"}),
-            ],
-            finish_reason="tool_calls",
-        )
-
-        assert result.content is None
-        assert len(result.tool_calls) == 1
-        assert result.tool_calls[0].name == "test_tool"
-        assert result.finish_reason == "tool_calls"
-
-
-class TestLLMToolCall:
-    """Test LLMToolCall model."""
-
-    def test_tool_call_basic(self):
-        """Test basic LLMToolCall creation."""
-        call = LLMToolCall(id="call_123", name="get_weather", arguments={"location": "Paris"})
-
-        assert call.id == "call_123"
-        assert call.name == "get_weather"
-        assert call.arguments == {"location": "Paris"}
-
-    def test_tool_call_empty_arguments(self):
-        """Test LLMToolCall with empty arguments."""
-        call = LLMToolCall(id="call_456", name="list_items", arguments={})
-
-        assert call.arguments == {}

hindsight-api/tests/test_load_large_batch.py

@@ -19,7 +19,6 @@ import pytest_asyncio
 from hindsight_api import MemoryEngine, LLMConfig, LocalSTEmbeddings, RequestContext
 from hindsight_api.engine.cross_encoder import LocalSTCrossEncoder
 from hindsight_api.engine.query_analyzer import DateparserQueryAnalyzer
-from hindsight_api.engine.task_backend import SyncTaskBackend
 from hindsight_api.engine.retain.fact_extraction import FactExtractionResponse, ExtractedFact
 from hindsight_api.engine.llm_wrapper import TokenUsage
 
@@ -107,7 +106,6 @@ class TestLargeBatchRetain:
             pool_max_size=10,
             run_migrations=False,
             skip_llm_verification=True,  # Skip LLM verification since we're mocking
-            task_backend=SyncTaskBackend(),  # Execute tasks immediately in tests
         )
         await mem.initialize()
         yield mem

hindsight-api/tests/test_main_module.py

@@ -1,400 +0,0 @@
-"""
-Tests for hindsight_api.main module (single-worker code path).
-
-The main.py module is used when running with a single worker:
-    hindsight-api  (or hindsight-api --workers 1)
-
-When workers=1, main.py creates the app directly and passes it to uvicorn.
-These tests ensure that extensions are properly loaded in this code path.
-
-Compare with test_server_module.py which tests the multi-worker path (workers > 1).
-"""
-
-import sys
-from unittest.mock import MagicMock, patch
-
-
-class TestMainModuleExtensionLoading:
-    """Tests that main.py correctly loads extensions when configured via environment."""
-
-    def test_main_loads_tenant_extension_when_configured(self, monkeypatch):
-        """
-        Verify that main.py loads tenant extension from HINDSIGHT_API_TENANT_EXTENSION.
-
-        This ensures extension loading works in the single-worker code path.
-        """
-        # Set up environment to configure a tenant extension
-        monkeypatch.setenv(
-            "HINDSIGHT_API_TENANT_EXTENSION",
-            "tests.test_main_module:MockTenantExtension",
-        )
-        # Ensure single worker mode
-        monkeypatch.setenv("HINDSIGHT_API_WORKERS", "1")
-
-        # Track what extensions were loaded via load_extension
-        loaded_extensions = {}
-
-        # Get the real load_extension function
-        from hindsight_api.extensions.loader import load_extension as real_load_extension
-
-        def tracking_load_extension(name, base_class):
-            """Track calls to load_extension and delegate to original."""
-            result = real_load_extension(name, base_class)
-            loaded_extensions[name] = result
-            return result
-
-        with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
-             patch("hindsight_api.main.create_app") as mock_create_app, \
-             patch("hindsight_api.main.get_config") as mock_get_config, \
-             patch("hindsight_api.main.load_extension", side_effect=tracking_load_extension), \
-             patch("hindsight_api.main.DefaultExtensionContext"), \
-             patch("hindsight_api.main.print_banner"), \
-             patch("uvicorn.run"):  # Don't actually start uvicorn
-
-            mock_config = MagicMock()
-            mock_config.host = "0.0.0.0"
-            mock_config.port = 8888
-            mock_config.log_level = "info"
-            mock_config.mcp_enabled = False
-            mock_config.run_migrations_on_startup = False
-            mock_config.database_url = "postgresql://test:test@localhost/test"
-            mock_get_config.return_value = mock_config
-            mock_engine.return_value = MagicMock()
-            mock_create_app.return_value = MagicMock()
-
-            # Mock sys.argv to simulate CLI invocation
-            with patch.object(sys, 'argv', ['hindsight-api']):
-                from hindsight_api.main import main
-                main()
-
-        # Verify TENANT extension was loaded
-        assert "TENANT" in loaded_extensions, \
-            "main.py did not call load_extension('TENANT', ...) - extensions not loaded!"
-        assert loaded_extensions["TENANT"] is not None, \
-            "load_extension('TENANT', ...) returned None despite env var being set"
-        assert isinstance(loaded_extensions["TENANT"], MockTenantExtension), \
-            f"Expected MockTenantExtension, got {type(loaded_extensions['TENANT'])}"
-
-    def test_main_loads_operation_validator_when_configured(self, monkeypatch):
-        """
-        Verify that main.py loads operation validator from HINDSIGHT_API_OPERATION_VALIDATOR_EXTENSION.
-        """
-        monkeypatch.setenv(
-            "HINDSIGHT_API_OPERATION_VALIDATOR_EXTENSION",
-            "tests.test_main_module:MockOperationValidator",
-        )
-        monkeypatch.setenv("HINDSIGHT_API_WORKERS", "1")
-
-        loaded_extensions = {}
-
-        from hindsight_api.extensions.loader import load_extension as real_load_extension
-
-        def tracking_load_extension(name, base_class):
-            result = real_load_extension(name, base_class)
-            loaded_extensions[name] = result
-            return result
-
-        with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
-             patch("hindsight_api.main.create_app") as mock_create_app, \
-             patch("hindsight_api.main.get_config") as mock_get_config, \
-             patch("hindsight_api.main.load_extension", side_effect=tracking_load_extension), \
-             patch("hindsight_api.main.DefaultExtensionContext"), \
-             patch("hindsight_api.main.print_banner"), \
-             patch("uvicorn.run"):
-
-            mock_config = MagicMock()
-            mock_config.host = "0.0.0.0"
-            mock_config.port = 8888
-            mock_config.log_level = "info"
-            mock_config.mcp_enabled = False
-            mock_config.run_migrations_on_startup = False
-            mock_config.database_url = "postgresql://test:test@localhost/test"
-            mock_get_config.return_value = mock_config
-            mock_engine.return_value = MagicMock()
-            mock_create_app.return_value = MagicMock()
-
-            with patch.object(sys, 'argv', ['hindsight-api']):
-                from hindsight_api.main import main
-                main()
-
-        assert "OPERATION_VALIDATOR" in loaded_extensions, \
-            "main.py did not call load_extension('OPERATION_VALIDATOR', ...)"
-        assert loaded_extensions["OPERATION_VALIDATOR"] is not None
-        assert isinstance(loaded_extensions["OPERATION_VALIDATOR"], MockOperationValidator)
-
-    def test_main_passes_extensions_to_memory_engine(self, monkeypatch):
-        """
-        Verify that main.py passes loaded extensions to MemoryEngine constructor.
-
-        This is the critical test - even if extensions are loaded, they must be
-        passed to MemoryEngine for authentication to work.
-        """
-        monkeypatch.setenv(
-            "HINDSIGHT_API_TENANT_EXTENSION",
-            "tests.test_main_module:MockTenantExtension",
-        )
-        monkeypatch.setenv("HINDSIGHT_API_WORKERS", "1")
-
-        memory_engine_calls = []
-
-        def capture_memory_engine(*args, **kwargs):
-            memory_engine_calls.append({"args": args, "kwargs": kwargs})
-            return MagicMock()
-
-        with patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine), \
-             patch("hindsight_api.main.create_app") as mock_create_app, \
-             patch("hindsight_api.main.get_config") as mock_get_config, \
-             patch("hindsight_api.main.DefaultExtensionContext"), \
-             patch("hindsight_api.main.print_banner"), \
-             patch("uvicorn.run"):
-
-            mock_config = MagicMock()
-            mock_config.host = "0.0.0.0"
-            mock_config.port = 8888
-            mock_config.log_level = "info"
-            mock_config.mcp_enabled = False
-            mock_config.run_migrations_on_startup = False
-            mock_config.database_url = "postgresql://test:test@localhost/test"
-            mock_get_config.return_value = mock_config
-            mock_create_app.return_value = MagicMock()
-
-            with patch.object(sys, 'argv', ['hindsight-api']):
-                from hindsight_api.main import main
-                main()
-
-        # Verify MemoryEngine was called
-        assert len(memory_engine_calls) == 1, "MemoryEngine should be called exactly once"
-
-        call_kwargs = memory_engine_calls[0]["kwargs"]
-
-        # THE CRITICAL ASSERTION: tenant_extension must be passed and not None
-        assert "tenant_extension" in call_kwargs, \
-            "MemoryEngine was not called with tenant_extension parameter!"
-        assert call_kwargs["tenant_extension"] is not None, \
-            "tenant_extension was None - main.py did not pass loaded extension to MemoryEngine!"
-
-    def test_main_sets_extension_context_on_tenant_extension(self, monkeypatch):
-        """
-        Verify that main.py sets the extension context on tenant extension.
-
-        This is required for tenant extensions that need to provision schemas.
-        """
-        monkeypatch.setenv(
-            "HINDSIGHT_API_TENANT_EXTENSION",
-            "tests.test_main_module:MockTenantExtension",
-        )
-        monkeypatch.setenv("HINDSIGHT_API_WORKERS", "1")
-
-        captured_tenant_ext = [None]
-
-        def capture_memory_engine(*args, **kwargs):
-            captured_tenant_ext[0] = kwargs.get("tenant_extension")
-            return MagicMock()
-
-        context_created = []
-
-        def capture_context(*args, **kwargs):
-            ctx = MagicMock()
-            context_created.append(ctx)
-            return ctx
-
-        with patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine), \
-             patch("hindsight_api.main.create_app") as mock_create_app, \
-             patch("hindsight_api.main.get_config") as mock_get_config, \
-             patch("hindsight_api.main.DefaultExtensionContext", side_effect=capture_context), \
-             patch("hindsight_api.main.print_banner"), \
-             patch("uvicorn.run"):
-
-            mock_config = MagicMock()
-            mock_config.host = "0.0.0.0"
-            mock_config.port = 8888
-            mock_config.log_level = "info"
-            mock_config.mcp_enabled = False
-            mock_config.run_migrations_on_startup = False
-            mock_config.database_url = "postgresql://test:test@localhost/test"
-            mock_get_config.return_value = mock_config
-            mock_create_app.return_value = MagicMock()
-
-            with patch.object(sys, 'argv', ['hindsight-api']):
-                from hindsight_api.main import main
-                main()
-
-        # Verify context was created and set
-        assert len(context_created) == 1, "DefaultExtensionContext should be created"
-        assert captured_tenant_ext[0] is not None, "Tenant extension should be captured"
-        assert captured_tenant_ext[0]._context_set, \
-            "set_context was not called on tenant extension"
-
-    def test_main_works_without_extensions(self, monkeypatch):
-        """
-        Verify that main.py works correctly when no extensions are configured.
-        """
-        # Ensure no extension env vars are set
-        monkeypatch.delenv("HINDSIGHT_API_TENANT_EXTENSION", raising=False)
-        monkeypatch.delenv("HINDSIGHT_API_OPERATION_VALIDATOR_EXTENSION", raising=False)
-        monkeypatch.setenv("HINDSIGHT_API_WORKERS", "1")
-
-        memory_engine_calls = []
-
-        def capture_memory_engine(*args, **kwargs):
-            memory_engine_calls.append({"args": args, "kwargs": kwargs})
-            return MagicMock()
-
-        with patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine), \
-             patch("hindsight_api.main.create_app") as mock_create_app, \
-             patch("hindsight_api.main.get_config") as mock_get_config, \
-             patch("hindsight_api.main.print_banner"), \
-             patch("uvicorn.run"):
-
-            mock_config = MagicMock()
-            mock_config.host = "0.0.0.0"
-            mock_config.port = 8888
-            mock_config.log_level = "info"
-            mock_config.mcp_enabled = False
-            mock_config.run_migrations_on_startup = False
-            mock_config.database_url = "postgresql://test:test@localhost/test"
-            mock_get_config.return_value = mock_config
-            mock_create_app.return_value = MagicMock()
-
-            with patch.object(sys, 'argv', ['hindsight-api']):
-                from hindsight_api.main import main
-                main()
-
-        # Should work without extensions
-        assert len(memory_engine_calls) == 1
-        call_kwargs = memory_engine_calls[0]["kwargs"]
-
-        # Extensions should be None when not configured
-        assert call_kwargs.get("tenant_extension") is None
-        assert call_kwargs.get("operation_validator") is None
-
-    def test_main_uses_app_object_for_single_worker(self, monkeypatch):
-        """
-        Verify that main.py passes the app object (not import string) when workers=1.
-
-        This is important because it means single-worker mode uses the app created
-        in main.py (with extensions loaded), not server.py.
-        """
-        monkeypatch.setenv("HINDSIGHT_API_WORKERS", "1")
-        monkeypatch.delenv("HINDSIGHT_API_TENANT_EXTENSION", raising=False)
-
-        uvicorn_calls = []
-
-        def capture_uvicorn_run(**kwargs):
-            uvicorn_calls.append(kwargs)
-
-        mock_app = MagicMock()
-
-        with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
-             patch("hindsight_api.main.create_app", return_value=mock_app), \
-             patch("hindsight_api.main.get_config") as mock_get_config, \
-             patch("hindsight_api.main.print_banner"), \
-             patch("uvicorn.run", side_effect=capture_uvicorn_run):
-
-            mock_config = MagicMock()
-            mock_config.host = "0.0.0.0"
-            mock_config.port = 8888
-            mock_config.log_level = "info"
-            mock_config.mcp_enabled = False
-            mock_config.run_migrations_on_startup = False
-            mock_config.database_url = "postgresql://test:test@localhost/test"
-            mock_get_config.return_value = mock_config
-            mock_engine.return_value = MagicMock()
-
-            with patch.object(sys, 'argv', ['hindsight-api', '--workers', '1']):
-                from hindsight_api.main import main
-                main()
-
-        assert len(uvicorn_calls) == 1
-        # With workers=1, should pass app object, not import string
-        assert uvicorn_calls[0]["app"] is mock_app, \
-            "main.py should pass app object (not import string) when workers=1"
-
-    def test_main_uses_import_string_for_multiple_workers(self, monkeypatch):
-        """
-        Verify that main.py uses import string when workers > 1.
-
-        This is important because multi-worker mode requires server.py to be imported
-        by each worker process.
-        """
-        monkeypatch.setenv("HINDSIGHT_API_WORKERS", "2")
-        monkeypatch.delenv("HINDSIGHT_API_TENANT_EXTENSION", raising=False)
-
-        uvicorn_calls = []
-
-        def capture_uvicorn_run(**kwargs):
-            uvicorn_calls.append(kwargs)
-
-        with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
-             patch("hindsight_api.main.create_app") as mock_create_app, \
-             patch("hindsight_api.main.get_config") as mock_get_config, \
-             patch("hindsight_api.main.print_banner"), \
-             patch("uvicorn.run", side_effect=capture_uvicorn_run):
-
-            mock_config = MagicMock()
-            mock_config.host = "0.0.0.0"
-            mock_config.port = 8888
-            mock_config.log_level = "info"
-            mock_config.mcp_enabled = False
-            mock_config.run_migrations_on_startup = False
-            mock_config.database_url = "postgresql://test:test@localhost/test"
-            mock_get_config.return_value = mock_config
-            mock_engine.return_value = MagicMock()
-            mock_create_app.return_value = MagicMock()
-
-            with patch.object(sys, 'argv', ['hindsight-api', '--workers', '2']):
-                from hindsight_api.main import main
-                main()
-
-        assert len(uvicorn_calls) == 1
-        # With workers > 1, should use import string
-        assert uvicorn_calls[0]["app"] == "hindsight_api.server:app", \
-            "main.py should use import string when workers > 1"
-        assert uvicorn_calls[0]["workers"] == 2
-
-
-# Mock extensions for testing
-from hindsight_api.extensions import (
-    TenantExtension,
-    TenantContext,
-    RequestContext,
-    OperationValidatorExtension,
-    ValidationResult,
-    RetainContext,
-    RecallContext,
-    ReflectContext,
-    RefreshMentalModelContext,
-)
-
-
-class MockTenantExtension(TenantExtension):
-    """Mock tenant extension for testing main.py extension loading."""
-
-    def __init__(self, config: dict):
-        super().__init__(config)
-        self._context_set = False
-
-    async def authenticate(self, request_context: RequestContext) -> TenantContext:
-        return TenantContext(schema_name="public")
-
-    def set_context(self, context) -> None:
-        self._context_set = True
-
-
-class MockOperationValidator(OperationValidatorExtension):
-    """Mock operation validator for testing main.py extension loading."""
-
-    def __init__(self, config: dict):
-        super().__init__(config)
-
-    async def validate_retain(self, ctx: RetainContext) -> ValidationResult:
-        return ValidationResult.accept()
-
-    async def validate_recall(self, ctx: RecallContext) -> ValidationResult:
-        return ValidationResult.accept()
-
-    async def validate_reflect(self, ctx: ReflectContext) -> ValidationResult:
-        return ValidationResult.accept()
-
-    async def validate_refresh_mental_model(self, ctx: RefreshMentalModelContext) -> ValidationResult:
-        return ValidationResult.accept()

hindsight-api/tests/test_mpfp_retrieval.py

@@ -20,7 +20,7 @@ from hindsight_api.engine.search.mpfp_retrieval import (
     MPFPGraphRetriever,
     PatternResult,
     SeedNode,
-    load_all_edges_for_frontier,
+    load_edges_for_frontier,
     mpfp_traverse_async,
     rrf_fusion,
 )
@@ -36,32 +36,32 @@ class TestEdgeCache:
         neighbors = cache.get_neighbors("semantic", "node-1")
         assert neighbors == []
 
-    def test_is_fully_loaded_false_for_uncached(self):
-        """is_fully_loaded should return False for nodes not yet loaded."""
+    def test_is_loaded_false_for_uncached(self):
+        """is_loaded should return False for nodes not yet loaded."""
         cache = EdgeCache()
-        assert cache.is_fully_loaded("node-1") is False
+        assert cache.is_loaded("semantic", "node-1") is False
 
-    def test_add_all_edges_marks_as_fully_loaded(self):
-        """Adding edges should mark nodes as fully loaded."""
+    def test_add_edges_marks_as_loaded(self):
+        """Adding edges should mark nodes as loaded."""
         cache = EdgeCache()
 
-        edges_by_type = {
-            "semantic": {"node-1": [EdgeTarget("node-2", 0.8), EdgeTarget("node-3", 0.6)]},
+        edges = {
+            "node-1": [EdgeTarget("node-2", 0.8), EdgeTarget("node-3", 0.6)],
         }
-        cache.add_all_edges(edges_by_type, ["node-1", "node-4"])  # node-4 has no edges
+        cache.add_edges("semantic", edges, ["node-1", "node-4"])  # node-4 has no edges
 
-        assert cache.is_fully_loaded("node-1") is True
-        assert cache.is_fully_loaded("node-4") is True  # Marked even with no edges
-        assert cache.is_fully_loaded("node-2") is False  # Target, not source
+        assert cache.is_loaded("semantic", "node-1") is True
+        assert cache.is_loaded("semantic", "node-4") is True  # Marked even with no edges
+        assert cache.is_loaded("semantic", "node-2") is False  # Target, not source
 
     def test_get_neighbors_returns_added_edges(self):
-        """get_neighbors should return edges after add_all_edges."""
+        """get_neighbors should return edges after add_edges."""
         cache = EdgeCache()
 
-        edges_by_type = {
-            "semantic": {"node-1": [EdgeTarget("node-2", 0.8), EdgeTarget("node-3", 0.6)]},
+        edges = {
+            "node-1": [EdgeTarget("node-2", 0.8), EdgeTarget("node-3", 0.6)],
         }
-        cache.add_all_edges(edges_by_type, ["node-1"])
+        cache.add_edges("semantic", edges, ["node-1"])
 
         neighbors = cache.get_neighbors("semantic", "node-1")
         assert len(neighbors) == 2
@@ -69,30 +69,28 @@ class TestEdgeCache:
         assert neighbors[0].weight == 0.8
 
     def test_get_uncached_filters_loaded_nodes(self):
-        """get_uncached should only return nodes not yet fully loaded."""
+        """get_uncached should only return nodes not yet loaded."""
         cache = EdgeCache()
 
-        # Load some nodes (all edge types)
-        cache.add_all_edges({"semantic": {"node-1": []}}, ["node-1", "node-2"])
+        # Load some nodes
+        cache.add_edges("semantic", {"node-1": []}, ["node-1", "node-2"])
 
         # Check uncached
-        uncached = cache.get_uncached(["node-1", "node-2", "node-3", "node-4"])
+        uncached = cache.get_uncached("semantic", ["node-1", "node-2", "node-3", "node-4"])
         assert set(uncached) == {"node-3", "node-4"}
 
     def test_get_normalized_neighbors_normalizes_weights(self):
         """get_normalized_neighbors should normalize weights to sum to 1."""
         cache = EdgeCache()
 
-        edges_by_type = {
-            "semantic": {
-                "node-1": [
-                    EdgeTarget("node-2", 0.8),
-                    EdgeTarget("node-3", 0.4),
-                    EdgeTarget("node-4", 0.2),
-                ],
-            },
+        edges = {
+            "node-1": [
+                EdgeTarget("node-2", 0.8),
+                EdgeTarget("node-3", 0.4),
+                EdgeTarget("node-4", 0.2),
+            ],
         }
-        cache.add_all_edges(edges_by_type, ["node-1"])
+        cache.add_edges("semantic", edges, ["node-1"])
 
         # Get top 2, normalized
         neighbors = cache.get_normalized_neighbors("semantic", "node-1", top_k=2)
@@ -113,11 +111,8 @@ class TestEdgeCache:
         """Different edge types should be stored separately."""
         cache = EdgeCache()
 
-        edges_by_type = {
-            "semantic": {"node-1": [EdgeTarget("node-2", 0.8)]},
-            "temporal": {"node-1": [EdgeTarget("node-3", 0.5)]},
-        }
-        cache.add_all_edges(edges_by_type, ["node-1"])
+        cache.add_edges("semantic", {"node-1": [EdgeTarget("node-2", 0.8)]}, ["node-1"])
+        cache.add_edges("temporal", {"node-1": [EdgeTarget("node-3", 0.5)]}, ["node-1"])
 
         semantic_neighbors = cache.get_neighbors("semantic", "node-1")
         temporal_neighbors = cache.get_neighbors("temporal", "node-1")
@@ -203,6 +198,7 @@ class TestMPFPTraverseAsync:
 
         result = await mpfp_traverse_async(
             pool=None,  # Not used when no seeds
+            bank_id="test",
             seeds=[],
             pattern=["semantic"],
             config=config,
@@ -217,18 +213,19 @@ class TestMPFPTraverseAsync:
         cache = EdgeCache()
         config = MPFPConfig(alpha=0.15, threshold=1e-6)
 
-        # Pre-populate cache with empty edges for seed (marks as fully loaded)
-        cache.add_all_edges({}, ["seed-1"])
+        # Pre-populate cache with empty edges for seed
+        cache.add_edges("semantic", {}, ["seed-1"])
 
         seeds = [SeedNode("seed-1", 1.0)]
 
         with patch(
-            "hindsight_api.engine.search.mpfp_retrieval.load_all_edges_for_frontier",
+            "hindsight_api.engine.search.mpfp_retrieval.load_edges_for_frontier",
             new_callable=AsyncMock,
             return_value={},
         ):
             result = await mpfp_traverse_async(
                 pool=MagicMock(),
+                bank_id="test",
                 seeds=seeds,
                 pattern=["semantic"],
                 config=config,
@@ -247,29 +244,24 @@ class TestMPFPTraverseAsync:
 
         seeds = [SeedNode("seed-1", 1.0)]
 
-        # Pre-populate cache with seed edges (mimics pre-warming in retrieve())
-        cache.add_all_edges(
-            {
-                "semantic": {
+        # Mock edge loading
+        async def mock_load_edges(pool, bank_id, edge_type, node_ids):
+            if "seed-1" in node_ids:
+                return {
                     "seed-1": [
                         EdgeTarget("neighbor-1", 0.8),
                         EdgeTarget("neighbor-2", 0.4),
                     ]
                 }
-            },
-            ["seed-1"],
-        )
-
-        # Mock for loading neighbor edges (after hop 0)
-        async def mock_load_all_edges(pool, node_ids, top_k=20):
             return {}
 
         with patch(
-            "hindsight_api.engine.search.mpfp_retrieval.load_all_edges_for_frontier",
-            side_effect=mock_load_all_edges,
+            "hindsight_api.engine.search.mpfp_retrieval.load_edges_for_frontier",
+            side_effect=mock_load_edges,
         ):
             result = await mpfp_traverse_async(
                 pool=MagicMock(),
+                bank_id="test",
                 seeds=seeds,
                 pattern=["semantic"],
                 config=config,
@@ -295,25 +287,22 @@ class TestMPFPTraverseAsync:
 
         seeds = [SeedNode("seed-1", 1.0)]
 
-        # Pre-populate cache with seed edges (mimics pre-warming in retrieve())
-        cache.add_all_edges(
-            {"semantic": {"seed-1": [EdgeTarget("hop1-node", 1.0)]}},
-            ["seed-1"],
-        )
-
-        # Mock edge loading for hop 1 nodes
-        async def mock_load_all_edges(pool, node_ids, top_k=20):
-            edges: dict[str, dict[str, list[EdgeTarget]]] = {"semantic": {}}
+        # Mock edge loading for two hops
+        async def mock_load_edges(pool, bank_id, edge_type, node_ids):
+            edges = {}
+            if "seed-1" in node_ids:
+                edges["seed-1"] = [EdgeTarget("hop1-node", 1.0)]
             if "hop1-node" in node_ids:
-                edges["semantic"]["hop1-node"] = [EdgeTarget("hop2-node", 1.0)]
+                edges["hop1-node"] = [EdgeTarget("hop2-node", 1.0)]
             return edges
 
         with patch(
-            "hindsight_api.engine.search.mpfp_retrieval.load_all_edges_for_frontier",
-            side_effect=mock_load_all_edges,
+            "hindsight_api.engine.search.mpfp_retrieval.load_edges_for_frontier",
+            side_effect=mock_load_edges,
         ):
             result = await mpfp_traverse_async(
                 pool=MagicMock(),
+                bank_id="test",
                 seeds=seeds,
                 pattern=["semantic", "semantic"],  # Two hops
                 config=config,
@@ -327,35 +316,31 @@ class TestMPFPTraverseAsync:
 
     @pytest.mark.asyncio
     async def test_cache_reuse(self):
-        """Cache should prevent redundant edge loading for already-cached nodes."""
+        """Cache should prevent redundant edge loading."""
         cache = EdgeCache()
         config = MPFPConfig(alpha=0.15, threshold=1e-6)
 
-        # Pre-load cache (marks seed-1 AND neighbor-1 as fully loaded)
-        # neighbor-1 is also cached because after hop 0, the frontier contains neighbor-1
-        # and the algorithm tries to pre-warm edges for the next hop
-        cache.add_all_edges(
-            {"semantic": {"seed-1": [EdgeTarget("neighbor-1", 1.0)], "neighbor-1": []}},
-            ["seed-1", "neighbor-1"],
-        )
+        # Pre-load cache
+        cache.add_edges("semantic", {"seed-1": [EdgeTarget("neighbor-1", 1.0)]}, ["seed-1"])
 
         seeds = [SeedNode("seed-1", 1.0)]
 
         load_mock = AsyncMock(return_value={})
 
         with patch(
-            "hindsight_api.engine.search.mpfp_retrieval.load_all_edges_for_frontier",
+            "hindsight_api.engine.search.mpfp_retrieval.load_edges_for_frontier",
             load_mock,
         ):
             await mpfp_traverse_async(
                 pool=MagicMock(),
+                bank_id="test",
                 seeds=seeds,
                 pattern=["semantic"],
                 config=config,
                 cache=cache,
             )
 
-        # Should not call load_all_edges_for_frontier since all nodes are already cached
+        # Should not call load_edges_for_frontier since seed-1 is already cached
         load_mock.assert_not_called()
 
 
@@ -369,9 +354,7 @@ class TestMPFPGraphRetriever:
 
     def test_default_config(self):
         """Default config should have expected patterns."""
-        # Use explicit config to avoid global config dependency
-        config = MPFPConfig()
-        retriever = MPFPGraphRetriever(config=config)
+        retriever = MPFPGraphRetriever()
 
         assert len(retriever.config.patterns_semantic) > 0
         assert len(retriever.config.patterns_temporal) > 0
@@ -413,9 +396,7 @@ class TestMPFPGraphRetriever:
     @pytest.mark.asyncio
     async def test_retrieve_no_seeds_returns_empty(self):
         """Retrieve with no seeds should return empty results."""
-        # Use explicit config to avoid global config dependency
-        config = MPFPConfig()
-        retriever = MPFPGraphRetriever(config=config)
+        retriever = MPFPGraphRetriever()
 
         # Mock _find_semantic_seeds to return empty
         with patch.object(retriever, "_find_semantic_seeds", new_callable=AsyncMock, return_value=[]):
@@ -434,18 +415,15 @@ class TestMPFPGraphRetriever:
     @pytest.mark.asyncio
     async def test_retrieve_with_semantic_seeds(self):
         """Retrieve with semantic seeds should run patterns and return results."""
-        # Use explicit config to avoid global config dependency
-        config = MPFPConfig()
-        retriever = MPFPGraphRetriever(config=config)
+        retriever = MPFPGraphRetriever()
 
         semantic_seeds = [
             RetrievalResult(id="seed-1", text="seed text", fact_type="world", similarity=0.9),
         ]
 
         # Mock the internal functions
-        # mpfp_traverse_hop_synchronized returns a list of PatternResult (one per pattern)
         async def mock_traverse(*args, **kwargs):
-            return [PatternResult(pattern=["semantic"], scores={"seed-1": 0.5, "result-1": 0.3})]
+            return PatternResult(pattern=["semantic"], scores={"seed-1": 0.5, "result-1": 0.3})
 
         async def mock_fetch(pool, node_ids, fact_type):
             return [
@@ -455,18 +433,13 @@ class TestMPFPGraphRetriever:
 
         with (
             patch(
-                "hindsight_api.engine.search.mpfp_retrieval.mpfp_traverse_hop_synchronized",
+                "hindsight_api.engine.search.mpfp_retrieval.mpfp_traverse_async",
                 side_effect=mock_traverse,
             ),
             patch(
                 "hindsight_api.engine.search.mpfp_retrieval.fetch_memory_units_by_ids",
                 side_effect=mock_fetch,
             ),
-            patch(
-                "hindsight_api.engine.search.mpfp_retrieval.load_all_edges_for_frontier",
-                new_callable=AsyncMock,
-                return_value={},
-            ),
         ):
             results, timings = await retriever.retrieve(
                 pool=MagicMock(),
@@ -578,242 +551,3 @@ async def test_mpfp_lazy_loading_efficiency(memory, request_context):
 
     finally:
         await memory.delete_bank(bank_id, request_context=request_context)
-
-
-# ============================================================================
-# MPFP Performance Benchmark Tests
-# ============================================================================
-# These tests require an external database with a large memory bank to be useful.
-# Set EXTERNAL_DATABASE_URL and BENCHMARK_BANK_ID environment variables to run.
-# Example:
-#   EXTERNAL_DATABASE_URL=postgresql://user:pass@host:port/db \
-#   BENCHMARK_BANK_ID=load-test \
-#   pytest tests/test_mpfp_retrieval.py::test_mpfp_edge_loading_performance -v -s
-
-
-import os
-import asyncpg
-
-EXTERNAL_DATABASE_URL = os.environ.get("EXTERNAL_DATABASE_URL")
-BENCHMARK_BANK_ID = os.environ.get("BENCHMARK_BANK_ID", "load-test")
-
-requires_external_db = pytest.mark.skipif(
-    EXTERNAL_DATABASE_URL is None,
-    reason="EXTERNAL_DATABASE_URL not set - skipping external DB benchmark",
-)
-
-
-@requires_external_db
-@pytest.mark.asyncio
-async def test_mpfp_edge_loading_performance():
-    """
-    Benchmark MPFP edge loading performance.
-
-    This test measures the performance of the LATERAL query optimization
-    for loading edges in the MPFP graph traversal algorithm.
-
-    Set EXTERNAL_DATABASE_URL to point to a database with existing data.
-    Set BENCHMARK_BANK_ID to specify which bank to query (default: load-test).
-
-    Example usage:
-        EXTERNAL_DATABASE_URL=postgresql://hindsight:hindsight@localhost:5435/hindsight \
-        BENCHMARK_BANK_ID=load-test \
-        pytest tests/test_mpfp_retrieval.py::test_mpfp_edge_loading_performance -v -s
-    """
-    import time
-
-    # Connect to external database
-    pool = await asyncpg.create_pool(EXTERNAL_DATABASE_URL, min_size=2, max_size=10)
-
-    try:
-        # Get some sample node IDs from the database
-        async with pool.acquire() as conn:
-            # First check how many links exist
-            stats = await conn.fetchrow("""
-                SELECT
-                    count(*) as total_links,
-                    count(DISTINCT from_unit_id) as unique_sources
-                FROM memory_links
-            """)
-            print(f"\n📊 Database Stats:")
-            print(f"   Total links: {stats['total_links']:,}")
-            print(f"   Unique sources: {stats['unique_sources']:,}")
-
-            # Get edge distribution by type
-            type_stats = await conn.fetch("""
-                SELECT link_type, count(*) as cnt,
-                       round(avg(weight)::numeric, 3) as avg_weight
-                FROM memory_links
-                GROUP BY link_type
-                ORDER BY cnt DESC
-            """)
-            print(f"\n   Edge distribution:")
-            for row in type_stats:
-                print(f"     - {row['link_type']}: {row['cnt']:,} (avg_weight={row['avg_weight']})")
-
-            # Get sample frontier nodes (from memory_units in the benchmark bank)
-            # bank_id is the text primary key in banks table
-            frontier_rows = await conn.fetch("""
-                SELECT id FROM memory_units
-                WHERE bank_id = $1
-                LIMIT 100
-            """, BENCHMARK_BANK_ID)
-
-            if not frontier_rows:
-                pytest.skip(f"No memory units found for bank '{BENCHMARK_BANK_ID}'")
-
-            frontier_node_ids = [str(row['id']) for row in frontier_rows]
-            print(f"\n🎯 Testing with {len(frontier_node_ids)} frontier nodes from bank '{BENCHMARK_BANK_ID}'")
-
-        # Test 1: Original query approach (all edges, no per-type limit)
-        async with pool.acquire() as conn:
-            start = time.time()
-            original_rows = await conn.fetch("""
-                SELECT ml.from_unit_id, ml.to_unit_id, ml.link_type, ml.weight
-                FROM memory_links ml
-                WHERE ml.from_unit_id = ANY($1::uuid[])
-                  AND ml.weight >= 0.1
-                ORDER BY ml.from_unit_id, ml.link_type, ml.weight DESC
-            """, frontier_node_ids)
-            original_time = time.time() - start
-            original_count = len(original_rows)
-
-        # Test 2: New LATERAL query approach (top-k per type)
-        async with pool.acquire() as conn:
-            start = time.time()
-            lateral_rows = await conn.fetch("""
-                WITH frontier(node_id) AS (SELECT unnest($1::uuid[]))
-                SELECT f.node_id as from_unit_id, lt.link_type, edges.to_unit_id, edges.weight
-                FROM frontier f
-                CROSS JOIN (VALUES ('semantic'), ('temporal'), ('entity'), ('causes'), ('caused_by')) AS lt(link_type)
-                CROSS JOIN LATERAL (
-                    SELECT ml.to_unit_id, ml.weight
-                    FROM memory_links ml
-                    WHERE ml.from_unit_id = f.node_id
-                      AND ml.link_type = lt.link_type
-                      AND ml.weight >= 0.1
-                    ORDER BY ml.weight DESC
-                    LIMIT 20
-                ) edges
-            """, frontier_node_ids)
-            lateral_time = time.time() - start
-            lateral_count = len(lateral_rows)
-
-        # Print results
-        print(f"\n⏱️  Performance Comparison ({len(frontier_node_ids)} nodes):")
-        print(f"\n   Original (all edges):")
-        print(f"     - Time: {original_time * 1000:.2f}ms")
-        print(f"     - Rows: {original_count:,}")
-        print(f"     - Rows/node: {original_count / len(frontier_node_ids):.1f}")
-
-        print(f"\n   LATERAL (top-20 per type):")
-        print(f"     - Time: {lateral_time * 1000:.2f}ms")
-        print(f"     - Rows: {lateral_count:,}")
-        print(f"     - Rows/node: {lateral_count / len(frontier_node_ids):.1f}")
-
-        speedup = original_time / lateral_time if lateral_time > 0 else float('inf')
-        reduction = (1 - lateral_count / original_count) * 100 if original_count > 0 else 0
-        print(f"\n   📈 Improvement:")
-        print(f"     - Speedup: {speedup:.2f}x faster")
-        print(f"     - Data reduction: {reduction:.1f}% fewer rows")
-
-        # Assert improvement (should be at least some improvement for large datasets)
-        if original_count > 1000:
-            # For large datasets, expect significant improvement
-            assert speedup >= 1.5, f"Expected at least 1.5x speedup, got {speedup:.2f}x"
-            assert reduction >= 30, f"Expected at least 30% data reduction, got {reduction:.1f}%"
-            print(f"\n✅ Performance test PASSED!")
-        else:
-            print(f"\n⚠️ Dataset too small ({original_count} rows) for meaningful performance comparison")
-
-    finally:
-        await pool.close()
-
-
-@requires_external_db
-@pytest.mark.asyncio
-async def test_mpfp_full_retrieval_performance():
-    """
-    Benchmark full MPFP retrieval including traversal and reranking.
-
-    This test measures end-to-end MPFP retrieval performance.
-    """
-    import time
-
-    pool = await asyncpg.create_pool(EXTERNAL_DATABASE_URL, min_size=2, max_size=10)
-
-    try:
-        # Get a sample query embedding from an existing memory unit
-        async with pool.acquire() as conn:
-            # Check if bank exists
-            bank_exists = await conn.fetchval("""
-                SELECT 1 FROM banks WHERE bank_id = $1
-            """, BENCHMARK_BANK_ID)
-            if not bank_exists:
-                pytest.skip(f"Bank '{BENCHMARK_BANK_ID}' not found")
-
-            sample = await conn.fetchrow("""
-                SELECT embedding::text as embedding_str
-                FROM memory_units
-                WHERE bank_id = $1
-                  AND embedding IS NOT NULL
-                LIMIT 1
-            """, BENCHMARK_BANK_ID)
-
-            if not sample:
-                pytest.skip("No memory units with embeddings found")
-
-            query_embedding_str = sample['embedding_str']
-
-        # Run MPFP retrieval
-        retriever = MPFPGraphRetriever()
-
-        print(f"\n🔍 Running MPFP retrieval benchmark on bank '{BENCHMARK_BANK_ID}'...")
-
-        # Warm-up run
-        await retriever.retrieve(
-            pool=pool,
-            query_embedding_str=query_embedding_str,
-            bank_id=BENCHMARK_BANK_ID,
-            fact_type="world",
-            budget=100,
-            query_text="test query",
-        )
-
-        # Timed runs
-        timings_list = []
-        for i in range(3):
-            start = time.time()
-            results, timings = await retriever.retrieve(
-                pool=pool,
-                query_embedding_str=query_embedding_str,
-                bank_id=BENCHMARK_BANK_ID,
-                fact_type="opinion",
-                budget=100,
-                query_text="What did I say about training models?",
-            )
-            elapsed = time.time() - start
-            timings_list.append((elapsed, timings, len(results)))
-
-        # Print results
-        print(f"\n⏱️  MPFP Retrieval Results (3 runs):")
-        for i, (elapsed, timings, count) in enumerate(timings_list):
-            print(f"\n   Run {i + 1}:")
-            print(f"     - Total: {elapsed * 1000:.2f}ms")
-            print(f"     - Results: {count}")
-            if timings:
-                print(f"     - Seeds: {timings.seeds_time * 1000:.2f}ms")
-                print(f"     - Patterns: {timings.pattern_count}")
-                print(f"     - Traverse: {timings.traverse * 1000:.2f}ms")
-                print(f"     - Edge load: {timings.edge_load_time * 1000:.2f}ms")
-                print(f"     - Edges: {timings.edge_count:,}")
-                print(f"     - DB queries: {timings.db_queries}")
-                print(f"     - Fusion: {timings.fusion * 1000:.2f}ms")
-                print(f"     - Fetch: {timings.fetch * 1000:.2f}ms")
-
-        avg_time = sum(t[0] for t in timings_list) / len(timings_list)
-        print(f"\n   📊 Average: {avg_time * 1000:.2f}ms")
-        print(f"\n✅ MPFP retrieval benchmark complete!")
-
-    finally:
-        await pool.close()

hindsight-api/tests/test_observation_trends.py

@@ -1,405 +0,0 @@
-"""Tests for observation trend computation and evidence-grounded models."""
-
-from datetime import datetime, timedelta, timezone
-
-import pytest
-
-from hindsight_api.engine.reflect.observations import (
-    CandidateObservation,
-    Observation,
-    ObservationEvidence,
-    Trend,
-    compute_trend,
-    verify_evidence_quotes,
-)
-
-
-class TestComputeTrend:
-    """Tests for the compute_trend function."""
-
-    def test_empty_evidence_returns_stale(self):
-        """No evidence should return STALE trend."""
-        trend = compute_trend([])
-        assert trend == Trend.STALE
-
-    def test_all_recent_evidence_returns_new(self):
-        """All evidence within recent window (30 days) should return NEW trend.
-
-        Scenario: User just started using the app and mentioned they like coffee twice.
-        Both mentions are within the last 2 weeks, so this is a NEW observation.
-        """
-        now = datetime.now(timezone.utc)
-        evidence = [
-            ObservationEvidence(
-                memory_id="mem-coffee-morning",
-                quote="I always start my day with a large black coffee",
-                relevance="Shows preference for coffee and morning routine",
-                timestamp=now - timedelta(days=5),
-            ),
-            ObservationEvidence(
-                memory_id="mem-coffee-meeting",
-                quote="grabbed coffee before the standup meeting",
-                relevance="Confirms regular coffee consumption",
-                timestamp=now - timedelta(days=10),
-            ),
-        ]
-
-        trend = compute_trend(evidence, now=now)
-        assert trend == Trend.NEW
-
-    def test_no_recent_evidence_returns_stale(self):
-        """No evidence in recent window should return STALE trend.
-
-        Scenario: User mentioned running 3 months ago but hasn't mentioned it since.
-        The observation about running as a hobby may no longer be accurate.
-        """
-        now = datetime.now(timezone.utc)
-        evidence = [
-            ObservationEvidence(
-                memory_id="mem-running-march",
-                quote="training for a half marathon in the spring",
-                relevance="Shows interest in running",
-                timestamp=now - timedelta(days=60),
-            ),
-            ObservationEvidence(
-                memory_id="mem-running-feb",
-                quote="went for a 10k run this morning",
-                relevance="Active runner",
-                timestamp=now - timedelta(days=100),
-            ),
-        ]
-
-        trend = compute_trend(evidence, now=now)
-        assert trend == Trend.STALE
-
-    def test_stable_evidence_distribution(self):
-        """Evidence spread evenly across time should return STABLE trend.
-
-        Scenario: User has consistently mentioned working remotely over 4 months.
-        Evidence is well-distributed, indicating a stable, ongoing preference.
-        """
-        now = datetime.now(timezone.utc)
-        evidence = [
-            # Recent (within 30 days)
-            ObservationEvidence(
-                memory_id="mem-remote-jan",
-                quote="working from my home office today",
-                relevance="Current remote work",
-                timestamp=now - timedelta(days=5),
-            ),
-            ObservationEvidence(
-                memory_id="mem-remote-dec",
-                quote="the flexibility of remote work is great",
-                relevance="Values remote work",
-                timestamp=now - timedelta(days=15),
-            ),
-            # Middle period (30-90 days)
-            ObservationEvidence(
-                memory_id="mem-remote-nov",
-                quote="set up a standing desk at home",
-                relevance="Invested in home office",
-                timestamp=now - timedelta(days=45),
-            ),
-            ObservationEvidence(
-                memory_id="mem-remote-oct",
-                quote="prefer async communication over meetings",
-                relevance="Remote work style preference",
-                timestamp=now - timedelta(days=60),
-            ),
-            # Older (90+ days)
-            ObservationEvidence(
-                memory_id="mem-remote-sep",
-                quote="switched to fully remote last quarter",
-                relevance="Original transition to remote",
-                timestamp=now - timedelta(days=100),
-            ),
-            ObservationEvidence(
-                memory_id="mem-remote-aug",
-                quote="negotiated remote work in my new contract",
-                relevance="Intentional choice for remote",
-                timestamp=now - timedelta(days=120),
-            ),
-        ]
-
-        trend = compute_trend(evidence, now=now)
-        assert trend == Trend.STABLE
-
-    def test_strengthening_trend(self):
-        """Much more recent evidence than older should return STRENGTHENING trend.
-
-        Scenario: User has been increasingly talking about learning Python recently
-        after mentioning it once months ago. Interest appears to be growing.
-        """
-        now = datetime.now(timezone.utc)
-        evidence = [
-            # Lots of recent evidence - actively learning
-            ObservationEvidence(
-                memory_id="mem-python-project",
-                quote="finished my first Python project - a web scraper",
-                relevance="Completed Python project",
-                timestamp=now - timedelta(days=2),
-            ),
-            ObservationEvidence(
-                memory_id="mem-python-course",
-                quote="halfway through the Python bootcamp",
-                relevance="Active learning",
-                timestamp=now - timedelta(days=5),
-            ),
-            ObservationEvidence(
-                memory_id="mem-python-book",
-                quote="reading Fluent Python, it's excellent",
-                relevance="Deepening knowledge",
-                timestamp=now - timedelta(days=10),
-            ),
-            ObservationEvidence(
-                memory_id="mem-python-practice",
-                quote="solved 50 LeetCode problems in Python",
-                relevance="Practicing skills",
-                timestamp=now - timedelta(days=15),
-            ),
-            ObservationEvidence(
-                memory_id="mem-python-ide",
-                quote="set up VS Code with all the Python extensions",
-                relevance="Setting up environment",
-                timestamp=now - timedelta(days=20),
-            ),
-            # Only one old mention - initial interest
-            ObservationEvidence(
-                memory_id="mem-python-start",
-                quote="thinking about learning Python someday",
-                relevance="Initial interest",
-                timestamp=now - timedelta(days=100),
-            ),
-        ]
-
-        trend = compute_trend(evidence, now=now)
-        assert trend == Trend.STRENGTHENING
-
-    def test_weakening_trend(self):
-        """Much less recent evidence than older should return WEAKENING trend.
-
-        Scenario: User was very active in a book club last year but mentions
-        have tapered off. The observation about being a book club member
-        may be becoming less relevant.
-        """
-        now = datetime.now(timezone.utc)
-        evidence = [
-            # Only one recent mention
-            ObservationEvidence(
-                memory_id="mem-book-recent",
-                quote="haven't had time for book club lately",
-                relevance="Reduced participation",
-                timestamp=now - timedelta(days=10),
-            ),
-            # Lots of older evidence - was very active
-            ObservationEvidence(
-                memory_id="mem-book-aug",
-                quote="hosting book club at my place next week",
-                relevance="Active organizer",
-                timestamp=now - timedelta(days=40),
-            ),
-            ObservationEvidence(
-                memory_id="mem-book-july",
-                quote="leading the discussion on 1984",
-                relevance="Active participant",
-                timestamp=now - timedelta(days=50),
-            ),
-            ObservationEvidence(
-                memory_id="mem-book-june",
-                quote="we picked The Midnight Library for June",
-                relevance="Regular member",
-                timestamp=now - timedelta(days=60),
-            ),
-            ObservationEvidence(
-                memory_id="mem-book-may",
-                quote="book club was amazing tonight",
-                relevance="Enthusiastic member",
-                timestamp=now - timedelta(days=100),
-            ),
-            ObservationEvidence(
-                memory_id="mem-book-april",
-                quote="joined a new book club in my neighborhood",
-                relevance="Started participation",
-                timestamp=now - timedelta(days=110),
-            ),
-            ObservationEvidence(
-                memory_id="mem-book-march",
-                quote="excited to finally join a book club",
-                relevance="Initial enthusiasm",
-                timestamp=now - timedelta(days=120),
-            ),
-        ]
-
-        trend = compute_trend(evidence, now=now)
-        assert trend == Trend.WEAKENING
-
-
-class TestObservationModel:
-    """Tests for the Observation model."""
-
-    def test_observation_computed_trend(self):
-        """Observation should have computed trend property based on evidence."""
-        now = datetime.now(timezone.utc)
-        obs = Observation(
-            title="Morning meeting preference",
-            content="Prefers morning meetings over afternoon ones",
-            evidence=[
-                ObservationEvidence(
-                    memory_id="mem-morning-standup",
-                    quote="I'm most productive in morning meetings",
-                    relevance="Direct preference statement",
-                    timestamp=now - timedelta(days=5),
-                ),
-            ],
-            created_at=now,
-        )
-
-        assert obs.trend == Trend.NEW
-        assert obs.evidence_count == 1
-
-    def test_observation_evidence_span(self):
-        """Observation should compute evidence span correctly.
-
-        The span shows the date range of supporting evidence, helping
-        understand how long this pattern has been observed.
-        """
-        now = datetime.now(timezone.utc)
-        old_time = now - timedelta(days=100)
-        recent_time = now - timedelta(days=5)
-
-        obs = Observation(
-            title="Values work-life balance",
-            content="Values work-life balance highly",
-            evidence=[
-                ObservationEvidence(
-                    memory_id="mem-balance-old",
-                    quote="turned down a promotion because of the hours",
-                    relevance="Prioritized balance over advancement",
-                    timestamp=old_time,
-                ),
-                ObservationEvidence(
-                    memory_id="mem-balance-recent",
-                    quote="always log off by 6pm no matter what",
-                    relevance="Maintains boundaries",
-                    timestamp=recent_time,
-                ),
-            ],
-            created_at=now,
-        )
-
-        evidence_span = obs.evidence_span
-        assert evidence_span["from"] == old_time.isoformat()
-        assert evidence_span["to"] == recent_time.isoformat()
-
-    def test_observation_empty_evidence_span(self):
-        """Observation with no evidence should have null span."""
-        obs = Observation(
-            title="Test observation",
-            content="Test observation without evidence",
-            evidence=[],
-        )
-
-        evidence_span = obs.evidence_span
-        assert evidence_span["from"] is None
-        assert evidence_span["to"] is None
-
-
-class TestVerifyEvidenceQuotes:
-    """Tests for evidence quote verification.
-
-    This ensures the LLM isn't hallucinating quotes - every quote
-    must actually appear in the source memory.
-    """
-
-    def test_valid_quotes(self):
-        """Should return True when quotes exist in their source memories."""
-        obs = Observation(
-            title="Enjoys hiking",
-            content="Enjoys hiking on weekends",
-            evidence=[
-                ObservationEvidence(
-                    memory_id="mem-hiking-trip",
-                    quote="went hiking at Mount Tam",
-                    relevance="Shows hiking activity",
-                    timestamp=datetime.now(timezone.utc),
-                ),
-            ],
-        )
-
-        memories = {
-            "mem-hiking-trip": "Had a great Saturday - went hiking at Mount Tam with friends and saw amazing views."
-        }
-        is_valid, errors = verify_evidence_quotes(obs, memories)
-
-        assert is_valid is True
-        assert len(errors) == 0
-
-    def test_invalid_quote(self):
-        """Should return False when quote doesn't exist in memory.
-
-        This catches LLM hallucinations where it fabricates quotes.
-        """
-        obs = Observation(
-            title="Loves spicy food",
-            content="Loves spicy food",
-            evidence=[
-                ObservationEvidence(
-                    memory_id="mem-dinner",
-                    quote="I love extra hot salsa",
-                    relevance="Shows spicy food preference",
-                    timestamp=datetime.now(timezone.utc),
-                ),
-            ],
-        )
-
-        memories = {"mem-dinner": "Had tacos for dinner. The guacamole was really fresh."}
-        is_valid, errors = verify_evidence_quotes(obs, memories)
-
-        assert is_valid is False
-        assert len(errors) == 1
-        assert "Quote not found" in errors[0]
-
-    def test_missing_memory(self):
-        """Should return False when referenced memory doesn't exist.
-
-        This catches cases where the LLM references a memory ID that
-        was never actually retrieved.
-        """
-        obs = Observation(
-            title="Has a dog named Max",
-            content="Has a dog named Max",
-            evidence=[
-                ObservationEvidence(
-                    memory_id="mem-pet-story",
-                    quote="took Max to the vet",
-                    relevance="Shows pet ownership",
-                    timestamp=datetime.now(timezone.utc),
-                ),
-            ],
-        )
-
-        memories = {"mem-different-id": "Some unrelated memory content"}
-        is_valid, errors = verify_evidence_quotes(obs, memories)
-
-        assert is_valid is False
-        assert len(errors) == 1
-        assert "not found" in errors[0]
-
-
-class TestCandidateObservation:
-    """Tests for candidate observation model.
-
-    Candidates are generated in the SEED phase and validated
-    before becoming full observations.
-    """
-
-    def test_create_candidate(self):
-        """Should create candidate with content and seed memories."""
-        candidate = CandidateObservation(
-            content="User prefers async communication over meetings",
-            seed_memory_ids=["mem-slack-pref", "mem-meeting-decline"],
-        )
-
-        assert candidate.content == "User prefers async communication over meetings"
-        assert len(candidate.seed_memory_ids) == 2
-        assert "mem-slack-pref" in candidate.seed_memory_ids

hindsight-api/tests/test_observations.py

@@ -1,9 +1,5 @@
 """
 Test observation generation and entity state functionality.
-
-NOTE: Observations are now stored as summaries on the entities table,
-not as separate memory_units. The observations list in EntityState is
-populated from the summary for backwards compatibility.
 """
 import pytest
 from hindsight_api.engine.memory_engine import Budget
@@ -12,16 +8,21 @@ from datetime import datetime, timezone
 
 
 @pytest.mark.asyncio
-async def test_entity_extraction_on_retain(memory, request_context):
+async def test_observation_generation_on_put(memory, request_context):
     """
-    Test that entities are extracted when new facts are added.
+    Test that observations are generated SYNCHRONOUSLY when new facts are added.
 
-    This test stores multiple facts and verifies entities are extracted.
+    Observations are generated during retain when:
+    - Entity has >= 5 facts (MIN_FACTS_THRESHOLD)
+    - Entity is in top 5 by mention count
+
+    This test stores enough facts to trigger automatic observation generation.
     """
-    bank_id = f"test_entity_extraction_{datetime.now(timezone.utc).timestamp()}"
+    bank_id = f"test_obs_{datetime.now(timezone.utc).timestamp()}"
 
     try:
-        # Store multiple facts about John
+        # Store multiple facts about John to reach the MIN_FACTS_THRESHOLD (5)
+        # Each retain call should extract at least one fact about John
         contents = [
             "John is a software engineer at Google.",
             "John is detail-oriented and methodical in his work.",
@@ -40,8 +41,9 @@ async def test_entity_extraction_on_retain(memory, request_context):
                 request_context=request_context,
             )
 
-        # Wait for background tasks
-        await memory.wait_for_background_tasks()
+        # Observations are generated SYNCHRONOUSLY during retain,
+        # so they should be available immediately after retain completes.
+        # No need to wait for background tasks for observations.
 
         # Find the John entity
         pool = await memory._get_pool()
@@ -56,7 +58,7 @@ async def test_entity_extraction_on_retain(memory, request_context):
                 bank_id
             )
 
-            # Check the fact count for this entity
+            # Also check the fact count for this entity
             if entity_row:
                 fact_count = await conn.fetchval(
                     """
@@ -68,9 +70,30 @@ async def test_entity_extraction_on_retain(memory, request_context):
                 print(f"Entity: {entity_row['canonical_name']} has {fact_count} linked facts")
 
         assert entity_row is not None, "John entity should have been extracted"
+
+        entity_id = str(entity_row['id'])
+        entity_name = entity_row['canonical_name']
         print(f"\n=== Found Entity ===")
-        print(f"Entity: {entity_row['canonical_name']} (id: {entity_row['id']})")
-        print(f"Entity was successfully extracted")
+        print(f"Entity: {entity_name} (id: {entity_id})")
+
+        # Get observations for the entity - should be available immediately
+        observations = await memory.get_entity_observations(bank_id, entity_id, limit=10, request_context=request_context)
+
+        print(f"\n=== Observations for {entity_name} ===")
+        print(f"Total observations: {len(observations)}")
+        for obs in observations:
+            print(f"  - {obs.text}")
+
+        # Verify observations were created (requires >= 5 facts)
+        assert len(observations) > 0, \
+            f"Observations should have been generated synchronously during retain (entity has {fact_count} facts, threshold is 5)"
+
+        # Check that observations mention relevant content
+        obs_texts = " ".join([o.text.lower() for o in observations])
+        assert any(keyword in obs_texts for keyword in ["google", "engineer", "ai", "machine learning", "detail"]), \
+            "Observations should contain relevant information about John"
+
+        print(f"✓ Observations were successfully generated synchronously during retain")
 
     finally:
         # Cleanup
@@ -83,7 +106,7 @@ async def test_entity_extraction_on_retain(memory, request_context):
 @pytest.mark.asyncio
 async def test_regenerate_entity_observations(memory, request_context):
     """
-    Test explicit regeneration of summary for an entity.
+    Test explicit regeneration of observations for an entity.
     """
     bank_id = f"test_regen_obs_{datetime.now(timezone.utc).timestamp()}"
 
@@ -116,7 +139,7 @@ async def test_regenerate_entity_observations(memory, request_context):
             entity_id = str(entity_row['id'])
             entity_name = entity_row['canonical_name']
 
-            # Manually regenerate summary (via observations API for backwards compat)
+            # Manually regenerate observations
             created_ids = await memory.regenerate_entity_observations(
                 bank_id=bank_id,
                 entity_id=entity_id,
@@ -124,25 +147,23 @@ async def test_regenerate_entity_observations(memory, request_context):
                 request_context=request_context,
             )
 
-            print(f"\n=== Regenerated Summary ===")
-            print(f"Created {len(created_ids)} summary for {entity_name}")
+            print(f"\n=== Regenerated Observations ===")
+            print(f"Created {len(created_ids)} observations for {entity_name}")
 
-            # Get entity state
-            state = await memory.get_entity_state(
-                bank_id, entity_id, entity_name, request_context=request_context
-            )
-            for obs in state.observations:
+            # Get the observations
+            observations = await memory.get_entity_observations(bank_id, entity_id, limit=10, request_context=request_context)
+            for obs in observations:
                 print(f"  - {obs.text}")
 
-            # Verify summary was created
+            # Verify observations were created
             if len(created_ids) > 0:
-                assert len(state.observations) == 1, "Should have exactly 1 observation (the summary)"
-                print(f"Summary regenerated successfully")
+                assert len(observations) == len(created_ids), "Should have same number of observations as created IDs"
+                print(f"✓ Observations regenerated successfully")
             else:
-                print(f"Note: No summary was regenerated")
+                print(f"⚠ Note: No observations were regenerated")
 
         else:
-            print(f"Note: No 'Sarah' entity was extracted")
+            print(f"⚠ Note: No 'Sarah' entity was extracted")
 
     finally:
         # Cleanup
@@ -153,14 +174,19 @@ async def test_regenerate_entity_observations(memory, request_context):
 
 
 @pytest.mark.asyncio
-async def test_entity_state_retrieval(memory, request_context):
+async def test_manual_regenerate_with_few_facts(memory, request_context):
     """
-    Test retrieving entity state with facts.
+    Test that manual regeneration works even with fewer than 5 facts.
+
+    This is important because:
+    - Automatic generation during retain requires MIN_FACTS_THRESHOLD (5)
+    - But manual regeneration via API should work with any number of facts
+    - The UI triggers manual regeneration, so it should work regardless of fact count
     """
-    bank_id = f"test_entity_state_{datetime.now(timezone.utc).timestamp()}"
+    bank_id = f"test_manual_regen_{datetime.now(timezone.utc).timestamp()}"
 
     try:
-        # Store facts
+        # Store only 2 facts - below the automatic threshold
         await memory.retain_async(
             bank_id=bank_id,
             content="Alice works at Google as a senior software engineer.",
@@ -194,25 +220,51 @@ async def test_entity_state_retrieval(memory, request_context):
         entity_id = str(entity_row['id'])
         entity_name = entity_row['canonical_name']
 
-        # Check fact count
+        # Check fact count - should be < 5
         async with pool.acquire() as conn:
             fact_count = await conn.fetchval(
                 "SELECT COUNT(*) FROM unit_entities WHERE entity_id = $1",
                 entity_row['id']
             )
 
-        print(f"\n=== Entity State Test ===")
+        print(f"\n=== Manual Regeneration Test ===")
         print(f"Entity: {entity_name} (id: {entity_id})")
         print(f"Linked facts: {fact_count}")
 
-        # Get entity state
-        state = await memory.get_entity_state(
-            bank_id, entity_id, entity_name, request_context=request_context
+        # Verify we're testing with fewer than the automatic threshold
+        assert fact_count < 5, f"Test requires < 5 facts, but entity has {fact_count}"
+
+        # Before regeneration - should have no observations (auto threshold not met)
+        obs_before = await memory.get_entity_observations(bank_id, entity_id, limit=10, request_context=request_context)
+        print(f"Observations before manual regenerate: {len(obs_before)}")
+
+        # Manually regenerate observations - this should work regardless of fact count
+        created_ids = await memory.regenerate_entity_observations(
+            bank_id=bank_id,
+            entity_id=entity_id,
+            entity_name=entity_name,
+            request_context=request_context,
         )
 
-        assert state.entity_id == entity_id
-        assert state.canonical_name == entity_name
-        print(f"Entity state retrieved successfully")
+        print(f"Observations created by manual regenerate: {len(created_ids)}")
+
+        # Get observations after regeneration
+        observations = await memory.get_entity_observations(bank_id, entity_id, limit=10, request_context=request_context)
+        print(f"Observations after manual regenerate: {len(observations)}")
+        for obs in observations:
+            print(f"  - {obs.text}")
+
+        # Manual regeneration should create observations even with < 5 facts
+        assert len(observations) > 0, \
+            f"Manual regeneration should create observations even with only {fact_count} facts. " \
+            f"The LLM should synthesize at least 1 observation from the available facts."
+
+        # Verify observations contain relevant content
+        obs_texts = " ".join([o.text.lower() for o in observations])
+        assert any(keyword in obs_texts for keyword in ["google", "engineer", "hiking", "photography", "alice"]), \
+            "Observations should contain relevant information about Alice"
+
+        print(f"✓ Manual regeneration works with {fact_count} facts (below automatic threshold of 5)")
 
     finally:
         # Cleanup
@@ -225,16 +277,16 @@ async def test_entity_state_retrieval(memory, request_context):
 @pytest.mark.asyncio
 async def test_search_with_include_entities(memory, request_context):
     """
-    Test that search with include_entities=True returns entity information.
+    Test that search with include_entities=True returns entity observations.
 
     This test verifies that:
-    1. Entities are extracted after retain
-    2. Entity info is returned in recall results with include_entities=True
+    1. Observations are generated during retain (when entity has >= 5 facts)
+    2. Observations are returned in recall results with include_entities=True
     """
     bank_id = f"test_search_ent_{datetime.now(timezone.utc).timestamp()}"
 
     try:
-        # Store facts about Alice
+        # Store enough facts about Alice to trigger observation generation (>= 5 facts)
         contents = [
             "Alice is a data scientist who works on recommendation systems at Netflix.",
             "Alice presented her research at the ML conference last month.",
@@ -253,8 +305,7 @@ async def test_search_with_include_entities(memory, request_context):
                 request_context=request_context,
             )
 
-        # Wait for background tasks
-        await memory.wait_for_background_tasks()
+        # Observations are generated synchronously during retain, no need to wait
 
         # Search with include_entities=True
         result = await memory.recall_async(
@@ -264,7 +315,7 @@ async def test_search_with_include_entities(memory, request_context):
             budget=Budget.LOW,
             max_tokens=2000,
             include_entities=True,
-            max_entity_tokens=5000,
+            max_entity_tokens=500,
             request_context=request_context,
         )
 
@@ -275,28 +326,40 @@ async def test_search_with_include_entities(memory, request_context):
             if fact.entities:
                 print(f"    Entities: {', '.join(fact.entities)}")
 
+        print(f"\n=== Entity Observations in Recall ===")
+        if result.entities:
+            for name, state in result.entities.items():
+                print(f"\n{name}:")
+                for obs in state.observations:
+                    print(f"  - {obs.text}")
+        else:
+            print("No entity observations returned")
+
         # Verify results
         assert len(result.results) > 0, "Should find some facts"
 
         # Check if entities are included in facts
         facts_with_entities = [f for f in result.results if f.entities]
         assert len(facts_with_entities) > 0, "Some facts should have entity information"
-        print(f"{len(facts_with_entities)} facts have entity information")
+        print(f"✓ {len(facts_with_entities)} facts have entity information")
 
-        # Check if entity info is returned
-        if result.entities:
-            print(f"Entity info included for {len(result.entities)} entities")
+        # Check if entity observations are included in recall
+        assert result.entities is not None and len(result.entities) > 0, \
+            "Entity observations should be included in recall results"
+        print(f"✓ Entity observations included for {len(result.entities)} entities")
 
-            # Verify Alice entity is in results
-            alice_found = False
-            for name, state in result.entities.items():
-                assert state.canonical_name == name, "Entity canonical_name should match key"
-                assert state.entity_id, "Entity should have an ID"
-                if "alice" in name.lower():
-                    alice_found = True
-                    print(f"Alice entity found: {name}")
+        # Verify Alice entity has observations
+        alice_found = False
+        for name, state in result.entities.items():
+            assert state.canonical_name == name, "Entity canonical_name should match key"
+            assert state.entity_id, "Entity should have an ID"
+            if "alice" in name.lower():
+                alice_found = True
+                assert len(state.observations) > 0, \
+                    "Alice should have observations (generated during retain)"
+                print(f"✓ Alice has {len(state.observations)} observations in recall result")
 
-            assert alice_found, "Alice entity should be in recall results"
+        assert alice_found, "Alice entity should be in recall results"
 
     finally:
         # Cleanup
@@ -372,10 +435,7 @@ async def test_get_entity_state(memory, request_context):
 @pytest.mark.asyncio
 async def test_observation_fact_type_in_database(memory, request_context):
     """
-    Test that observations are NOT stored as memory_units with fact_type='observation'.
-
-    NOTE: Observations are now handled via mental models, not as memory_units
-    or entity summaries.
+    Test that observations are stored with correct fact_type in database.
     """
     bank_id = f"test_obs_db_{datetime.now(timezone.utc).timestamp()}"
 
@@ -391,7 +451,7 @@ async def test_observation_fact_type_in_database(memory, request_context):
 
         await memory.wait_for_background_tasks()
 
-        # Check that NO observations exist in memory_units
+        # Check that observations have correct fact_type
         pool = await memory._get_pool()
         async with pool.acquire() as conn:
             observations = await conn.fetch(
@@ -403,11 +463,17 @@ async def test_observation_fact_type_in_database(memory, request_context):
                 bank_id
             )
 
-        print(f"\n=== Observation Records in memory_units ===")
-        print(f"Found {len(observations)} observation records (should be 0)")
+        print(f"\n=== Observation Records in Database ===")
+        print(f"Found {len(observations)} observation records")
+        for obs in observations:
+            print(f"  - fact_type: {obs['fact_type']}")
+            print(f"    text: {obs['text']}")
+            print(f"    context: {obs['context']}")
 
-        # Observations are no longer stored as memory_units
-        assert len(observations) == 0, "Observations should NOT be stored as memory_units"
+        if len(observations) > 0:
+            for obs in observations:
+                assert obs['fact_type'] == 'observation', "All observation records should have fact_type='observation'"
+            print(f"✓ All observations have correct fact_type")
 
     finally:
         # Cleanup
@@ -418,183 +484,23 @@ async def test_observation_fact_type_in_database(memory, request_context):
 
 
 @pytest.mark.asyncio
-async def test_entity_mention_counts(memory, request_context):
+async def test_user_entity_prioritized_for_observations(memory, request_context):
     """
-    Test that entity mention counts are tracked correctly.
+    Test that the 'user' entity gets observations even when many other entities exist.
 
-    This test creates entities with varying mention counts and verifies
-    that the counts are accurate.
+    The retain pipeline only regenerates observations for TOP_N_ENTITIES (5) entities,
+    sorted by mention count. This test verifies that the most mentioned entity ('user')
+    gets prioritized and receives observations.
+
+    This is critical because 'user' is often the most important entity in personal memory.
     """
-    bank_id = f"test_mention_counts_{datetime.now(timezone.utc).timestamp()}"
+    bank_id = f"test_user_priority_{datetime.now(timezone.utc).timestamp()}"
 
     try:
-        # Create content with varying entity mention counts:
-        # - "HighMention Corp" mentioned 10+ times
-        # - "LowMention Ltd" mentioned 1 time
-        contents = [
-            # High mentions - HighMention Corp
-            "HighMention Corp is a tech company based in San Francisco.",
-            "HighMention Corp was founded in 2010 by experienced entrepreneurs.",
-            "HighMention Corp has over 500 employees worldwide.",
-            "HighMention Corp specializes in cloud computing solutions.",
-            "HighMention Corp recently raised $50 million in Series C funding.",
-            "HighMention Corp has partnerships with major tech companies.",
-            "HighMention Corp is known for its innovative culture.",
-            "HighMention Corp offers competitive salaries and benefits.",
-            "HighMention Corp has offices in 5 countries.",
-            "HighMention Corp won the best workplace award last year.",
-            # Low mentions - LowMention Ltd
-            "LowMention Ltd is a small consulting firm.",
-        ]
-
-        for i, content in enumerate(contents):
-            await memory.retain_async(
-                bank_id=bank_id,
-                content=content,
-                context="company info",
-                event_date=datetime(2024, 1, 15 + i, tzinfo=timezone.utc),
-                request_context=request_context,
-            )
-
-        # Wait for background tasks
-        await memory.wait_for_background_tasks()
-
-        # Check entity mention counts
-        pool = await memory._get_pool()
-        async with pool.acquire() as conn:
-            entities = await conn.fetch(
-                """
-                SELECT e.id, e.canonical_name, e.mention_count
-                FROM entities e
-                WHERE e.bank_id = $1
-                ORDER BY e.mention_count DESC
-                """,
-                bank_id
-            )
-
-        print(f"\n=== Entity Mention Counts Test ===")
-        print(f"Total entities: {len(entities)}")
-
-        high_mention_entity = None
-        low_mention_entity = None
-
-        for entity in entities:
-            name = entity['canonical_name'].lower()
-            mention_count = entity['mention_count']
-
-            print(f"  {entity['canonical_name']}: mentions={mention_count}")
-
-            if "highmention" in name:
-                high_mention_entity = entity
-            elif "lowmention" in name:
-                low_mention_entity = entity
-
-        # Verify HighMention Corp has higher mention count
-        if high_mention_entity and low_mention_entity:
-            assert high_mention_entity['mention_count'] > low_mention_entity['mention_count'], \
-                "HighMention Corp should have more mentions than LowMention Ltd"
-            print("PASS: Entity mention counts are tracked correctly")
-
-    finally:
-        # Cleanup
-        pool = await memory._get_pool()
-        async with pool.acquire() as conn:
-            await conn.execute("DELETE FROM memory_units WHERE bank_id = $1", bank_id)
-            await conn.execute("DELETE FROM entities WHERE bank_id = $1", bank_id)
-
-
-@pytest.mark.asyncio
-async def test_entity_mention_ranking(memory, request_context):
-    """
-    Test that entity mention counts correctly rank entities.
-
-    This test:
-    1. Creates an entity with 6 mentions
-    2. Adds more entities with higher mention counts
-    3. Verifies entities are ranked correctly by mention count
-    """
-    bank_id = f"test_ranking_{datetime.now(timezone.utc).timestamp()}"
-
-    try:
-        # Phase 1: Create "OriginalEntity" with 6 mentions
-        print("\n=== Phase 1: Create OriginalEntity with 6 mentions ===")
-        for i in range(6):
-            await memory.retain_async(
-                bank_id=bank_id,
-                content=f"OriginalEntity is mentioned here in fact {i+1}.",
-                context="test",
-                event_date=datetime(2024, 1, 1 + i, tzinfo=timezone.utc),
-                request_context=request_context,
-            )
-
-        await memory.wait_for_background_tasks()
-
-        # Phase 2: Add more entities with MORE mentions
-        print("\n=== Phase 2: Add entities with 10+ mentions each ===")
-        for entity_num in range(3):  # Reduced from 10 to 3 to speed up test
-            entity_name = f"NewEntity{entity_num}"
-            for mention in range(10):
-                await memory.retain_async(
-                    bank_id=bank_id,
-                    content=f"{entity_name} is a very important entity, mention {mention+1}.",
-                    context="test",
-                    event_date=datetime(2024, 2, 1 + mention, tzinfo=timezone.utc),
-                    request_context=request_context,
-                )
-
-        await memory.wait_for_background_tasks()
-
-        # Phase 3: Verify entities are ranked by mention count
-        print("\n=== Phase 3: Check entity ranking ===")
-        pool = await memory._get_pool()
-        async with pool.acquire() as conn:
-            all_entities = await conn.fetch(
-                """
-                SELECT canonical_name, mention_count
-                FROM entities
-                WHERE bank_id = $1
-                ORDER BY mention_count DESC
-                """,
-                bank_id
-            )
-
-        print(f"\nAll entities by mention count:")
-        for e in all_entities:
-            print(f"  {e['canonical_name']}: mentions={e['mention_count']}")
-
-        # Verify new entities have higher counts than OriginalEntity
-        original = next((e for e in all_entities if 'originalentity' in e['canonical_name'].lower()), None)
-        new_entities = [e for e in all_entities if 'newentity' in e['canonical_name'].lower()]
-
-        assert original is not None, "OriginalEntity should exist"
-        assert len(new_entities) > 0, "NewEntity entities should exist"
-
-        # Verify entities are created and have mention counts
-        # Note: LLM may merge mentions, so we just check that new entities exist
-        print(f"OriginalEntity mentions: {original['mention_count']}")
-        for new_entity in new_entities:
-            print(f"{new_entity['canonical_name']} mentions: {new_entity['mention_count']}")
-
-        print("PASS: Entities are created with mention counts tracked")
-
-    finally:
-        # Cleanup
-        pool = await memory._get_pool()
-        async with pool.acquire() as conn:
-            await conn.execute("DELETE FROM memory_units WHERE bank_id = $1", bank_id)
-            await conn.execute("DELETE FROM entities WHERE bank_id = $1", bank_id)
-
-
-@pytest.mark.asyncio
-async def test_user_entity_extraction(memory, request_context):
-    """
-    Test that the 'user' entity is correctly extracted when mentioned frequently.
-    """
-    bank_id = f"test_user_entity_{datetime.now(timezone.utc).timestamp()}"
-
-    try:
-        # Create content where 'user' is mentioned many times
+        # Create content where 'user' (the user) is mentioned many times
+        # along with several other entities
         contents = [
+            # User mentioned frequently
             "The user loves hiking in the mountains during summer.",
             "The user works as a software engineer at Microsoft.",
             "The user has a dog named Max who is a golden retriever.",
@@ -604,8 +510,11 @@ async def test_user_entity_extraction(memory, request_context):
             # Other entities mentioned fewer times
             "Sarah is a friend who works at Google.",
             "Bob is a colleague from the data science team.",
+            "Tokyo is a city the user visited last year.",
+            "Python is the user's favorite programming language.",
         ]
 
+        # Retain all content in a single batch for efficiency
         for i, content in enumerate(contents):
             await memory.retain_async(
                 bank_id=bank_id,
@@ -615,12 +524,12 @@ async def test_user_entity_extraction(memory, request_context):
                 request_context=request_context,
             )
 
-        # Wait for background tasks
-        await memory.wait_for_background_tasks()
+        # Observations are generated synchronously during retain
 
         # Find the 'user' entity
         pool = await memory._get_pool()
         async with pool.acquire() as conn:
+            # Find user entity (may be named "user", "the user", etc.)
             user_entity = await conn.fetchrow(
                 """
                 SELECT e.id, e.canonical_name,
@@ -635,7 +544,7 @@ async def test_user_entity_extraction(memory, request_context):
                 bank_id
             )
 
-            # Get all entities with their fact counts
+            # Get all entities with their fact counts to verify prioritization
             all_entities = await conn.fetch(
                 """
                 SELECT e.id, e.canonical_name,
@@ -655,10 +564,41 @@ async def test_user_entity_extraction(memory, request_context):
 
         # Verify user entity exists
         assert user_entity is not None, "User entity should have been extracted"
+        user_entity_id = str(user_entity['id'])
+        user_entity_name = user_entity['canonical_name']
+        user_fact_count = user_entity['fact_count']
+
         print(f"\n=== User Entity ===")
-        print(f"Entity: {user_entity['canonical_name']} (id: {user_entity['id']})")
-        print(f"Fact count: {user_entity['fact_count']}")
-        print(f"User entity was successfully extracted")
+        print(f"Entity: {user_entity_name} (id: {user_entity_id})")
+        print(f"Fact count: {user_fact_count}")
+
+        # Verify user has enough facts for observations (>= MIN_FACTS_THRESHOLD of 5)
+        assert user_fact_count >= 5, \
+            f"User entity should have at least 5 facts, but has {user_fact_count}"
+
+        # Get observations for user entity
+        observations = await memory.get_entity_observations(bank_id, user_entity_id, limit=10, request_context=request_context)
+
+        print(f"\n=== User Entity Observations ===")
+        print(f"Total observations: {len(observations)}")
+        for obs in observations:
+            print(f"  - {obs.text}")
+
+        # Verify observations were generated for user (critical assertion)
+        assert len(observations) > 0, \
+            f"User entity should have observations (has {user_fact_count} facts, threshold is 5). " \
+            f"This may indicate that 'user' is not being prioritized in the top 5 entities by mention count."
+
+        # Verify observations mention relevant content about the user
+        obs_texts = " ".join([o.text.lower() for o in observations])
+        user_keywords = ["hiking", "software", "engineer", "dog", "max", "cooking",
+                        "italian", "mit", "dune", "microsoft"]
+        matching_keywords = [k for k in user_keywords if k in obs_texts]
+        assert len(matching_keywords) > 0, \
+            f"Observations should contain relevant information about the user. Keywords found: {matching_keywords}"
+
+        print(f"✓ User entity was prioritized and received {len(observations)} observations")
+        print(f"✓ Observations contain relevant keywords: {matching_keywords}")
 
     finally:
         # Cleanup

hindsight-api/tests/test_retain.py

@@ -465,7 +465,7 @@ async def test_occurred_dates_not_defaulted(memory, request_context):
             query="Tell me about Alice",
             budget=Budget.LOW,
             max_tokens=500,
-            fact_type=["world", "experience"],
+            fact_type=["world", "opinion"],
             request_context=request_context,
         )
 
@@ -2058,26 +2058,3 @@ async def test_user_provided_entities(memory, request_context):
 
     finally:
         await memory.delete_bank(bank_id, request_context=request_context)
-
-
-def test_recall_result_model_empty_construction():
-    """
-    Test that RecallResultModel can be constructed with empty results.
-
-    This is a regression test for the bug where constructing an empty RecallResultModel
-    would cause an UnboundLocalError because RecallResult was imported as RecallResultModel
-    but the code mistakenly used the wrong name.
-
-    The fix ensures RecallResultModel is used consistently throughout memory_engine.py.
-    """
-    from hindsight_api.engine.response_models import RecallResult
-
-    # This should not raise any errors
-    result = RecallResult(results=[], entities={}, chunks={})
-
-    assert result is not None, "Should create a result object"
-    assert result.results == [], "Should have empty results"
-    assert result.entities == {}, "Should have empty entities"
-    assert result.chunks == {}, "Should have empty chunks"
-
-    logger.info("✓ RecallResult empty construction works correctly")

hindsight-api/tests/test_server_module.py

@@ -1,294 +0,0 @@
-"""
-Tests for hindsight_api.server module (multi-worker code path).
-
-The server.py module is used when running with multiple workers:
-    uvicorn hindsight_api.server:app --workers 2
-
-This module executes code at import time, creating the app at module level.
-These tests ensure that extensions are properly loaded in this code path,
-which was previously a regression that caused authentication bypass in production.
-"""
-
-import importlib
-import sys
-from unittest.mock import MagicMock, patch
-
-
-def _clean_server_module():
-    """Remove hindsight_api.server from sys.modules for fresh import."""
-    modules_to_remove = [k for k in sys.modules.keys() if k.startswith("hindsight_api.server")]
-    for mod in modules_to_remove:
-        del sys.modules[mod]
-
-
-class TestServerModuleExtensionLoading:
-    """Tests that server.py correctly loads extensions when configured via environment."""
-
-    def test_server_loads_tenant_extension_when_configured(self, monkeypatch):
-        """
-        Verify that server.py loads tenant extension from HINDSIGHT_API_TENANT_EXTENSION.
-
-        This test catches the regression where server.py didn't call load_extension(),
-        causing authentication to be bypassed in multi-worker deployments.
-        """
-        # Set up environment to configure a tenant extension
-        monkeypatch.setenv(
-            "HINDSIGHT_API_TENANT_EXTENSION",
-            "tests.test_server_module:MockTenantExtension",
-        )
-
-        _clean_server_module()
-
-        # Track what extensions were loaded via load_extension
-        loaded_extensions = {}
-
-        # Get the real load_extension function
-        from hindsight_api.extensions.loader import load_extension as real_load_extension
-
-        def tracking_load_extension(name, base_class):
-            """Track calls to load_extension and delegate to original."""
-            result = real_load_extension(name, base_class)
-            loaded_extensions[name] = result
-            return result
-
-        # Patch at source level BEFORE importing server
-        # Note: We patch the entire hindsight_api module namespace
-        with patch("hindsight_api.MemoryEngine") as mock_engine, \
-             patch("hindsight_api.api.create_app") as mock_create_app, \
-             patch("hindsight_api.config.get_config") as mock_get_config, \
-             patch("hindsight_api.extensions.load_extension", side_effect=tracking_load_extension), \
-             patch("hindsight_api.extensions.DefaultExtensionContext"):
-
-            mock_config = MagicMock()
-            mock_config.mcp_enabled = False
-            mock_config.run_migrations_on_startup = False
-            mock_config.database_url = "postgresql://test:test@localhost/test"
-            mock_get_config.return_value = mock_config
-            mock_engine.return_value = MagicMock()
-            mock_create_app.return_value = MagicMock()
-
-            # Now import server - this triggers module-level code
-            import hindsight_api.server
-
-        # Verify TENANT extension was loaded
-        assert "TENANT" in loaded_extensions, \
-            "server.py did not call load_extension('TENANT', ...) - extensions not loaded!"
-        assert loaded_extensions["TENANT"] is not None, \
-            "load_extension('TENANT', ...) returned None despite env var being set"
-        assert isinstance(loaded_extensions["TENANT"], MockTenantExtension), \
-            f"Expected MockTenantExtension, got {type(loaded_extensions['TENANT'])}"
-
-    def test_server_loads_operation_validator_when_configured(self, monkeypatch):
-        """
-        Verify that server.py loads operation validator from HINDSIGHT_API_OPERATION_VALIDATOR_EXTENSION.
-        """
-        monkeypatch.setenv(
-            "HINDSIGHT_API_OPERATION_VALIDATOR_EXTENSION",
-            "tests.test_server_module:MockOperationValidator",
-        )
-
-        _clean_server_module()
-
-        loaded_extensions = {}
-
-        from hindsight_api.extensions.loader import load_extension as real_load_extension
-
-        def tracking_load_extension(name, base_class):
-            result = real_load_extension(name, base_class)
-            loaded_extensions[name] = result
-            return result
-
-        with patch("hindsight_api.MemoryEngine") as mock_engine, \
-             patch("hindsight_api.api.create_app") as mock_create_app, \
-             patch("hindsight_api.config.get_config") as mock_get_config, \
-             patch("hindsight_api.extensions.load_extension", side_effect=tracking_load_extension), \
-             patch("hindsight_api.extensions.DefaultExtensionContext"):
-
-            mock_config = MagicMock()
-            mock_config.mcp_enabled = False
-            mock_config.run_migrations_on_startup = False
-            mock_config.database_url = "postgresql://test:test@localhost/test"
-            mock_get_config.return_value = mock_config
-            mock_engine.return_value = MagicMock()
-            mock_create_app.return_value = MagicMock()
-
-            import hindsight_api.server
-
-        assert "OPERATION_VALIDATOR" in loaded_extensions, \
-            "server.py did not call load_extension('OPERATION_VALIDATOR', ...)"
-        assert loaded_extensions["OPERATION_VALIDATOR"] is not None
-        assert isinstance(loaded_extensions["OPERATION_VALIDATOR"], MockOperationValidator)
-
-    def test_server_passes_extensions_to_memory_engine(self, monkeypatch):
-        """
-        Verify that server.py passes loaded extensions to MemoryEngine constructor.
-
-        This is the critical test - even if extensions are loaded, they must be
-        passed to MemoryEngine for authentication to work.
-        """
-        monkeypatch.setenv(
-            "HINDSIGHT_API_TENANT_EXTENSION",
-            "tests.test_server_module:MockTenantExtension",
-        )
-
-        _clean_server_module()
-
-        memory_engine_calls = []
-
-        def capture_memory_engine(*args, **kwargs):
-            memory_engine_calls.append({"args": args, "kwargs": kwargs})
-            return MagicMock()
-
-        with patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine), \
-             patch("hindsight_api.api.create_app") as mock_create_app, \
-             patch("hindsight_api.config.get_config") as mock_get_config, \
-             patch("hindsight_api.extensions.DefaultExtensionContext"):
-
-            mock_config = MagicMock()
-            mock_config.mcp_enabled = False
-            mock_config.run_migrations_on_startup = False
-            mock_config.database_url = "postgresql://test:test@localhost/test"
-            mock_get_config.return_value = mock_config
-            mock_create_app.return_value = MagicMock()
-
-            import hindsight_api.server
-
-        # Verify MemoryEngine was called
-        assert len(memory_engine_calls) == 1, "MemoryEngine should be called exactly once"
-
-        call_kwargs = memory_engine_calls[0]["kwargs"]
-
-        # THE CRITICAL ASSERTION: tenant_extension must be passed and not None
-        assert "tenant_extension" in call_kwargs, \
-            "MemoryEngine was not called with tenant_extension parameter!"
-        assert call_kwargs["tenant_extension"] is not None, \
-            "tenant_extension was None - server.py did not pass loaded extension to MemoryEngine!"
-
-    def test_server_sets_extension_context_on_tenant_extension(self, monkeypatch):
-        """
-        Verify that server.py sets the extension context on tenant extension.
-
-        This is required for tenant extensions that need to provision schemas.
-        """
-        monkeypatch.setenv(
-            "HINDSIGHT_API_TENANT_EXTENSION",
-            "tests.test_server_module:MockTenantExtension",
-        )
-
-        _clean_server_module()
-
-        context_set_calls = []
-        captured_tenant_ext = [None]
-
-        def capture_memory_engine(*args, **kwargs):
-            captured_tenant_ext[0] = kwargs.get("tenant_extension")
-            return MagicMock()
-
-        def capture_context(*args, **kwargs):
-            ctx = MagicMock()
-            context_set_calls.append(ctx)
-            return ctx
-
-        with patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine), \
-             patch("hindsight_api.api.create_app") as mock_create_app, \
-             patch("hindsight_api.config.get_config") as mock_get_config, \
-             patch("hindsight_api.extensions.DefaultExtensionContext", side_effect=capture_context):
-
-            mock_config = MagicMock()
-            mock_config.mcp_enabled = False
-            mock_config.run_migrations_on_startup = False
-            mock_config.database_url = "postgresql://test:test@localhost/test"
-            mock_get_config.return_value = mock_config
-            mock_create_app.return_value = MagicMock()
-
-            import hindsight_api.server
-
-        # Verify context was created and set
-        assert len(context_set_calls) == 1, "DefaultExtensionContext should be created"
-        assert captured_tenant_ext[0] is not None, "Tenant extension should be captured"
-        assert captured_tenant_ext[0]._context_set, \
-            "set_context was not called on tenant extension"
-
-    def test_server_works_without_extensions(self, monkeypatch):
-        """
-        Verify that server.py works correctly when no extensions are configured.
-        """
-        # Ensure no extension env vars are set
-        monkeypatch.delenv("HINDSIGHT_API_TENANT_EXTENSION", raising=False)
-        monkeypatch.delenv("HINDSIGHT_API_OPERATION_VALIDATOR_EXTENSION", raising=False)
-
-        _clean_server_module()
-
-        memory_engine_calls = []
-
-        def capture_memory_engine(*args, **kwargs):
-            memory_engine_calls.append({"args": args, "kwargs": kwargs})
-            return MagicMock()
-
-        with patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine), \
-             patch("hindsight_api.api.create_app") as mock_create_app, \
-             patch("hindsight_api.config.get_config") as mock_get_config:
-
-            mock_config = MagicMock()
-            mock_config.mcp_enabled = False
-            mock_config.run_migrations_on_startup = False
-            mock_config.database_url = "postgresql://test:test@localhost/test"
-            mock_get_config.return_value = mock_config
-            mock_create_app.return_value = MagicMock()
-
-            import hindsight_api.server
-
-        # Should work without extensions
-        assert len(memory_engine_calls) == 1
-        call_kwargs = memory_engine_calls[0]["kwargs"]
-
-        # Extensions should be None when not configured
-        assert call_kwargs.get("tenant_extension") is None
-        assert call_kwargs.get("operation_validator") is None
-
-
-# Mock extensions for testing
-from hindsight_api.extensions import (
-    TenantExtension,
-    TenantContext,
-    RequestContext,
-    OperationValidatorExtension,
-    ValidationResult,
-    RetainContext,
-    RecallContext,
-    ReflectContext,
-    RefreshMentalModelContext,
-)
-
-
-class MockTenantExtension(TenantExtension):
-    """Mock tenant extension for testing server.py extension loading."""
-
-    def __init__(self, config: dict):
-        super().__init__(config)
-        self._context_set = False
-
-    async def authenticate(self, request_context: RequestContext) -> TenantContext:
-        return TenantContext(schema_name="public")
-
-    def set_context(self, context) -> None:
-        self._context_set = True
-
-
-class MockOperationValidator(OperationValidatorExtension):
-    """Mock operation validator for testing server.py extension loading."""
-
-    def __init__(self, config: dict):
-        super().__init__(config)
-
-    async def validate_retain(self, ctx: RetainContext) -> ValidationResult:
-        return ValidationResult.accept()
-
-    async def validate_recall(self, ctx: RecallContext) -> ValidationResult:
-        return ValidationResult.accept()
-
-    async def validate_reflect(self, ctx: ReflectContext) -> ValidationResult:
-        return ValidationResult.accept()
-
-    async def validate_refresh_mental_model(self, ctx: RefreshMentalModelContext) -> ValidationResult:
-        return ValidationResult.accept()

hindsight-api/tests/test_tags_visibility.py

@@ -1,886 +0,0 @@
-"""
-Tests for tags-based visibility scoping.
-
-This module tests the tags feature which allows filtering memories by visibility tags.
-Use cases:
-- Multi-user agent: Agent has a single memory bank, users should only see memories from
-  conversations they participated in
-- Student tracking: Teacher tracks students, students should only see their own data
-
-The tags use OR-based matching: a memory matches if ANY of its tags overlap with the request tags.
-"""
-from datetime import datetime
-
-import httpx
-import pytest
-import pytest_asyncio
-
-from hindsight_api.api import create_app
-from hindsight_api.engine.search.tags import build_tags_where_clause_simple, filter_results_by_tags
-
-# ============================================================================
-# Unit Tests for tags SQL builder
-# ============================================================================
-
-
-class TestTagsWhereClauseBuilder:
-    """Unit tests for the tags WHERE clause SQL builder."""
-
-    def test_no_tags_returns_empty_string(self):
-        """When tags is None, should return empty string (no filtering)."""
-        result = build_tags_where_clause_simple(None, 5)
-        assert result == ""
-
-    def test_empty_tags_list_returns_empty_string(self):
-        """When tags is an empty list, should return empty string (no filtering)."""
-        result = build_tags_where_clause_simple([], 5)
-        assert result == ""
-
-    def test_tags_with_different_param_num(self):
-        """Should use the provided parameter number."""
-        result = build_tags_where_clause_simple(["user_a", "user_b"], 3)
-        # Default is "any" which includes untagged
-        assert "$3" in result
-
-    def test_tags_with_table_alias(self):
-        """Should include table alias when provided."""
-        result = build_tags_where_clause_simple(["user_a"], 5, table_alias="mu.")
-        assert "mu.tags" in result
-
-    # ---- Test "any" mode (OR, includes untagged - default) ----
-
-    def test_tags_match_any_includes_untagged(self):
-        """When match='any', should include untagged memories (NULL or empty)."""
-        result = build_tags_where_clause_simple(["user_a"], 5, match="any")
-        # Should use OR with NULL/empty check
-        assert "IS NULL" in result
-        assert "= '{}'" in result
-        assert "&&" in result  # overlap operator
-
-    def test_tags_match_any_uses_overlap(self):
-        """When match='any', should use overlap operator (&&)."""
-        result = build_tags_where_clause_simple(["user_a"], 5, match="any")
-        assert "&&" in result
-
-    # ---- Test "all" mode (AND, includes untagged) ----
-
-    def test_tags_match_all_includes_untagged(self):
-        """When match='all', should include untagged memories (NULL or empty)."""
-        result = build_tags_where_clause_simple(["user_a"], 5, match="all")
-        # Should use OR with NULL/empty check
-        assert "IS NULL" in result
-        assert "= '{}'" in result
-        assert "@>" in result  # contains operator
-
-    def test_tags_match_all_uses_contains(self):
-        """When match='all', should use contains operator (@>)."""
-        result = build_tags_where_clause_simple(["user_a"], 5, match="all")
-        assert "@>" in result
-
-    # ---- Test "any_strict" mode (OR, excludes untagged) ----
-
-    def test_tags_match_any_strict_excludes_untagged(self):
-        """When match='any_strict', should exclude untagged memories."""
-        result = build_tags_where_clause_simple(["user_a"], 5, match="any_strict")
-        # Should require tags to be NOT NULL and not empty
-        assert "IS NOT NULL" in result
-        assert "!= '{}'" in result
-        assert "&&" in result  # overlap operator
-
-    def test_tags_match_any_strict_uses_overlap(self):
-        """When match='any_strict', should use overlap operator (&&)."""
-        result = build_tags_where_clause_simple(["user_a"], 5, match="any_strict")
-        assert "&&" in result
-        # Should NOT include untagged
-        assert "IS NULL" not in result or "IS NOT NULL" in result
-
-    # ---- Test "all_strict" mode (AND, excludes untagged) ----
-
-    def test_tags_match_all_strict_excludes_untagged(self):
-        """When match='all_strict', should exclude untagged memories."""
-        result = build_tags_where_clause_simple(["user_a"], 5, match="all_strict")
-        # Should require tags to be NOT NULL and not empty
-        assert "IS NOT NULL" in result
-        assert "!= '{}'" in result
-        assert "@>" in result  # contains operator
-
-    def test_tags_match_all_strict_uses_contains(self):
-        """When match='all_strict', should use contains operator (@>)."""
-        result = build_tags_where_clause_simple(["user_a"], 5, match="all_strict")
-        assert "@>" in result
-
-    # ---- Test table alias with all modes ----
-
-    def test_tags_match_any_with_table_alias(self):
-        """Should include table alias with any mode."""
-        result = build_tags_where_clause_simple(["user_a"], 3, table_alias="mu.", match="any")
-        assert "mu.tags" in result
-
-    def test_tags_match_all_strict_with_table_alias(self):
-        """Should include table alias with all_strict mode."""
-        result = build_tags_where_clause_simple(["user_a", "user_b"], 3, table_alias="mu.", match="all_strict")
-        assert "mu.tags" in result
-        assert "@>" in result
-        assert "IS NOT NULL" in result
-
-
-# ============================================================================
-# Unit Tests for filter_results_by_tags (Python-side filtering)
-# ============================================================================
-
-
-class MockResult:
-    """Mock result object for testing filter_results_by_tags."""
-
-    def __init__(self, tags):
-        self.tags = tags
-
-
-class TestFilterResultsByTags:
-    """Unit tests for the Python-side tags filter function."""
-
-    def test_no_tags_returns_all(self):
-        """When tags is None, should return all results."""
-        results = [MockResult(["a"]), MockResult(["b"]), MockResult(None)]
-        filtered = filter_results_by_tags(results, None)
-        assert len(filtered) == 3
-
-    def test_empty_tags_returns_all(self):
-        """When tags is empty list, should return all results."""
-        results = [MockResult(["a"]), MockResult(["b"]), MockResult(None)]
-        filtered = filter_results_by_tags(results, [])
-        assert len(filtered) == 3
-
-    # ---- Test "any" mode (OR, includes untagged) ----
-
-    def test_any_mode_includes_matching_tags(self):
-        """'any' mode should include results with matching tags."""
-        results = [MockResult(["a"]), MockResult(["b"]), MockResult(["c"])]
-        filtered = filter_results_by_tags(results, ["a", "b"], match="any")
-        # "a" and "b" match, "c" doesn't match and isn't untagged, so excluded
-        assert len(filtered) == 2
-        tags_found = [r.tags[0] for r in filtered if r.tags]
-        assert "a" in tags_found
-        assert "b" in tags_found
-        assert "c" not in tags_found
-
-    def test_any_mode_includes_untagged(self):
-        """'any' mode should include untagged results."""
-        results = [MockResult(["a"]), MockResult(None), MockResult([])]
-        filtered = filter_results_by_tags(results, ["a"], match="any")
-        assert len(filtered) == 3  # a matches, None is untagged, [] is untagged
-
-    def test_any_mode_includes_partial_overlap(self):
-        """'any' mode should include results with ANY overlapping tag."""
-        results = [MockResult(["a", "x"]), MockResult(["b", "y"])]
-        filtered = filter_results_by_tags(results, ["a"], match="any")
-        # ["a", "x"] matches, ["b", "y"] doesn't, but untagged would be included
-        tags_found = [r.tags for r in filtered]
-        assert ["a", "x"] in tags_found
-
-    # ---- Test "any_strict" mode (OR, excludes untagged) ----
-
-    def test_any_strict_excludes_untagged(self):
-        """'any_strict' mode should exclude untagged results."""
-        results = [MockResult(["a"]), MockResult(None), MockResult([])]
-        filtered = filter_results_by_tags(results, ["a"], match="any_strict")
-        assert len(filtered) == 1  # Only ["a"] matches
-        assert filtered[0].tags == ["a"]
-
-    def test_any_strict_excludes_non_matching(self):
-        """'any_strict' mode should exclude non-matching tagged results."""
-        results = [MockResult(["a"]), MockResult(["b"]), MockResult(["c"])]
-        filtered = filter_results_by_tags(results, ["a"], match="any_strict")
-        assert len(filtered) == 1
-        assert filtered[0].tags == ["a"]
-
-    # ---- Test "all" mode (AND, includes untagged) ----
-
-    def test_all_mode_requires_all_tags(self):
-        """'all' mode should require ALL requested tags to be present."""
-        results = [MockResult(["a", "b"]), MockResult(["a"]), MockResult(["b"])]
-        filtered = filter_results_by_tags(results, ["a", "b"], match="all")
-        # Only ["a", "b"] has both tags, but untagged would also be included
-        tags_found = [r.tags for r in filtered]
-        assert ["a", "b"] in tags_found
-
-    def test_all_mode_includes_untagged(self):
-        """'all' mode should include untagged results."""
-        results = [MockResult(["a", "b"]), MockResult(None), MockResult([])]
-        filtered = filter_results_by_tags(results, ["a", "b"], match="all")
-        assert len(filtered) == 3  # ["a", "b"] matches, None is untagged, [] is untagged
-
-    # ---- Test "all_strict" mode (AND, excludes untagged) ----
-
-    def test_all_strict_requires_all_tags(self):
-        """'all_strict' mode should require ALL requested tags."""
-        results = [MockResult(["a", "b"]), MockResult(["a"]), MockResult(["b"])]
-        filtered = filter_results_by_tags(results, ["a", "b"], match="all_strict")
-        assert len(filtered) == 1
-        assert filtered[0].tags == ["a", "b"]
-
-    def test_all_strict_excludes_untagged(self):
-        """'all_strict' mode should exclude untagged results."""
-        results = [MockResult(["a", "b"]), MockResult(None), MockResult([])]
-        filtered = filter_results_by_tags(results, ["a", "b"], match="all_strict")
-        assert len(filtered) == 1
-        assert filtered[0].tags == ["a", "b"]
-
-    def test_all_strict_allows_superset(self):
-        """'all_strict' mode should allow results with MORE tags than requested."""
-        results = [MockResult(["a", "b", "c"]), MockResult(["a"])]
-        filtered = filter_results_by_tags(results, ["a", "b"], match="all_strict")
-        assert len(filtered) == 1
-        assert filtered[0].tags == ["a", "b", "c"]  # Has a, b, AND c
-
-
-# ============================================================================
-# Integration Tests for tags in retain/recall/reflect
-# ============================================================================
-
-
-@pytest_asyncio.fixture
-async def api_client(memory):
-    """Create an async test client for the FastAPI app."""
-    app = create_app(memory, initialize_memory=False)
-    transport = httpx.ASGITransport(app=app)
-    async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
-        yield client
-
-
-@pytest.fixture
-def test_bank_id():
-    """Provide a unique bank ID for this test run."""
-    return f"tags_test_{datetime.now().timestamp()}"
-
-
-@pytest.mark.asyncio
-async def test_retain_with_tags(api_client, test_bank_id):
-    """Test that memories can be stored with tags."""
-    # Store memory with tags
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories",
-        json={
-            "items": [
-                {
-                    "content": "Alice loves hiking in the mountains.",
-                    "tags": ["user_alice"]
-                }
-            ]
-        }
-    )
-    assert response.status_code == 200
-    result = response.json()
-    assert result["success"] is True
-    assert result["items_count"] == 1
-
-
-@pytest.mark.asyncio
-async def test_retain_with_document_tags(api_client, test_bank_id):
-    """Test that document-level tags are applied to all items."""
-    # Store memories with document-level tags
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories",
-        json={
-            "document_tags": ["session_123"],
-            "items": [
-                {"content": "Bob discussed the quarterly report."},
-                {"content": "Charlie mentioned the new product launch."}
-            ]
-        }
-    )
-    assert response.status_code == 200
-    result = response.json()
-    assert result["success"] is True
-    assert result["items_count"] == 2
-
-
-@pytest.mark.asyncio
-async def test_retain_merges_document_and_item_tags(api_client, test_bank_id):
-    """Test that document tags and item tags are merged."""
-    # Store memory with both document and item tags
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories",
-        json={
-            "document_tags": ["session_abc"],
-            "items": [
-                {
-                    "content": "Dave talked about machine learning.",
-                    "tags": ["user_dave"]
-                }
-            ]
-        }
-    )
-    assert response.status_code == 200
-    result = response.json()
-    assert result["success"] is True
-
-
-@pytest.mark.asyncio
-async def test_recall_without_tags_returns_all_memories(api_client, test_bank_id):
-    """Test that recall without tags returns all memories (no filtering)."""
-    # Store memories for different users
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories",
-        json={
-            "items": [
-                {"content": "Eve works on natural language processing.", "tags": ["user_eve"]},
-                {"content": "Frank specializes in computer vision.", "tags": ["user_frank"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # Recall without tags - should return all
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories/recall",
-        json={"query": "Who works on what?", "budget": "low"}
-    )
-    assert response.status_code == 200
-    results = response.json()["results"]
-
-    # Should find both Eve and Frank
-    texts = [r["text"] for r in results]
-    assert any("Eve" in t for t in texts), "Should find Eve"
-    assert any("Frank" in t for t in texts), "Should find Frank"
-
-
-@pytest.mark.asyncio
-async def test_recall_with_tags_filters_memories(api_client, test_bank_id):
-    """Test that recall with tags only returns matching memories."""
-    # Store memories for different users
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories",
-        json={
-            "items": [
-                {"content": "Grace is a data scientist at Google.", "tags": ["user_grace"]},
-                {"content": "Henry is a software engineer at Meta.", "tags": ["user_henry"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # Recall with user_grace tag - should only return Grace's memory
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories/recall",
-        json={"query": "Who works at which company?", "budget": "low", "tags": ["user_grace"]}
-    )
-    assert response.status_code == 200
-    results = response.json()["results"]
-
-    # Should find Grace but not Henry
-    texts = [r["text"] for r in results]
-    assert any("Grace" in t for t in texts), "Should find Grace with user_grace tag"
-    # Henry should NOT be found since he has user_henry tag
-    assert not any("Henry" in t for t in texts), "Should NOT find Henry (different tag)"
-
-
-@pytest.mark.asyncio
-async def test_recall_with_multiple_tags_uses_or_matching(api_client, test_bank_id):
-    """Test that multiple tags use OR matching (any match returns the memory)."""
-    # Store memories for different users
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories",
-        json={
-            "items": [
-                {"content": "Ivan leads the security team.", "tags": ["user_ivan"]},
-                {"content": "Julia manages the design team.", "tags": ["user_julia"]},
-                {"content": "Karl oversees the marketing team.", "tags": ["user_karl"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # Recall with user_ivan OR user_julia - should return both Ivan and Julia, but not Karl
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories/recall",
-        json={"query": "Who leads which team?", "budget": "low", "tags": ["user_ivan", "user_julia"]}
-    )
-    assert response.status_code == 200
-    results = response.json()["results"]
-
-    texts = [r["text"] for r in results]
-    assert any("Ivan" in t for t in texts), "Should find Ivan (tag matches)"
-    assert any("Julia" in t for t in texts), "Should find Julia (tag matches)"
-    assert not any("Karl" in t for t in texts), "Should NOT find Karl (tag doesn't match)"
-
-
-@pytest.mark.asyncio
-async def test_recall_returns_memories_with_any_overlapping_tag(api_client, test_bank_id):
-    """Test that memories with multiple tags are returned if ANY tag matches."""
-    # Store memory with multiple tags
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories",
-        json={
-            "items": [
-                {
-                    "content": "Lisa and Mike discussed the budget in a group chat.",
-                    "tags": ["user_lisa", "user_mike"]  # Memory visible to both
-                },
-                {"content": "Nancy reviewed the budget alone.", "tags": ["user_nancy"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # Recall with user_lisa - should return the group chat memory
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories/recall",
-        json={"query": "What was discussed about the budget?", "budget": "low", "tags": ["user_lisa"]}
-    )
-    assert response.status_code == 200
-    results = response.json()["results"]
-
-    texts = [r["text"] for r in results]
-    assert any("Lisa" in t and "Mike" in t for t in texts), "Should find group chat (Lisa is in tags)"
-    assert not any("Nancy" in t for t in texts), "Should NOT find Nancy's memory"
-
-
-@pytest.mark.asyncio
-async def test_reflect_with_tags_filters_memories(api_client, test_bank_id):
-    """Test that reflect with tags only uses matching memories for reasoning."""
-    # Store different memories for different users
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories",
-        json={
-            "items": [
-                {"content": "Oscar's favorite color is blue.", "tags": ["user_oscar"]},
-                {"content": "Peter's favorite color is red.", "tags": ["user_peter"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # Reflect with user_oscar tag - should only use Oscar's memories
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/reflect",
-        json={
-            "query": "What is the favorite color?",
-            "budget": "low",
-            "tags": ["user_oscar"],
-            "include": {"facts": {}}  # Request facts to verify what was used
-        }
-    )
-    assert response.status_code == 200
-    result = response.json()
-
-    # The response should mention Oscar's color (blue), not Peter's (red)
-    # Note: We can check based_on facts if they're returned
-    if result.get("based_on"):
-        based_on = result["based_on"]
-        memories = based_on.get("memories", []) if isinstance(based_on, dict) else []
-        fact_texts = [f["text"] for f in memories]
-        # Should use Oscar's memory (if facts are included)
-        if fact_texts:
-            assert any("Oscar" in t or "blue" in t for t in fact_texts), "Should use Oscar's memory"
-
-
-@pytest.mark.asyncio
-async def test_recall_with_empty_tags_returns_all(api_client, test_bank_id):
-    """Test that empty tags list behaves same as no tags (returns all)."""
-    # Store memories
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories",
-        json={
-            "items": [
-                {"content": "Quinn studies mathematics.", "tags": ["user_quinn"]},
-                {"content": "Rachel studies physics.", "tags": ["user_rachel"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # Recall with empty tags list - should return all
-    response = await api_client.post(
-        f"/v1/default/banks/{test_bank_id}/memories/recall",
-        json={"query": "Who studies what?", "budget": "low", "tags": []}
-    )
-    assert response.status_code == 200
-    results = response.json()["results"]
-
-    texts = [r["text"] for r in results]
-    assert any("Quinn" in t for t in texts), "Should find Quinn"
-    assert any("Rachel" in t for t in texts), "Should find Rachel"
-
-
-@pytest.mark.asyncio
-async def test_multi_user_agent_visibility(api_client):
-    """
-    Test multi-user agent visibility scoping.
-
-    Scenario:
-    - Agent has one memory bank
-    - Agent chats with User A (room 1) and User B (room 2) separately
-    - Agent also hosts a group chat with both users (room 3)
-    - User A should only see memories from rooms 1 and 3
-    - User B should only see memories from rooms 2 and 3
-    - Agent (no filter) should see all memories
-    """
-    bank_id = f"multi_user_test_{datetime.now().timestamp()}"
-
-    # Store memories from different chat rooms
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories",
-        json={
-            "items": [
-                # Room 1: Agent + User A private chat
-                {"content": "User A said they prefer morning meetings.", "tags": ["user_a"]},
-                # Room 2: Agent + User B private chat
-                {"content": "User B mentioned they like afternoon meetings.", "tags": ["user_b"]},
-                # Room 3: Group chat with both users
-                {"content": "In the group meeting, they agreed to meet at noon.", "tags": ["user_a", "user_b"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # User A queries - should see their private chat and group chat
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories/recall",
-        json={"query": "What meeting time preferences were discussed?", "budget": "low", "tags": ["user_a"]}
-    )
-    assert response.status_code == 200
-    user_a_results = response.json()["results"]
-    user_a_texts = [r["text"] for r in user_a_results]
-
-    assert any("morning" in t for t in user_a_texts), "User A should see their own preference (morning)"
-    assert any("noon" in t for t in user_a_texts), "User A should see group chat (noon)"
-    assert not any("afternoon" in t for t in user_a_texts), "User A should NOT see User B's private preference"
-
-    # User B queries - should see their private chat and group chat
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories/recall",
-        json={"query": "What meeting time preferences were discussed?", "budget": "low", "tags": ["user_b"]}
-    )
-    assert response.status_code == 200
-    user_b_results = response.json()["results"]
-    user_b_texts = [r["text"] for r in user_b_results]
-
-    assert any("afternoon" in t for t in user_b_texts), "User B should see their own preference (afternoon)"
-    assert any("noon" in t for t in user_b_texts), "User B should see group chat (noon)"
-    assert not any("morning" in t for t in user_b_texts), "User B should NOT see User A's private preference"
-
-    # Agent queries (no filter) - should see everything
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories/recall",
-        json={"query": "What meeting time preferences were discussed?", "budget": "low"}  # No tags
-    )
-    assert response.status_code == 200
-    agent_results = response.json()["results"]
-    agent_texts = [r["text"] for r in agent_results]
-
-    assert any("morning" in t for t in agent_texts), "Agent should see User A's preference"
-    assert any("afternoon" in t for t in agent_texts), "Agent should see User B's preference"
-    assert any("noon" in t for t in agent_texts), "Agent should see group chat"
-
-
-@pytest.mark.asyncio
-async def test_student_tracking_visibility(api_client):
-    """
-    Test student tracking visibility scoping.
-
-    Scenario:
-    - Teacher bot has one memory bank
-    - Teacher records observations for Student A, Student B
-    - Student A should only see their own data
-    - Teacher (no filter) should see all student data
-    """
-    bank_id = f"student_test_{datetime.now().timestamp()}"
-
-    # Store memories for different students
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories",
-        json={
-            "items": [
-                {"content": "Student A showed improvement in algebra today.", "tags": ["student_a"]},
-                {"content": "Student B struggled with geometry concepts.", "tags": ["student_b"]},
-                {"content": "Student A participated actively in class discussion.", "tags": ["student_a"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # Student A queries - should only see their own data
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories/recall",
-        json={"query": "How am I doing in class?", "budget": "low", "tags": ["student_a"]}
-    )
-    assert response.status_code == 200
-    student_a_results = response.json()["results"]
-    student_a_texts = [r["text"] for r in student_a_results]
-
-    assert any("algebra" in t for t in student_a_texts), "Student A should see their algebra progress"
-    assert any("participated" in t for t in student_a_texts), "Student A should see their participation"
-    assert not any("Student B" in t or "geometry" in t for t in student_a_texts), "Student A should NOT see Student B's data"
-
-    # Teacher queries (no filter) - should see all students
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories/recall",
-        json={"query": "Which students need help?", "budget": "low"}  # No tags
-    )
-    assert response.status_code == 200
-    teacher_results = response.json()["results"]
-    teacher_texts = [r["text"] for r in teacher_results]
-
-    assert any("Student A" in t for t in teacher_texts), "Teacher should see Student A's data"
-    assert any("Student B" in t for t in teacher_texts), "Teacher should see Student B's data"
-
-
-# ============================================================================
-# Tests for list_tags API endpoint
-# ============================================================================
-
-
-@pytest.mark.asyncio
-async def test_list_tags_returns_all_tags(api_client):
-    """Test that list_tags returns all unique tags with counts."""
-    bank_id = f"list_tags_test_{datetime.now().timestamp()}"
-
-    # Store memories with various tags
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories",
-        json={
-            "items": [
-                {"content": "Memory 1 for user alice.", "tags": ["user:alice"]},
-                {"content": "Memory 2 for user alice.", "tags": ["user:alice"]},
-                {"content": "Memory 3 for user bob.", "tags": ["user:bob"]},
-                {"content": "Memory 4 in session 123.", "tags": ["session:123"]},
-                {"content": "Memory 5 for alice in session 456.", "tags": ["user:alice", "session:456"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # List all tags
-    response = await api_client.get(f"/v1/default/banks/{bank_id}/tags")
-    assert response.status_code == 200
-    result = response.json()
-
-    # Verify structure
-    assert "items" in result
-    assert "total" in result
-    assert "limit" in result
-    assert "offset" in result
-
-    # Verify tags and counts
-    tags_map = {item["tag"]: item["count"] for item in result["items"]}
-    assert "user:alice" in tags_map
-    assert tags_map["user:alice"] == 3  # 3 memories have this tag
-    assert "user:bob" in tags_map
-    assert tags_map["user:bob"] == 1
-    assert "session:123" in tags_map
-    assert tags_map["session:123"] == 1
-    assert "session:456" in tags_map
-    assert tags_map["session:456"] == 1
-
-    assert result["total"] == 4  # 4 unique tags
-
-
-@pytest.mark.asyncio
-async def test_list_tags_with_wildcard_prefix(api_client):
-    """Test that list_tags filters with prefix wildcard pattern (user:*)."""
-    bank_id = f"list_tags_wildcard_test_{datetime.now().timestamp()}"
-
-    # Store memories with various tags
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories",
-        json={
-            "items": [
-                {"content": "Memory for alice who works at tech.", "tags": ["user:alice"]},
-                {"content": "Memory for bob who is an engineer.", "tags": ["user:bob"]},
-                {"content": "Memory for charlie the designer.", "tags": ["user:charlie"]},
-                {"content": "Session memory about the meeting.", "tags": ["session:abc"]},
-                {"content": "Room memory for conference room.", "tags": ["room:123"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # List tags with 'user:*' wildcard pattern
-    response = await api_client.get(f"/v1/default/banks/{bank_id}/tags", params={"q": "user:*"})
-    assert response.status_code == 200
-    result = response.json()
-
-    # Should only return user:* tags
-    tags = [item["tag"] for item in result["items"]]
-    assert "user:alice" in tags
-    assert "user:bob" in tags
-    assert "user:charlie" in tags
-    assert "session:abc" not in tags
-    assert "room:123" not in tags
-    assert result["total"] == 3
-
-
-@pytest.mark.asyncio
-async def test_list_tags_with_wildcard_suffix(api_client):
-    """Test that list_tags filters with suffix wildcard pattern (*-admin)."""
-    bank_id = f"list_tags_suffix_test_{datetime.now().timestamp()}"
-
-    # Store memories with various tags - use meaningful content for reliable fact extraction
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories",
-        json={
-            "items": [
-                {"content": "John has the role-admin permission and can manage user accounts.", "tags": ["role-admin"]},
-                {"content": "Sarah has super-admin access and can modify system settings.", "tags": ["super-admin"]},
-                {"content": "Mike is a standard role-user who can only view content.", "tags": ["role-user"]},
-                {"content": "Alice is a role-guest visitor with limited read access.", "tags": ["role-guest"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # List tags with '*-admin' wildcard pattern (suffix match)
-    response = await api_client.get(f"/v1/default/banks/{bank_id}/tags", params={"q": "*-admin"})
-    assert response.status_code == 200
-    result = response.json()
-
-    # Should only return *-admin tags
-    tags = [item["tag"] for item in result["items"]]
-    assert "role-admin" in tags
-    assert "super-admin" in tags
-    assert "role-user" not in tags
-    assert "role-guest" not in tags
-    assert result["total"] == 2
-
-
-@pytest.mark.asyncio
-async def test_list_tags_with_wildcard_middle(api_client):
-    """Test that list_tags filters with middle wildcard pattern (env*-prod)."""
-    bank_id = f"list_tags_middle_test_{datetime.now().timestamp()}"
-
-    # Store memories with various tags - use meaningful content for fact extraction
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories",
-        json={
-            "items": [
-                {"content": "The production environment is configured with high availability and uses AWS infrastructure.", "tags": ["env-prod"]},
-                {"content": "The enterprise environment for production runs on dedicated servers with 24/7 monitoring.", "tags": ["environment-prod"]},
-                {"content": "The staging environment mirrors production but uses smaller instance sizes.", "tags": ["env-staging"]},
-                {"content": "The development environment allows developers to test their code locally.", "tags": ["env-dev"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # List tags with 'env*-prod' wildcard pattern (middle match)
-    response = await api_client.get(f"/v1/default/banks/{bank_id}/tags", params={"q": "env*-prod"})
-    assert response.status_code == 200
-    result = response.json()
-
-    # Should only return env*-prod tags
-    tags = [item["tag"] for item in result["items"]]
-    assert "env-prod" in tags
-    assert "environment-prod" in tags
-    assert "env-staging" not in tags
-    assert "env-dev" not in tags
-    assert result["total"] == 2
-
-
-@pytest.mark.asyncio
-async def test_list_tags_case_insensitive(api_client):
-    """Test that list_tags wildcard matching is case-insensitive."""
-    bank_id = f"list_tags_case_test_{datetime.now().timestamp()}"
-
-    # Store memories with mixed case tags - use meaningful content
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories",
-        json={
-            "items": [
-                {"content": "Alice is a software engineer who specializes in machine learning algorithms.", "tags": ["User:Alice"]},
-                {"content": "Bob works as a data scientist at a large technology company.", "tags": ["user:bob"]},
-                {"content": "Charlie is the lead designer responsible for the user interface.", "tags": ["USER:CHARLIE"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # List tags with lowercase pattern - should match all cases
-    response = await api_client.get(f"/v1/default/banks/{bank_id}/tags", params={"q": "user:*"})
-    assert response.status_code == 200
-    result = response.json()
-
-    # Should match all user tags regardless of case
-    tags = [item["tag"] for item in result["items"]]
-    assert len(tags) == 3
-    assert result["total"] == 3
-
-
-@pytest.mark.asyncio
-async def test_list_tags_pagination(api_client):
-    """Test that list_tags supports pagination."""
-    bank_id = f"list_tags_pagination_test_{datetime.now().timestamp()}"
-
-    # Store memories with many tags - use meaningful content for fact extraction
-    names = ["Alice", "Bob", "Charlie", "Diana", "Eve", "Frank", "Grace", "Henry", "Ivan", "Julia"]
-    items = [
-        {"content": f"{name} works as a software engineer at company {i}.", "tags": [f"tag:{i:03d}"]}
-        for i, name in enumerate(names)
-    ]
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories",
-        json={"items": items}
-    )
-    assert response.status_code == 200
-
-    # Get first page (limit 3)
-    response = await api_client.get(f"/v1/default/banks/{bank_id}/tags", params={"limit": 3, "offset": 0})
-    assert response.status_code == 200
-    result = response.json()
-    assert len(result["items"]) == 3
-    assert result["total"] == 10
-    assert result["limit"] == 3
-    assert result["offset"] == 0
-
-    # Get second page
-    response = await api_client.get(f"/v1/default/banks/{bank_id}/tags", params={"limit": 3, "offset": 3})
-    assert response.status_code == 200
-    result = response.json()
-    assert len(result["items"]) == 3
-    assert result["offset"] == 3
-
-
-@pytest.mark.asyncio
-async def test_list_tags_empty_bank(api_client):
-    """Test that list_tags returns empty for bank with no tags."""
-    bank_id = f"list_tags_empty_test_{datetime.now().timestamp()}"
-
-    # List tags without storing anything
-    response = await api_client.get(f"/v1/default/banks/{bank_id}/tags")
-    assert response.status_code == 200
-    result = response.json()
-
-    assert result["items"] == []
-    assert result["total"] == 0
-
-
-@pytest.mark.asyncio
-async def test_list_tags_ordered_by_count(api_client):
-    """Test that list_tags returns tags ordered by frequency (most used first)."""
-    bank_id = f"list_tags_order_test_{datetime.now().timestamp()}"
-
-    # Store memories with tags having different frequencies - use meaningful content
-    response = await api_client.post(
-        f"/v1/default/banks/{bank_id}/memories",
-        json={
-            "items": [
-                {"content": "Alice works at a startup company as a developer.", "tags": ["rare"]},
-                {"content": "Bob is a senior engineer at Google.", "tags": ["common"]},
-                {"content": "Charlie manages the marketing team at Microsoft.", "tags": ["common"]},
-                {"content": "Diana leads the design department at Apple.", "tags": ["common"]},
-                {"content": "Eve is a data scientist at Amazon.", "tags": ["medium"]},
-                {"content": "Frank handles customer support at Meta.", "tags": ["medium"]},
-            ]
-        }
-    )
-    assert response.status_code == 200
-
-    # List tags - should be ordered by count descending
-    response = await api_client.get(f"/v1/default/banks/{bank_id}/tags")
-    assert response.status_code == 200
-    result = response.json()
-
-    tags = [item["tag"] for item in result["items"]]
-    # common (3) should come before medium (2) which should come before rare (1)
-    assert tags.index("common") < tags.index("medium")
-    assert tags.index("medium") < tags.index("rare")

hindsight-api/tests/test_tei_cross_encoder.py

@@ -544,243 +544,3 @@ class TestRemoteTEICrossEncoderConfig:
             assert encoder.base_url == "http://test:9000"
             assert encoder.batch_size == 256
             assert encoder.max_concurrent == 16
-
-
-# ============================================================================
-# TEI Reranker Performance Benchmark Tests
-# ============================================================================
-# These tests require a running TEI server to measure actual performance.
-# Set TEI_RERANKER_URL environment variable to run.
-# Example:
-#   TEI_RERANKER_URL=http://localhost:8000 \
-#   pytest tests/test_tei_cross_encoder.py::test_tei_reranker_performance -v -s -n0
-
-import os
-
-TEI_RERANKER_URL = os.environ.get("TEI_RERANKER_URL")
-
-requires_tei_server = pytest.mark.skipif(
-    TEI_RERANKER_URL is None,
-    reason="TEI_RERANKER_URL not set - skipping TEI performance benchmark",
-)
-
-
-@requires_tei_server
-@pytest.mark.asyncio
-async def test_tei_reranker_performance():
-    """
-    Benchmark TEI reranker performance with different configurations.
-
-    This test measures latency for different batch sizes and concurrency levels
-    to find the optimal configuration for your TEI server.
-
-    Example usage:
-        TEI_RERANKER_URL=http://localhost:8000 \
-        pytest tests/test_tei_cross_encoder.py::test_tei_reranker_performance -v -s -n0
-    """
-    import httpx
-
-    # Get server info
-    async with httpx.AsyncClient() as client:
-        response = await client.get(f"{TEI_RERANKER_URL}/info")
-        info = response.json()
-        print(f"\n📊 TEI Server Info:")
-        print(f"   URL: {TEI_RERANKER_URL}")
-        print(f"   Model: {info.get('model_id', 'unknown')}")
-        if "reranker_model" in info:
-            print(f"   Reranker Model: {info['reranker_model']}")
-
-    # Generate test data (800 pairs to simulate real workload)
-    num_pairs = 800
-    query = "What did I say about training machine learning models and artificial intelligence?"
-    test_pairs = [
-        (query, f"Document {i} about machine learning, neural networks, and AI training techniques.")
-        for i in range(num_pairs)
-    ]
-
-    # Test configurations: (batch_size, max_concurrent)
-    configs = [
-        (128, 8),   # Default
-        (256, 4),   # Larger batches, fewer concurrent
-        (256, 8),   # Larger batches, same concurrent
-        (512, 2),   # Very large batches, few concurrent
-        (512, 4),   # Very large batches, moderate concurrent
-        (64, 16),   # Smaller batches, more concurrent
-        (800, 1),   # Single batch (all at once)
-    ]
-
-    results = []
-    print(f"\n⏱️  Benchmarking {num_pairs} pairs with different configurations:\n")
-
-    for batch_size, max_concurrent in configs:
-        encoder = RemoteTEICrossEncoder(
-            base_url=TEI_RERANKER_URL,
-            batch_size=batch_size,
-            max_concurrent=max_concurrent,
-            timeout=60.0,
-        )
-        await encoder.initialize()
-
-        # Warm-up run
-        await encoder.predict(test_pairs[:100])
-
-        # Timed runs (3 iterations)
-        times = []
-        for _ in range(3):
-            start = time.time()
-            scores = await encoder.predict(test_pairs)
-            elapsed = time.time() - start
-            times.append(elapsed)
-            assert len(scores) == num_pairs
-
-        avg_time = sum(times) / len(times)
-        min_time = min(times)
-        results.append({
-            "batch_size": batch_size,
-            "max_concurrent": max_concurrent,
-            "avg_ms": avg_time * 1000,
-            "min_ms": min_time * 1000,
-            "num_batches": (num_pairs + batch_size - 1) // batch_size,
-        })
-
-        print(f"   batch_size={batch_size:4d}, max_concurrent={max_concurrent:2d}: "
-              f"avg={avg_time * 1000:6.1f}ms, min={min_time * 1000:6.1f}ms "
-              f"({results[-1]['num_batches']} batches)")
-
-    # Find best configuration
-    best = min(results, key=lambda x: x["avg_ms"])
-    print(f"\n🏆 Best Configuration:")
-    print(f"   batch_size={best['batch_size']}, max_concurrent={best['max_concurrent']}")
-    print(f"   Average: {best['avg_ms']:.1f}ms, Min: {best['min_ms']:.1f}ms")
-
-    # Performance target check
-    target_ms = 100
-    if best["avg_ms"] <= target_ms:
-        print(f"\n✅ Target met! Average {best['avg_ms']:.1f}ms <= {target_ms}ms")
-    else:
-        print(f"\n⚠️ Target NOT met. Average {best['avg_ms']:.1f}ms > {target_ms}ms")
-        print(f"   Consider: larger batch size, GPU optimization, or faster network")
-
-
-@requires_tei_server
-@pytest.mark.asyncio
-async def test_tei_reranker_concurrent_requests():
-    """
-    Test TEI reranker performance under concurrent request load.
-
-    This simulates multiple parallel recall requests hitting the reranker
-    at the same time.
-    """
-    # Smaller batches to simulate typical recall workload
-    num_pairs_per_request = 200
-    num_concurrent_requests = 4
-
-    query = "Tell me about machine learning and AI training"
-    test_pairs = [
-        (query, f"Document {i} about ML and training.")
-        for i in range(num_pairs_per_request)
-    ]
-
-    # Test configurations
-    configs = [
-        (128, 8),   # Default
-        (256, 4),   # Larger batches
-        (512, 2),   # Very large batches
-        (200, 1),   # Single batch per request
-    ]
-
-    print(f"\n⏱️  Concurrent Load Test: {num_concurrent_requests} parallel requests, "
-          f"{num_pairs_per_request} pairs each:\n")
-
-    for batch_size, max_concurrent in configs:
-        encoder = RemoteTEICrossEncoder(
-            base_url=TEI_RERANKER_URL,
-            batch_size=batch_size,
-            max_concurrent=max_concurrent,
-            timeout=60.0,
-        )
-        await encoder.initialize()
-
-        # Warm-up
-        await encoder.predict(test_pairs[:50])
-
-        async def run_single_request():
-            start = time.time()
-            scores = await encoder.predict(test_pairs)
-            return time.time() - start, len(scores)
-
-        # Run concurrent requests
-        times = []
-        for _ in range(3):  # 3 iterations
-            start = time.time()
-            results = await asyncio.gather(*[run_single_request() for _ in range(num_concurrent_requests)])
-            total_time = time.time() - start
-
-            individual_times = [r[0] for r in results]
-            times.append({
-                "total": total_time,
-                "max_individual": max(individual_times),
-                "avg_individual": sum(individual_times) / len(individual_times),
-            })
-
-        avg_total = sum(t["total"] for t in times) / len(times)
-        avg_max_individual = sum(t["max_individual"] for t in times) / len(times)
-
-        print(f"   batch_size={batch_size:4d}, max_concurrent={max_concurrent:2d}: "
-              f"total={avg_total * 1000:6.1f}ms, slowest_req={avg_max_individual * 1000:6.1f}ms")
-
-
-@requires_tei_server
-@pytest.mark.asyncio
-async def test_tei_reranker_latency_breakdown():
-    """
-    Measure latency breakdown for TEI reranker requests.
-
-    This helps identify where time is spent: network vs processing.
-    """
-    import httpx
-
-    print(f"\n⏱️  Latency Breakdown Test:\n")
-
-    # Test single document latency (network overhead)
-    async with httpx.AsyncClient(timeout=30.0) as client:
-        times = []
-        for _ in range(10):
-            start = time.time()
-            await client.post(
-                f"{TEI_RERANKER_URL}/rerank",
-                json={
-                    "query": "test query",
-                    "texts": ["test document"],
-                    "return_text": False,
-                },
-            )
-            times.append((time.time() - start) * 1000)
-
-        avg_single = sum(times) / len(times)
-        print(f"   Single doc latency (raw HTTP): {avg_single:.2f}ms")
-
-    # Test batch latencies
-    batch_sizes = [10, 50, 100, 200, 500]
-    for batch_size in batch_sizes:
-        texts = [f"Document {i} about machine learning" for i in range(batch_size)]
-        async with httpx.AsyncClient(timeout=30.0) as client:
-            times = []
-            for _ in range(5):
-                start = time.time()
-                await client.post(
-                    f"{TEI_RERANKER_URL}/rerank",
-                    json={
-                        "query": "What about machine learning?",
-                        "texts": texts,
-                        "return_text": False,
-                    },
-                )
-                times.append((time.time() - start) * 1000)
-
-            avg = sum(times) / len(times)
-            per_doc = avg / batch_size
-            print(f"   Batch size {batch_size:4d}: {avg:6.1f}ms total, {per_doc:.2f}ms/doc")
-
-    print(f"\n   💡 Insight: Higher per-doc time at small batches = network overhead dominant")
-    print(f"   💡 Insight: Lower per-doc time at large batches = GPU efficiently utilized")

hindsight-api/tests/test_worker.py

@@ -1,592 +0,0 @@
-"""
-Tests for the distributed worker system.
-
-Tests cover:
-- BrokerTaskBackend task submission and storage
-- WorkerPoller task claiming with FOR UPDATE SKIP LOCKED
-- Concurrent workers claiming different tasks (no duplicates)
-- Task completion and failure handling
-- Retry mechanism
-- Worker decommissioning
-"""
-
-import asyncio
-import json
-import uuid
-
-import pytest
-import pytest_asyncio
-
-from hindsight_api.engine.task_backend import BrokerTaskBackend, SyncTaskBackend
-
-
-# Use loadgroup to ensure these tests run in the same worker
-# since they share database state
-pytestmark = pytest.mark.xdist_group("worker_tests")
-
-
-@pytest_asyncio.fixture
-async def pool(pg0_db_url):
-    """Create a dedicated connection pool for worker tests."""
-    import asyncpg
-
-    from hindsight_api.pg0 import resolve_database_url
-
-    # Resolve pg0:// URL to postgresql:// URL if needed
-    resolved_url = await resolve_database_url(pg0_db_url)
-
-    pool = await asyncpg.create_pool(
-        resolved_url,
-        min_size=2,
-        max_size=10,
-        command_timeout=30,
-    )
-    yield pool
-    await pool.close()
-
-
-@pytest_asyncio.fixture
-async def clean_operations(pool):
-    """Clean up async_operations table before and after tests."""
-    # Clean before test
-    await pool.execute("DELETE FROM async_operations WHERE bank_id LIKE 'test-worker-%'")
-    yield
-    # Clean after test
-    await pool.execute("DELETE FROM async_operations WHERE bank_id LIKE 'test-worker-%'")
-
-
-class TestBrokerTaskBackend:
-    """Tests for BrokerTaskBackend task storage."""
-
-    @pytest.mark.asyncio
-    async def test_submit_task_updates_existing_operation(self, pool, clean_operations):
-        """Test that submit_task updates task_payload for existing operations."""
-        # Create an operation record first
-        operation_id = uuid.uuid4()
-        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-
-        await pool.execute(
-            """
-            INSERT INTO async_operations (operation_id, bank_id, operation_type, status)
-            VALUES ($1, $2, 'test_operation', 'pending')
-            """,
-            operation_id,
-            bank_id,
-        )
-
-        # Submit task with same operation_id
-        backend = BrokerTaskBackend(pool_getter=lambda: pool)
-        await backend.initialize()
-
-        task_dict = {
-            "operation_id": str(operation_id),
-            "type": "test_task",
-            "bank_id": bank_id,
-            "data": {"key": "value"},
-        }
-        await backend.submit_task(task_dict)
-
-        # Verify task_payload was stored
-        row = await pool.fetchrow(
-            "SELECT task_payload, status FROM async_operations WHERE operation_id = $1",
-            operation_id,
-        )
-        assert row is not None
-        assert row["status"] == "pending"
-        payload = json.loads(row["task_payload"])
-        assert payload["type"] == "test_task"
-        assert payload["data"] == {"key": "value"}
-
-    @pytest.mark.asyncio
-    async def test_submit_task_creates_new_operation(self, pool, clean_operations):
-        """Test that submit_task creates new operation when no operation_id provided."""
-        backend = BrokerTaskBackend(pool_getter=lambda: pool)
-        await backend.initialize()
-
-        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-        task_dict = {
-            "type": "access_count_update",
-            "bank_id": bank_id,
-            "node_ids": ["node1", "node2"],
-        }
-        await backend.submit_task(task_dict)
-
-        # Verify new operation was created
-        row = await pool.fetchrow(
-            "SELECT operation_type, status, task_payload FROM async_operations WHERE bank_id = $1",
-            bank_id,
-        )
-        assert row is not None
-        assert row["operation_type"] == "access_count_update"
-        assert row["status"] == "pending"
-        payload = json.loads(row["task_payload"])
-        assert payload["node_ids"] == ["node1", "node2"]
-
-
-class TestWorkerPoller:
-    """Tests for WorkerPoller task claiming and execution."""
-
-    @pytest.mark.asyncio
-    async def test_claim_batch_claims_pending_tasks(self, pool, clean_operations):
-        """Test that claim_batch claims pending tasks with task_payload."""
-        from hindsight_api.worker import WorkerPoller
-
-        # Create some pending tasks
-        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-        for i in range(3):
-            op_id = uuid.uuid4()
-            payload = json.dumps({"type": "test_task", "index": i, "bank_id": bank_id})
-            await pool.execute(
-                """
-                INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
-                VALUES ($1, $2, 'test', 'pending', $3::jsonb)
-                """,
-                op_id,
-                bank_id,
-                payload,
-            )
-
-        # Create poller and claim tasks
-        executed_tasks = []
-
-        async def mock_executor(task_dict):
-            executed_tasks.append(task_dict)
-
-        poller = WorkerPoller(
-            pool=pool,
-            worker_id="test-worker-1",
-            executor=mock_executor,
-            batch_size=10,
-        )
-
-        claimed = await poller.claim_batch()
-        assert len(claimed) == 3
-
-        # Verify tasks are marked as processing with worker_id
-        rows = await pool.fetch(
-            "SELECT status, worker_id FROM async_operations WHERE bank_id = $1",
-            bank_id,
-        )
-        for row in rows:
-            assert row["status"] == "processing"
-            assert row["worker_id"] == "test-worker-1"
-
-    @pytest.mark.asyncio
-    async def test_claim_batch_respects_batch_size(self, pool, clean_operations):
-        """Test that claim_batch respects the batch_size limit."""
-        from hindsight_api.worker import WorkerPoller
-
-        # Create 10 pending tasks
-        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-        for i in range(10):
-            op_id = uuid.uuid4()
-            payload = json.dumps({"type": "test_task", "index": i, "bank_id": bank_id})
-            await pool.execute(
-                """
-                INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
-                VALUES ($1, $2, 'test', 'pending', $3::jsonb)
-                """,
-                op_id,
-                bank_id,
-                payload,
-            )
-
-        # Claim with batch_size=3
-        poller = WorkerPoller(
-            pool=pool,
-            worker_id="test-worker-1",
-            executor=lambda x: None,
-            batch_size=3,
-        )
-
-        claimed = await poller.claim_batch()
-        assert len(claimed) == 3
-
-    @pytest.mark.asyncio
-    async def test_execute_task_marks_completed(self, pool, clean_operations):
-        """Test that successful task execution marks task as completed."""
-        from hindsight_api.worker import WorkerPoller
-
-        # Create a pending task
-        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-        op_id = uuid.uuid4()
-        payload = json.dumps({"type": "test_task", "operation_id": str(op_id), "bank_id": bank_id})
-        await pool.execute(
-            """
-            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id)
-            VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'test-worker-1')
-            """,
-            op_id,
-            bank_id,
-            payload,
-        )
-
-        executed = []
-
-        async def mock_executor(task_dict):
-            executed.append(task_dict)
-
-        poller = WorkerPoller(
-            pool=pool,
-            worker_id="test-worker-1",
-            executor=mock_executor,
-        )
-
-        # Execute the task
-        task_dict = json.loads(payload)
-        await poller.execute_task(str(op_id), task_dict)
-
-        assert len(executed) == 1
-
-        # Verify task is marked as completed
-        row = await pool.fetchrow(
-            "SELECT status, completed_at FROM async_operations WHERE operation_id = $1",
-            op_id,
-        )
-        assert row["status"] == "completed"
-        assert row["completed_at"] is not None
-
-    @pytest.mark.asyncio
-    async def test_execute_task_retries_on_failure(self, pool, clean_operations):
-        """Test that failed task execution triggers retry mechanism."""
-        from hindsight_api.worker import WorkerPoller
-
-        # Create a pending task with retry_count=0
-        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-        op_id = uuid.uuid4()
-        payload = json.dumps({"type": "test_task", "operation_id": str(op_id), "bank_id": bank_id})
-        await pool.execute(
-            """
-            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id, retry_count)
-            VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'test-worker-1', 0)
-            """,
-            op_id,
-            bank_id,
-            payload,
-        )
-
-        async def failing_executor(task_dict):
-            raise ValueError("Simulated failure")
-
-        poller = WorkerPoller(
-            pool=pool,
-            worker_id="test-worker-1",
-            executor=failing_executor,
-            max_retries=3,
-        )
-
-        # Execute (should fail and retry)
-        task_dict = json.loads(payload)
-        await poller.execute_task(str(op_id), task_dict)
-
-        # Verify task is back to pending with incremented retry_count
-        row = await pool.fetchrow(
-            "SELECT status, retry_count, worker_id FROM async_operations WHERE operation_id = $1",
-            op_id,
-        )
-        assert row["status"] == "pending"
-        assert row["retry_count"] == 1
-        assert row["worker_id"] is None  # Worker ID cleared for retry
-
-    @pytest.mark.asyncio
-    async def test_execute_task_fails_after_max_retries(self, pool, clean_operations):
-        """Test that task is marked failed after exceeding max retries."""
-        from hindsight_api.worker import WorkerPoller
-
-        # Create a task that has already used all retries
-        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-        op_id = uuid.uuid4()
-        payload = json.dumps({"type": "test_task", "operation_id": str(op_id), "bank_id": bank_id})
-        await pool.execute(
-            """
-            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id, retry_count)
-            VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'test-worker-1', 3)
-            """,
-            op_id,
-            bank_id,
-            payload,
-        )
-
-        async def failing_executor(task_dict):
-            raise ValueError("Simulated failure")
-
-        poller = WorkerPoller(
-            pool=pool,
-            worker_id="test-worker-1",
-            executor=failing_executor,
-            max_retries=3,
-        )
-
-        # Execute (should fail permanently)
-        task_dict = json.loads(payload)
-        await poller.execute_task(str(op_id), task_dict)
-
-        # Verify task is marked as failed
-        row = await pool.fetchrow(
-            "SELECT status, error_message FROM async_operations WHERE operation_id = $1",
-            op_id,
-        )
-        assert row["status"] == "failed"
-        assert "Max retries" in row["error_message"]
-
-
-class TestConcurrentWorkers:
-    """Tests for concurrent worker task claiming (FOR UPDATE SKIP LOCKED)."""
-
-    @pytest.mark.asyncio
-    async def test_concurrent_workers_claim_different_tasks(self, pool, clean_operations):
-        """Test that multiple workers claim different tasks (no duplicates)."""
-        from hindsight_api.worker import WorkerPoller
-
-        # Create 10 pending tasks
-        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-        task_ids = []
-        for i in range(10):
-            op_id = uuid.uuid4()
-            task_ids.append(op_id)
-            payload = json.dumps({"type": "test_task", "index": i, "bank_id": bank_id, "operation_id": str(op_id)})
-            await pool.execute(
-                """
-                INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
-                VALUES ($1, $2, 'test', 'pending', $3::jsonb)
-                """,
-                op_id,
-                bank_id,
-                payload,
-            )
-
-        # Create 3 workers that will claim tasks concurrently
-        workers_claimed: dict[str, list[str]] = {"worker-1": [], "worker-2": [], "worker-3": []}
-
-        async def claim_for_worker(worker_id: str):
-            poller = WorkerPoller(
-                pool=pool,
-                worker_id=worker_id,
-                executor=lambda x: None,
-                batch_size=5,  # Each worker tries to claim 5
-            )
-            claimed = await poller.claim_batch()
-            workers_claimed[worker_id] = [op_id for op_id, _ in claimed]
-
-        # Run all workers concurrently
-        await asyncio.gather(
-            claim_for_worker("worker-1"),
-            claim_for_worker("worker-2"),
-            claim_for_worker("worker-3"),
-        )
-
-        # Verify no duplicates - each task claimed by exactly one worker
-        all_claimed = workers_claimed["worker-1"] + workers_claimed["worker-2"] + workers_claimed["worker-3"]
-        assert len(all_claimed) == len(set(all_claimed)), "Duplicate task claimed by multiple workers!"
-
-        # Verify total claimed equals available tasks (10)
-        assert len(all_claimed) == 10, f"Expected 10 tasks claimed, got {len(all_claimed)}"
-
-        # Verify each task is assigned to exactly one worker in DB
-        rows = await pool.fetch(
-            "SELECT operation_id, worker_id FROM async_operations WHERE bank_id = $1",
-            bank_id,
-        )
-        worker_assignments = {str(row["operation_id"]): row["worker_id"] for row in rows}
-
-        # With FOR UPDATE SKIP LOCKED, it's a race condition which workers get tasks.
-        # The important invariant is no duplicates and all tasks claimed, which we verified above.
-        # Just verify that at least 1 worker got tasks and all tasks have a worker assigned.
-        assert len(set(worker_assignments.values())) >= 1, "At least one worker should have claimed tasks"
-        assert all(w is not None for w in worker_assignments.values()), "All tasks should have a worker assigned"
-
-    @pytest.mark.asyncio
-    async def test_workers_do_not_claim_already_processing_tasks(self, pool, clean_operations):
-        """Test that workers skip tasks already being processed by another worker."""
-        from hindsight_api.worker import WorkerPoller
-
-        # Create tasks - some pending, some already processing
-        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-
-        # Create 3 pending tasks
-        for i in range(3):
-            op_id = uuid.uuid4()
-            payload = json.dumps({"type": "test_task", "index": i, "bank_id": bank_id})
-            await pool.execute(
-                """
-                INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
-                VALUES ($1, $2, 'test', 'pending', $3::jsonb)
-                """,
-                op_id,
-                bank_id,
-                payload,
-            )
-
-        # Create 2 already-processing tasks owned by another worker
-        for i in range(2):
-            op_id = uuid.uuid4()
-            payload = json.dumps({"type": "test_task", "index": i + 10, "bank_id": bank_id})
-            await pool.execute(
-                """
-                INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id)
-                VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'other-worker')
-                """,
-                op_id,
-                bank_id,
-                payload,
-            )
-
-        # New worker should only claim the 3 pending tasks
-        poller = WorkerPoller(
-            pool=pool,
-            worker_id="new-worker",
-            executor=lambda x: None,
-            batch_size=10,
-        )
-
-        claimed = await poller.claim_batch()
-        assert len(claimed) == 3, "Worker should only claim pending tasks"
-
-        # Verify other worker's tasks are still owned by them
-        row = await pool.fetchrow(
-            "SELECT COUNT(*) as count FROM async_operations WHERE bank_id = $1 AND worker_id = 'other-worker'",
-            bank_id,
-        )
-        assert row["count"] == 2
-
-
-class TestWorkerDecommission:
-    """Tests for worker decommissioning functionality."""
-
-    @pytest.mark.asyncio
-    async def test_decommission_releases_worker_tasks(self, pool, clean_operations):
-        """Test that decommissioning a worker releases all its processing tasks."""
-        # Create tasks being processed by a worker
-        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-        worker_id = "worker-to-decommission"
-
-        for i in range(5):
-            op_id = uuid.uuid4()
-            payload = json.dumps({"type": "test_task", "index": i, "bank_id": bank_id})
-            await pool.execute(
-                """
-                INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id, claimed_at)
-                VALUES ($1, $2, 'test', 'processing', $3::jsonb, $4, now())
-                """,
-                op_id,
-                bank_id,
-                payload,
-                worker_id,
-            )
-
-        # Run decommission
-        result = await pool.fetch(
-            """
-            UPDATE async_operations
-            SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
-            WHERE worker_id = $1 AND status = 'processing'
-            RETURNING operation_id
-            """,
-            worker_id,
-        )
-
-        assert len(result) == 5
-
-        # Verify all tasks are back to pending
-        rows = await pool.fetch(
-            "SELECT status, worker_id, claimed_at FROM async_operations WHERE bank_id = $1",
-            bank_id,
-        )
-        for row in rows:
-            assert row["status"] == "pending"
-            assert row["worker_id"] is None
-            assert row["claimed_at"] is None
-
-    @pytest.mark.asyncio
-    async def test_decommission_does_not_affect_other_workers(self, pool, clean_operations):
-        """Test that decommissioning one worker doesn't affect another worker's tasks."""
-        bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-
-        # Create tasks for worker-1
-        for i in range(3):
-            op_id = uuid.uuid4()
-            payload = json.dumps({"type": "test_task", "index": i, "bank_id": bank_id})
-            await pool.execute(
-                """
-                INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id)
-                VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'worker-1')
-                """,
-                op_id,
-                bank_id,
-                payload,
-            )
-
-        # Create tasks for worker-2
-        for i in range(3):
-            op_id = uuid.uuid4()
-            payload = json.dumps({"type": "test_task", "index": i + 10, "bank_id": bank_id})
-            await pool.execute(
-                """
-                INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id)
-                VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'worker-2')
-                """,
-                op_id,
-                bank_id,
-                payload,
-            )
-
-        # Decommission worker-1 only
-        await pool.execute(
-            """
-            UPDATE async_operations
-            SET status = 'pending', worker_id = NULL, claimed_at = NULL
-            WHERE worker_id = 'worker-1' AND status = 'processing'
-            """
-        )
-
-        # Verify worker-1 tasks are released
-        worker1_rows = await pool.fetch(
-            "SELECT status, worker_id FROM async_operations WHERE bank_id = $1 AND worker_id IS NULL",
-            bank_id,
-        )
-        assert len(worker1_rows) == 3
-
-        # Verify worker-2 tasks are unaffected
-        worker2_rows = await pool.fetch(
-            "SELECT status, worker_id FROM async_operations WHERE bank_id = $1 AND worker_id = 'worker-2'",
-            bank_id,
-        )
-        assert len(worker2_rows) == 3
-        for row in worker2_rows:
-            assert row["status"] == "processing"
-
-
-class TestSyncTaskBackend:
-    """Tests for SyncTaskBackend (used in tests and embedded mode)."""
-
-    @pytest.mark.asyncio
-    async def test_sync_backend_executes_immediately(self):
-        """Test that SyncTaskBackend executes tasks immediately."""
-        executed = []
-
-        async def mock_executor(task_dict):
-            executed.append(task_dict)
-
-        backend = SyncTaskBackend()
-        backend.set_executor(mock_executor)
-        await backend.initialize()
-
-        task_dict = {"type": "test", "data": "value"}
-        await backend.submit_task(task_dict)
-
-        assert len(executed) == 1
-        assert executed[0] == task_dict
-
-    @pytest.mark.asyncio
-    async def test_sync_backend_handles_errors(self):
-        """Test that SyncTaskBackend handles executor errors gracefully."""
-
-        async def failing_executor(task_dict):
-            raise ValueError("Test error")
-
-        backend = SyncTaskBackend()
-        backend.set_executor(failing_executor)
-        await backend.initialize()
-
-        # Should not raise, error is logged
-        await backend.submit_task({"type": "test"})

hindsight-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hindsight-cli"
-version = "0.3.0"
+version = "0.2.1"
 edition = "2021"
 authors = ["Hindsight Team"]
 description = "A beautiful CLI for Hindsight - semantic memory system"

hindsight-cli/src/api.rs

@@ -55,7 +55,6 @@ pub struct MemoryPutResult {
     pub items_count: i64,
     pub message: String,
     pub is_async: bool,
-    pub operation_id: Option<String>,
 }
 
 #[derive(Clone)]
@@ -104,7 +103,7 @@ impl ApiClient {
 
     pub fn get_stats(&self, agent_id: &str, _verbose: bool) -> Result<AgentStats> {
         self.runtime.block_on(async {
-            let response = self.client.get_agent_stats(agent_id, None).await?;
+            let response = self.client.get_agent_stats(agent_id).await?;
             let value = response.into_inner();
             // Convert to JSON Value first, then parse into our type
             let json_value = serde_json::to_value(&value)?;
@@ -117,7 +116,6 @@ impl ApiClient {
         self.runtime.block_on(async {
             let request = types::CreateBankRequest {
                 name: Some(name.to_string()),
-                mission: None,
                 background: None,
                 disposition: None,
             };
@@ -163,54 +161,10 @@ impl ApiClient {
                 items_count: result.items_count,
                 message: format!("Stored {} memory units", result.items_count),
                 is_async: result.async_,
-                operation_id: result.operation_id,
             })
         })
     }
 
-    /// Poll an operation until it completes or fails.
-    /// Returns Ok(true) if completed successfully, Ok(false) if failed, Err if polling error.
-    pub fn poll_operation(&self, agent_id: &str, operation_id: &str, verbose: bool) -> Result<(bool, Option<String>)> {
-        self.runtime.block_on(async {
-            loop {
-                let response = self.client.list_operations(agent_id, None).await?;
-                let ops = response.into_inner();
-
-                // Find our operation
-                let op = ops.operations.iter().find(|o| o.id == operation_id);
-
-                match op {
-                    Some(operation) => {
-                        if verbose {
-                            eprintln!("Operation {} status: {}", operation_id, operation.status);
-                        }
-                        match operation.status.as_str() {
-                            "pending" => {
-                                // Still running, wait and poll again
-                                tokio::time::sleep(std::time::Duration::from_millis(500)).await;
-                            }
-                            "completed" => {
-                                // Operation completed successfully
-                                return Ok((true, None));
-                            }
-                            "failed" => {
-                                return Ok((false, operation.error_message.clone()));
-                            }
-                            _ => {
-                                // Unknown status, treat as failed
-                                return Ok((false, Some(format!("Unknown status: {}", operation.status))));
-                            }
-                        }
-                    }
-                    None => {
-                        // Operation not in list means it completed successfully (removed from pending/failed)
-                        return Ok((true, None));
-                    }
-                }
-            }
-        })
-    }
-
     pub fn delete_memory(&self, _agent_id: &str, _unit_id: &str, _verbose: bool) -> Result<types::DeleteResponse> {
         // Note: Individual memory deletion is no longer supported in the API
         anyhow::bail!("Individual memory deletion is no longer supported. Use 'memory clear' to clear all memories.")
@@ -287,9 +241,9 @@ impl ApiClient {
         })
     }
 
-    pub fn list_entities(&self, bank_id: &str, limit: Option<i64>, offset: Option<i64>, _verbose: bool) -> Result<types::EntityListResponse> {
+    pub fn list_entities(&self, bank_id: &str, limit: Option<i64>, _verbose: bool) -> Result<types::EntityListResponse> {
         self.runtime.block_on(async {
-            let response = self.client.list_entities(bank_id, limit, offset, None).await?;
+            let response = self.client.list_entities(bank_id, limit, None).await?;
             Ok(response.into_inner())
         })
     }
@@ -327,105 +281,3 @@ pub use types::{
     ReflectResponse,
     RetainRequest,
 };
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_operation_deserialize() {
-        let json = r#"{
-            "id": "test-op-123",
-            "task_type": "retain",
-            "items_count": 5,
-            "document_id": "doc-456",
-            "created_at": "2024-01-15T10:00:00Z",
-            "status": "pending",
-            "error_message": null
-        }"#;
-        let op: Operation = serde_json::from_str(json).unwrap();
-        assert_eq!(op.id, "test-op-123");
-        assert_eq!(op.task_type, "retain");
-        assert_eq!(op.items_count, 5);
-        assert_eq!(op.document_id, Some("doc-456".to_string()));
-        assert_eq!(op.status, "pending");
-        assert!(op.error_message.is_none());
-    }
-
-    #[test]
-    fn test_operation_deserialize_with_error() {
-        let json = r#"{
-            "id": "test-op-456",
-            "task_type": "retain",
-            "items_count": 3,
-            "document_id": null,
-            "created_at": "2024-01-15T10:00:00Z",
-            "status": "failed",
-            "error_message": "Something went wrong"
-        }"#;
-        let op: Operation = serde_json::from_str(json).unwrap();
-        assert_eq!(op.status, "failed");
-        assert_eq!(op.error_message, Some("Something went wrong".to_string()));
-    }
-
-    #[test]
-    fn test_memory_put_result_serialize() {
-        let result = MemoryPutResult {
-            success: true,
-            items_count: 10,
-            message: "Stored 10 memory units".to_string(),
-            is_async: true,
-            operation_id: Some("op-789".to_string()),
-        };
-        let json = serde_json::to_string(&result).unwrap();
-        assert!(json.contains("\"success\":true"));
-        assert!(json.contains("\"items_count\":10"));
-        assert!(json.contains("\"is_async\":true"));
-        assert!(json.contains("\"operation_id\":\"op-789\""));
-    }
-
-    #[test]
-    fn test_memory_put_result_without_operation_id() {
-        let result = MemoryPutResult {
-            success: true,
-            items_count: 5,
-            message: "Stored 5 memory units".to_string(),
-            is_async: false,
-            operation_id: None,
-        };
-        let json = serde_json::to_string(&result).unwrap();
-        assert!(json.contains("\"operation_id\":null"));
-    }
-
-    #[test]
-    fn test_operations_response_deserialize() {
-        let json = r#"{
-            "bank_id": "test-bank",
-            "operations": [
-                {
-                    "id": "op-1",
-                    "task_type": "retain",
-                    "items_count": 2,
-                    "document_id": null,
-                    "created_at": "2024-01-15T10:00:00Z",
-                    "status": "pending",
-                    "error_message": null
-                },
-                {
-                    "id": "op-2",
-                    "task_type": "retain",
-                    "items_count": 3,
-                    "document_id": "doc-123",
-                    "created_at": "2024-01-15T11:00:00Z",
-                    "status": "completed",
-                    "error_message": null
-                }
-            ]
-        }"#;
-        let ops: OperationsResponse = serde_json::from_str(json).unwrap();
-        assert_eq!(ops.bank_id, "test-bank");
-        assert_eq!(ops.operations.len(), 2);
-        assert_eq!(ops.operations[0].status, "pending");
-        assert_eq!(ops.operations[1].status, "completed");
-    }
-}

hindsight-cli/src/commands/bank.rs

@@ -201,7 +201,7 @@ pub fn update_background(
         Ok(profile) => {
             if output_format == OutputFormat::Pretty {
                 ui::print_success("Background updated successfully");
-                println!("\n{}", profile.mission);
+                println!("\n{}", profile.background);
 
                 if !no_update_disposition {
                     if let (Some(old_p), Some(new_p)) =

hindsight-cli/src/commands/entity.rs

@@ -16,7 +16,7 @@ pub fn list(
         None
     };
 
-    let response = client.list_entities(bank_id, Some(limit), None, verbose)?;
+    let response = client.list_entities(bank_id, Some(limit), verbose)?;
 
     if let Some(mut sp) = spinner {
         sp.finish();

hindsight-cli/src/commands/explore.rs

@@ -5,7 +5,7 @@ use crossterm::{
     execute,
     terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen},
 };
-use hindsight_client::types::{BankListItem, RecallResult, EntityListItem, Budget, TagsMatch};
+use hindsight_client::types::{BankListItem, RecallResult, EntityListItem, Budget};
 use serde_json::{Map, Value};
 use ratatui::{
     backend::{Backend, CrosstermBackend},
@@ -283,7 +283,7 @@ impl App {
     }
 
     fn load_entities(&mut self, bank_id: &str) -> Result<()> {
-        let response = self.client.list_entities(bank_id, Some(100), None, false)?;
+        let response = self.client.list_entities(bank_id, Some(100), false)?;
         self.entities = response.items;
 
         if !self.entities.is_empty() && self.entities_state.selected().is_none() {
@@ -341,8 +341,6 @@ impl App {
                             trace: false,
                             query_timestamp: None,
                             include: None,
-                            tags: None,
-                            tags_match: TagsMatch::Any,
                         };
 
                         let result = client.recall(&bank_id, &request, false)
@@ -359,8 +357,6 @@ impl App {
                             max_tokens: 4096,
                             include: None,
                             response_schema: None,
-                            tags: None,
-                            tags_match: TagsMatch::Any,
                         };
 
                         let result = client.reflect(&bank_id, &request, false)

hindsight-cli/src/commands/memory.rs

@@ -9,7 +9,7 @@ use crate::output::{self, OutputFormat};
 use crate::ui;
 
 // Import types from generated client
-use hindsight_client::types::{Budget, ChunkIncludeOptions, IncludeOptions, TagsMatch};
+use hindsight_client::types::{Budget, ChunkIncludeOptions, IncludeOptions};
 use serde_json;
 
 // Helper function to parse budget string to Budget enum
@@ -21,17 +21,6 @@ fn parse_budget(budget: &str) -> Budget {
     }
 }
 
-// Helper function to check if a file has a text-based extension
-fn is_text_file(path: &std::path::Path) -> bool {
-    const TEXT_EXTENSIONS: &[&str] = &[
-        "txt", "md", "json", "yaml", "yml", "toml", "xml", "csv", "log", "rst", "adoc",
-    ];
-    path.extension()
-        .and_then(|ext| ext.to_str())
-        .map(|ext| TEXT_EXTENSIONS.contains(&ext.to_lowercase().as_str()))
-        .unwrap_or(false)
-}
-
 pub fn recall(
     client: &ApiClient,
     agent_id: &str,
@@ -71,8 +60,6 @@ pub fn recall(
         trace,
         query_timestamp: None,
         include,
-        tags: None,
-        tags_match: TagsMatch::Any,
     };
 
     let response = client.recall(agent_id, &request, verbose);
@@ -129,8 +116,6 @@ pub fn reflect(
         max_tokens: max_tokens.unwrap_or(4096),
         include: None,
         response_schema,
-        tags: None,
-        tags_match: TagsMatch::Any,
     };
 
     let response = client.reflect(agent_id, &request, verbose);
@@ -177,13 +162,11 @@ pub fn retain(
         timestamp: None,
         document_id: Some(doc_id.clone()),
         entities: None,
-        tags: None,
     };
 
     let request = RetainRequest {
         items: vec![item],
         async_: r#async,
-        document_tags: None,
     };
 
     let response = client.retain(agent_id, &request, r#async, verbose);
@@ -240,23 +223,29 @@ pub fn retain_files(
                 .filter(|e| e.file_type().is_file())
             {
                 let path = entry.path();
-                if is_text_file(&path) {
-                    files.push(path.to_path_buf());
+                if let Some(ext) = path.extension() {
+                    if ext == "txt" || ext == "md" {
+                        files.push(path.to_path_buf());
+                    }
                 }
             }
         } else {
             for entry in fs::read_dir(&path)? {
                 let entry = entry?;
                 let path = entry.path();
-                if path.is_file() && is_text_file(&path) {
-                    files.push(path);
+                if path.is_file() {
+                    if let Some(ext) = path.extension() {
+                        if ext == "txt" || ext == "md" {
+                            files.push(path);
+                        }
+                    }
                 }
             }
         }
     }
 
     if files.is_empty() {
-        ui::print_warning("No text files found (supported: txt, md, json, yaml, yml, toml, xml, csv, log, rst, adoc)");
+        ui::print_warning("No .txt or .md files found");
         return Ok(());
     }
 
@@ -283,7 +272,6 @@ pub fn retain_files(
             timestamp: None,
             document_id: Some(doc_id),
             entities: None,
-            tags: None,
         });
 
         pb.inc(1);
@@ -291,20 +279,18 @@ pub fn retain_files(
 
     pb.finish_with_message("Files processed");
 
-    // Always use async mode for the API call
-    let request = RetainRequest {
-        items,
-        async_: true,
-        document_tags: None,
-    };
-
     let spinner = if output_format == OutputFormat::Pretty {
-        Some(ui::create_spinner("Submitting retain request..."))
+        Some(ui::create_spinner("Retaining memories..."))
     } else {
         None
     };
 
-    let response = client.retain(agent_id, &request, true, verbose);
+    let request = RetainRequest {
+        items,
+        async_: r#async,
+    };
+
+    let response = client.retain(agent_id, &request, r#async, verbose);
 
     if let Some(mut sp) = spinner {
         sp.finish();
@@ -312,55 +298,16 @@ pub fn retain_files(
 
     match response {
         Ok(result) => {
-            if r#async {
-                // User requested async mode - return immediately
-                if output_format == OutputFormat::Pretty {
-                    ui::print_success("Files queued for processing");
+            if output_format == OutputFormat::Pretty {
+                ui::print_success("Files retained successfully");
+                if result.is_async {
+                    println!("  Status: queued for background processing");
                     println!("  Items: {}", result.items_count);
-                    if let Some(op_id) = &result.operation_id {
-                        println!("  Operation ID: {}", op_id);
-                    }
                 } else {
-                    output::print_output(&result, output_format)?;
+                    println!("  Total units created: {}", result.items_count);
                 }
             } else {
-                // Poll until completion
-                if let Some(operation_id) = &result.operation_id {
-                    let poll_spinner = if output_format == OutputFormat::Pretty {
-                        Some(ui::create_spinner("Processing memories..."))
-                    } else {
-                        None
-                    };
-
-                    let (success, error_msg) = client.poll_operation(agent_id, operation_id, verbose)?;
-
-                    if let Some(mut sp) = poll_spinner {
-                        sp.finish();
-                    }
-
-                    if success {
-                        if output_format == OutputFormat::Pretty {
-                            ui::print_success("Files retained successfully");
-                            println!("  Items processed: {}", result.items_count);
-                        } else {
-                            output::print_output(&result, output_format)?;
-                        }
-                    } else {
-                        let msg = error_msg.unwrap_or_else(|| "Unknown error".to_string());
-                        if output_format == OutputFormat::Pretty {
-                            ui::print_error(&format!("Retain operation failed: {}", msg));
-                        }
-                        anyhow::bail!("Retain operation failed: {}", msg);
-                    }
-                } else {
-                    // No operation ID returned, shouldn't happen with async=true
-                    if output_format == OutputFormat::Pretty {
-                        ui::print_success("Files retained successfully");
-                        println!("  Items processed: {}", result.items_count);
-                    } else {
-                        output::print_output(&result, output_format)?;
-                    }
-                }
+                output::print_output(&result, output_format)?;
             }
             Ok(())
         }
@@ -473,84 +420,3 @@ pub fn clear(
         Err(e) => Err(e)
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::path::Path;
-
-    #[test]
-    fn test_is_text_file_supported_extensions() {
-        let supported = [
-            "file.txt", "file.md", "file.json", "file.yaml", "file.yml",
-            "file.toml", "file.xml", "file.csv", "file.log", "file.rst", "file.adoc",
-        ];
-        for filename in supported {
-            assert!(
-                is_text_file(Path::new(filename)),
-                "{} should be recognized as a text file",
-                filename
-            );
-        }
-    }
-
-    #[test]
-    fn test_is_text_file_case_insensitive() {
-        assert!(is_text_file(Path::new("file.JSON")));
-        assert!(is_text_file(Path::new("file.TXT")));
-        assert!(is_text_file(Path::new("file.Md")));
-        assert!(is_text_file(Path::new("file.YAML")));
-    }
-
-    #[test]
-    fn test_is_text_file_unsupported_extensions() {
-        let unsupported = [
-            "file.pdf", "file.doc", "file.docx", "file.png", "file.jpg",
-            "file.exe", "file.bin", "file.zip", "file.tar", "file.gz",
-        ];
-        for filename in unsupported {
-            assert!(
-                !is_text_file(Path::new(filename)),
-                "{} should NOT be recognized as a text file",
-                filename
-            );
-        }
-    }
-
-    #[test]
-    fn test_is_text_file_no_extension() {
-        assert!(!is_text_file(Path::new("README")));
-        assert!(!is_text_file(Path::new("Makefile")));
-        assert!(!is_text_file(Path::new(".gitignore")));
-    }
-
-    #[test]
-    fn test_is_text_file_with_path() {
-        assert!(is_text_file(Path::new("/some/path/to/file.json")));
-        assert!(is_text_file(Path::new("../relative/path/file.md")));
-        assert!(!is_text_file(Path::new("/path/to/image.png")));
-    }
-
-    #[test]
-    fn test_parse_budget_valid_values() {
-        assert!(matches!(parse_budget("low"), Budget::Low));
-        assert!(matches!(parse_budget("mid"), Budget::Mid));
-        assert!(matches!(parse_budget("high"), Budget::High));
-    }
-
-    #[test]
-    fn test_parse_budget_case_insensitive() {
-        assert!(matches!(parse_budget("LOW"), Budget::Low));
-        assert!(matches!(parse_budget("MID"), Budget::Mid));
-        assert!(matches!(parse_budget("HIGH"), Budget::High));
-        assert!(matches!(parse_budget("Low"), Budget::Low));
-        assert!(matches!(parse_budget("High"), Budget::High));
-    }
-
-    #[test]
-    fn test_parse_budget_defaults_to_mid() {
-        assert!(matches!(parse_budget("invalid"), Budget::Mid));
-        assert!(matches!(parse_budget(""), Budget::Mid));
-        assert!(matches!(parse_budget("unknown"), Budget::Mid));
-    }
-}

hindsight-cli/src/config.rs

@@ -8,7 +8,6 @@ const DEFAULT_API_URL: &str = "http://localhost:8888";
 const CONFIG_FILE_NAME: &str = "config";
 const CONFIG_DIR_NAME: &str = ".hindsight";
 
-#[derive(Debug)]
 pub struct Config {
     pub api_url: String,
     pub api_key: Option<String>,
@@ -175,156 +174,3 @@ pub fn generate_doc_id() -> String {
     let now = chrono::Local::now();
     format!("cli_put_{}", now.format("%Y%m%d_%H%M%S"))
 }
-
-/// Parse a simple TOML-like config line and extract value.
-/// Handles both quoted and unquoted values.
-pub fn parse_config_value(line: &str, key: &str) -> Option<String> {
-    let line = line.trim();
-    if !line.starts_with(key) {
-        return None;
-    }
-    line.split('=').nth(1).map(|value| {
-        value.trim().trim_matches('"').trim_matches('\'').to_string()
-    }).filter(|v| !v.is_empty())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_config_source_display() {
-        assert_eq!(format!("{}", ConfigSource::LocalFile), "config file");
-        assert_eq!(format!("{}", ConfigSource::Environment), "environment variable");
-        assert_eq!(format!("{}", ConfigSource::Default), "default");
-    }
-
-    #[test]
-    fn test_validate_and_create_valid_http() {
-        let config = Config::validate_and_create(
-            "http://localhost:8888".to_string(),
-            None,
-            ConfigSource::Default,
-        );
-        assert!(config.is_ok());
-        let config = config.unwrap();
-        assert_eq!(config.api_url, "http://localhost:8888");
-        assert_eq!(config.source, ConfigSource::Default);
-    }
-
-    #[test]
-    fn test_validate_and_create_valid_https() {
-        let config = Config::validate_and_create(
-            "https://api.example.com".to_string(),
-            Some("secret-key".to_string()),
-            ConfigSource::Environment,
-        );
-        assert!(config.is_ok());
-        let config = config.unwrap();
-        assert_eq!(config.api_url, "https://api.example.com");
-        assert_eq!(config.api_key, Some("secret-key".to_string()));
-        assert_eq!(config.source, ConfigSource::Environment);
-    }
-
-    #[test]
-    fn test_validate_and_create_invalid_url() {
-        let config = Config::validate_and_create(
-            "localhost:8888".to_string(),
-            None,
-            ConfigSource::Default,
-        );
-        assert!(config.is_err());
-        let err = config.unwrap_err().to_string();
-        assert!(err.contains("Invalid API URL"));
-        assert!(err.contains("Must start with http:// or https://"));
-    }
-
-    #[test]
-    fn test_validate_and_create_ftp_url() {
-        let config = Config::validate_and_create(
-            "ftp://example.com".to_string(),
-            None,
-            ConfigSource::Default,
-        );
-        assert!(config.is_err());
-    }
-
-    #[test]
-    fn test_generate_doc_id_format() {
-        let doc_id = generate_doc_id();
-        assert!(doc_id.starts_with("cli_put_"));
-        // Should be cli_put_YYYYMMDD_HHMMSS format
-        assert!(doc_id.len() > 20); // cli_put_ (8) + date (8) + _ (1) + time (6) = 23
-    }
-
-    #[test]
-    fn test_generate_doc_id_uniqueness() {
-        let id1 = generate_doc_id();
-        std::thread::sleep(std::time::Duration::from_secs(1));
-        let id2 = generate_doc_id();
-        // IDs generated at different times should be different
-        assert_ne!(id1, id2);
-    }
-
-    #[test]
-    fn test_parse_config_value_quoted() {
-        assert_eq!(
-            parse_config_value(r#"api_url = "http://localhost:8888""#, "api_url"),
-            Some("http://localhost:8888".to_string())
-        );
-    }
-
-    #[test]
-    fn test_parse_config_value_single_quoted() {
-        assert_eq!(
-            parse_config_value("api_url = 'http://localhost:8888'", "api_url"),
-            Some("http://localhost:8888".to_string())
-        );
-    }
-
-    #[test]
-    fn test_parse_config_value_unquoted() {
-        assert_eq!(
-            parse_config_value("api_url = http://localhost:8888", "api_url"),
-            Some("http://localhost:8888".to_string())
-        );
-    }
-
-    #[test]
-    fn test_parse_config_value_with_spaces() {
-        assert_eq!(
-            parse_config_value("  api_url  =  \"http://localhost:8888\"  ", "api_url"),
-            Some("http://localhost:8888".to_string())
-        );
-    }
-
-    #[test]
-    fn test_parse_config_value_wrong_key() {
-        assert_eq!(
-            parse_config_value("api_key = secret", "api_url"),
-            None
-        );
-    }
-
-    #[test]
-    fn test_parse_config_value_empty() {
-        assert_eq!(
-            parse_config_value("api_url = ", "api_url"),
-            None
-        );
-        assert_eq!(
-            parse_config_value("api_url = \"\"", "api_url"),
-            None
-        );
-    }
-
-    #[test]
-    fn test_config_api_url_accessor() {
-        let config = Config {
-            api_url: "http://test:8080".to_string(),
-            api_key: None,
-            source: ConfigSource::Default,
-        };
-        assert_eq!(config.api_url(), "http://test:8080");
-    }
-}

hindsight-cli/src/output.rs

@@ -8,35 +8,13 @@ pub enum OutputFormat {
     Yaml,
 }
 
-impl OutputFormat {
-    /// Parse output format from string
-    pub fn from_str(s: &str) -> Option<Self> {
-        match s.to_lowercase().as_str() {
-            "json" => Some(OutputFormat::Json),
-            "yaml" | "yml" => Some(OutputFormat::Yaml),
-            "pretty" | "text" => Some(OutputFormat::Pretty),
-            _ => None,
-        }
-    }
-}
-
-/// Format data as JSON string
-pub fn to_json<T: Serialize>(data: &T) -> Result<String> {
-    Ok(serde_json::to_string_pretty(data)?)
-}
-
-/// Format data as YAML string
-pub fn to_yaml<T: Serialize>(data: &T) -> Result<String> {
-    Ok(serde_yaml::to_string(data)?)
-}
-
 pub fn print_output<T: Serialize>(data: &T, format: OutputFormat) -> Result<()> {
     match format {
         OutputFormat::Json => {
-            println!("{}", to_json(data)?);
+            println!("{}", serde_json::to_string_pretty(data)?);
         }
         OutputFormat::Yaml => {
-            println!("{}", to_yaml(data)?);
+            println!("{}", serde_yaml::to_string(data)?);
         }
         OutputFormat::Pretty => {
             // This should not be called - pretty printing is handled in ui.rs
@@ -45,121 +23,3 @@ pub fn print_output<T: Serialize>(data: &T, format: OutputFormat) -> Result<()>
     }
     Ok(())
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use serde::{Deserialize, Serialize};
-
-    #[derive(Debug, Serialize, Deserialize, PartialEq)]
-    struct TestData {
-        name: String,
-        count: i32,
-        active: bool,
-    }
-
-    #[test]
-    fn test_output_format_from_str_json() {
-        assert_eq!(OutputFormat::from_str("json"), Some(OutputFormat::Json));
-        assert_eq!(OutputFormat::from_str("JSON"), Some(OutputFormat::Json));
-        assert_eq!(OutputFormat::from_str("Json"), Some(OutputFormat::Json));
-    }
-
-    #[test]
-    fn test_output_format_from_str_yaml() {
-        assert_eq!(OutputFormat::from_str("yaml"), Some(OutputFormat::Yaml));
-        assert_eq!(OutputFormat::from_str("YAML"), Some(OutputFormat::Yaml));
-        assert_eq!(OutputFormat::from_str("yml"), Some(OutputFormat::Yaml));
-        assert_eq!(OutputFormat::from_str("YML"), Some(OutputFormat::Yaml));
-    }
-
-    #[test]
-    fn test_output_format_from_str_pretty() {
-        assert_eq!(OutputFormat::from_str("pretty"), Some(OutputFormat::Pretty));
-        assert_eq!(OutputFormat::from_str("PRETTY"), Some(OutputFormat::Pretty));
-        assert_eq!(OutputFormat::from_str("text"), Some(OutputFormat::Pretty));
-    }
-
-    #[test]
-    fn test_output_format_from_str_invalid() {
-        assert_eq!(OutputFormat::from_str("xml"), None);
-        assert_eq!(OutputFormat::from_str("csv"), None);
-        assert_eq!(OutputFormat::from_str(""), None);
-    }
-
-    #[test]
-    fn test_to_json() {
-        let data = TestData {
-            name: "test".to_string(),
-            count: 42,
-            active: true,
-        };
-        let json = to_json(&data).unwrap();
-        assert!(json.contains("\"name\": \"test\""));
-        assert!(json.contains("\"count\": 42"));
-        assert!(json.contains("\"active\": true"));
-    }
-
-    #[test]
-    fn test_to_yaml() {
-        let data = TestData {
-            name: "test".to_string(),
-            count: 42,
-            active: true,
-        };
-        let yaml = to_yaml(&data).unwrap();
-        assert!(yaml.contains("name: test"));
-        assert!(yaml.contains("count: 42"));
-        assert!(yaml.contains("active: true"));
-    }
-
-    #[test]
-    fn test_to_json_array() {
-        let data = vec![
-            TestData { name: "a".to_string(), count: 1, active: true },
-            TestData { name: "b".to_string(), count: 2, active: false },
-        ];
-        let json = to_json(&data).unwrap();
-        assert!(json.contains("\"name\": \"a\""));
-        assert!(json.contains("\"name\": \"b\""));
-    }
-
-    #[test]
-    fn test_to_yaml_array() {
-        let data = vec![
-            TestData { name: "a".to_string(), count: 1, active: true },
-            TestData { name: "b".to_string(), count: 2, active: false },
-        ];
-        let yaml = to_yaml(&data).unwrap();
-        assert!(yaml.contains("name: a"));
-        assert!(yaml.contains("name: b"));
-    }
-
-    #[test]
-    fn test_output_format_equality() {
-        assert_eq!(OutputFormat::Json, OutputFormat::Json);
-        assert_ne!(OutputFormat::Json, OutputFormat::Yaml);
-        assert_ne!(OutputFormat::Yaml, OutputFormat::Pretty);
-    }
-
-    #[test]
-    fn test_output_format_clone() {
-        let format = OutputFormat::Json;
-        let cloned = format.clone();
-        assert_eq!(format, cloned);
-    }
-
-    #[test]
-    fn test_to_json_special_chars() {
-        let data = TestData {
-            name: "test\"with\\special\nchars".to_string(),
-            count: 0,
-            active: false,
-        };
-        let json = to_json(&data).unwrap();
-        // JSON should properly escape special characters
-        assert!(json.contains("\\\""));
-        assert!(json.contains("\\\\"));
-        assert!(json.contains("\\n"));
-    }
-}

hindsight-cli/src/ui.rs

@@ -172,11 +172,8 @@ pub fn print_think_response(response: &ReflectResponse) {
     println!("{}", response.text);
     println!();
 
-    if let Some(based_on) = &response.based_on {
-        let count = based_on.memories.len() + based_on.mental_models.len();
-        if count > 0 {
-            println!("{}", dim(&format!("Based on {} memory units", count)));
-        }
+    if !response.based_on.is_empty() {
+        println!("{}", dim(&format!("Based on {} memory units", response.based_on.len())));
     }
 
     // Display structured output if present
@@ -325,10 +322,10 @@ pub fn print_disposition(profile: &BankProfileResponse) {
     println!("{} {}", dim("Name:"), gradient_start(&profile.name));
     println!();
 
-    // Print mission if available
-    if !profile.mission.is_empty() {
-        println!("{}", gradient_mid("Mission:"));
-        for line in profile.mission.lines() {
+    // Print background if available
+    if !profile.background.is_empty() {
+        println!("{}", gradient_mid("Background:"));
+        for line in profile.background.lines() {
             println!("{}", line);
         }
         println!();

hindsight-clients/python/.openapi-generator/FILES

@@ -4,7 +4,6 @@ hindsight_client_api/api/banks_api.py
 hindsight_client_api/api/documents_api.py
 hindsight_client_api/api/entities_api.py
 hindsight_client_api/api/memory_api.py
-hindsight_client_api/api/mental_models_api.py
 hindsight_client_api/api/monitoring_api.py
 hindsight_client_api/api/operations_api.py
 hindsight_client_api/api_client.py
@@ -13,7 +12,6 @@ hindsight_client_api/configuration.py
 hindsight_client_api/exceptions.py
 hindsight_client_api/models/__init__.py
 hindsight_client_api/models/add_background_request.py
-hindsight_client_api/models/async_operation_submit_response.py
 hindsight_client_api/models/background_response.py
 hindsight_client_api/models/bank_list_item.py
 hindsight_client_api/models/bank_list_response.py
@@ -25,8 +23,6 @@ hindsight_client_api/models/chunk_data.py
 hindsight_client_api/models/chunk_include_options.py
 hindsight_client_api/models/chunk_response.py
 hindsight_client_api/models/create_bank_request.py
-hindsight_client_api/models/create_mental_model_request.py
-hindsight_client_api/models/created_mental_model.py
 hindsight_client_api/models/delete_document_response.py
 hindsight_client_api/models/delete_response.py
 hindsight_client_api/models/disposition_traits.py
@@ -43,37 +39,20 @@ hindsight_client_api/models/http_validation_error.py
 hindsight_client_api/models/include_options.py
 hindsight_client_api/models/list_documents_response.py
 hindsight_client_api/models/list_memory_units_response.py
-hindsight_client_api/models/list_tags_response.py
 hindsight_client_api/models/memory_item.py
-hindsight_client_api/models/mental_model_freshness_response.py
-hindsight_client_api/models/mental_model_list_response.py
-hindsight_client_api/models/mental_model_observation_response.py
-hindsight_client_api/models/mental_model_response.py
-hindsight_client_api/models/observation_evidence_response.py
-hindsight_client_api/models/observation_input.py
 hindsight_client_api/models/operation_response.py
-hindsight_client_api/models/operation_status_response.py
 hindsight_client_api/models/operations_list_response.py
 hindsight_client_api/models/recall_request.py
 hindsight_client_api/models/recall_response.py
 hindsight_client_api/models/recall_result.py
-hindsight_client_api/models/reflect_based_on.py
 hindsight_client_api/models/reflect_fact.py
 hindsight_client_api/models/reflect_include_options.py
-hindsight_client_api/models/reflect_llm_call.py
-hindsight_client_api/models/reflect_mental_model.py
 hindsight_client_api/models/reflect_request.py
 hindsight_client_api/models/reflect_response.py
-hindsight_client_api/models/reflect_tool_call.py
-hindsight_client_api/models/reflect_trace.py
-hindsight_client_api/models/refresh_mental_models_request.py
 hindsight_client_api/models/retain_request.py
 hindsight_client_api/models/retain_response.py
-hindsight_client_api/models/tag_item.py
 hindsight_client_api/models/token_usage.py
-hindsight_client_api/models/tool_calls_include_options.py
 hindsight_client_api/models/update_disposition_request.py
-hindsight_client_api/models/update_mental_model_request.py
 hindsight_client_api/models/validation_error.py
 hindsight_client_api/models/validation_error_loc_inner.py
 hindsight_client_api/rest.py