Compare commits

..
Author SHA1 Message Date
Nicolò Boschi d25222ba9c fix: embed get-skill installer 2025-12-23 10:33:23 +01:00
1341 changed files with 47220 additions and 287183 deletions
+1 -53
View File
@@ -2,55 +2,18 @@
# Copy this file to .env and fill in your values
# LLM Configuration (Required)
# Supported providers: openai, groq, ollama, gemini, anthropic, lmstudio, vertexai, minimax
HINDSIGHT_API_LLM_PROVIDER=openai
HINDSIGHT_API_LLM_API_KEY=your-api-key-here
HINDSIGHT_API_LLM_MODEL=gpt-4o-mini
HINDSIGHT_API_LLM_MODEL=o3-mini
HINDSIGHT_API_LLM_BASE_URL=https://api.openai.com/v1
# Example: Anthropic Claude configuration
# HINDSIGHT_API_LLM_PROVIDER=anthropic
# HINDSIGHT_API_LLM_API_KEY=your-anthropic-api-key
# HINDSIGHT_API_LLM_MODEL=claude-sonnet-4-20250514
# Example: Google Vertex AI configuration
# HINDSIGHT_API_LLM_PROVIDER=vertexai
# HINDSIGHT_API_LLM_MODEL=google/gemini-2.0-flash-001
# HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=your-gcp-project-id
# HINDSIGHT_API_LLM_VERTEXAI_REGION=us-central1
# HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY=/path/to/service-account-key.json # Optional, uses ADC if not set
# Example: MiniMax configuration (1M context window)
# HINDSIGHT_API_LLM_PROVIDER=minimax
# HINDSIGHT_API_LLM_API_KEY=your-minimax-api-key
# HINDSIGHT_API_LLM_MODEL=MiniMax-M2.7
# Example: LM Studio local configuration (Qwen 2.5 32B recommended)
# HINDSIGHT_API_LLM_PROVIDER=lmstudio
# HINDSIGHT_API_LLM_API_KEY=lmstudio
# HINDSIGHT_API_LLM_BASE_URL=http://localhost:1234/v1
# HINDSIGHT_API_LLM_MODEL=qwen2.5-32b-instruct
# API Configuration (Optional)
HINDSIGHT_API_HOST=0.0.0.0
HINDSIGHT_API_PORT=8888
HINDSIGHT_API_LOG_LEVEL=info
# Base Path / Reverse Proxy Support (Optional)
# Set these when deploying behind a reverse proxy with path-based routing
# Example: To deploy at example.com/hindsight/, set both to "/hindsight"
# HINDSIGHT_API_BASE_PATH=/hindsight
# NEXT_PUBLIC_BASE_PATH=/hindsight
# Database (Optional - uses embedded pg0 by default)
# HINDSIGHT_API_DATABASE_URL=postgresql://user:pass@host:5432/db
# HINDSIGHT_API_DATABASE_SCHEMA=public # PostgreSQL schema name (default: public)
# Vector Extension (Optional - uses pgvector by default)
# Options: "pgvector" (default), "vchord", "pgvectorscale" (DiskANN)
# HINDSIGHT_API_VECTOR_EXTENSION=pgvector
# For Azure PostgreSQL with DiskANN:
# HINDSIGHT_API_VECTOR_EXTENSION=pgvectorscale # Auto-detects pg_diskann on Azure
# Embeddings Configuration (Optional - uses local by default)
# Provider: "local" (default) or "tei" (HuggingFace Text Embeddings Inference)
@@ -67,18 +30,3 @@ HINDSIGHT_API_LOG_LEVEL=info
# HINDSIGHT_API_RERANKER_LOCAL_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2
# For TEI provider:
# HINDSIGHT_API_RERANKER_TEI_URL=http://localhost:8081
# Observability & Tracing (Optional - disabled by default)
# Enable OpenTelemetry tracing for LLM calls (GenAI semantic conventions)
# HINDSIGHT_API_OTEL_TRACES_ENABLED=true
#
# Local development with Grafana LGTM stack (recommended - see scripts/dev/grafana/README.md)
# HINDSIGHT_API_OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
#
# Cloud backends (Grafana Cloud, Langfuse, DataDog, etc.)
# HINDSIGHT_API_OTEL_EXPORTER_OTLP_ENDPOINT=https://your-backend-url
# HINDSIGHT_API_OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer your-token"
#
# Custom service name and environment (optional, defaults: hindsight-api, development)
# HINDSIGHT_API_OTEL_SERVICE_NAME=hindsight-production
# HINDSIGHT_API_OTEL_DEPLOYMENT_ENVIRONMENT=production
-71
View File
@@ -1,71 +0,0 @@
name: Bug Report
description: Report a bug or unexpected behavior
labels: ["bug", "triage"]
body:
- type: markdown
attributes:
value: |
Thanks for taking the time to report a bug! Please fill out the sections below.
- type: textarea
id: description
attributes:
label: Bug Description
description: A clear and concise description of the bug
placeholder: What happened?
validations:
required: true
- type: textarea
id: reproduction
attributes:
label: Steps to Reproduce
description: Steps to reproduce the behavior
placeholder: |
1. Configure '...'
2. Call '...'
3. See error
validations:
required: true
- type: textarea
id: expected
attributes:
label: Expected Behavior
description: What did you expect to happen?
validations:
required: true
- type: textarea
id: actual
attributes:
label: Actual Behavior
description: What actually happened?
validations:
required: true
- type: input
id: version
attributes:
label: Version
description: What version are you using?
placeholder: e.g., 0.1.0 or commit hash
validations:
required: false
- type: dropdown
id: llm-provider
attributes:
label: LLM Provider
description: Which LLM provider are you using?
options:
- OpenAI
- Anthropic
- Gemini
- Groq
- Ollama
- LM Studio
- Other
validations:
required: false
-8
View File
@@ -1,8 +0,0 @@
blank_issues_enabled: false
contact_links:
- name: Questions & Help
url: https://github.com/vectorize-io/hindsight/discussions/categories/q-a
about: Please ask questions and get help in Discussions instead of opening an issue.
- name: Ideas & Feedback
url: https://github.com/vectorize-io/hindsight/discussions/categories/ideas
about: Share ideas or give feedback in Discussions.
@@ -1,82 +0,0 @@
name: Feature Request
description: Suggest a new feature or enhancement
labels: ["enhancement", "triage"]
body:
- type: markdown
attributes:
value: |
Thanks for suggesting a feature! Please describe what you'd like to see added.
- type: textarea
id: use-case
attributes:
label: Use Case
description: Describe your specific use case. What are you building? What's your goal?
placeholder: |
I'm building an AI agent that needs to...
My application handles...
validations:
required: true
- type: textarea
id: problem
attributes:
label: Problem Statement
description: What problem are you facing? What's missing or difficult today?
placeholder: Currently I have to... which causes...
validations:
required: true
- type: textarea
id: benefit
attributes:
label: How This Feature Would Help
description: Explain how this feature would improve your workflow or solve your problem
placeholder: With this feature, I would be able to...
validations:
required: true
- type: textarea
id: solution
attributes:
label: Proposed Solution
description: Describe your ideal solution (optional - we may have ideas too!)
placeholder: It would be great if Hindsight could...
validations:
required: false
- type: textarea
id: alternatives
attributes:
label: Alternatives Considered
description: Have you considered any alternative solutions or workarounds?
validations:
required: false
- type: dropdown
id: priority
attributes:
label: Priority
description: How important is this feature to you?
options:
- Nice to have
- Important - affects my workflow
- Critical - blocking my use case
validations:
required: true
- type: textarea
id: additional
attributes:
label: Additional Context
description: Any other context, mockups, or examples?
validations:
required: false
- type: checkboxes
id: checklist
attributes:
label: Checklist
options:
- label: I would be willing to contribute this feature
required: false
-6
View File
@@ -1,6 +0,0 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
+4 -7
View File
@@ -21,20 +21,17 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/setup-node@v6
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: npm
cache-dependency-path: package-lock.json
- uses: astral-sh/setup-uv@v7
- uses: astral-sh/setup-uv@v4
- run: npm ci --workspace=hindsight-docs
- run: uv run generate-llms-full
- run: npm run build --workspace=hindsight-docs
env:
UMAMI_URL: https://analytics.hindsight.vectorize.io
UMAMI_WEBSITE_ID: ${{ secrets.UMAMI_WEBSITE_ID }}
- uses: actions/upload-pages-artifact@v4
- uses: actions/upload-pages-artifact@v3
with:
path: hindsight-docs/build
deploy:
-99
View File
@@ -1,99 +0,0 @@
name: Release Integration
on:
push:
tags:
- 'integrations/**'
jobs:
publish:
runs-on: ubuntu-latest
permissions:
id-token: write # for PyPI trusted publishing
steps:
- uses: actions/checkout@v6
- name: Extract integration info
id: info
run: |
# refs/tags/integrations/litellm/v0.1.0 → integration=litellm, version=0.1.0
TAG="${GITHUB_REF#refs/tags/}"
INTEGRATION=$(echo "$TAG" | cut -d'/' -f2)
VERSION=$(echo "$TAG" | cut -d'/' -f3 | sed 's/^v//')
echo "integration=$INTEGRATION" >> $GITHUB_OUTPUT
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "tag=$TAG" >> $GITHUB_OUTPUT
echo "Integration: $INTEGRATION, Version: $VERSION"
- name: Detect integration type
id: type
run: |
if [ -f "hindsight-integrations/${{ steps.info.outputs.integration }}/pyproject.toml" ]; then
echo "type=python" >> $GITHUB_OUTPUT
else
echo "type=typescript" >> $GITHUB_OUTPUT
fi
# ── Python integrations (litellm, pydantic-ai, crewai) ──────────────────
- name: Install uv
if: steps.type.outputs.type == 'python'
uses: astral-sh/setup-uv@v7
with:
enable-cache: true
- name: Set up Python
if: steps.type.outputs.type == 'python'
uses: actions/setup-python@v5
with:
python-version-file: ".python-version"
- name: Build Python package
if: steps.type.outputs.type == 'python'
working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
run: uv build --out-dir dist
- name: Publish Python package to PyPI
if: steps.type.outputs.type == 'python'
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: ./hindsight-integrations/${{ steps.info.outputs.integration }}/dist
skip-existing: true
# ── TypeScript integrations (ai-sdk, chat, openclaw) ────────────────────
- name: Set up Node.js
if: steps.type.outputs.type == 'typescript'
uses: actions/setup-node@v6
with:
node-version: '22'
registry-url: 'https://registry.npmjs.org'
- name: Install dependencies
if: steps.type.outputs.type == 'typescript'
working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
run: npm ci
- name: Build TypeScript package
if: steps.type.outputs.type == 'typescript'
working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
run: npm run build
- name: Publish TypeScript package to npm
if: steps.type.outputs.type == 'typescript'
working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
run: |
set +e
OUTPUT=$(npm publish --access public 2>&1)
EXIT_CODE=$?
echo "$OUTPUT"
if [ $EXIT_CODE -ne 0 ]; then
if echo "$OUTPUT" | grep -q "cannot publish over"; then
echo "Package version already published, skipping..."
exit 0
fi
exit $EXIT_CODE
fi
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+42 -84
View File
@@ -13,15 +13,15 @@ jobs:
id-token: write
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v7
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
- name: Set up Python
uses: actions/setup-python@v6
uses: actions/setup-python@v5
with:
python-version-file: ".python-version"
@@ -30,39 +30,29 @@ jobs:
working-directory: ./hindsight-clients/python
run: uv build --out-dir dist
- name: Build hindsight-api-slim
working-directory: ./hindsight-api-slim
run: uv build --out-dir dist
- name: Build hindsight-api
working-directory: ./hindsight-api
run: uv build --out-dir dist
- name: Build hindsight-all
working-directory: ./hindsight-all
working-directory: ./hindsight
run: uv build --out-dir dist
- name: Build hindsight-all-slim
working-directory: ./hindsight-all-slim
- name: Build hindsight-litellm
working-directory: ./hindsight-integrations/litellm
run: uv build --out-dir dist
- name: Build hindsight-embed
working-directory: ./hindsight-embed
run: uv build --out-dir dist
# Publish in order (client and api-slim first, then api/all wrappers which depend on them)
# Publish in order (client and api first, then hindsight-all which depends on them)
- name: Publish hindsight-client to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: ./hindsight-clients/python/dist
skip-existing: true
- name: Publish hindsight-api-slim to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: ./hindsight-api-slim/dist
skip-existing: true
- name: Publish hindsight-api to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
@@ -72,13 +62,13 @@ jobs:
- name: Publish hindsight-all to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: ./hindsight-all/dist
packages-dir: ./hindsight/dist
skip-existing: true
- name: Publish hindsight-all-slim to PyPI
- name: Publish hindsight-litellm to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: ./hindsight-all-slim/dist
packages-dir: ./hindsight-integrations/litellm/dist
skip-existing: true
- name: Publish hindsight-embed to PyPI
@@ -89,15 +79,14 @@ jobs:
# Upload artifacts for GitHub release
- name: Upload artifacts
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
with:
name: python-packages
path: |
hindsight-clients/python/dist/*
hindsight-api-slim/dist/*
hindsight-api/dist/*
hindsight-all/dist/*
hindsight-all-slim/dist/*
hindsight/dist/*
hindsight-integrations/litellm/dist/*
hindsight-embed/dist/*
retention-days: 1
@@ -106,10 +95,10 @@ jobs:
environment: npm
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Set up Node.js
uses: actions/setup-node@v6
uses: actions/setup-node@v4
with:
node-version: '20'
registry-url: 'https://registry.npmjs.org'
@@ -144,7 +133,7 @@ jobs:
run: npm pack
- name: Upload artifacts
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
with:
name: typescript-client
path: hindsight-clients/typescript/*.tgz
@@ -155,10 +144,10 @@ jobs:
environment: npm
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Set up Node.js
uses: actions/setup-node@v6
uses: actions/setup-node@v4
with:
node-version: '20'
registry-url: 'https://registry.npmjs.org'
@@ -181,14 +170,11 @@ jobs:
- name: Build
run: npm run build --workspace=hindsight-control-plane
- name: Verify standalone build
run: test -f hindsight-control-plane/standalone/server.js || (echo 'standalone/server.js missing - build failed' && exit 1)
- name: Publish to npm
working-directory: ./hindsight-control-plane
run: |
set +e
OUTPUT=$(npm publish --access public --ignore-scripts 2>&1)
OUTPUT=$(npm publish --access public 2>&1)
EXIT_CODE=$?
echo "$OUTPUT"
if [ $EXIT_CODE -ne 0 ]; then
@@ -206,7 +192,7 @@ jobs:
run: npm pack
- name: Upload artifacts
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
with:
name: control-plane
path: hindsight-control-plane/*.tgz
@@ -229,13 +215,9 @@ jobs:
target: aarch64-apple-darwin
artifact_name: hindsight
asset_name: hindsight-darwin-arm64
- os: ubuntu-24.04-arm
target: aarch64-unknown-linux-gnu
artifact_name: hindsight
asset_name: hindsight-linux-arm64
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
@@ -253,14 +235,13 @@ jobs:
chmod +x artifacts/${{ matrix.asset_name }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
with:
name: rust-cli-${{ matrix.asset_name }}
path: artifacts/${{ matrix.asset_name }}
retention-days: 1
release-docker-images:
name: Release Docker (${{ matrix.image_name }}${{ matrix.tag_suffix }})
runs-on: ubuntu-latest
permissions:
contents: read
@@ -270,31 +251,13 @@ jobs:
include:
- target: api-only
image_name: hindsight-api
tag_suffix: ""
build_args: ""
- target: api-only
image_name: hindsight-api
tag_suffix: "-slim"
build_args: |
INCLUDE_LOCAL_MODELS=false
PRELOAD_ML_MODELS=false
- target: cp-only
image_name: hindsight-control-plane
tag_suffix: ""
build_args: ""
- target: standalone
image_name: hindsight
tag_suffix: ""
build_args: ""
- target: standalone
image_name: hindsight
tag_suffix: "-slim"
build_args: |
INCLUDE_LOCAL_MODELS=false
PRELOAD_ML_MODELS=false
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Free Disk Space
uses: jlumbroso/free-disk-space@main
@@ -308,13 +271,13 @@ jobs:
swap-storage: true
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Log in to GitHub Container Registry
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
@@ -326,12 +289,9 @@ jobs:
- name: Extract metadata for release tags
id: meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@v5
with:
images: ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}
flavor: |
latest=auto
suffix=${{ matrix.tag_suffix }}
tags: |
type=semver,pattern={{version}},value=${{ steps.get_version.outputs.VERSION }}
type=semver,pattern={{major}}.{{minor}},value=${{ steps.get_version.outputs.VERSION }}
@@ -342,7 +302,7 @@ jobs:
# # Step 1: Build for local testing (single platform, no push)
# # This creates an identical image to what will be released, just for one platform
# - name: Build image for testing
# uses: docker/build-push-action@v7
# uses: docker/build-push-action@v6
# with:
# context: .
# file: docker/standalone/Dockerfile
@@ -357,16 +317,15 @@ jobs:
# - name: Smoke test - verify container starts
# env:
# GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
# run: ./docker/test-image.sh "${{ matrix.image_name }}:test" "${{ matrix.target }}"
# run: ./scripts/docker-smoke-test.sh "${{ matrix.image_name }}:test" "${{ matrix.target }}"
# Build multi-platform and push to release tags
- name: Build and push release images
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
file: docker/standalone/Dockerfile
target: ${{ matrix.target }}
build-args: ${{ matrix.build_args }}
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }}
@@ -379,7 +338,7 @@ jobs:
packages: write
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Install Helm
uses: azure/setup-helm@v4
@@ -399,7 +358,7 @@ jobs:
run: helm push helm-packages/*.tgz oci://ghcr.io/${{ github.repository_owner }}/charts
- name: Upload artifacts
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
with:
name: helm-chart
path: helm-packages/*.tgz
@@ -412,50 +371,50 @@ jobs:
contents: write
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Extract version from tag
id: get_version
run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
- name: Download Python packages
uses: actions/download-artifact@v8
uses: actions/download-artifact@v4
with:
name: python-packages
path: ./artifacts/python-packages
- name: Download TypeScript client
uses: actions/download-artifact@v8
uses: actions/download-artifact@v4
with:
name: typescript-client
path: ./artifacts/typescript-client
- name: Download Control Plane
uses: actions/download-artifact@v8
uses: actions/download-artifact@v4
with:
name: control-plane
path: ./artifacts/control-plane
- name: Download Rust CLI (Linux)
uses: actions/download-artifact@v8
uses: actions/download-artifact@v4
with:
name: rust-cli-hindsight-linux-amd64
path: ./artifacts/rust-cli-linux
- name: Download Rust CLI (macOS Intel)
uses: actions/download-artifact@v8
uses: actions/download-artifact@v4
with:
name: rust-cli-hindsight-darwin-amd64
path: ./artifacts/rust-cli-darwin-amd64
- name: Download Rust CLI (macOS ARM)
uses: actions/download-artifact@v8
uses: actions/download-artifact@v4
with:
name: rust-cli-hindsight-darwin-arm64
path: ./artifacts/rust-cli-darwin-arm64
- name: Download Helm chart
uses: actions/download-artifact@v8
uses: actions/download-artifact@v4
with:
name: helm-chart
path: ./artifacts/helm-chart
@@ -465,10 +424,9 @@ jobs:
mkdir -p release-assets
# Python packages
cp artifacts/python-packages/hindsight-clients/python/dist/* release-assets/ || true
cp artifacts/python-packages/hindsight-api-slim/dist/* release-assets/ || true
cp artifacts/python-packages/hindsight-api/dist/* release-assets/ || true
cp artifacts/python-packages/hindsight-all/dist/* release-assets/ || true
cp artifacts/python-packages/hindsight-all-slim/dist/* release-assets/ || true
cp artifacts/python-packages/hindsight/dist/* release-assets/ || true
cp artifacts/python-packages/hindsight-integrations/litellm/dist/* release-assets/ || true
cp artifacts/python-packages/hindsight-embed/dist/* release-assets/ || true
# TypeScript client
cp artifacts/typescript-client/*.tgz release-assets/ || true
+177 -1301
View File
File diff suppressed because it is too large Load Diff
+3 -20
View File
@@ -5,18 +5,15 @@ build/
dist/
wheels/
*.egg-info
.mcp.json
.osgrep
# Virtual environments
.venv
# Node
node_modules/
# Environment variables and local config
# Environment variables
.env
docker-compose.yml
docker-compose.override.yml
# IDE
.idea/
@@ -27,10 +24,6 @@ docker-compose.override.yml
# NLTK data (will be downloaded automatically)
nltk_data/
# Monitoring stack (Prometheus/Grafana binaries and data)
.monitoring/
.pgbouncer/
# Large benchmark datasets (will be downloaded automatically)
**/longmemeval_s_cleaned.json
@@ -45,15 +38,5 @@ hindsight-docs/static/llms-full.txt
hindsight-dev/benchmarks/locomo/results/
hindsight-dev/benchmarks/longmemeval/results/
hindsight-dev/benchmarks/consolidation/results/
hindsight-dev/benchmarks/perf/results/
benchmarks/results/
hindsight-cli/target
hindsight-clients/rust/target
.claude
whats-next.md
TASK.md
# Changelog is now tracked in hindsight-docs/src/pages/changelog.md
# CHANGELOG.md
blog-post*
hindsight-clients/rust/target
+151 -1
View File
@@ -1,3 +1,153 @@
# AGENTS.md
See [CLAUDE.md](./CLAUDE.md) for project documentation and coding conventions.
This document captures architectural decisions and coding conventions for the Hindsight project.
## Documentation
- **Main documentation**: [hindsight-docs/docs/developer/](./hindsight-docs/docs/developer/)
- **Use case patterns**: [hindsight-docs/docs/cookbook/](./hindsight-docs/docs/cookbook/)
- **API reference**: Auto-generated from OpenAPI spec
## Project Structure
```
hindsight/ # Python package for embedded usage
hindsight-api/ # FastAPI server (core memory engine)
hindsight-cli/ # Rust CLI client
hindsight-embed/ # Embedded CLI (no server needed)
hindsight-control-plane/ # Next.js admin UI
hindsight-docs/ # Docusaurus documentation site
hindsight-dev/ # Development tools and benchmarks
hindsight-integrations/ # Framework integrations (LangChain, etc.)
hindsight-clients/ # Generated API clients (Python, TypeScript, Rust)
```
## Core Concepts
### Memory Banks
- Each bank is an isolated memory store (like a "brain" for one user/agent)
- Banks contain: memory units (facts), entities, documents, entity links
- Banks have a **disposition** (personality traits) and **background** (context)
- Bank isolation is strict - no cross-bank data leakage
### Memory Types
- **World facts**: General knowledge ("The sky is blue")
- **Experience facts**: Personal experiences ("I visited Paris in 2023")
- **Opinion facts**: Beliefs with confidence scores ("Paris is beautiful" - 0.9 confidence)
### Operations
- **Retain**: Store new memories (extracts facts, entities, relationships)
- **Recall**: Retrieve memories (semantic, BM25, graph, temporal search)
- **Reflect**: Deep analysis to form new insights/opinions
## API Design Decisions
### Single Bank Per Request
- All API endpoints (`recall`, `reflect`, `retain`) operate on a single bank
- Multi-bank queries are the **client/agent's responsibility** to orchestrate
- This keeps the API simple and the isolation model clear
### Disposition Traits (3-trait system)
- **Skepticism** (1-5): How skeptical vs trusting when forming opinions
- **Literalism** (1-5): How literally to interpret information
- **Empathy** (1-5): How much to consider emotional context
- These influence the `reflect` operation, not `recall`
- Background info also only affects `reflect` (opinion formation)
## Multi-Bank Architecture Patterns
See [hindsight-docs/docs/cookbook/](./hindsight-docs/docs/cookbook/) for detailed guides:
- **Per-User Memory**: One bank per user, simplest pattern
- **Support Agent + Shared Knowledge**: User bank + shared docs bank, client orchestrates
## Developer Guide
### Running the API Server
```bash
# From project root
./scripts/dev/start-api.sh
# With options
./scripts/dev/start-api.sh --reload --port 8888 --log-level debug
```
### Running Tests
```bash
# API tests
cd hindsight-api
uv run pytest tests/
# Specific test
uv run pytest tests/test_http_api_integration.py -v
```
### Generating OpenAPI Spec
After changing API endpoints, regenerate the OpenAPI spec and docs:
```bash
./scripts/generate-openapi.sh
```
This will:
1. Generate `openapi.json` at project root
2. Copy to `hindsight-docs/openapi.json`
3. Regenerate API reference documentation
### Generating API Clients
After updating the OpenAPI spec, regenerate all clients:
```bash
./scripts/generate-clients.sh
```
This generates:
- **Rust client**: `hindsight-clients/rust/` (via progenitor in build.rs)
- **Python client**: `hindsight-clients/python/` (via openapi-generator Docker)
- **TypeScript client**: `hindsight-clients/typescript/` (via @hey-api/openapi-ts)
Note: The maintained wrapper `hindsight_client.py` and `README.md` are preserved during regeneration.
### Running the Documentation Site
```bash
./scripts/dev/start-docs.sh
```
### Running the Control Plane
```bash
./scripts/dev/start-control-plane.sh
```
## Code Style
### Python (hindsight-api)
- Use `uv` for package management
- Async throughout (asyncpg, async FastAPI endpoints)
- Pydantic models for request/response validation
- No py files at project root - maintain clean directory structure
### TypeScript (control-plane, clients)
- Next.js with App Router for control plane
- Tailwind CSS with shadcn/ui components
### Rust (CLI)
- Async with tokio
- reqwest for HTTP client
- progenitor for API client generation
## Database
- PostgreSQL with pgvector extension
- Schema managed via Alembic migrations in `hindsight-api/alembic/`, db migrations happen during api startup, no manual commands
- Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
# Branding
## Colors
- Primary: gradient from #0074d9 to #009296
-326
View File
@@ -1,326 +0,0 @@
# CLAUDE.md
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Project Overview
Hindsight is an agent memory system that provides long-term memory for AI agents using biomimetic data structures. Memories are organized as:
- **World facts**: General knowledge ("The sky is blue")
- **Experience facts**: Personal experiences ("I visited Paris in 2023")
- **Mental models**: Consolidated knowledge synthesized from facts ("User prefers functional programming patterns")
## Development Commands
### API Server (Python/FastAPI)
```bash
# Start API server (loads .env automatically)
./scripts/dev/start-api.sh
# Run all tests (parallelized with pytest-xdist)
cd hindsight-api-slim && uv run pytest tests/
# Run specific test file
cd hindsight-api-slim && uv run pytest tests/test_http_api_integration.py -v
# Run single test function
cd hindsight-api-slim && uv run pytest tests/test_retain.py::test_retain_simple -v
# Lint and format
cd hindsight-api-slim && uv run ruff check .
cd hindsight-api-slim && uv run ruff format .
# Type checking (uses ty - extremely fast type checker from Astral)
cd hindsight-api-slim && uv run ty check hindsight_api/
```
### Control Plane (Next.js)
```bash
./scripts/dev/start-control-plane.sh
# Or manually:
cd hindsight-control-plane && npm run dev
```
### Documentation Site (Docusaurus)
```bash
./scripts/dev/start-docs.sh
```
### Generating Clients/OpenAPI
```bash
# Regenerate OpenAPI spec after API changes (REQUIRED after changing endpoints)
./scripts/generate-openapi.sh
# Regenerate all client SDKs (Python, TypeScript, Rust)
./scripts/generate-clients.sh
```
### Benchmarks
```bash
# Accuracy benchmarks
./scripts/benchmarks/run-longmemeval.sh
./scripts/benchmarks/run-locomo.sh
# Performance benchmarks
./scripts/benchmarks/run-consolidation.sh
./scripts/benchmarks/run-retain-perf.sh --document <path> # Requires API server running
# Results viewer
./scripts/benchmarks/start-visualizer.sh # View results at localhost:8001
```
## Architecture
### Monorepo Structure
- **hindsight-api-slim/**: Core FastAPI server with memory engine (Python, uv)
- **hindsight/**: Embedded Python bundle (hindsight-all package)
- **hindsight-control-plane/**: Admin UI (Next.js, npm)
- **hindsight-cli/**: CLI tool (Rust, cargo, uses progenitor for API client)
- **hindsight-clients/**: Generated SDK clients (Python, TypeScript, Rust)
- **hindsight-docs/**: Docusaurus documentation site
- **hindsight-integrations/**: Framework integrations (LiteLLM, OpenAI)
- **hindsight-dev/**: Development tools and benchmarks
### Core Engine (hindsight-api-slim/hindsight_api/engine/)
- `memory_engine.py`: Main orchestrator (~170KB) for retain/recall/reflect operations
- `llm_wrapper.py`: LLM abstraction supporting OpenAI, Anthropic, Gemini, Groq, MiniMax, Ollama, LM Studio
- `embeddings.py`: Embedding generation (local sentence-transformers or TEI)
- `cross_encoder.py`: Reranking (local or TEI)
- `entity_resolver.py`: Entity extraction and normalization
- `query_analyzer.py`: Query intent analysis
**retain/**: Memory ingestion pipeline
- `orchestrator.py`: Coordinates the retain flow
- `fact_extraction.py`: LLM-based fact extraction from content
- `link_utils.py`: Entity link creation and management
**search/**: Multi-strategy retrieval
- `retrieval.py`: Main retrieval orchestrator
- `graph_retrieval.py`: Entity/relationship graph traversal
- `mpfp_retrieval.py`: Multi-Path Fact Propagation retrieval
- `fusion.py`: Reciprocal rank fusion for combining results
- `reranking.py`: Cross-encoder reranking
### API Layer (hindsight-api-slim/hindsight_api/api/)
- `http.py`: FastAPI HTTP routers (~80KB) for all REST endpoints
- `mcp.py`: Model Context Protocol server implementation
Main operations:
- **Retain**: Store memories, extracts facts/entities/relationships
- **Recall**: Retrieve memories via 4 parallel strategies (semantic, BM25, graph, temporal) + reranking
- **Reflect**: Disposition-aware reasoning using memories and mental models.
### Database
PostgreSQL with pgvector. Schema managed via Alembic migrations in `hindsight-api-slim/hindsight_api/alembic/`. Migrations run automatically on API startup.
Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
### Adding Database Migrations
1. **Create a new migration file** in `hindsight-api-slim/hindsight_api/alembic/versions/`:
- File name format: `<revision_id>_<description>.py` (e.g., `f1a2b3c4d5e6_add_new_index.py`)
- Use a unique hex revision ID (12 chars)
- Set `down_revision` to the previous migration's revision ID
2. **Migration template**:
```python
"""Description of the migration
Revision ID: f1a2b3c4d5e6
Revises: <previous_revision_id>
Create Date: YYYY-MM-DD
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "f1a2b3c4d5e6"
down_revision: str | Sequence[str] | None = "<previous_revision_id>"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"CREATE INDEX ... ON {schema}table_name(...)")
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}index_name")
```
3. **Run migrations locally**:
```bash
# Set database URL and run migrations for the base schema plus all tenants
uv run hindsight-admin run-db-migration
# Run on a specific tenant schema
uv run hindsight-admin run-db-migration --schema tenant_xyz
```
## Key Conventions
### Code Quality
**Always run the lint script after making Python or TypeScript/Node changes:**
```bash
./scripts/hooks/lint.sh
```
This runs the same checks as the pre-commit hook (Ruff for Python, ESLint/Prettier for TypeScript).
### Memory Banks
- Each bank is an isolated memory store (like a "brain" for one user/agent)
- Banks have dispositions (skepticism, literalism, empathy traits 1-5) affecting reflect
- Banks can have background context
- Bank isolation is strict - no cross-bank data leakage
### API Design
- All endpoints operate on a single bank per request
- Multi-bank queries are client responsibility to orchestrate
- Disposition traits only affect reflect, not recall
### Control Plane API Routes
When adding or modifying parameters in the dataplane API (hindsight-api), you must also update the control plane routes that proxy to it:
1. **API Routes** (`hindsight-control-plane/src/app/api/`):
- `recall/route.ts` - proxies to `/v1/default/banks/{bank_id}/memories/recall`
- `reflect/route.ts` - proxies to `/v1/default/banks/{bank_id}/reflect`
- `memories/retain/route.ts` - proxies to `/v1/default/banks/{bank_id}/memories/retain`
- Other routes follow the same pattern
2. **Client types** (`hindsight-control-plane/src/lib/api.ts`):
- Update the TypeScript type definitions for `recall()`, `reflect()`, `retain()` etc.
3. **Checklist when adding new API parameters**:
- Add parameter extraction in the route handler (destructure from `body`)
- Pass the parameter to the SDK call
- Update the client type definition in `lib/api.ts`
- Update any UI components that need to use the new parameter
### Python Style
- Python 3.11+, type hints required
- Async throughout (asyncpg, async FastAPI)
- Pydantic models for request/response
- Ruff for linting (line-length 120)
- No Python files at project root - maintain clean directory structure
- **Never use multi-item tuple return values** - prefer dataclass or Pydantic model for structured returns
### Type Safety with Pydantic Models
**NEVER use raw `dict` types for structured data.** Always use Pydantic models:
- Use Pydantic `BaseModel` for all data structures passed between functions
- Add `@field_validator` for type coercion (e.g., ensuring datetimes are timezone-aware)
- Avoid `dict.get()` patterns - use typed model attributes instead
- Parse external data (JSON, API responses) into Pydantic models at the boundary
- This catches type errors at parse time, not deep in business logic
```python
# BAD - error-prone dict access
def process(data: dict) -> str:
return data.get("name", "") # No validation, silent failures
# GOOD - typed and validated
class UserData(BaseModel):
name: str
created_at: datetime
@field_validator("created_at", mode="before")
@classmethod
def ensure_tz_aware(cls, v):
if isinstance(v, str):
v = datetime.fromisoformat(v.replace("Z", "+00:00"))
if v.tzinfo is None:
return v.replace(tzinfo=timezone.utc)
return v
def process(data: UserData) -> str:
return data.name # Type-safe, validated at construction
```
### TypeScript Style
- Next.js App Router for control plane
- Tailwind CSS with shadcn/ui components
### Adding New API Configuration Flags
Configuration follows a hierarchical system: **Global (env vars) → Tenant (via extension) → Bank (database)**.
Fields must be categorized as either **hierarchical** (can be overridden per-tenant/bank) or **static** (server-level only).
#### Adding a New Configuration Field
1. **config.py** (`hindsight-api-slim/hindsight_api/config.py`):
- Add `ENV_*` constant for the environment variable name (e.g., `ENV_MY_SETTING = "HINDSIGHT_API_MY_SETTING"`)
- Add `DEFAULT_*` constant for the default value
- Add field to `HindsightConfig` dataclass with type annotation
- **Mark as hierarchical or static** by adding to `_HIERARCHICAL_FIELDS` set (hierarchical) or leaving it out (static)
- Add initialization in `from_env()` method
```python
# Hierarchical field (can be overridden per-bank)
_HIERARCHICAL_FIELDS = {
...,
"my_setting", # Add here for hierarchical
}
# Static field - just don't add to _HIERARCHICAL_FIELDS
```
2. **main.py** (`hindsight-api-slim/hindsight_api/main.py`):
- Add field to the manual `HindsightConfig()` constructor call (search for "CLI override")
3. **Use hierarchical config in MemoryEngine**:
```python
# Config is resolved automatically per bank via ConfigResolver
config_dict = await self._config_resolver.get_bank_config(bank_id, context)
value = config_dict["my_setting"]
```
4. **Use static config** (non-hierarchical):
```python
from ...config import get_config
config = get_config()
value = config.my_static_field
```
5. **Documentation** (`hindsight-docs/docs/developer/configuration.md`):
- Add to appropriate section table with Variable, Description, Default
- Mark if it's hierarchical (can be overridden per-bank)
#### Hierarchical vs Static Guidelines
**Hierarchical** (per-bank overridable):
- LLM settings (provider, model, API key, base URL)
- Operation-specific settings (retain mode, chunk size, etc.)
- Feature flags that vary by customer/bank
**Static** (server-level only):
- Infrastructure settings (database URL, port, host)
- Global limits (max concurrent operations)
- System-wide feature flags
## Environment Setup
```bash
cp .env.example .env
# Edit .env with LLM API key
# Python deps
uv sync --directory hindsight-api-slim/
# Node deps (uses npm workspaces)
npm install
```
Required env vars:
- `HINDSIGHT_API_LLM_PROVIDER`: openai, anthropic, gemini, groq, minimax, ollama, lmstudio
- `HINDSIGHT_API_LLM_API_KEY`: Your API key
- `HINDSIGHT_API_LLM_MODEL`: Model name (e.g., gpt-4o-mini, claude-sonnet-4-20250514)
Optional (uses local models by default):
- `HINDSIGHT_API_EMBEDDINGS_PROVIDER`: local (default) or tei
- `HINDSIGHT_API_RERANKER_PROVIDER`: local (default) or tei
- `HINDSIGHT_API_DATABASE_URL`: External PostgreSQL (uses embedded pg0 by default)
- `HINDSIGHT_API_ENABLE_BANK_CONFIG_API`: Enable per-bank config API (default: true)
+1 -58
View File
@@ -51,36 +51,7 @@ cd hindsight-api
uv run pytest tests/
```
### Code Style
We use [Ruff](https://docs.astral.sh/ruff/) for Python linting and formatting, and ESLint/Prettier for TypeScript.
#### Setting up git hooks (recommended)
Set up git hooks to automatically lint and format code before each commit:
```bash
./scripts/setup-hooks.sh
```
This configures git to use the hooks in `.githooks/`, which run all scripts in `scripts/hooks/` on commit. The lint hook runs in parallel:
- **Python**: `ruff check --fix`, `ruff format`, `ty check`
- **TypeScript**: `eslint --fix`, `prettier`
#### Manual linting and formatting
```bash
# Run all lints (same as pre-commit)
./scripts/hooks/lint.sh
# Or run individually for Python:
cd hindsight-api
uv run ruff check --fix . # Lint and auto-fix
uv run ruff format . # Format code
uv run ty check hindsight_api # Type check
```
#### Style guidelines
### Code style
- Use Python type hints
- Follow existing code patterns
@@ -93,34 +64,6 @@ uv run ty check hindsight_api # Type check
3. Run tests to ensure nothing breaks
4. Submit a PR with a clear description of changes
## Release Process
The project uses `scripts/release.sh` for creating releases. This script automates the entire release workflow:
1. Bumps version in all components (API, clients, CLI, control plane, Helm)
2. **Regenerates OpenAPI spec and client SDKs** (Python, TypeScript, Rust)
3. Updates documentation versioning
4. Creates a commit and git tag
5. Pushes to GitHub (triggers CI/CD to publish packages)
### Usage
```bash
./scripts/release.sh <version>
```
**Example:**
```bash
./scripts/release.sh 0.5.0
```
### Important for Developers
- During development, version bumps in `__init__.py` do NOT require client regeneration
- Clients are only regenerated during releases
- Do not manually run `./scripts/generate-clients.sh` unless testing generation changes
- Client version comments will reflect the API version from the latest release
## Reporting Issues
Open an issue on GitHub with:
+61 -111
View File
@@ -1,93 +1,90 @@
<div align="center">
![Hindsight Banner](./hindsight-docs/static/img/hindsight-github-banner.png)
![Hindsight Banner](./hindsight-docs/static/img/banner.svg)
[Documentation](https://hindsight.vectorize.io) • [Paper](https://arxiv.org/abs/2512.12818) • [Cookbook](https://hindsight.vectorize.io/cookbook) • [Hindsight Cloud](https://ui.hindsight.vectorize.io/signup)
[Documentation](https://hindsight.vectorize.io) • [Paper](https://arxiv.org/abs/2512.12818) • [Cookbook](https://hindsight.vectorize.io/cookbook) • [Hindsight Cloud](https://vectorize.io/hindsight/cloud)
[![CI](https://github.com/vectorize-io/hindsight/actions/workflows/release.yml/badge.svg)](https://github.com/vectorize-io/hindsight/actions/workflows/release.yml)
[![Slack Community](https://img.shields.io/badge/Slack-Join%20Community-4A154B?logo=slack)](https://join.slack.com/t/hindsight-space/shared_invite/zt-3nhbm4w29-LeSJ5Ixi6j8PdiYOCPlOgg)
[![Slack Community](https://img.shields.io/badge/Slack-Join%20Community-4A154B?logo=slack)](https://join.slack.com/t/hindsight-space/shared_invite/zt-3klo21kua-VUCC_zHP5rIcXFB1_5yw6A)
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
![PyPI - Downloads](https://img.shields.io/pypi/dm/hindsight-api?label=PyPI)
![NPM Downloads](https://img.shields.io/npm/dm/%40vectorize-io%2Fhindsight-client?logoColor=orange&label=NPM&color=blue&link=https%3A%2F%2Fwww.npmjs.com%2Fpackage%2F%40vectorize-io%2Fhindsight-client)
<br/>
<a href="https://trendshift.io/repositories/15603" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15603" alt="vectorize-io%2Fhindsight | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
</div>
---
## What is Hindsight?
Hindsight™ is an agent memory system built to create smarter agents that learn over time. Most agent memory systems focus on recalling conversation history. Hindsight is focused on making agents that learn, not just remember.
Hindsight™ is an agent memory system built to create smarter agents that learn over time. It eliminates the shortcomings of alternative techniques such as RAG and knowledge graph and delivers state-of-the-art performance on long term memory tasks.
Hindsight addresses common challenges that have frustrated AI engineers building agents to automate tasks and assist users with conversational interfaces. Many of these challenges stem directly from a lack of memory.
<video src="https://github.com/user-attachments/assets/923b798d-3581-4897-bb62-9cfa5a931682" controls></video>
- **Inconsistency:** Agents complete tasks successfully one time, then fail when asked to complete the same task again. Memory gives the agent a mechanism to remember what worked and what didn't and to use that information to reduce errors and improve consistency.
- **Hallucinations:** Long term memory can be seeded with external knowledge to ground agent behavior in reliable sources to augment training data.
- **Cognitive Overload:** As workflows get complex, retrievals, tool calls, user messages and agent responses can grow to fill the context window leading to context rot. Short term memory optimization allows agents to reduce tokens and focus context by removing irrelevant details.
It eliminates the shortcomings of alternative techniques such as RAG and knowledge graph and delivers state-of-the-art performance on long term memory tasks.
## How is Hindsight Different From Other Memory Systems?
![Overview](./hindsight-docs/static/img/hindsight-overview.webp)
Most agent memory implementation rely on basic vector search or sometimes use a knowledge graph. Hindsight uses biomimetic data structures to organize agent memories in a way that is more like how human memory works:
- **World:** Facts about the world ("The stove gets hot")
- **Experiences:** Agent's own experiences ("I touched the stove and it really hurt")
- **Opinion:** Beliefs with confidence scores ("I shouldn't touch the stove again" - .99 confidence)
- **Observation:** Complex mental models derived by reflecting on facts and experiences ("Curling irons, ovens, and fire are also hot. I shouldn't touch those either.")
Memories in Hindsight are stored in banks (i.e. memory banks). When memories are added to Hindsight, they are pushed into either the world facts or experiences memory pathway. They are then represented as a combination of entities, relationships, and time series with sparse/dense vector representations to aid in later recall.
Hindsight provides three simple methods to interact with the system:
- **Retain:** Provide information to Hindsight that you want it to remember
- **Recall:** Retrieve memories from Hindsight
- **Reflect:** Reflect on memories and experiences to generate new observations and insights from existing memories.
### Agent Memory That Learns
A key goal of Hindsight is to build agent memory that enables agents to learn and improve over time. This is the role of the `reflect` operation which provides the agent to form broader opinions and observations over time.
For example, imagine a product support agent that is helping a user troubleshoot a problem. It uses a `search-documentation` tool it found on an MCP server. Later in the conversation, the agent discovers that the documentation returned from the tool wasn't for the product the user was asking about. The agent now has an experience in its memory bank. And just like humans, we want that agent to learn from its experience.
As the agent gains more experiences, `reflect` allows the agent to form observations about what worked, what didn't, and what to do differently the next time it encounters a similar task.
---
## Memory Performance & Accuracy
Hindsight is the most accurate agent memory system ever tested according to benchmark performance. It has achieved state-of-the-art performance on the LongMemEval benchmark, widely used to assess memory system performance across a variety of conversational AI scenarios. The current reported performance of Hindsight and other agent memory solutions as of January 2026 is shown here:
Hindsight has achieved state-of-the-art performance on the LongMemEval benchmark, widely used to assess memory system performance across a variety of conversational
AI scenarios. The current reported performance of Hindsight and other agent memory solutions as of December 2025 is shown here:
![Overview](./hindsight-docs/static/img/hindsight-bench.jpg)
The benchmark performance data for Hindsight has been independently reproduced by research collaborators at the Virginia Tech [Sanghani Center for Artificial Intelligence and Data Analytics](https://sanghani.cs.vt.edu/) and The Washington Post. Other scores are self-reported by software vendors.
The benchmark performance data for Hindsight and GPT-4o (full context) have been reproduced by research collaborators at the Virginia Tech [Sanghani Center for Artificial Intelligence and Data Analytics](https://sanghani.cs.vt.edu/) and The Washington Post. Other scores are self-reported by software vendors.
Hindsight is being used in production at Fortune 500 enterprises and by a growing number of AI startups.
## Adding Hindsight to Your AI Agents
The easiest way to use Hindsight with an existing agent is with the LLM Wrapper. You can add memory to your agent with 2 lines of code. That will swap your current LLM client out with the Hindsight wrapper. After that, memories will be stored and retrieved automatically as you make LLM calls.
If you need more control over how and when your agent stores and recalls memories, there's also a simple API you can integrate with using the SDKs or directly via HTTP.
![Hindsight Banner](./hindsight-docs/static/img/migration-code.png)
---
> 🤖 **Using a coding agent?** Install the Hindsight documentation skill for instant access to docs while you code:
> ```bash
> npx skills add https://github.com/vectorize-io/hindsight --skill hindsight-docs
> ```
> Works with Claude Code, Cursor, and other AI coding assistants.
---
A thorough examination of the techniques implemented in Hindsight and detailed breakdowns of benchmark performance are [available on arXiv](https://arxiv.org/abs/2512.12818). This research is currently being prepared for conference submission and the wider peer review process.
The benchmark results from this research can be inspected in our [visual benchmark explorer](https://hindsight-benchmarks.vercel.app). As additional improvements are made to Hindsight, new benchmark data will be available for review using this same tool.
## Quick Start
### Docker (recommended)
```bash
export OPENAI_API_KEY=sk-xxx
export OPENAI_API_KEY=your-key
docker run --rm -it --pull always -p 8888:8888 -p 9999:9999 \
-e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \
-e HINDSIGHT_API_LLM_MODEL=o3-mini \
-v $HOME/.hindsight-docker:/home/hindsight/.pg0 \
ghcr.io/vectorize-io/hindsight:latest
```
>API: http://localhost:8888
>UI: http://localhost:9999
API: http://localhost:8888
UI: http://localhost:9999
You can modify the LLM provider by setting `HINDSIGHT_API_LLM_PROVIDER`. Valid options are `openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`, and `minimax`. The documentation provides more details on [supported models](https://hindsight.vectorize.io/developer/models).
### Docker (external PostgreSQL)
```bash
export OPENAI_API_KEY=sk-xxx
export HINDSIGHT_DB_PASSWORD=choose-a-password
cd docker/docker-compose
docker compose up
```
>API: http://localhost:8888
>UI: http://localhost:9999
### Client
Install client:
```bash
pip install hindsight-client -U
@@ -95,7 +92,7 @@ pip install hindsight-client -U
npm install @vectorize-io/hindsight-client
```
#### Python
Python example:
```python
from hindsight_client import Hindsight
@@ -112,29 +109,7 @@ client.recall(bank_id="my-bank", query="What does Alice do?")
client.reflect(bank_id="my-bank", query="Tell me about Alice")
```
#### Node.js / TypeScript
```bash
npm install @vectorize-io/hindsight-client
```
```javascript
const { HindsightClient } = require('@vectorize-io/hindsight-client');
const main = async () => {
const client = new HindsightClient({ baseUrl: 'http://localhost:8888' });
await client.retain('my-bank', 'Alice loves hiking in Yosemite');
const results = await client.recall('my-bank', 'What does Alice like?');
console.log(results);
}
main();
```
### Python Embedded (no server required)
### Python (embedded, no Docker)
```bash
pip install hindsight-all -U
@@ -154,48 +129,25 @@ with HindsightServer(
results = client.recall(bank_id="my-bank", query="Where does Alice work?")
```
### Node.js / TypeScript
---
```bash
npm install @vectorize-io/hindsight-client
```
## Use Cases
```javascript
const { HindsightClient } = require('@vectorize-io/hindsight-client');
const client = new HindsightClient({ baseUrl: 'http://localhost:8888' });
Hindsight is built to support conversational AI agents as well as agents that are intended to perform tasks autonomously. The ideal use case for Hindsight are agents that require a blend of these features such as AI employees that need to handle open-ended tasks, change behavior based on user feedback, and learn to perform complex tasks to automate work at a level that approximates a human work. Hindsight can be used with simple AI workflows like those built with n8n and other similar tools, but may be overkill for such applications.
### Per-User Memories and Chat History
One of the simpler use cases you can use Hindsight for is to personalize AI chatbots and other conversational agents by storing and recalling memories associated with individual users.
The requirements for this use case usually look something like this:
![Per-User Memories](./hindsight-docs/static/img/per-user-memory-requirements.png)
<video src="https://github.com/user-attachments/assets/4805e8e1-e7d1-47c6-a4f8-2344a5ec8906" controls></video>
Satisfying these requirements in Hindsight is straightforward. When new user inputs and tool calls are ingested into Hindsight using the retain operation, custom metadata can be used to enrich the new memories. Metadata provides a convenient way to isolate memories that need to be restricted to a given user. Once these are fed into the retain operation, any raw memories and mental models that get created can be filtered when retrieving relevant memories.
![Per-User Memories](./hindsight-docs/static/img/per-user-memory-howto.png)
await client.retain('my-bank', 'Alice loves hiking in Yosemite');
await client.recall('my-bank', 'What does Alice like?');
```
---
## Architecture & Operations
![Overview](./hindsight-docs/static/img/hindsight-overview.webp)
Most agent memory implementations rely on basic vector search or sometimes use a knowledge graph. Hindsight uses biomimetic data structures to organize agent memories in a way that is more like how human memory works:
- **World:** Facts about the world ("The stove gets hot")
- **Experiences:** Agent's own experiences ("I touched the stove and it really hurt")
- **Mental Models:** Learned understanding of the agent's world formed by reflecting on raw memories and experiences.
Memories in Hindsight are stored in banks (i.e. memory banks). When memories are added to Hindsight, they are pushed into either the world facts or experiences memory pathway. They are then represented as a combination of entities, relationships, and time series with sparse/dense vector representations to aid in later recall.
Hindsight provides three simple methods to interact with the system:
- **Retain:** Provide information to Hindsight that you want it to remember
- **Recall:** Retrieve memories from Hindsight
- **Reflect:** Reflect on memories and experiences to generate new observations and insights from existing memories.
### Retain
The `retain` operation is used to push new memories into Hindsight. It tells Hindsight to _retain_ the information you pass in as an input.
@@ -254,7 +206,7 @@ The final output is trimmed as needed to fit within the token limit.
### Reflect
The reflect operation is used to perform a more thorough analysis of existing memories. This allows the agent to form new connections between memories and build a more thorough understanding of its world.
The reflect operation is used to perform a more thorough analysis of existing memories. This allows the agent to form new connections between memories which are then persisted as opinions and/or observations. When building agents, the reflect operation is a key capability to enable the agent to learn from its experiences.
For example, the `reflect` operation can be used to support use cases such as:
@@ -288,7 +240,7 @@ client.reflect(bank_id="my-bank", query="What should I know about Alice?")
- [CLI](https://hindsight.vectorize.io/sdks/cli)
**Community:**
- [Slack](https://join.slack.com/t/hindsight-space/shared_invite/zt-3nhbm4w29-LeSJ5Ixi6j8PdiYOCPlOgg)
- [Slack](https://join.slack.com/t/hindsight-space/shared_invite/zt-3klo21kua-VUCC_zHP5rIcXFB1_5yw6A)
- [GitHub Issues](https://github.com/vectorize-io/hindsight/issues)
---
@@ -308,5 +260,3 @@ MIT — see [LICENSE](./LICENSE)
---
Built by [Vectorize.io](https://vectorize.io)
<img src="https://umami-pixel.chris-latimer.workers.dev/?id=a8b043e6-6964-454d-80df-69b69d3f0d50&host=github.com&url=/vectorize-io/hindsight" width="1" height="1" alt="" />
Generated
-139
View File
@@ -1,139 +0,0 @@
{
"version": "5",
"specifiers": {
"jsr:@std/assert@^1.0.17": "1.0.19",
"jsr:@std/assert@^1.0.19": "1.0.19",
"jsr:@std/expect@*": "1.0.18",
"jsr:@std/internal@^1.0.12": "1.0.12",
"jsr:@std/path@^1.1.4": "1.1.4",
"jsr:@std/testing@*": "1.0.17"
},
"jsr": {
"@std/[email protected]": {
"integrity": "eaada96ee120cb980bc47e040f82814d786fe8162ecc53c91d8df60b8755991e",
"dependencies": [
"jsr:@std/internal"
]
},
"@std/[email protected]": {
"integrity": "8566eab35200466f8609eb7e7aed062ed0db314e9a258d5d201b1b8997ce801a",
"dependencies": [
"jsr:@std/assert@^1.0.19",
"jsr:@std/internal",
"jsr:@std/path"
]
},
"@std/[email protected]": {
"integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027"
},
"@std/[email protected]": {
"integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5",
"dependencies": [
"jsr:@std/internal"
]
},
"@std/[email protected]": {
"integrity": "87bdc2700fa98249d48a17cd72413352d3d3680dcfbdb64947fd0982d6bbf681",
"dependencies": [
"jsr:@std/assert@^1.0.17",
"jsr:@std/internal"
]
}
},
"workspace": {
"members": {
"hindsight-clients/typescript": {
"packageJson": {
"dependencies": [
"npm:@hey-api/[email protected]",
"npm:@types/jest@29",
"npm:@types/node@20",
"npm:jest@29",
"npm:ts-jest@29",
"npm:tsup@^8.5.1",
"npm:typescript@5"
]
}
},
"hindsight-control-plane": {
"packageJson": {
"dependencies": [
"npm:@eslint/eslintrc@^3.3.3",
"npm:@eslint/js@^9.39.2",
"npm:@radix-ui/react-alert-dialog@^1.1.15",
"npm:@radix-ui/react-checkbox@^1.3.3",
"npm:@radix-ui/react-dialog@^1.1.15",
"npm:@radix-ui/react-dropdown-menu@^2.1.16",
"npm:@radix-ui/react-label@^2.1.8",
"npm:@radix-ui/react-popover@^1.1.15",
"npm:@radix-ui/react-radio-group@^1.3.8",
"npm:@radix-ui/react-select@^2.2.6",
"npm:@radix-ui/react-slider@^1.3.6",
"npm:@radix-ui/react-slot@^1.2.4",
"npm:@radix-ui/react-switch@^1.2.6",
"npm:@radix-ui/react-tabs@^1.1.13",
"npm:@radix-ui/react-tooltip@^1.2.8",
"npm:@tailwindcss/postcss@^4.1.17",
"npm:@tailwindcss/typography@~0.5.19",
"npm:@types/cytoscape@^3.21.9",
"npm:@types/node@^24.10.0",
"npm:@types/react-dom@^19.2.2",
"npm:@types/react@^19.2.2",
"npm:autoprefixer@^10.4.21",
"npm:class-variance-authority@~0.7.1",
"npm:clsx@^2.1.1",
"npm:cmdk@^1.1.1",
"npm:cytoscape-fcose@^2.2.0",
"npm:cytoscape@^3.33.1",
"npm:eslint-config-next@^16.0.1",
"npm:eslint-plugin-react-hooks@^7.0.1",
"npm:eslint-plugin-react@^7.37.5",
"npm:eslint@^9.39.1",
"npm:[email protected]",
"npm:next-themes@~0.4.6",
"npm:next@^16.1.6",
"npm:postcss@^8.5.6",
"npm:prettier@^3.7.4",
"npm:react-chrono@^2.9.1",
"npm:react-dom@^19.2.0",
"npm:react-markdown@^10.1.0",
"npm:react18-json-view@~0.2.9",
"npm:react@^19.2.0",
"npm:recharts@^3.5.1",
"npm:remark-gfm@^4.0.1",
"npm:sonner@^2.0.7",
"npm:tailwind-merge@^3.4.0",
"npm:tailwindcss-animate@^1.0.7",
"npm:tailwindcss@^4.1.17",
"npm:[email protected]",
"npm:typescript-eslint@^8.50.0",
"npm:typescript@^5.9.3"
]
}
},
"hindsight-docs": {
"packageJson": {
"dependencies": [
"npm:@docusaurus/[email protected]",
"npm:@docusaurus/[email protected]",
"npm:@docusaurus/[email protected]",
"npm:@docusaurus/theme-common@^3.9.2",
"npm:@docusaurus/theme-mermaid@^3.9.2",
"npm:@docusaurus/[email protected]",
"npm:@docusaurus/[email protected]",
"npm:@easyops-cn/docusaurus-search-local@~0.52.2",
"npm:@mdx-js/react@3",
"npm:clsx@2",
"npm:prism-react-renderer@^2.3.0",
"npm:raw-loader@^4.0.2",
"npm:react-dom@19",
"npm:react-icons@^5.6.0",
"npm:react@19",
"npm:redocusaurus@^2.5.0",
"npm:typescript@~5.6.2"
]
}
}
}
}
}
@@ -1,54 +0,0 @@
# Docker Compose file for Hindsight with PostgreSQL and pgvector
#
# Make sure to set the required environment variables before running:
# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
# - Configure LLM provider variables as needed (see below in the hindsight service)
#
# Usage:
# docker compose up -d
#
# Optional environment variables with defaults:
# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
# - HINDSIGHT_DB_VERSION: PostgreSQL version (default: 18)
services:
db:
# Use a PostgreSQL-Image with pgvector extension pre-installed
# see https://hub.docker.com/r/pgvector/pgvector
image: pgvector/pgvector:pg${HINDSIGHT_DB_VERSION:-18}
container_name: hindsight-db
restart: always
# Expose PostgreSQL port
# ports:
# - "5432:5432"
environment:
POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}
POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
volumes:
- pg_data:/var/lib/postgresql/${HINDSIGHT_DB_VERSION:-18}/docker
networks:
- hindsight-net
hindsight:
image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
container_name: hindsight-app
ports:
- "8888:8888"
- "9999:9999"
environment:
- HINDSIGHT_API_LLM_API_KEY=${OPENAI_API_KEY?Please set the OPENAI_API_KEY env variable}
- HINDSIGHT_API_DATABASE_URL=postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
depends_on:
- db
networks:
- hindsight-net
networks:
hindsight-net:
driver: bridge
volumes:
pg_data:
-96
View File
@@ -1,96 +0,0 @@
# Nginx Reverse Proxy with Custom Base Path
Deploy Hindsight API under `/hindsight` (or any custom path) using Nginx reverse proxy.
## Quick Start (Published Image - API Only)
```bash
docker-compose up
```
- **API:** http://localhost:8080/hindsight/docs
- **Control Plane:** http://localhost:9999 (direct access, not proxied)
## Full Stack with Custom Base Path (Requires Build)
**Important:** You cannot rebuild from the published image with build args. You must build from source.
### Build from Source with Custom Base Path
1. **Clone the repository** (if you haven't):
```bash
git clone https://github.com/vectorize-io/hindsight.git
cd hindsight
```
2. **Build with base path**:
```bash
docker build \
--build-arg NEXT_PUBLIC_BASE_PATH=/hindsight \
-f docker/standalone/Dockerfile \
-t hindsight:custom \
.
```
3. **Update docker-compose.yml** to use your built image:
```yaml
services:
hindsight:
image: hindsight:custom # ← Change this
environment:
HINDSIGHT_API_BASE_PATH: /hindsight
NEXT_PUBLIC_BASE_PATH: /hindsight
```
4. **Update nginx.conf** to handle Control Plane routes (see below)
5. **Run**:
```bash
docker-compose up
```
### Required nginx.conf for Full Stack
Replace the current `nginx.conf` with this to proxy both API and Control Plane:
```nginx
events { worker_connections 1024; }
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
upstream hindsight_api { server hindsight:8888; }
upstream hindsight_cp { server hindsight:9999; }
server {
listen 80;
# API
location ~ ^/hindsight/(docs|openapi\.json|health|metrics|v1|mcp) {
proxy_pass http://hindsight_api;
proxy_set_header Host $http_host;
}
# Control Plane static files
location ~ ^/hindsight/_next/ {
proxy_pass http://hindsight_cp;
proxy_set_header Host $http_host;
}
# Control Plane UI
location /hindsight {
proxy_pass http://hindsight_cp;
proxy_set_header Host $http_host;
}
location = / { return 301 /hindsight; }
}
}
```
### Why Build is Required
Next.js requires `basePath` at **build time**. The published image was built without a custom base path, so you must rebuild from source with the `NEXT_PUBLIC_BASE_PATH` build arg to deploy the Control Plane under a subpath.
The API works without rebuild because `HINDSIGHT_API_BASE_PATH` is a runtime environment variable.
@@ -1,88 +0,0 @@
# Hindsight API deployment with Nginx reverse proxy (API-only)
#
# This example deploys Hindsight API under the path /hindsight with:
# - Hindsight standalone image (API + Control Plane + embedded pg0)
# - Nginx reverse proxy (API only)
#
# Quick Start:
# docker-compose -f docker/docker-compose/nginx/docker-compose.yml up
#
# Access:
# API (via nginx): http://localhost:8080/hindsight/docs
# Control Plane (direct): http://localhost:9999
#
# For full stack deployment (API + Control Plane both under /hindsight):
# See README.md in this directory for instructions on building with basePath.
#
# Note: This configuration uses the published image (no build required).
# Control Plane is served directly because Next.js basePath requires
# build-time configuration. See README.md for the full stack option.
services:
# Hindsight (API + Control Plane + embedded pg0)
hindsight:
image: ghcr.io/vectorize-io/hindsight:latest
ports:
- "9999:9999" # Control Plane (direct access, not proxied)
environment:
# API base path for reverse proxy
HINDSIGHT_API_BASE_PATH: /hindsight
# LLM configuration
# Using mock provider for testing (no API key needed)
# For production, set OPENAI_API_KEY or ANTHROPIC_API_KEY and use a real provider
HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-mock}
HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-not-needed-for-mock}
HINDSIGHT_API_LLM_MODEL: ${HINDSIGHT_API_LLM_MODEL:-mock-model}
# Production examples (uncomment and set appropriate API key):
# HINDSIGHT_API_LLM_PROVIDER: openai
# HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY}
# HINDSIGHT_API_LLM_MODEL: gpt-4o-mini
# HINDSIGHT_API_LLM_PROVIDER: anthropic
# HINDSIGHT_API_LLM_API_KEY: ${ANTHROPIC_API_KEY}
# HINDSIGHT_API_LLM_MODEL: claude-sonnet-4-20250514
# Server config
HINDSIGHT_API_HOST: 0.0.0.0
HINDSIGHT_API_PORT: 8888
HINDSIGHT_API_LOG_LEVEL: info
# Control Plane config
HINDSIGHT_CP_DATAPLANE_API_URL: http://localhost:8888
volumes:
# Persist embedded pg0 database
- hindsight_data:/app/data
# Note: Ports not exposed - access via Nginx at localhost:8080/hindsight/
# To debug directly, uncomment these ports:
# ports:
# - "8888:8888" # API
# - "9999:9999" # Control Plane
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8888/hindsight/health"]
interval: 10s
timeout: 5s
retries: 3
start_period: 30s
networks:
- hindsight
# Nginx reverse proxy
nginx:
image: nginx:alpine
ports:
- "8080:80"
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf:ro
depends_on:
hindsight:
condition: service_healthy
networks:
- hindsight
volumes:
hindsight_data:
networks:
hindsight:
-40
View File
@@ -1,40 +0,0 @@
# Nginx configuration for API-only reverse proxy
# Control Plane accessed directly (not through nginx)
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
# Logging
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
# Upstream - Hindsight API
upstream hindsight_api {
server hindsight:8888;
}
server {
listen 80;
server_name _;
# API endpoints - forward with /hindsight prefix
location /hindsight/ {
proxy_pass http://hindsight_api;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
# Redirect root to API docs
location = / {
return 301 /hindsight/docs;
}
}
}
@@ -1,32 +0,0 @@
# PostgreSQL with pgvector and pg_textsearch extensions
# Note: pg_textsearch requires PostgreSQL 17+
FROM postgres:17
# Install build dependencies
RUN apt-get update && apt-get install -y \
build-essential \
git \
postgresql-server-dev-17 \
libpq-dev \
&& rm -rf /var/lib/apt/lists/*
# Install pgvector
RUN cd /tmp && \
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git && \
cd pgvector && \
make && \
make install
# Install pg_textsearch
RUN cd /tmp && \
git clone https://github.com/timescale/pg_textsearch.git && \
cd pg_textsearch && \
make && \
make install
# Clean up source files and build dependencies
RUN rm -rf /tmp/pgvector /tmp/pg_textsearch && \
apt-get purge -y --auto-remove build-essential git postgresql-server-dev-17
# Ensure extensions are preloaded
RUN echo "shared_preload_libraries = 'pg_textsearch'" >> /usr/share/postgresql/postgresql.conf.sample
@@ -1,91 +0,0 @@
name: hindsight
# Docker Compose file for Hindsight with PostgreSQL and Timescale pg_textsearch
# docker compose -f docker/docker-compose/pg_textsearch/docker-compose.yaml down && sleep 2 && docker compose -f docker/docker-compose/pg_textsearch/docker-compose.yaml up -d
# Make sure to set the required environment variables before running:
# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
# - Configure LLM provider variables as needed (see below in the hindsight service)
#
# Usage:
# docker compose up -d
#
# Optional environment variables with defaults:
# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
services:
db:
# Use custom PostgreSQL image with pgvector and pg_textsearch extensions
build:
context: .
dockerfile: Dockerfile
container_name: hindsight-db
restart: always
# Expose PostgreSQL port
ports:
- "5437:5432"
environment:
POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:-hindsight_password}
POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
volumes:
- pg_data:/var/lib/postgresql/data
networks:
- hindsight-net
pg-textsearch-init:
build:
context: .
dockerfile: Dockerfile
depends_on:
- db
environment:
- PGPASSWORD=${HINDSIGHT_DB_PASSWORD:-hindsight_password}
command: >
bash -c "
echo 'Waiting for PostgreSQL to be ready...';
until pg_isready -h hindsight-db -p 5432 -U hindsight_user; do
echo 'PostgreSQL is unavailable - sleeping';
sleep 2;
done;
echo 'PostgreSQL is ready - creating hindsight_db database';
psql -h hindsight-db -p 5432 -U hindsight_user -c 'CREATE DATABASE hindsight_db;' 2>/dev/null || echo 'Database already exists';
echo 'Creating extensions in hindsight_db database';
psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vector CASCADE;';
psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS pg_textsearch CASCADE;';
echo 'Database and extensions created successfully';
"
restart: "no"
networks:
- hindsight-net
hindsight:
image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
container_name: hindsight-app
ports:
- "8888:8888"
- "9999:9999"
environment:
# LLM Configuration
HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai}
HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key}
# Database Configuration
HINDSIGHT_API_DATABASE_URL: postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:-hindsight_password}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
# Vector and Text Search Extensions
HINDSIGHT_API_VECTOR_EXTENSION: pgvector
HINDSIGHT_API_TEXT_SEARCH_EXTENSION: pg_textsearch
depends_on:
- db
networks:
- hindsight-net
networks:
hindsight-net:
driver: bridge
volumes:
pg_data:
@@ -1,83 +0,0 @@
# Docker Compose file for Hindsight with S3 file storage (SeaweedFS)
#
# SeaweedFS (Apache 2.0) provides an S3-compatible object storage backend
# for storing uploaded files instead of PostgreSQL BYTEA storage.
#
# Make sure to set the required environment variables before running:
# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
# - Configure LLM provider variables as needed (see below in the hindsight service)
#
# Usage:
# docker compose up -d
#
# Optional environment variables with defaults:
# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
# - HINDSIGHT_DB_VERSION: PostgreSQL version (default: 18)
# - SEAWEEDFS_S3_ACCESS_KEY: S3 access key (default: hindsight_s3_key)
# - SEAWEEDFS_S3_SECRET_KEY: S3 secret key (default: hindsight_s3_secret)
services:
db:
image: pgvector/pgvector:pg${HINDSIGHT_DB_VERSION:-18}
container_name: hindsight-db
restart: always
environment:
POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}
POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
volumes:
- pg_data:/var/lib/postgresql/${HINDSIGHT_DB_VERSION:-18}/docker
networks:
- hindsight-net
seaweedfs:
image: chrislusf/seaweedfs:latest
container_name: hindsight-seaweedfs
restart: always
# Single-node mode: master + volume + filer + S3 gateway all in one process
command: >
server
-s3
-s3.port=8333
-s3.config=/etc/seaweedfs/s3.json
-ip.bind=0.0.0.0
volumes:
- seaweedfs_data:/data
- ./s3.json:/etc/seaweedfs/s3.json:ro
# Expose S3 API port (uncomment to access from host)
# ports:
# - "8333:8333"
networks:
- hindsight-net
hindsight:
image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
container_name: hindsight-app
ports:
- "8888:8888"
- "9999:9999"
environment:
- HINDSIGHT_API_LLM_API_KEY=${OPENAI_API_KEY?Please set the OPENAI_API_KEY env variable}
- HINDSIGHT_API_DATABASE_URL=postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
# S3 file storage configuration (SeaweedFS)
- HINDSIGHT_API_FILE_STORAGE_TYPE=s3
- HINDSIGHT_API_FILE_STORAGE_S3_BUCKET=hindsight
- HINDSIGHT_API_FILE_STORAGE_S3_ENDPOINT=http://seaweedfs:8333
- HINDSIGHT_API_FILE_STORAGE_S3_REGION=us-east-1
- HINDSIGHT_API_FILE_STORAGE_S3_ACCESS_KEY_ID=${SEAWEEDFS_S3_ACCESS_KEY:-hindsight_s3_key}
- HINDSIGHT_API_FILE_STORAGE_S3_SECRET_ACCESS_KEY=${SEAWEEDFS_S3_SECRET_KEY:-hindsight_s3_secret}
depends_on:
- db
- seaweedfs
networks:
- hindsight-net
networks:
hindsight-net:
driver: bridge
volumes:
pg_data:
seaweedfs_data:
@@ -1,19 +0,0 @@
{
"identities": [
{
"name": "hindsight",
"credentials": [
{
"accessKey": "hindsight_s3_key",
"secretKey": "hindsight_s3_secret"
}
],
"actions": [
"Admin",
"Read",
"Write",
"List"
]
}
]
}
@@ -1,16 +0,0 @@
# Git
.git
.gitignore
.gitattributes
# Docker
docker-compose.yaml
.dockerignore
# Documentation
README.md
*.md
# Environment
.env
.env.example
@@ -1,25 +0,0 @@
# PostgreSQL Configuration
HINDSIGHT_DB_USER=hindsight_user
HINDSIGHT_DB_PASSWORD=change-me-to-secure-password
HINDSIGHT_DB_NAME=hindsight_db
# Hindsight Version
HINDSIGHT_VERSION=latest
# LLM Configuration
HINDSIGHT_API_LLM_PROVIDER=openai
OPENAI_API_KEY=your-openai-api-key-here
# Alternative LLM providers (uncomment and configure as needed):
# HINDSIGHT_API_LLM_PROVIDER=anthropic
# ANTHROPIC_API_KEY=your-anthropic-api-key
# HINDSIGHT_API_LLM_PROVIDER=gemini
# GEMINI_API_KEY=your-gemini-api-key
# HINDSIGHT_API_LLM_PROVIDER=groq
# GROQ_API_KEY=your-groq-api-key
# Vector and Text Search (already configured in docker-compose.yaml)
# HINDSIGHT_API_VECTOR_EXTENSION=pgvectorscale
# HINDSIGHT_API_TEXT_SEARCH_EXTENSION=pg_textsearch
@@ -1,55 +0,0 @@
# PostgreSQL with pgvector, pgvectorscale, and pg_textsearch extensions
# All three extensions from Timescale/pgvector for high-performance vector and text search
# Note: Requires PostgreSQL 16+
FROM postgres:17
# Install build dependencies and Rust toolchain
RUN apt-get update && apt-get install -y \
build-essential \
git \
postgresql-server-dev-17 \
libpq-dev \
cmake \
curl \
pkg-config \
libssl-dev \
&& rm -rf /var/lib/apt/lists/*
# Install Rust toolchain (required for pgvectorscale)
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
# Install pgvector (required by pgvectorscale)
RUN cd /tmp && \
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git && \
cd pgvector && \
make && \
make install && \
rm -rf /tmp/pgvector
# Install cargo-pgrx (PostgreSQL extension framework for Rust)
RUN cargo install cargo-pgrx --version 0.12.5 --locked && \
cargo pgrx init --pg17 /usr/bin/pg_config
# Install pgvectorscale (DiskANN index support)
RUN cd /tmp && \
git clone --branch 0.5.1 https://github.com/timescale/pgvectorscale.git && \
cd pgvectorscale/pgvectorscale && \
cargo pgrx install --release && \
rm -rf /tmp/pgvectorscale
# Install pg_textsearch (BM25 text search)
RUN cd /tmp && \
git clone https://github.com/timescale/pg_textsearch.git && \
cd pg_textsearch && \
make && \
make install && \
rm -rf /tmp/pg_textsearch
# Clean up build dependencies (keep runtime dependencies)
RUN apt-get purge -y --auto-remove git cmake curl && \
rm -rf /root/.cargo/registry /root/.cargo/git
# Ensure extensions are preloaded (pg_textsearch requires preloading)
RUN echo "shared_preload_libraries = 'pg_textsearch'" >> /usr/share/postgresql/postgresql.conf.sample
-101
View File
@@ -1,101 +0,0 @@
# Hindsight with Timescale Extensions
This Docker Compose setup provides a complete Hindsight deployment with **Timescale extensions**:
- **pgvectorscale** - DiskANN algorithm for disk-based scalable vector search
- **pg_textsearch** - High-performance BM25 text search
Both extensions are from [Timescale](https://github.com/timescale) and provide production-grade performance.
## Prerequisites
- Docker and Docker Compose installed
- OpenAI API key (or another LLM provider)
## Quick Start
```bash
# Set environment variables
export HINDSIGHT_DB_PASSWORD="your-secure-password"
export OPENAI_API_KEY="your-openai-api-key"
# Build and start
docker compose -f docker/docker-compose/timescale/docker-compose.yaml up -d --build
# Check logs
docker compose -f docker/docker-compose/timescale/docker-compose.yaml logs -f
```
**Access:**
- API: http://localhost:8888
- Control Plane: http://localhost:9999
## Stop and Clean Up
```bash
# Stop services
docker compose -f docker/docker-compose/timescale/docker-compose.yaml down
# Remove volumes (deletes all data)
docker compose -f docker/docker-compose/timescale/docker-compose.yaml down -v
```
## Configuration
### Environment Variables
| Variable | Description | Default |
|----------|-------------|---------|
| `HINDSIGHT_DB_PASSWORD` | PostgreSQL password | `hindsight_password` |
| `HINDSIGHT_DB_USER` | PostgreSQL username | `hindsight_user` |
| `HINDSIGHT_DB_NAME` | Database name | `hindsight_db` |
| `HINDSIGHT_VERSION` | Hindsight Docker image version | `latest` |
| `OPENAI_API_KEY` | OpenAI API key | (required) |
| `HINDSIGHT_API_LLM_PROVIDER` | LLM provider | `openai` |
### Why Timescale Extensions?
**pgvectorscale (DiskANN):**
- 28x lower p95 latency vs dedicated vector databases
- 16x higher query throughput at 99% recall
- 60-75% cost reduction (disk is cheaper than RAM)
- Best for large datasets (10M+ vectors)
**pg_textsearch (BM25):**
- High-performance keyword retrieval
- Native BM25 ranking algorithm
- Optimized for full-text search
## Troubleshooting
### Extensions not installed
Check if extensions are available:
```bash
docker exec -it hindsight-db-timescale psql -U hindsight_user -d hindsight_db -c "\dx"
```
You should see:
- `vector` (pgvector)
- `vectorscale` (pgvectorscale/DiskANN)
- `pg_textsearch` (BM25 search)
### Build fails
If the Docker build fails during pgvectorscale compilation:
1. Ensure you have sufficient memory (recommended: 4GB+)
2. Check Docker build logs for Rust compilation errors
3. Try building with more resources: `docker compose build --no-cache --memory 4g`
### Port conflicts
If port 5438 is already in use, modify the `ports` section in docker-compose.yaml.
## Learn More
- [pgvectorscale GitHub](https://github.com/timescale/pgvectorscale)
- [pg_textsearch GitHub](https://github.com/timescale/pg_textsearch)
- [HNSW vs DiskANN](https://www.tigerdata.com/learn/hnsw-vs-diskann)
- [Hindsight Documentation](https://hindsight.dev)
@@ -1,108 +0,0 @@
name: hindsight
# Docker Compose file for Hindsight with Timescale extensions
# - pgvectorscale: DiskANN vector search (disk-based, scalable)
# - pg_textsearch: BM25 text search (high-performance keyword retrieval)
#
# Quick start:
# docker compose -f docker/docker-compose/timescale/docker-compose.yaml up -d --build
#
# Required environment variables:
# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
# - OPENAI_API_KEY (or configure another LLM provider)
#
# Optional environment variables with defaults:
# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
services:
db:
# Custom PostgreSQL image with Timescale extensions (pgvectorscale + pg_textsearch)
build:
context: .
dockerfile: Dockerfile
container_name: hindsight-db-timescale
restart: always
# Expose PostgreSQL port (using 5438 to avoid conflicts with other setups)
ports:
- "5438:5432"
environment:
POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:-hindsight_password}
POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
volumes:
- pg_data:/var/lib/postgresql/data
networks:
- hindsight-net
# Health check to ensure database is ready
healthcheck:
test: ["CMD-SHELL", "pg_isready -U hindsight_user"]
interval: 5s
timeout: 5s
retries: 5
timescale-init:
build:
context: .
dockerfile: Dockerfile
depends_on:
db:
condition: service_healthy
environment:
- PGPASSWORD=${HINDSIGHT_DB_PASSWORD:-hindsight_password}
command: >
bash -c "
echo 'PostgreSQL is ready - creating hindsight_db database';
psql -h hindsight-db-timescale -p 5432 -U hindsight_user -c 'CREATE DATABASE hindsight_db;' 2>/dev/null || echo 'Database already exists';
echo 'Installing Timescale extensions...';
echo '1/3: Installing pgvector (required by pgvectorscale)...';
psql -h hindsight-db-timescale -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vector CASCADE;';
echo '2/3: Installing pgvectorscale (DiskANN vector search)...';
psql -h hindsight-db-timescale -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vectorscale CASCADE;';
echo '3/3: Installing pg_textsearch (BM25 text search)...';
psql -h hindsight-db-timescale -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS pg_textsearch CASCADE;';
echo '';
echo '✅ Timescale extensions installed successfully';
echo '';
echo 'Installed extensions:';
psql -h hindsight-db-timescale -p 5432 -U hindsight_user -d hindsight_db -c \"\\dx\" | grep -E '(vector|vectorscale|pg_textsearch)';
"
restart: "no"
networks:
- hindsight-net
hindsight:
image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
container_name: hindsight-app-timescale
ports:
- "8888:8888"
- "9999:9999"
environment:
# LLM Configuration
HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai}
HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key}
# Database Configuration
HINDSIGHT_API_DATABASE_URL: postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:-hindsight_password}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
# Timescale Extensions
# pgvectorscale: DiskANN algorithm for disk-based scalable vector search
HINDSIGHT_API_VECTOR_EXTENSION: pgvectorscale
# pg_textsearch: High-performance BM25 text search
HINDSIGHT_API_TEXT_SEARCH_EXTENSION: pg_textsearch
depends_on:
db:
condition: service_healthy
timescale-init:
condition: service_completed_successfully
networks:
- hindsight-net
networks:
hindsight-net:
driver: bridge
volumes:
pg_data:
@@ -1,93 +0,0 @@
name: hindsight
# Docker Compose file for Hindsight with PostgreSQL and vectorchord
# docker compose -f docker/docker-compose/docker-compose.yaml down && sleep 2 && docker compose -f docker/docker-compose/docker-compose.yaml up -d
# Make sure to set the required environment variables before running:
# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
# - Configure LLM provider variables as needed (see below in the hindsight service)
#
# Usage:
# docker compose up -d
#
# Optional environment variables with defaults:
# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
# - HINDSIGHT_DB_VERSION: PostgreSQL version (default: 18)
services:
db:
# Use a PostgreSQL-Image with vectorchord extension pre-installed
image: tensorchord/vchord-suite:pg${HINDSIGHT_DB_VERSION:-18-latest}
container_name: hindsight-db
restart: always
# Expose PostgreSQL port
ports:
- "5436:5432"
environment:
POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:-hindsight_password}
POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
volumes:
- pg_data:/var/lib/postgresql/${HINDSIGHT_DB_VERSION:-18}/docker
networks:
- hindsight-net
vectorchord-init:
image: tensorchord/vchord-suite:pg18-latest
#container_name: vectorchord-init
depends_on:
- db
environment:
- PGPASSWORD=${HINDSIGHT_DB_PASSWORD:-hindsight_password}
command: >
bash -c "
echo 'Waiting for PostgreSQL to be ready...';
until pg_isready -h hindsight-db -p 5432 -U hindsight_user; do
echo 'PostgreSQL is unavailable - sleeping';
sleep 2;
done;
echo 'PostgreSQL is ready - creating hindsight_db database';
psql -h hindsight-db -p 5432 -U hindsight_user -c 'CREATE DATABASE hindsight_db;' 2>/dev/null || echo 'Database already exists';
echo 'Creating extensions in hindsight_db database';
psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vchord CASCADE;';
psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS pg_tokenizer CASCADE;';
psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vchord_bm25 CASCADE;';
echo 'Creating llmlingua2 tokenizer';
psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c \"SELECT create_tokenizer('llmlingua2', \\$\\$ model = \\\"llmlingua2\\\" \\$\\$);\" 2>/dev/null || echo 'Tokenizer already exists or creation skipped';
echo 'Database and extensions created successfully';
"
restart: "no"
networks:
- hindsight-net
hindsight:
image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
container_name: hindsight-app
ports:
- "8888:8888"
- "9999:9999"
environment:
# LLM Configuration (uses OpenAI for testing vchord)
# LLM configuration
HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai}
HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key}
# Database Configuration
HINDSIGHT_API_DATABASE_URL: postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:-hindsight_password}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
# Vector and Text Search Extensions
HINDSIGHT_API_VECTOR_EXTENSION: vchord
HINDSIGHT_API_TEXT_SEARCH_EXTENSION: vchord
depends_on:
- db
networks:
- hindsight-net
networks:
hindsight-net:
driver: bridge
volumes:
pg_data:
+41 -134
View File
@@ -2,25 +2,19 @@
# Supports building API-only, Control Plane-only, or both
#
# Build args:
# INCLUDE_API=true/false - Include API (default: true)
# INCLUDE_CP=true/false - Include Control Plane (default: true)
# INCLUDE_LOCAL_MODELS=true/false - Include local ML models for embeddings/reranking (default: true)
# Set to false when using external providers (TEI, OpenAI, Cohere)
# PRELOAD_ML_MODELS=true/false - Pre-download ML models during build (default: true)
# Only effective when INCLUDE_LOCAL_MODELS=true
# NOTE: tiktoken encodings are ALWAYS preloaded (required for air-gapped deployments)
# INCLUDE_API=true/false - Include API (default: true)
# INCLUDE_CP=true/false - Include Control Plane (default: true)
# PRELOAD_ML_MODELS=true/false - Pre-download ML models during build (default: true)
#
# Examples:
# docker build -t hindsight . # Both (standalone)
# docker build -t hindsight-api --build-arg INCLUDE_CP=false . # API only
# docker build -t hindsight-cp --build-arg INCLUDE_API=false . # Control Plane only
# docker build -t hindsight --build-arg PRELOAD_ML_MODELS=false . # Skip ML model preload
# docker build -t hindsight --build-arg INCLUDE_LOCAL_MODELS=false . # Skip local ML deps (for external providers)
ARG INCLUDE_API=true
ARG INCLUDE_CP=true
ARG PRELOAD_ML_MODELS=true
ARG INCLUDE_LOCAL_MODELS=true
# =============================================================================
# Stage: API Builder
@@ -28,7 +22,6 @@ ARG INCLUDE_LOCAL_MODELS=true
FROM python:3.11-slim AS api-builder
ARG INCLUDE_API
ARG INCLUDE_LOCAL_MODELS
RUN if [ "$INCLUDE_API" != "true" ]; then echo "Skipping API build" && exit 0; fi
WORKDIR /app
@@ -42,22 +35,16 @@ RUN apt-get update && apt-get install -y \
&& pip install --no-cache-dir uv
# Copy dependency files and README (required by pyproject.toml)
COPY hindsight-api-slim/pyproject.toml ./api/
COPY hindsight-api-slim/README.md ./api/
COPY hindsight-api/pyproject.toml ./api/
COPY hindsight-api/README.md ./api/
WORKDIR /app/api
# Sync dependencies using appropriate extras based on INCLUDE_LOCAL_MODELS
# local-ml: torch, sentence-transformers, transformers, einops, flashrank, mlx (optional)
# embedded-db: pg0-embedded (always included for embedded PostgreSQL support)
RUN if [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
uv sync --extra local-ml --extra embedded-db; \
else \
uv sync --extra embedded-db; \
fi
# Sync dependencies (will create lock file if needed)
RUN uv sync
# Copy source code (alembic migrations are inside hindsight_api/)
COPY hindsight-api-slim/hindsight_api ./hindsight_api
COPY hindsight-api/hindsight_api ./hindsight_api
# Install the local package (uv sync only installed dependencies, not the package itself)
RUN uv pip install -e .
@@ -109,10 +96,6 @@ RUN rm -f package-lock.json && sed -i '/"@vectorize-io\/hindsight-client":/d' pa
# Copy built SDK directly into node_modules (more reliable than npm link in Docker)
COPY --from=sdk-builder /app/hindsight-clients/typescript ./node_modules/@vectorize-io/hindsight-client
# Accept base path as build argument for reverse proxy deployments
# Usage: docker build --build-arg NEXT_PUBLIC_BASE_PATH=/hindsight ...
ARG NEXT_PUBLIC_BASE_PATH=""
# Build Control Plane - run next build first, then custom standalone copy
# (The build:standalone script expects a specific path structure that differs in Docker)
RUN npm exec -- next build
@@ -142,6 +125,7 @@ FROM python:3.11-slim AS api-only
WORKDIR /app
# Install pg0 dependencies (procps provides 'kill' command needed by pg0)
# Note: libicu version varies by Debian version - try common versions in order
RUN apt-get update && apt-get install -y \
curl \
@@ -154,6 +138,7 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/* \
&& pip install --no-cache-dir uv
# Create non-root user (PostgreSQL cannot run as root)
RUN useradd -m -s /bin/bash hindsight
# Copy API with virtual environment from builder
@@ -163,69 +148,30 @@ COPY --from=api-builder /app/api /app/api
COPY docker/standalone/start-all.sh /app/start-all.sh
RUN chmod +x /app/start-all.sh
RUN chown -R hindsight:hindsight /app
# Create data directory for pg0 and set ownership
RUN mkdir -p /app/data && chown -R hindsight:hindsight /app
# Switch to non-root user
USER hindsight
# Create pg0 data directory as hindsight user so that Docker seeds new named
# volumes with correct ownership (UID 1000) on first use, avoiding the
# "Permission denied" error when mounting a fresh root-owned volume.
RUN mkdir -p /home/hindsight/.pg0
# Set PATH for hindsight user
ENV PATH="/app/api/.venv/bin:${PATH}"
# Pre-download tiktoken encoding (ALWAYS - required for token counting even in air-gapped envs)
# Tiktoken is a core runtime dependency, not an optional ML model
RUN MAX_RETRIES=3; \
RETRY_DELAY=5; \
for i in $(seq 1 $MAX_RETRIES); do \
echo "Attempt $i/$MAX_RETRIES: Downloading tiktoken encoding..."; \
/app/api/.venv/bin/python -c "\
import tiktoken; \
print('Downloading cl100k_base encoding...'); \
tiktoken.get_encoding('cl100k_base'); \
print('Tiktoken encoding cached successfully')" && break; \
if [ $i -lt $MAX_RETRIES ]; then \
echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
sleep $RETRY_DELAY; \
RETRY_DELAY=$((RETRY_DELAY * 2)); \
fi; \
done; \
if [ $i -eq $MAX_RETRIES ]; then \
echo "ERROR: Failed to download tiktoken encoding after $MAX_RETRIES attempts"; \
exit 1; \
fi
# Pre-cache PostgreSQL binaries by starting/stopping pg0-embedded
ENV PG0_HOME=/home/hindsight/.pg0-cache
ENV PG0_HOME=/home/hindsight/.pg0
# Pre-download ML models to avoid runtime download (conditional)
# Only runs if both PRELOAD_ML_MODELS=true AND INCLUDE_LOCAL_MODELS=true
# Includes retry logic with exponential backoff for transient network failures
ARG PRELOAD_ML_MODELS
ARG INCLUDE_LOCAL_MODELS
ENV HF_HUB_DOWNLOAD_TIMEOUT=600
RUN if [ "$PRELOAD_ML_MODELS" = "true" ] && [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
MAX_RETRIES=3; \
RETRY_DELAY=10; \
for i in $(seq 1 $MAX_RETRIES); do \
echo "Attempt $i/$MAX_RETRIES: Downloading ML models..."; \
/app/api/.venv/bin/python -c "\
import os; os.environ['HF_HUB_DOWNLOAD_TIMEOUT'] = '600'; \
RUN if [ "$PRELOAD_ML_MODELS" = "true" ]; then \
/app/api/.venv/bin/python -c "\
from sentence_transformers import SentenceTransformer, CrossEncoder; \
print('Downloading embedding model...'); \
SentenceTransformer('BAAI/bge-small-en-v1.5'); \
print('Downloading cross-encoder model...'); \
CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'); \
print('Models cached successfully')" && break; \
if [ $i -lt $MAX_RETRIES ]; then \
echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
sleep $RETRY_DELAY; \
RETRY_DELAY=$((RETRY_DELAY * 2)); \
fi; \
done; \
if [ $i -eq $MAX_RETRIES ] && ! /app/api/.venv/bin/python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('BAAI/bge-small-en-v1.5')" 2>/dev/null; then \
echo "ERROR: Failed to download models after $MAX_RETRIES attempts"; \
exit 1; \
fi; \
elif [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then echo "Skipping ML model preload (local-models not included)"; \
print('Models cached successfully')"; \
else echo "Skipping ML model preload"; fi
EXPOSE 8888
@@ -236,10 +182,6 @@ ENV HINDSIGHT_API_LOG_LEVEL=info
ENV HINDSIGHT_ENABLE_API=true
ENV HINDSIGHT_ENABLE_CP=false
ENV PYTHONUNBUFFERED=1
# Suppress verbose transformers/HuggingFace model loading warnings
ENV TRANSFORMERS_VERBOSITY=error
ENV HF_HUB_VERBOSITY=error
ENV TOKENIZERS_PARALLELISM=false
CMD ["/app/start-all.sh"]
@@ -284,7 +226,7 @@ FROM python:3.11-slim AS standalone
WORKDIR /app
# Install Node.js, curl, uv, and system dependencies
# Install Node.js, curl, uv, and pg0 dependencies (procps provides 'kill' command needed by pg0)
# Note: libicu version varies by Debian version - try common versions in order
RUN apt-get update && apt-get install -y \
curl \
@@ -299,6 +241,7 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/* \
&& pip install --no-cache-dir uv
# Create non-root user (PostgreSQL cannot run as root)
RUN useradd -m -s /bin/bash hindsight
# Copy API with virtual environment from builder
@@ -319,69 +262,37 @@ WORKDIR /app
COPY docker/standalone/start-all.sh /app/start-all.sh
RUN chmod +x /app/start-all.sh
RUN chown -R hindsight:hindsight /app
# Create data directory for pg0 and set ownership
RUN mkdir -p /app/data && chown -R hindsight:hindsight /app
# Switch to non-root user
USER hindsight
# Create pg0 data directory as hindsight user so that Docker seeds new named
# volumes with correct ownership (UID 1000) on first use, avoiding the
# "Permission denied" error when mounting a fresh root-owned volume.
RUN mkdir -p /home/hindsight/.pg0
# Set PATH for hindsight user
ENV PATH="/app/api/.venv/bin:${PATH}"
# Pre-download tiktoken encoding (ALWAYS - required for token counting even in air-gapped envs)
# Tiktoken is a core runtime dependency, not an optional ML model
RUN MAX_RETRIES=3; \
RETRY_DELAY=5; \
for i in $(seq 1 $MAX_RETRIES); do \
echo "Attempt $i/$MAX_RETRIES: Downloading tiktoken encoding..."; \
/app/api/.venv/bin/python -c "\
import tiktoken; \
print('Downloading cl100k_base encoding...'); \
tiktoken.get_encoding('cl100k_base'); \
print('Tiktoken encoding cached successfully')" && break; \
if [ $i -lt $MAX_RETRIES ]; then \
echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
sleep $RETRY_DELAY; \
RETRY_DELAY=$((RETRY_DELAY * 2)); \
fi; \
done; \
if [ $i -eq $MAX_RETRIES ]; then \
echo "ERROR: Failed to download tiktoken encoding after $MAX_RETRIES attempts"; \
exit 1; \
fi
# Pre-cache PostgreSQL binaries by starting/stopping pg0-embedded
ENV PG0_HOME=/home/hindsight/.pg0-cache
RUN /app/api/.venv/bin/python -c "\
from pg0 import Pg0; \
print('Pre-caching PostgreSQL binaries...'); \
pg = Pg0(name='hindsight', port=5555, username='hindsight', password='hindsight', database='hindsight'); \
pg.start(); \
pg.stop(); \
print('PostgreSQL pre-cached to PG0_HOME')" || echo "Pre-download skipped"
ENV PG0_HOME=/home/hindsight/.pg0
# Pre-download ML models to avoid runtime download (conditional)
# Only runs if both PRELOAD_ML_MODELS=true AND INCLUDE_LOCAL_MODELS=true
# Includes retry logic with exponential backoff for transient network failures
ARG PRELOAD_ML_MODELS
ARG INCLUDE_LOCAL_MODELS
ENV HF_HUB_DOWNLOAD_TIMEOUT=600
RUN if [ "$PRELOAD_ML_MODELS" = "true" ] && [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
MAX_RETRIES=3; \
RETRY_DELAY=10; \
for i in $(seq 1 $MAX_RETRIES); do \
echo "Attempt $i/$MAX_RETRIES: Downloading ML models..."; \
/app/api/.venv/bin/python -c "\
import os; os.environ['HF_HUB_DOWNLOAD_TIMEOUT'] = '600'; \
RUN if [ "$PRELOAD_ML_MODELS" = "true" ]; then \
/app/api/.venv/bin/python -c "\
from sentence_transformers import SentenceTransformer, CrossEncoder; \
print('Downloading embedding model...'); \
SentenceTransformer('BAAI/bge-small-en-v1.5'); \
print('Downloading cross-encoder model...'); \
CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'); \
print('Models cached successfully')" && break; \
if [ $i -lt $MAX_RETRIES ]; then \
echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
sleep $RETRY_DELAY; \
RETRY_DELAY=$((RETRY_DELAY * 2)); \
fi; \
done; \
if [ $i -eq $MAX_RETRIES ] && ! /app/api/.venv/bin/python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('BAAI/bge-small-en-v1.5')" 2>/dev/null; then \
echo "ERROR: Failed to download models after $MAX_RETRIES attempts"; \
exit 1; \
fi; \
elif [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then echo "Skipping ML model preload (local-models not included)"; \
print('Models cached successfully')"; \
else echo "Skipping ML model preload"; fi
EXPOSE 8888 9999
@@ -394,10 +305,6 @@ ENV HINDSIGHT_CP_DATAPLANE_API_URL=http://localhost:8888
ENV HINDSIGHT_ENABLE_API=true
ENV HINDSIGHT_ENABLE_CP=true
ENV PYTHONUNBUFFERED=1
# Suppress verbose transformers/HuggingFace model loading warnings
ENV TRANSFORMERS_VERBOSITY=error
ENV HF_HUB_VERBOSITY=error
ENV TOKENIZERS_PARALLELISM=false
CMD ["/app/start-all.sh"]
+13 -82
View File
@@ -5,70 +5,16 @@ set -e
ENABLE_API="${HINDSIGHT_ENABLE_API:-true}"
ENABLE_CP="${HINDSIGHT_ENABLE_CP:-true}"
# =============================================================================
# Dependency waiting (opt-in via HINDSIGHT_WAIT_FOR_DEPS=true)
#
# Problem: When running with LM Studio, the LLM may take time to load models.
# If Hindsight starts before LM Studio is ready, it fails on LLM verification.
# This wait loop ensures dependencies are ready before starting.
# =============================================================================
if [ "${HINDSIGHT_WAIT_FOR_DEPS:-false}" = "true" ]; then
LLM_BASE_URL="${HINDSIGHT_API_LLM_BASE_URL:-http://host.docker.internal:1234/v1}"
MAX_RETRIES="${HINDSIGHT_RETRY_MAX:-0}" # 0 = infinite
RETRY_INTERVAL="${HINDSIGHT_RETRY_INTERVAL:-10}"
# Check if external database is configured (skip check for embedded pg0)
SKIP_DB_CHECK=false
if [ -z "${HINDSIGHT_API_DATABASE_URL}" ]; then
SKIP_DB_CHECK=true
else
DB_CHECK_HOST=$(echo "$HINDSIGHT_API_DATABASE_URL" | sed -E 's|.*@([^:/]+):([0-9]+)/.*|\1 \2|')
# Copy pre-cached PostgreSQL data if runtime directory is empty (first run with volume)
if [ "$ENABLE_API" = "true" ]; then
PG0_CACHE="/home/hindsight/.pg0-cache"
PG0_HOME="/home/hindsight/.pg0"
if [ -d "$PG0_CACHE" ] && [ "$(ls -A $PG0_CACHE 2>/dev/null)" ]; then
if [ ! "$(ls -A $PG0_HOME 2>/dev/null)" ]; then
echo "📦 Copying pre-cached PostgreSQL data..."
cp -r "$PG0_CACHE"/* "$PG0_HOME"/ 2>/dev/null || true
fi
fi
check_db() {
if $SKIP_DB_CHECK; then
return 0
fi
if command -v pg_isready &> /dev/null; then
pg_isready -h $(echo $DB_CHECK_HOST | cut -d' ' -f1) -p $(echo $DB_CHECK_HOST | cut -d' ' -f2) &>/dev/null
else
python3 -c "import socket; s=socket.socket(); s.settimeout(5); exit(0 if s.connect_ex(('$(echo $DB_CHECK_HOST | cut -d' ' -f1)', $(echo $DB_CHECK_HOST | cut -d' ' -f2))) == 0 else 1)" 2>/dev/null
fi
}
check_llm() {
curl -sf "${LLM_BASE_URL}/models" --connect-timeout 5 &>/dev/null
}
echo "⏳ Waiting for dependencies to be ready..."
attempt=1
while true; do
db_ok=false
llm_ok=false
if check_db; then
db_ok=true
fi
if check_llm; then
llm_ok=true
fi
if $db_ok && $llm_ok; then
echo "✅ Dependencies ready!"
break
fi
if [ "$MAX_RETRIES" -ne 0 ] && [ "$attempt" -ge "$MAX_RETRIES" ]; then
echo "❌ Max retries ($MAX_RETRIES) reached. Dependencies not available."
exit 1
fi
echo " Attempt $attempt: DB=$( $db_ok && echo 'ok' || echo 'waiting' ), LLM=$( $llm_ok && echo 'ok' || echo 'waiting' )"
sleep "$RETRY_INTERVAL"
((attempt++))
done
fi
# Track PIDs for wait
@@ -77,32 +23,18 @@ PIDS=()
# Start API if enabled
if [ "$ENABLE_API" = "true" ]; then
cd /app/api
API_HEALTH_URL="${HINDSIGHT_API_HEALTH_URL:-http://localhost:8888/health}"
API_STARTUP_WAIT_SECONDS="${HINDSIGHT_API_STARTUP_WAIT_SECONDS:-300}"
# Run API directly - Python's PYTHONUNBUFFERED=1 handles output buffering
hindsight-api &
API_PID=$!
PIDS+=($API_PID)
# Wait for API to be ready
api_ready=false
for ((i=1; i<=API_STARTUP_WAIT_SECONDS; i++)); do
if ! kill -0 "$API_PID" 2>/dev/null; then
wait "$API_PID"
exit $?
fi
if curl -sf "$API_HEALTH_URL" &>/dev/null; then
api_ready=true
for i in {1..60}; do
if curl -sf http://localhost:8888/health &>/dev/null; then
break
fi
sleep 1
done
if [ "$api_ready" != "true" ]; then
echo "❌ API did not become healthy within ${API_STARTUP_WAIT_SECONDS}s"
exit 1
fi
else
echo "API disabled (HINDSIGHT_ENABLE_API=false)"
fi
@@ -111,8 +43,7 @@ fi
if [ "$ENABLE_CP" = "true" ]; then
echo "🎛️ Starting Control Plane..."
cd /app/control-plane
export HOSTNAME="${HINDSIGHT_CP_HOSTNAME:-0.0.0.0}"
PORT="${HINDSIGHT_CP_PORT:-9999}" node server.js &
PORT=9999 node server.js &
CP_PID=$!
PIDS+=($CP_PID)
else
@@ -125,7 +56,7 @@ echo "✅ Hindsight is running!"
echo ""
echo "📍 Access:"
if [ "$ENABLE_CP" = "true" ]; then
echo " Control Plane: http://localhost:${HINDSIGHT_CP_PORT:-9999}"
echo " Control Plane: http://localhost:9999"
fi
if [ "$ENABLE_API" = "true" ]; then
echo " API: http://localhost:8888"
-235
View File
@@ -1,235 +0,0 @@
#!/bin/bash
#
# Docker Smoke Test Script
#
# Tests that a Hindsight Docker image starts correctly and becomes healthy.
# Can be run locally or in CI pipelines.
#
# Usage:
# ./docker/test-image.sh <image> [target]
#
# Arguments:
# image - Docker image to test (e.g., hindsight-api:test, ghcr.io/vectorize-io/hindsight:latest)
# target - Optional: 'cp-only' for control plane, otherwise assumes API image (default: api)
#
# Environment variables:
# HINDSIGHT_API_LLM_API_KEY - Required for API/standalone images (LLM verification)
# HINDSIGHT_API_LLM_PROVIDER - LLM provider (default: openai)
# HINDSIGHT_API_LLM_MODEL - LLM model (default: gpt-4o-mini)
# HINDSIGHT_API_EMBEDDINGS_PROVIDER - Embeddings provider (optional, for slim images: openai, cohere, tei)
# HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY - OpenAI API key for embeddings (optional)
# HINDSIGHT_API_RERANKER_PROVIDER - Reranker provider (optional, for slim images: cohere, tei)
# HINDSIGHT_API_COHERE_API_KEY - Cohere API key for reranking (optional)
# SMOKE_TEST_TIMEOUT - Timeout in seconds (default: 120)
# SMOKE_TEST_CONTAINER_NAME - Container name (default: hindsight-smoke-test)
#
# Examples:
# # Test a locally built full image
# ./docker/test-image.sh hindsight-api:test
#
# # Test a released image
# ./docker/test-image.sh ghcr.io/vectorize-io/hindsight:latest
#
# # Test control plane image
# ./docker/test-image.sh hindsight-control-plane:test cp-only
#
# # Test slim image with external providers
# export HINDSIGHT_API_LLM_API_KEY=sk_xxx
# export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai
# export HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=sk-xxx
# export HINDSIGHT_API_RERANKER_PROVIDER=cohere
# export HINDSIGHT_API_COHERE_API_KEY=xxx
# ./docker/test-image.sh hindsight-slim:test
#
# Exit codes:
# 0 - Success (container healthy)
# 1 - Failure (container not healthy within timeout)
# 2 - Invalid arguments
#
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(dirname "$SCRIPT_DIR")"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
NC='\033[0m' # No Color
# Configuration
IMAGE="${1:-}"
TARGET="${2:-api}"
TIMEOUT="${SMOKE_TEST_TIMEOUT:-120}"
CONTAINER_NAME="${SMOKE_TEST_CONTAINER_NAME:-hindsight-smoke-test}"
LLM_PROVIDER="${HINDSIGHT_API_LLM_PROVIDER:-openai}"
LLM_MODEL="${HINDSIGHT_API_LLM_MODEL:-gpt-4o-mini}"
# Validate arguments
if [ -z "$IMAGE" ]; then
echo -e "${RED}Error: Image argument is required${NC}"
echo ""
echo "Usage: $0 <image> [target]"
echo ""
echo "Examples:"
echo " $0 hindsight-api:test"
echo " $0 ghcr.io/vectorize-io/hindsight:latest"
echo " $0 hindsight-control-plane:test cp-only"
exit 2
fi
# Determine health endpoint based on target
if [ "$TARGET" = "cp-only" ]; then
HEALTH_PORT=9999
HEALTH_PATH="/api/health"
NEEDS_LLM=false
else
HEALTH_PORT=8888
HEALTH_PATH="/health"
NEEDS_LLM=true
fi
# Check for required environment variables
if [ "$NEEDS_LLM" = true ] && [ "$LLM_PROVIDER" != "vertexai" ] && [ -z "${HINDSIGHT_API_LLM_API_KEY:-}" ]; then
echo -e "${RED}Error: HINDSIGHT_API_LLM_API_KEY environment variable is required for API/standalone images${NC}"
echo "Set it with: export HINDSIGHT_API_LLM_API_KEY=your-api-key"
exit 2
fi
# Cleanup function
cleanup() {
echo "Cleaning up..."
docker stop "$CONTAINER_NAME" 2>/dev/null || true
docker rm "$CONTAINER_NAME" 2>/dev/null || true
}
# Set trap to cleanup on exit
trap cleanup EXIT
echo -e "${YELLOW}Starting smoke test for: ${IMAGE}${NC}"
echo " Target: $TARGET"
echo " Health endpoint: http://localhost:${HEALTH_PORT}${HEALTH_PATH}"
echo " Timeout: ${TIMEOUT}s"
echo ""
# Remove any existing container with the same name
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
# Start container based on target type
echo "Starting container..."
if [ "$TARGET" = "cp-only" ]; then
docker run -d --name "$CONTAINER_NAME" \
-p "${HEALTH_PORT}:${HEALTH_PORT}" \
"$IMAGE"
else
# Build docker run command with required and optional env vars
DOCKER_CMD="docker run -d --name $CONTAINER_NAME"
DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_PROVIDER=$LLM_PROVIDER"
if [ -n "${HINDSIGHT_API_LLM_API_KEY:-}" ]; then
DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_API_KEY=${HINDSIGHT_API_LLM_API_KEY}"
fi
DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_MODEL=$LLM_MODEL"
# Add Vertex AI config if provider is vertexai
if [ "$LLM_PROVIDER" = "vertexai" ]; then
if [ -n "${HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY:-}" ]; then
DOCKER_CMD="$DOCKER_CMD -v ${HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY}:/tmp/gcp-credentials.json:ro"
DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY=/tmp/gcp-credentials.json"
fi
if [ -n "${HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID:-}" ]; then
DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=${HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID}"
fi
if [ -n "${HINDSIGHT_API_LLM_VERTEXAI_REGION:-}" ]; then
DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_LLM_VERTEXAI_REGION=${HINDSIGHT_API_LLM_VERTEXAI_REGION}"
fi
fi
# Add optional embeddings provider config
if [ -n "${HINDSIGHT_API_EMBEDDINGS_PROVIDER:-}" ]; then
DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_EMBEDDINGS_PROVIDER=${HINDSIGHT_API_EMBEDDINGS_PROVIDER}"
fi
if [ -n "${HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY:-}" ]; then
DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=${HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY}"
fi
# Add optional reranker provider config
if [ -n "${HINDSIGHT_API_RERANKER_PROVIDER:-}" ]; then
DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_RERANKER_PROVIDER=${HINDSIGHT_API_RERANKER_PROVIDER}"
fi
if [ -n "${HINDSIGHT_API_COHERE_API_KEY:-}" ]; then
DOCKER_CMD="$DOCKER_CMD -e HINDSIGHT_API_COHERE_API_KEY=${HINDSIGHT_API_COHERE_API_KEY}"
fi
DOCKER_CMD="$DOCKER_CMD -p ${HEALTH_PORT}:${HEALTH_PORT}"
DOCKER_CMD="$DOCKER_CMD $IMAGE"
eval $DOCKER_CMD
fi
# Wait for health endpoint
echo "Waiting for health endpoint at http://localhost:${HEALTH_PORT}${HEALTH_PATH}..."
start_time=$(date +%s)
for i in $(seq 1 "$TIMEOUT"); do
if curl -sf "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" > /dev/null 2>&1; then
end_time=$(date +%s)
duration=$((end_time - start_time))
echo ""
echo -e "${GREEN}Container is healthy after ${duration}s${NC}"
echo ""
echo "=== Health Response ==="
curl -s "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" | python3 -m json.tool 2>/dev/null || curl -s "http://localhost:${HEALTH_PORT}${HEALTH_PATH}"
echo ""
# Run retain/recall smoke test for API targets
if [ "$TARGET" != "cp-only" ]; then
echo ""
echo "=== Retain/Recall Smoke Test ==="
if ! "$REPO_ROOT/scripts/smoke-test-slim.sh" "http://localhost:${HEALTH_PORT}"; then
echo ""
echo "=== Container Logs (last 50 lines) ==="
docker logs "$CONTAINER_NAME" 2>&1 | tail -50
echo ""
echo -e "${RED}Smoke test FAILED${NC}"
exit 1
fi
fi
echo ""
echo "=== Container Logs (last 50 lines) ==="
docker logs "$CONTAINER_NAME" 2>&1 | tail -50
echo ""
echo -e "${GREEN}Smoke test PASSED${NC}"
exit 0
fi
# Show progress every 10 seconds
if [ $((i % 10)) -eq 0 ]; then
echo " Still waiting... (${i}s)"
fi
# Check if container is still running
if ! docker ps -q -f "name=$CONTAINER_NAME" | grep -q .; then
echo ""
echo -e "${RED}Container exited unexpectedly!${NC}"
echo ""
echo "=== Container Logs ==="
docker logs "$CONTAINER_NAME" 2>&1
echo ""
echo -e "${RED}Smoke test FAILED${NC}"
exit 1
fi
sleep 1
done
# Timeout reached
echo ""
echo -e "${RED}Container failed to become healthy after ${TIMEOUT}s${NC}"
echo ""
echo "=== Container Logs ==="
docker logs "$CONTAINER_NAME" 2>&1
echo ""
echo -e "${RED}Smoke test FAILED${NC}"
exit 1
-47
View File
@@ -1,47 +0,0 @@
#!/bin/bash
#
# Local Test Script for Slim Docker Images
#
# This script makes it easy to test slim images locally with external providers.
# It expects API keys to be set in environment variables.
#
# Usage:
# export OPENAI_API_KEY=sk-xxx
# export COHERE_API_KEY=xxx
# ./docker/test-slim-local.sh
#
# Or inline:
# OPENAI_API_KEY=sk_xxx COHERE_API_KEY=xxx ./docker/test-slim-local.sh
#
set -euo pipefail
# Check for required API keys
if [ -z "${OPENAI_API_KEY:-}" ]; then
echo "❌ Error: OPENAI_API_KEY environment variable is required"
echo "Set it with: export OPENAI_API_KEY=sk-xxx"
exit 1
fi
if [ -z "${COHERE_API_KEY:-}" ]; then
echo "❌ Error: COHERE_API_KEY environment variable is required"
echo "Set it with: export COHERE_API_KEY=xxx"
exit 1
fi
# Configuration
IMAGE="${1:-hindsight-slim:test}"
echo "Testing image: $IMAGE"
echo ""
# Set up LLM and external providers
export HINDSIGHT_API_LLM_PROVIDER=openai
export HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY
export HINDSIGHT_API_LLM_MODEL=gpt-4o-mini
export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai
export HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=$OPENAI_API_KEY
export HINDSIGHT_API_RERANKER_PROVIDER=cohere
export HINDSIGHT_API_COHERE_API_KEY=$COHERE_API_KEY
# Run the test
exec "$(dirname "$0")/test-image.sh" "$IMAGE" standalone
+2 -2
View File
@@ -2,8 +2,8 @@ apiVersion: v2
name: hindsight
description: Hindsight helm chart
type: application
version: 0.4.19
appVersion: "0.4.19"
version: 0.1.13
appVersion: "0.1.13"
keywords:
- ai
- memory
-59
View File
@@ -80,22 +80,6 @@ Control plane selector labels
app.kubernetes.io/component: control-plane
{{- end }}
{{/*
Worker labels
*/}}
{{- define "hindsight.worker.labels" -}}
{{ include "hindsight.labels" . }}
app.kubernetes.io/component: worker
{{- end }}
{{/*
Worker selector labels
*/}}
{{- define "hindsight.worker.selectorLabels" -}}
{{ include "hindsight.selectorLabels" . }}
app.kubernetes.io/component: worker
{{- end }}
{{/*
Create the name of the service account to use
*/}}
@@ -126,46 +110,3 @@ API URL for control plane
{{- define "hindsight.apiUrl" -}}
{{- printf "http://%s-api:%d" (include "hindsight.fullname" .) (.Values.api.service.port | int) }}
{{- end }}
{{/*
TEI reranker labels
*/}}
{{- define "hindsight.tei.reranker.labels" -}}
{{ include "hindsight.labels" . }}
app.kubernetes.io/component: tei-reranker
{{- end }}
{{/*
TEI reranker selector labels
*/}}
{{- define "hindsight.tei.reranker.selectorLabels" -}}
{{ include "hindsight.selectorLabels" . }}
app.kubernetes.io/component: tei-reranker
{{- end }}
{{/*
TEI embedding labels
*/}}
{{- define "hindsight.tei.embedding.labels" -}}
{{ include "hindsight.labels" . }}
app.kubernetes.io/component: tei-embedding
{{- end }}
{{/*
TEI embedding selector labels
*/}}
{{- define "hindsight.tei.embedding.selectorLabels" -}}
{{ include "hindsight.selectorLabels" . }}
app.kubernetes.io/component: tei-embedding
{{- end }}
{{/*
Get the name of the secret to use
*/}}
{{- define "hindsight.secretName" -}}
{{- if .Values.existingSecret }}
{{- .Values.existingSecret }}
{{- else }}
{{- printf "%s-secret" (include "hindsight.fullname" .) }}
{{- end }}
{{- end }}
+6 -37
View File
@@ -15,9 +15,7 @@ spec:
template:
metadata:
annotations:
{{- if not .Values.existingSecret }}
checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
@@ -33,62 +31,33 @@ spec:
- name: api
securityContext:
{{- toYaml .Values.securityContext | nindent 10 }}
image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag | default .Values.version | default .Chart.AppVersion }}"
image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag | default .Values.version }}"
imagePullPolicy: {{ .Values.api.image.pullPolicy }}
ports:
- name: http
containerPort: {{ .Values.api.service.targetPort }}
protocol: TCP
{{- if .Values.existingSecret }}
envFrom:
- secretRef:
name: {{ .Values.existingSecret }}
{{- end }}
env:
{{- /* POSTGRES_PASSWORD must be defined before DATABASE_URL for $(VAR) interpolation */}}
- name: HINDSIGHT_API_DATABASE_URL
value: {{ include "hindsight.databaseUrl" . | quote }}
{{- if not .Values.postgresql.enabled }}
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "hindsight.secretName" . }}
name: {{ include "hindsight.fullname" . }}-secret
key: postgres-password
{{- end }}
- name: HINDSIGHT_API_DATABASE_URL
value: {{ include "hindsight.databaseUrl" . | quote }}
{{- /* Disable internal worker when dedicated workers are enabled */}}
{{- if .Values.worker.enabled }}
- name: HINDSIGHT_API_WORKER_ENABLED
value: "false"
{{- end }}
{{- /* Explicitly set port to override K8s service discovery env var (HINDSIGHT_API_PORT) */}}
- name: HINDSIGHT_API_PORT
value: {{ .Values.api.service.targetPort | quote }}
{{- range $key, $value := .Values.api.env }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
{{- if .Values.tei.reranker.enabled }}
- name: HINDSIGHT_API_RERANKER_PROVIDER
value: "tei"
- name: HINDSIGHT_API_RERANKER_TEI_URL
value: "http://{{ include "hindsight.fullname" . }}-tei-reranker:{{ .Values.tei.reranker.port }}"
{{- end }}
{{- if .Values.tei.embedding.enabled }}
- name: HINDSIGHT_API_EMBEDDINGS_PROVIDER
value: "tei"
- name: HINDSIGHT_API_EMBEDDINGS_TEI_URL
value: "http://{{ include "hindsight.fullname" . }}-tei-embedding:{{ .Values.tei.embedding.port }}"
{{- end }}
{{- /* Only use api.secrets when not using existingSecret (for chart-managed secrets) */}}
{{- if not .Values.existingSecret }}
{{- range $key, $value := .Values.api.secrets }}
- name: {{ $key }}
valueFrom:
secretKeyRef:
name: {{ include "hindsight.secretName" $ }}
name: {{ include "hindsight.fullname" $ }}-secret
key: {{ $key }}
{{- end }}
{{- end }}
livenessProbe:
{{- toYaml .Values.api.livenessProbe | nindent 10 }}
readinessProbe:
@@ -99,7 +68,7 @@ spec:
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with (.Values.api.affinity | default .Values.affinity) }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
@@ -15,9 +15,7 @@ spec:
template:
metadata:
annotations:
{{- if not .Values.existingSecret }}
checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
@@ -33,17 +31,12 @@ spec:
- name: control-plane
securityContext:
{{- toYaml .Values.securityContext | nindent 10 }}
image: "{{ .Values.controlPlane.image.repository }}:{{ .Values.controlPlane.image.tag | default .Values.version | default .Chart.AppVersion }}"
image: "{{ .Values.controlPlane.image.repository }}:{{ .Values.controlPlane.image.tag | default .Values.version }}"
imagePullPolicy: {{ .Values.controlPlane.image.pullPolicy }}
ports:
- name: http
containerPort: {{ .Values.controlPlane.service.targetPort }}
protocol: TCP
{{- if .Values.existingSecret }}
envFrom:
- secretRef:
name: {{ .Values.existingSecret }}
{{- end }}
env:
- name: HINDSIGHT_CP_DATAPLANE_API_URL
value: {{ include "hindsight.apiUrl" . | quote }}
@@ -51,16 +44,13 @@ spec:
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
{{- /* Only use controlPlane.secrets when not using existingSecret (for chart-managed secrets) */}}
{{- if not .Values.existingSecret }}
{{- range $key, $value := .Values.controlPlane.secrets }}
- name: {{ $key }}
valueFrom:
secretKeyRef:
name: {{ include "hindsight.secretName" $ }}
name: {{ include "hindsight.fullname" $ }}-secret
key: {{ $key }}
{{- end }}
{{- end }}
livenessProbe:
{{- toYaml .Values.controlPlane.livenessProbe | nindent 10 }}
readinessProbe:
@@ -71,7 +61,7 @@ spec:
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with (.Values.controlPlane.affinity | default .Values.affinity) }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
-56
View File
@@ -1,56 +0,0 @@
{{- if and .Values.api.enabled .Values.api.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "hindsight.fullname" . }}-api
labels:
{{- include "hindsight.api.labels" . | nindent 4 }}
spec:
{{- if .Values.api.podDisruptionBudget.minAvailable }}
minAvailable: {{ .Values.api.podDisruptionBudget.minAvailable }}
{{- end }}
{{- if .Values.api.podDisruptionBudget.maxUnavailable }}
maxUnavailable: {{ .Values.api.podDisruptionBudget.maxUnavailable }}
{{- end }}
selector:
matchLabels:
{{- include "hindsight.api.selectorLabels" . | nindent 6 }}
{{- end }}
---
{{- if and .Values.controlPlane.enabled .Values.controlPlane.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "hindsight.fullname" . }}-control-plane
labels:
{{- include "hindsight.controlPlane.labels" . | nindent 4 }}
spec:
{{- if .Values.controlPlane.podDisruptionBudget.minAvailable }}
minAvailable: {{ .Values.controlPlane.podDisruptionBudget.minAvailable }}
{{- end }}
{{- if .Values.controlPlane.podDisruptionBudget.maxUnavailable }}
maxUnavailable: {{ .Values.controlPlane.podDisruptionBudget.maxUnavailable }}
{{- end }}
selector:
matchLabels:
{{- include "hindsight.controlPlane.selectorLabels" . | nindent 6 }}
{{- end }}
---
{{- if and .Values.worker.enabled .Values.worker.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "hindsight.fullname" . }}-worker
labels:
{{- include "hindsight.worker.labels" . | nindent 4 }}
spec:
{{- if .Values.worker.podDisruptionBudget.minAvailable }}
minAvailable: {{ .Values.worker.podDisruptionBudget.minAvailable }}
{{- end }}
{{- if .Values.worker.podDisruptionBudget.maxUnavailable }}
maxUnavailable: {{ .Values.worker.podDisruptionBudget.maxUnavailable }}
{{- end }}
selector:
matchLabels:
{{- include "hindsight.worker.selectorLabels" . | nindent 6 }}
{{- end }}
+1 -3
View File
@@ -1,8 +1,7 @@
{{- if not .Values.existingSecret }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "hindsight.secretName" . }}
name: {{ include "hindsight.fullname" . }}-secret
labels:
{{- include "hindsight.labels" . | nindent 4 }}
type: Opaque
@@ -16,4 +15,3 @@ data:
{{- if and (not .Values.postgresql.enabled) .Values.postgresql.external.password }}
postgres-password: {{ .Values.postgresql.external.password | b64enc | quote }}
{{- end }}
{{- end }}
@@ -1,76 +0,0 @@
{{- if .Values.tei.embedding.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "hindsight.fullname" . }}-tei-embedding
labels:
{{- include "hindsight.tei.embedding.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.tei.embedding.replicaCount }}
selector:
matchLabels:
{{- include "hindsight.tei.embedding.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "hindsight.tei.embedding.selectorLabels" . | nindent 8 }}
spec:
{{- if .Values.serviceAccount.create }}
serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
{{- end }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: tei-embedding
securityContext:
{{- toYaml .Values.securityContext | nindent 10 }}
image: "{{ .Values.tei.embedding.image.repository }}:{{ .Values.tei.embedding.image.tag }}"
imagePullPolicy: {{ .Values.tei.embedding.image.pullPolicy }}
args:
- "--model-id"
- {{ .Values.tei.embedding.model | quote }}
- "--hostname"
- "0.0.0.0"
{{- range .Values.tei.embedding.args }}
- {{ . | quote }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.tei.embedding.port }}
protocol: TCP
env:
- name: PORT
value: {{ .Values.tei.embedding.port | quote }}
{{- range $key, $value := .Values.tei.embedding.env }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
livenessProbe:
{{- toYaml .Values.tei.embedding.livenessProbe | nindent 10 }}
readinessProbe:
{{- toYaml .Values.tei.embedding.readinessProbe | nindent 10 }}
resources:
{{- toYaml .Values.tei.embedding.resources | nindent 10 }}
volumeMounts:
- name: model-cache
mountPath: /data
volumes:
- name: model-cache
emptyDir: {}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
@@ -1,17 +0,0 @@
{{- if .Values.tei.embedding.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "hindsight.fullname" . }}-tei-embedding
labels:
{{- include "hindsight.tei.embedding.labels" . | nindent 4 }}
spec:
type: ClusterIP
ports:
- port: {{ .Values.tei.embedding.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "hindsight.tei.embedding.selectorLabels" . | nindent 4 }}
{{- end }}
@@ -1,76 +0,0 @@
{{- if .Values.tei.reranker.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "hindsight.fullname" . }}-tei-reranker
labels:
{{- include "hindsight.tei.reranker.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.tei.reranker.replicaCount }}
selector:
matchLabels:
{{- include "hindsight.tei.reranker.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "hindsight.tei.reranker.selectorLabels" . | nindent 8 }}
spec:
{{- if .Values.serviceAccount.create }}
serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
{{- end }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: tei-reranker
securityContext:
{{- toYaml .Values.securityContext | nindent 10 }}
image: "{{ .Values.tei.reranker.image.repository }}:{{ .Values.tei.reranker.image.tag }}"
imagePullPolicy: {{ .Values.tei.reranker.image.pullPolicy }}
args:
- "--model-id"
- {{ .Values.tei.reranker.model | quote }}
- "--hostname"
- "0.0.0.0"
{{- range .Values.tei.reranker.args }}
- {{ . | quote }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.tei.reranker.port }}
protocol: TCP
env:
- name: PORT
value: {{ .Values.tei.reranker.port | quote }}
{{- range $key, $value := .Values.tei.reranker.env }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
livenessProbe:
{{- toYaml .Values.tei.reranker.livenessProbe | nindent 10 }}
readinessProbe:
{{- toYaml .Values.tei.reranker.readinessProbe | nindent 10 }}
resources:
{{- toYaml .Values.tei.reranker.resources | nindent 10 }}
volumeMounts:
- name: model-cache
mountPath: /data
volumes:
- name: model-cache
emptyDir: {}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
@@ -1,17 +0,0 @@
{{- if .Values.tei.reranker.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "hindsight.fullname" . }}-tei-reranker
labels:
{{- include "hindsight.tei.reranker.labels" . | nindent 4 }}
spec:
type: ClusterIP
ports:
- port: {{ .Values.tei.reranker.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "hindsight.tei.reranker.selectorLabels" . | nindent 4 }}
{{- end }}
@@ -1,25 +0,0 @@
{{- if .Values.worker.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "hindsight.fullname" . }}-worker
labels:
{{- include "hindsight.worker.labels" . | nindent 4 }}
{{- if .Values.podAnnotations }}
annotations:
{{- /* Common Prometheus annotations for metrics scraping */}}
prometheus.io/scrape: "true"
prometheus.io/port: {{ .Values.worker.service.port | quote }}
prometheus.io/path: "/metrics"
{{- end }}
spec:
# Headless service for StatefulSet (enables stable DNS names like worker-0.worker.namespace)
clusterIP: None
ports:
- port: {{ .Values.worker.service.port }}
targetPort: {{ .Values.worker.service.targetPort }}
protocol: TCP
name: http
selector:
{{- include "hindsight.worker.selectorLabels" . | nindent 4 }}
{{- end }}
@@ -1,110 +0,0 @@
{{- if .Values.worker.enabled }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ include "hindsight.fullname" . }}-worker
labels:
{{- include "hindsight.worker.labels" . | nindent 4 }}
spec:
serviceName: {{ include "hindsight.fullname" . }}-worker
replicas: {{ .Values.worker.replicaCount }}
selector:
matchLabels:
{{- include "hindsight.worker.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
{{- if not .Values.existingSecret }}
checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "hindsight.worker.selectorLabels" . | nindent 8 }}
spec:
{{- if .Values.serviceAccount.create }}
serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
{{- end }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: worker
securityContext:
{{- toYaml .Values.securityContext | nindent 10 }}
image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Values.version | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
command: ["hindsight-worker"]
ports:
- name: http
containerPort: {{ .Values.worker.service.targetPort }}
protocol: TCP
{{- if .Values.existingSecret }}
envFrom:
- secretRef:
name: {{ .Values.existingSecret }}
{{- end }}
env:
{{- /* POSTGRES_PASSWORD must be defined before DATABASE_URL for $(VAR) interpolation */}}
{{- if not .Values.postgresql.enabled }}
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "hindsight.secretName" . }}
key: postgres-password
{{- end }}
- name: HINDSIGHT_API_DATABASE_URL
value: {{ include "hindsight.databaseUrl" . | quote }}
{{- /* Worker ID uses pod name (StatefulSet provides stable names like worker-0, worker-1) */}}
- name: HINDSIGHT_API_WORKER_ID
valueFrom:
fieldRef:
fieldPath: metadata.name
{{- /* Inherit LLM config from api.env */}}
{{- range $key, $value := .Values.api.env }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
{{- /* Worker-specific env vars */}}
{{- range $key, $value := .Values.worker.env }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
{{- /* Only use secrets when not using existingSecret */}}
{{- if not .Values.existingSecret }}
{{- /* Inherit secrets from api.secrets */}}
{{- range $key, $value := .Values.api.secrets }}
- name: {{ $key }}
valueFrom:
secretKeyRef:
name: {{ include "hindsight.secretName" $ }}
key: {{ $key }}
{{- end }}
{{- /* Worker-specific secrets (can override api.secrets) */}}
{{- range $key, $value := .Values.worker.secrets }}
- name: {{ $key }}
valueFrom:
secretKeyRef:
name: {{ include "hindsight.secretName" $ }}
key: {{ $key }}
{{- end }}
{{- end }}
livenessProbe:
{{- toYaml .Values.worker.livenessProbe | nindent 10 }}
readinessProbe:
{{- toYaml .Values.worker.readinessProbe | nindent 10 }}
resources:
{{- toYaml .Values.worker.resources | nindent 10 }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with (.Values.worker.affinity | default .Values.affinity) }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
+3 -175
View File
@@ -1,17 +1,7 @@
# Default values for hindsight
# Global version override - use this to set a consistent image tag across all components
# If not set, defaults to Chart.appVersion from Chart.yaml
# version: ""
# Use an existing secret instead of creating one from values
# When set, all keys from this secret are injected as environment variables via envFrom
# Required keys:
# - postgres-password: PostgreSQL password (when postgresql.enabled=false)
# Optional keys (any key becomes an env var):
# - HINDSIGHT_API_LLM_API_KEY: API key for LLM provider
# - Any other env vars you want to inject
# existingSecret: "my-hindsight-secret"
# Chart version - use this to set a consistent image tag across all components
version: "0.1.1"
# Global settings
replicaCount: 1
@@ -58,15 +48,6 @@ api:
timeoutSeconds: 3
failureThreshold: 3
# Pod disruption budget
podDisruptionBudget:
enabled: false
minAvailable: 1
# maxUnavailable: 1
# Pod affinity/anti-affinity (overrides global affinity for this component)
# affinity: {}
# Environment variables
env:
#HINDSIGHT_API_LLM_PROVIDER: "groq"
@@ -77,72 +58,6 @@ api:
# HINDSIGHT_API_LLM_API_KEY: "your-api-key"
# HINDSIGHT_API_LLM_BASE_URL: "https://api.groq.com/openai/v1"
# Worker settings (distributed task processing)
# When enabled, dedicated worker pods process tasks and the API's internal worker is disabled
worker:
enabled: false
replicaCount: 2
image:
repository: ghcr.io/vectorize-io/hindsight-api
pullPolicy: IfNotPresent
# tag: "" # defaults to .Values.version, then Chart.appVersion if not specified
service:
# Service for metrics scraping (headless for StatefulSet)
port: 8889
targetPort: 8889
# Resource limits and requests
resources:
limits:
cpu: 2000m
memory: 4Gi
requests:
cpu: 500m
memory: 1Gi
# Liveness and readiness probes
livenessProbe:
httpGet:
path: /health
port: 8889
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
readinessProbe:
httpGet:
path: /health
port: 8889
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 3
# Worker-specific environment variables
env:
# Poll interval in milliseconds (how often to check for new tasks)
HINDSIGHT_API_WORKER_POLL_INTERVAL_MS: "500"
# Number of tasks to claim per poll cycle
HINDSIGHT_API_WORKER_BATCH_SIZE: "10"
# Max retries before marking a task as failed
HINDSIGHT_API_WORKER_MAX_RETRIES: "3"
# HTTP port for metrics/health (matches service.targetPort)
HINDSIGHT_API_WORKER_HTTP_PORT: "8889"
# Pod disruption budget
podDisruptionBudget:
enabled: false
minAvailable: 1
# maxUnavailable: 1
# Pod affinity/anti-affinity (overrides global affinity for this component)
# affinity: {}
# Secret environment variables (inherited from api.secrets if not specified)
secrets: {}
# Image settings for control plane
controlPlane:
enabled: true
@@ -183,15 +98,6 @@ controlPlane:
timeoutSeconds: 3
failureThreshold: 3
# Pod disruption budget
podDisruptionBudget:
enabled: false
minAvailable: 1
# maxUnavailable: 1
# Pod affinity/anti-affinity (overrides global affinity for this component)
# affinity: {}
# Environment variables
env:
NODE_ENV: "production"
@@ -290,87 +196,9 @@ nodeSelector: {}
# Tolerations
tolerations: []
# Affinity (applied to all components unless overridden per-component)
# Affinity
affinity: {}
# TEI (Text Embeddings Inference) - optional standalone deployments
# for reranking and/or embedding models
tei:
reranker:
enabled: false
replicaCount: 1
image:
repository: ghcr.io/huggingface/text-embeddings-inference
tag: cpu-1.8.3
pullPolicy: IfNotPresent
model: "cross-encoder/ms-marco-MiniLM-L-6-v2"
port: 8090
args:
- "--auto-truncate"
env:
PAYLOAD_LIMIT: "10000000"
MAX_CLIENT_BATCH_SIZE: "256"
resources:
limits:
cpu: 2000m
memory: 2Gi
requests:
cpu: 500m
memory: 1Gi
livenessProbe:
httpGet:
path: /health
port: 8090
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
readinessProbe:
httpGet:
path: /health
port: 8090
initialDelaySeconds: 15
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 3
embedding:
enabled: false
replicaCount: 1
image:
repository: ghcr.io/huggingface/text-embeddings-inference
tag: cpu-1.8.3
pullPolicy: IfNotPresent
model: "sentence-transformers/all-MiniLM-L6-v2"
port: 8091
args: []
env:
PAYLOAD_LIMIT: "10000000"
MAX_CLIENT_BATCH_SIZE: "256"
resources:
limits:
cpu: 2000m
memory: 2Gi
requests:
cpu: 500m
memory: 1Gi
livenessProbe:
httpGet:
path: /health
port: 8091
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
readinessProbe:
httpGet:
path: /health
port: 8091
initialDelaySeconds: 15
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 3
# Autoscaling
autoscaling:
enabled: false
-33
View File
@@ -1,33 +0,0 @@
[build-system]
requires = ["setuptools>=61"]
build-backend = "setuptools.build_meta"
[project]
name = "hindsight-all-slim"
version = "0.4.19"
description = "Hindsight: Agent Memory That Works Like Human Memory - Slim All-in-One Bundle"
readme = "README.md"
requires-python = ">=3.11"
dependencies = [
"hindsight-api-slim>=0.4.17",
"hindsight-client>=0.0.7",
"hindsight-embed>=0.1.0",
]
[tool.uv.sources]
hindsight-api-slim = { workspace = true }
hindsight-client = { workspace = true }
hindsight-embed = { workspace = true }
[project.optional-dependencies]
test = [
"pytest>=7.0.0",
"pytest-asyncio>=0.21.0",
]
[tool.setuptools]
packages = []
[tool.pytest.ini_options]
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
-48
View File
@@ -1,48 +0,0 @@
# hindsight-all
All-in-one package for Hindsight - Agent Memory That Works Like Human Memory
## Quick Start
```python
from hindsight import start_server, HindsightClient
# Start server with embedded PostgreSQL
server = start_server(
llm_provider="groq",
llm_api_key="your-api-key",
llm_model="openai/gpt-oss-120b"
)
# Create client
client = HindsightClient(base_url=server.url)
# Store memories
client.put(agent_id="assistant", content="User prefers Python for data analysis")
# Search memories
results = client.search(agent_id="assistant", query="programming preferences")
# Generate contextual response
response = client.think(agent_id="assistant", query="What languages should I recommend?")
# Stop server when done
server.stop()
```
## Using Context Manager
```python
from hindsight import HindsightServer, HindsightClient
with HindsightServer(llm_provider="groq", llm_api_key="...") as server:
client = HindsightClient(base_url=server.url)
# ... use client ...
# Server automatically stops
```
## Installation
```bash
pip install hindsight-all
```
-193
View File
@@ -1,193 +0,0 @@
"""
API namespace classes for organizing client methods.
These classes provide organized access to different parts of the Hindsight API
while ensuring the daemon is running before each call.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from .embedded import HindsightEmbedded
class BanksAPI:
"""Namespace for bank-related operations."""
def __init__(self, embedded: "HindsightEmbedded"):
self._embedded = embedded
def create(
self,
bank_id: str,
name: str | None = None,
mission: str | None = None,
disposition: dict[str, Any] | None = None,
):
"""Create a new bank."""
self._embedded._ensure_started()
return self._embedded._client.create_bank(
bank_id=bank_id,
name=name,
mission=mission,
disposition=disposition,
)
def delete(self, bank_id: str):
"""Delete a bank."""
self._embedded._ensure_started()
return self._embedded._client.delete_bank(bank_id=bank_id)
def set_mission(self, bank_id: str, mission: str):
"""Set or update the mission for a bank."""
self._embedded._ensure_started()
return self._embedded._client.set_mission(bank_id=bank_id, mission=mission)
def set_disposition(self, bank_id: str, disposition: dict[str, Any]):
"""Set or update the disposition for a bank."""
self._embedded._ensure_started()
return self._embedded._client.set_disposition(bank_id=bank_id, disposition=disposition)
class MentalModelsAPI:
"""Namespace for mental model operations."""
def __init__(self, embedded: "HindsightEmbedded"):
self._embedded = embedded
def create(
self,
bank_id: str,
name: str,
content: str,
tags: list[str] | None = None,
):
"""Create a new mental model."""
self._embedded._ensure_started()
return self._embedded._client.create_mental_model(
bank_id=bank_id,
name=name,
content=content,
tags=tags,
)
def list(self, bank_id: str, tags: list[str] | None = None):
"""List all mental models for a bank."""
self._embedded._ensure_started()
return self._embedded._client.list_mental_models(bank_id=bank_id, tags=tags)
def get(self, bank_id: str, mental_model_id: str):
"""Get a specific mental model."""
self._embedded._ensure_started()
return self._embedded._client.get_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
def refresh(self, bank_id: str, mental_model_id: str):
"""Refresh a mental model."""
self._embedded._ensure_started()
return self._embedded._client.refresh_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
def update(
self,
bank_id: str,
mental_model_id: str,
name: str | None = None,
content: str | None = None,
tags: list[str] | None = None,
):
"""Update a mental model."""
self._embedded._ensure_started()
return self._embedded._client.update_mental_model(
bank_id=bank_id,
mental_model_id=mental_model_id,
name=name,
content=content,
tags=tags,
)
def delete(self, bank_id: str, mental_model_id: str):
"""Delete a mental model."""
self._embedded._ensure_started()
return self._embedded._client.delete_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
class DirectivesAPI:
"""Namespace for directive operations."""
def __init__(self, embedded: "HindsightEmbedded"):
self._embedded = embedded
def create(
self,
bank_id: str,
name: str,
content: str,
tags: list[str] | None = None,
):
"""Create a new directive."""
self._embedded._ensure_started()
return self._embedded._client.create_directive(
bank_id=bank_id,
name=name,
content=content,
tags=tags,
)
def list(self, bank_id: str, tags: list[str] | None = None):
"""List all directives for a bank."""
self._embedded._ensure_started()
return self._embedded._client.list_directives(bank_id=bank_id, tags=tags)
def get(self, bank_id: str, directive_id: str):
"""Get a specific directive."""
self._embedded._ensure_started()
return self._embedded._client.get_directive(bank_id=bank_id, directive_id=directive_id)
def update(
self,
bank_id: str,
directive_id: str,
name: str | None = None,
content: str | None = None,
tags: list[str] | None = None,
):
"""Update a directive."""
self._embedded._ensure_started()
return self._embedded._client.update_directive(
bank_id=bank_id,
directive_id=directive_id,
name=name,
content=content,
tags=tags,
)
def delete(self, bank_id: str, directive_id: str):
"""Delete a directive."""
self._embedded._ensure_started()
return self._embedded._client.delete_directive(bank_id=bank_id, directive_id=directive_id)
class MemoriesAPI:
"""Namespace for memory operations."""
def __init__(self, embedded: "HindsightEmbedded"):
self._embedded = embedded
def list(
self,
bank_id: str,
type: str | None = None,
search_query: str | None = None,
limit: int = 100,
offset: int = 0,
):
"""List memories in a bank."""
self._embedded._ensure_started()
return self._embedded._client.list_memories(
bank_id=bank_id,
type=type,
search_query=search_query,
limit=limit,
offset=offset,
)
-423
View File
@@ -1,423 +0,0 @@
"""
Wrapper for Hindsight client that adds API namespaces.
Provides organized access to different parts of the Hindsight API through
namespaces like .banks, .mental_models, etc.
"""
from __future__ import annotations
from typing import Any
from hindsight_client import Hindsight
class BanksAPI:
"""Namespace for bank-related operations.
Provides methods to create, delete, and manage memory banks.
"""
def __init__(self, client: Hindsight):
self._client = client
def create(
self,
bank_id: str,
name: str | None = None,
mission: str | None = None,
disposition: dict[str, Any] | None = None,
) -> Any:
"""Create a new bank.
Args:
bank_id: Unique identifier for the bank.
name: Optional display name for the bank.
mission: Optional mission statement for the bank.
disposition: Optional disposition configuration dict.
Returns:
Bank creation response from the API.
"""
return self._client.create_bank(
bank_id=bank_id,
name=name,
mission=mission,
disposition=disposition,
)
def delete(self, bank_id: str) -> Any:
"""Delete a bank.
Args:
bank_id: The ID of the bank to delete.
Returns:
Deletion response from the API.
"""
return self._client.delete_bank(bank_id=bank_id)
def set_mission(self, bank_id: str, mission: str) -> Any:
"""Set or update the mission for a bank.
Args:
bank_id: The ID of the bank.
mission: The mission statement to set.
Returns:
API response confirming the update.
"""
return self._client.set_mission(bank_id=bank_id, mission=mission)
def set_disposition(self, bank_id: str, disposition: dict[str, Any]) -> Any:
"""Set or update the disposition for a bank.
Args:
bank_id: The ID of the bank.
disposition: The disposition configuration dict.
Returns:
API response confirming the update.
"""
return self._client.set_disposition(bank_id=bank_id, disposition=disposition)
def list(self) -> Any:
"""List all banks.
Returns:
List of banks from the API.
"""
from hindsight_client.hindsight_client import _run_async
return _run_async(self._client._banks_api.list_banks())
class MentalModelsAPI:
"""Namespace for mental model operations.
Mental models are reusable knowledge structures that guide agent behavior.
"""
def __init__(self, client: Hindsight):
self._client = client
def create(
self,
bank_id: str,
name: str,
content: str,
tags: list[str] | None = None,
) -> Any:
"""Create a new mental model.
Args:
bank_id: The ID of the bank to add the model to.
name: Name for the mental model.
content: The content/instructions for the mental model.
tags: Optional list of tags for categorization.
Returns:
Creation response from the API.
"""
return self._client.create_mental_model(
bank_id=bank_id,
name=name,
content=content,
tags=tags,
)
def list(self, bank_id: str, tags: list[str] | None = None) -> Any:
"""List all mental models for a bank.
Args:
bank_id: The ID of the bank.
tags: Optional filter by tags.
Returns:
List of mental models.
"""
return self._client.list_mental_models(bank_id=bank_id, tags=tags)
def get(self, bank_id: str, mental_model_id: str) -> Any:
"""Get a specific mental model.
Args:
bank_id: The ID of the bank.
mental_model_id: The ID of the mental model.
Returns:
The mental model details.
"""
return self._client.get_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
def refresh(self, bank_id: str, mental_model_id: str) -> Any:
"""Refresh a mental model.
Args:
bank_id: The ID of the bank.
mental_model_id: The ID of the mental model to refresh.
Returns:
Refresh response from the API.
"""
return self._client.refresh_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
def update(
self,
bank_id: str,
mental_model_id: str,
name: str | None = None,
content: str | None = None,
tags: list[str] | None = None,
) -> Any:
"""Update a mental model.
Args:
bank_id: The ID of the bank.
mental_model_id: The ID of the mental model to update.
name: Optional new name.
content: Optional new content.
tags: Optional new tags list.
Returns:
Update response from the API.
"""
return self._client.update_mental_model(
bank_id=bank_id,
mental_model_id=mental_model_id,
name=name,
content=content,
tags=tags,
)
def delete(self, bank_id: str, mental_model_id: str) -> Any:
"""Delete a mental model.
Args:
bank_id: The ID of the bank.
mental_model_id: The ID of the mental model to delete.
Returns:
Deletion response from the API.
"""
return self._client.delete_mental_model(bank_id=bank_id, mental_model_id=mental_model_id)
class DirectivesAPI:
"""Namespace for directive operations.
Directives are explicit instructions that guide agent behavior.
"""
def __init__(self, client: Hindsight):
self._client = client
def create(
self,
bank_id: str,
name: str,
content: str,
tags: list[str] | None = None,
) -> Any:
"""Create a new directive.
Args:
bank_id: The ID of the bank to add the directive to.
name: Name for the directive.
content: The directive content/instructions.
tags: Optional list of tags for categorization.
Returns:
Creation response from the API.
"""
return self._client.create_directive(
bank_id=bank_id,
name=name,
content=content,
tags=tags,
)
def list(self, bank_id: str, tags: list[str] | None = None) -> Any:
"""List all directives for a bank.
Args:
bank_id: The ID of the bank.
tags: Optional filter by tags.
Returns:
List of directives.
"""
return self._client.list_directives(bank_id=bank_id, tags=tags)
def get(self, bank_id: str, directive_id: str) -> Any:
"""Get a specific directive.
Args:
bank_id: The ID of the bank.
directive_id: The ID of the directive.
Returns:
The directive details.
"""
return self._client.get_directive(bank_id=bank_id, directive_id=directive_id)
def update(
self,
bank_id: str,
directive_id: str,
name: str | None = None,
content: str | None = None,
tags: list[str] | None = None,
) -> Any:
"""Update a directive.
Args:
bank_id: The ID of the bank.
directive_id: The ID of the directive to update.
name: Optional new name.
content: Optional new content.
tags: Optional new tags list.
Returns:
Update response from the API.
"""
return self._client.update_directive(
bank_id=bank_id,
directive_id=directive_id,
name=name,
content=content,
tags=tags,
)
def delete(self, bank_id: str, directive_id: str) -> Any:
"""Delete a directive.
Args:
bank_id: The ID of the bank.
directive_id: The ID of the directive to delete.
Returns:
Deletion response from the API.
"""
return self._client.delete_directive(bank_id=bank_id, directive_id=directive_id)
class MemoriesAPI:
"""Namespace for memory operations.
Provides methods to query and retrieve stored memories.
"""
def __init__(self, client: Hindsight):
self._client = client
def list(
self,
bank_id: str,
type: str | None = None,
search_query: str | None = None,
limit: int = 100,
offset: int = 0,
) -> Any:
"""List memories in a bank.
Args:
bank_id: The ID of the bank to query.
type: Optional filter by memory type.
search_query: Optional search query for filtering.
limit: Maximum number of results to return (default: 100).
offset: Number of results to skip for pagination (default: 0).
Returns:
List of memories matching the criteria.
"""
return self._client.list_memories(
bank_id=bank_id,
type=type,
search_query=search_query,
limit=limit,
offset=offset,
)
class HindsightClient(Hindsight):
"""
Enhanced Hindsight client with organized API namespaces.
This wrapper extends the auto-generated Hindsight client with organized
access to different parts of the API through namespaces.
Example:
```python
from hindsight import HindsightClient
client = HindsightClient(base_url="http://localhost:8888")
# Core operations (inherited from Hindsight)
client.retain(bank_id="test", content="Hello")
results = client.recall(bank_id="test", query="Hello")
# Organized API access through namespaces
client.banks.create(bank_id="test", name="Test Bank")
models = client.mental_models.list(bank_id="test")
directives = client.directives.list(bank_id="test")
memories = client.memories.list(bank_id="test")
```
Attributes:
banks: Namespace for bank management operations.
mental_models: Namespace for mental model operations.
directives: Namespace for directive operations.
memories: Namespace for memory listing operations.
"""
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self._banks_namespace: BanksAPI | None = None
self._mental_models_namespace: MentalModelsAPI | None = None
self._directives_namespace: DirectivesAPI | None = None
self._memories_namespace: MemoriesAPI | None = None
@property
def banks(self) -> BanksAPI:
"""Access bank management operations.
Returns:
BanksAPI instance for bank operations.
"""
if self._banks_namespace is None:
self._banks_namespace = BanksAPI(self)
return self._banks_namespace
@property
def mental_models(self) -> MentalModelsAPI:
"""Access mental model operations.
Returns:
MentalModelsAPI instance for mental model operations.
"""
if self._mental_models_namespace is None:
self._mental_models_namespace = MentalModelsAPI(self)
return self._mental_models_namespace
@property
def directives(self) -> DirectivesAPI:
"""Access directive operations.
Returns:
DirectivesAPI instance for directive operations.
"""
if self._directives_namespace is None:
self._directives_namespace = DirectivesAPI(self)
return self._directives_namespace
@property
def memories(self) -> MemoriesAPI:
"""Access memory listing operations.
Returns:
MemoriesAPI instance for memory operations.
"""
if self._memories_namespace is None:
self._memories_namespace = MemoriesAPI(self)
return self._memories_namespace
-377
View File
@@ -1,377 +0,0 @@
"""
Embedded Hindsight client with automatic daemon lifecycle management.
This module provides HindsightEmbedded, a client that uses the same daemon
management interface as hindsight-embed CLI, ensuring full compatibility.
Example:
```python
from hindsight import HindsightEmbedded
# Daemon starts automatically on first use
client = HindsightEmbedded(
profile="myapp",
llm_provider="groq",
llm_api_key="your-api-key",
)
# Use just like HindsightClient
client.retain(bank_id="alice", content="Alice loves AI")
results = client.recall(bank_id="alice", query="What does Alice like?")
# Optional cleanup
client.close()
```
Using context manager:
```python
from hindsight import HindsightEmbedded
with HindsightEmbedded(profile="myapp") as client:
client.retain(bank_id="alice", content="Alice loves AI")
# Daemon managed automatically
```
"""
import logging
import os
import threading
from typing import Optional
from hindsight_client import Hindsight
from hindsight_embed import get_embed_manager
from .api_namespaces import BanksAPI, DirectivesAPI, MemoriesAPI, MentalModelsAPI
logger = logging.getLogger(__name__)
class HindsightEmbedded:
"""
Hindsight client with automatic daemon lifecycle management.
This client uses the same daemon management interface as hindsight-embed CLI,
ensuring full compatibility and shared profiles. The daemon is started automatically
on first use and manages profile-specific databases.
Profile data is stored in: ~/.pg0/instances/hindsight-embed-{profile}/
All methods from HindsightClient are available:
- retain(), retain_batch()
- recall()
- reflect()
- create_bank(), set_mission(), delete_bank()
- create_mental_model(), list_mental_models(), etc.
- create_directive(), list_directives(), etc.
- And all async variants (aretain, arecall, areflect, etc.)
Args:
profile: Profile name for data isolation (default: "default")
llm_provider: LLM provider ("groq", "openai", "ollama", "gemini", "anthropic", "lmstudio")
llm_api_key: API key for the LLM provider
llm_model: Model name to use
llm_base_url: Optional custom base URL for LLM API
database_url: Optional database URL override (default: profile-specific pg0)
idle_timeout: Seconds before daemon auto-exits when idle (default: 300)
log_level: Daemon log level (default: "info")
"""
def __init__(
self,
profile: str = "default",
llm_provider: str = "groq",
llm_api_key: str = "",
llm_model: str = "openai/gpt-oss-120b",
llm_base_url: Optional[str] = None,
database_url: Optional[str] = None,
idle_timeout: int = 300,
log_level: str = "info",
):
"""
Initialize the embedded client (daemon starts on first use).
Args:
profile: Profile name for data isolation
llm_provider: LLM provider
llm_api_key: API key for the LLM provider
llm_model: Model name to use
llm_base_url: Optional custom base URL for LLM API
database_url: Optional database URL override
idle_timeout: Seconds before daemon auto-exits when idle
log_level: Daemon log level
"""
self.profile = profile
# Build config dict for daemon (matches CLI format)
self.config = {
"HINDSIGHT_API_LLM_PROVIDER": llm_provider,
"HINDSIGHT_API_LLM_API_KEY": llm_api_key,
"HINDSIGHT_API_LLM_MODEL": llm_model,
"HINDSIGHT_API_LOG_LEVEL": log_level,
"HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT": str(idle_timeout),
}
if llm_base_url:
self.config["HINDSIGHT_API_LLM_BASE_URL"] = llm_base_url
if database_url:
self.config["HINDSIGHT_EMBED_API_DATABASE_URL"] = database_url
self._client: Optional[Hindsight] = None
self._lock = threading.Lock()
self._started = False
self._closed = False
self._manager = get_embed_manager()
# API namespaces (initialized once, lazily)
self._banks_api: Optional[BanksAPI] = None
self._mental_models_api: Optional[MentalModelsAPI] = None
self._directives_api: Optional[DirectivesAPI] = None
self._memories_api: Optional[MemoriesAPI] = None
def _ensure_started(self):
"""Ensure daemon is running (thread-safe)."""
if self._started and self._client is not None:
return
with self._lock:
# Double-check after acquiring lock
if self._started and self._client is not None:
return
if self._closed:
raise RuntimeError("Cannot use HindsightEmbedded after it has been closed")
# Use embed manager interface for daemon management
logger.info(f"Ensuring daemon is running for profile '{self.profile}'...")
success = self._manager.ensure_running(self.config, self.profile)
if not success:
raise RuntimeError(f"Failed to start daemon for profile '{self.profile}'")
# Get daemon URL and create client
daemon_url = self._manager.get_url(self.profile)
self._client = Hindsight(base_url=daemon_url)
self._started = True
logger.info(f"Connected to daemon at {daemon_url}")
def _cleanup(self, stop_daemon_on_close: bool = False):
"""
Cleanup client resources (idempotent).
Args:
stop_daemon_on_close: If True, stops the daemon. Otherwise, daemon continues
running (it will auto-stop after idle timeout).
"""
if self._closed:
return
with self._lock:
if self._closed:
return
if self._client is not None:
self._client.close()
self._client = None
# Optionally stop daemon (daemon has idle timeout, so not required)
if stop_daemon_on_close and self._started:
logger.info(f"Stopping daemon for profile '{self.profile}'...")
self._manager.stop(self.profile)
self._closed = True
def close(self, stop_daemon: bool = False):
"""
Explicitly close the client.
Args:
stop_daemon: If True, stops the daemon. Otherwise, daemon continues running
and will auto-stop after idle timeout (default: False).
Note:
The daemon may be shared with other clients or the CLI, so stopping it
might affect other users. By default, we rely on the daemon's idle timeout.
"""
self._cleanup(stop_daemon_on_close=stop_daemon)
def __getattr__(self, name: str):
"""
Proxy all method calls to the underlying Hindsight client.
This allows HindsightEmbedded to expose all HindsightClient methods
without manually wrapping each one.
"""
# Ensure server is started before proxying
self._ensure_started()
# Get the attribute from the underlying client
attr = getattr(self._client, name)
# If it's a callable, wrap it to ensure server is started
# (shouldn't be needed since _ensure_started already called, but defensive)
if callable(attr):
def wrapper(*args, **kwargs):
self._ensure_started()
return attr(*args, **kwargs)
return wrapper
return attr
def __enter__(self):
"""Context manager entry - ensures server is started."""
self._ensure_started()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""Context manager exit - stops the server."""
self.close()
def __del__(self):
"""Cleanup on garbage collection."""
self._cleanup()
@property
def banks(self) -> BanksAPI:
"""
Access bank management operations.
Each method call ensures the daemon is running before executing.
Example:
```python
from hindsight import HindsightEmbedded
embedded = HindsightEmbedded(profile="myapp", ...)
# Create a bank
embedded.banks.create(bank_id="test", name="Test Bank")
# Set mission
embedded.banks.set_mission(bank_id="test", mission="Help users")
```
"""
if self._banks_api is None:
self._banks_api = BanksAPI(self)
return self._banks_api
@property
def mental_models(self) -> MentalModelsAPI:
"""
Access mental model operations.
Each method call ensures the daemon is running before executing.
Example:
```python
from hindsight import HindsightEmbedded
embedded = HindsightEmbedded(profile="myapp", ...)
# Create a mental model
embedded.mental_models.create(
bank_id="test",
name="User Preferences",
content="User prefers dark mode"
)
# List mental models
models = embedded.mental_models.list(bank_id="test")
```
"""
if self._mental_models_api is None:
self._mental_models_api = MentalModelsAPI(self)
return self._mental_models_api
@property
def directives(self) -> DirectivesAPI:
"""
Access directive operations.
Each method call ensures the daemon is running before executing.
Example:
```python
from hindsight import HindsightEmbedded
embedded = HindsightEmbedded(profile="myapp", ...)
# Create a directive
embedded.directives.create(
bank_id="test",
name="Response Style",
content="Always be concise and friendly"
)
# List directives
directives = embedded.directives.list(bank_id="test")
```
"""
if self._directives_api is None:
self._directives_api = DirectivesAPI(self)
return self._directives_api
@property
def memories(self) -> MemoriesAPI:
"""
Access memory listing operations.
Each method call ensures the daemon is running before executing.
Example:
```python
from hindsight import HindsightEmbedded
embedded = HindsightEmbedded(profile="myapp", ...)
# List memories
memories = embedded.memories.list(
bank_id="test",
type="world",
limit=50
)
```
"""
if self._memories_api is None:
self._memories_api = MemoriesAPI(self)
return self._memories_api
@property
def client(self) -> Hindsight:
"""
Get the underlying Hindsight client for direct access.
WARNING: Using this property directly means daemon restarts won't be
handled automatically. Prefer using the API namespaces (banks, mental_models,
directives, memories) or direct method calls on HindsightEmbedded instead.
Ensures daemon is started before returning the client.
Returns:
Hindsight: The underlying client instance
Example:
```python
from hindsight import HindsightEmbedded
embedded = HindsightEmbedded(profile="myapp", ...)
# Direct access (not recommended - daemon crashes won't be handled)
client = embedded.client
banks = client.list_banks() # If daemon crashes, this will fail
```
"""
self._ensure_started()
return self._client
@property
def url(self) -> str:
"""Get the daemon URL (starts daemon if needed)."""
self._ensure_started()
return self._manager.get_url(self.profile)
@property
def is_running(self) -> bool:
"""Check if the client is initialized."""
return self._started and not self._closed and self._client is not None
-338
View File
@@ -1,338 +0,0 @@
"""
Integration tests for HindsightEmbedded client.
Tests the embedded client with automatic server lifecycle management:
1. Lazy server startup on first use
2. Server reuse across multiple operations
3. Context manager support
4. Method proxying to underlying HindsightClient
5. Proper cleanup
Note: Each test uses random bank_ids to avoid conflicts and allow safe parallel execution.
"""
import os
import uuid
import pytest
from hindsight import HindsightEmbedded
@pytest.fixture(scope="session")
def llm_config():
"""Get LLM configuration from environment (session-scoped)."""
# Try both naming conventions
provider = os.getenv("HINDSIGHT_API_LLM_PROVIDER") or os.getenv("HINDSIGHT_LLM_PROVIDER", "groq")
api_key = os.getenv("HINDSIGHT_API_LLM_API_KEY") or os.getenv("HINDSIGHT_LLM_API_KEY", "")
model = os.getenv("HINDSIGHT_API_LLM_MODEL") or os.getenv("HINDSIGHT_LLM_MODEL", "openai/gpt-oss-120b")
if not api_key:
pytest.skip("LLM API key not configured. Set HINDSIGHT_API_LLM_API_KEY or HINDSIGHT_LLM_API_KEY.")
return {
"llm_provider": provider,
"llm_api_key": api_key,
"llm_model": model,
}
def test_embedded_lazy_start(llm_config):
"""
Test that HindsightEmbedded starts server lazily on first use.
"""
profile = f"test_lazy_{uuid.uuid4().hex[:8]}"
bank_id = f"bank_{uuid.uuid4().hex[:8]}"
# Create client - should NOT start server yet
client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
assert not client.is_running, "Server should not be running after initialization"
# First call should start server
result = client.retain(bank_id=bank_id, content="Test content for lazy start")
# Verify server is now running
assert client.is_running, "Server should be running after first call"
assert result.success, "Retain should succeed"
assert result.items_count >= 1, "Should have stored at least 1 item"
# Cleanup
client.close()
assert not client.is_running, "Server should stop after close()"
def test_embedded_context_manager(llm_config):
"""
Test HindsightEmbedded with context manager.
"""
profile = f"test_ctx_{uuid.uuid4().hex[:8]}"
bank_id = f"bank_{uuid.uuid4().hex[:8]}"
# Use context manager
with HindsightEmbedded(profile=profile, log_level="info", **llm_config) as client:
assert client.is_running, "Server should be running inside context"
# Store memory
result = client.retain(bank_id=bank_id, content="Testing context manager")
assert result.success, "Retain should succeed"
# Recall memory
recall_results = client.recall(bank_id=bank_id, query="context")
assert isinstance(recall_results.results, list), "Recall should return results list"
# Server should be stopped after context exit
# Note: We can't check client.is_running here as client is out of scope
def test_embedded_complete_workflow(llm_config):
"""
Test complete workflow with HindsightEmbedded.
This test:
1. Creates a client with lazy start
2. Creates a memory bank
3. Stores multiple memories
4. Recalls memories
5. Reflects on memories
6. Tests cleanup
"""
profile = f"test_workflow_{uuid.uuid4().hex[:8]}"
bank_id = f"assistant_{uuid.uuid4().hex[:8]}"
client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
try:
# Step 1: Create a memory bank
print(f"\n1. Creating memory bank: {bank_id}")
bank_response = client.create_bank(
bank_id=bank_id, name="Test Assistant", mission="Help with programming tasks"
)
assert bank_response.bank_id == bank_id
# Step 2: Store memories (single)
print("\n2. Storing single memory...")
retain_response = client.retain(
bank_id=bank_id,
content="User prefers Python for data analysis.",
context="Programming preferences",
)
assert retain_response.success
assert retain_response.items_count >= 1
# Step 3: Store batch memories
print("\n3. Storing batch memories...")
batch_response = client.retain_batch(
bank_id=bank_id,
items=[
{"content": "User works with pandas and numpy."},
{"content": "User likes matplotlib for visualization."},
{"content": "User is interested in machine learning with scikit-learn."},
],
)
assert batch_response.success
assert batch_response.items_count >= 3
# Step 4: Recall memories
print("\n4. Recalling memories...")
recall_response = client.recall(bank_id=bank_id, query="What tools does the user prefer?", max_tokens=2000)
assert isinstance(recall_response.results, list)
assert len(recall_response.results) > 0
print(f" Found {len(recall_response.results)} relevant memories")
# Step 5: Reflect on memories
print("\n5. Reflecting on memories...")
reflect_response = client.reflect(
bank_id=bank_id,
query="What programming tools should I recommend?",
budget="low",
)
assert reflect_response.text
assert len(reflect_response.text) > 0
print(f" Answer: {reflect_response.text[:150]}...")
# Verify answer mentions relevant tools
answer_lower = reflect_response.text.lower()
assert any(term in answer_lower for term in ["python", "pandas", "numpy", "data"])
# Step 6: List memories
print("\n6. Listing memories...")
list_response = client.list_memories(bank_id=bank_id, limit=10)
assert len(list_response.items) > 0
print(f" Listed {len(list_response.items)} memories")
finally:
# Cleanup
client.close()
def test_embedded_server_reuse(llm_config):
"""
Test that the same server is reused across multiple calls.
"""
profile = f"test_reuse_{uuid.uuid4().hex[:8]}"
bank_id = f"bank_{uuid.uuid4().hex[:8]}"
client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
try:
# First call starts server
result1 = client.retain(bank_id=bank_id, content="First message")
url1 = client.url
assert client.is_running
# Second call should reuse the same server
result2 = client.retain(bank_id=bank_id, content="Second message")
url2 = client.url
# URLs should be identical (same server)
assert url1 == url2, "Server URL should remain the same across calls"
assert result1.success and result2.success
# Third call should also reuse
recall_result = client.recall(bank_id=bank_id, query="message")
url3 = client.url
assert url3 == url1, "Server URL should remain the same for recall"
assert isinstance(recall_result.results, list)
finally:
client.close()
def test_embedded_method_proxying(llm_config):
"""
Test that all HindsightClient methods are properly proxied.
This ensures __getattr__ proxying works for various method types.
"""
profile = f"test_proxy_{uuid.uuid4().hex[:8]}"
bank_id = f"bank_{uuid.uuid4().hex[:8]}"
client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
try:
# Test bank operations
bank = client.create_bank(bank_id=bank_id, name="Proxy Test")
assert bank.bank_id == bank_id
# Test mission setting
mission_response = client.set_mission(bank_id=bank_id, mission="Test mission for proxying")
assert mission_response.bank_id == bank_id
# Test retain
retain_result = client.retain(bank_id=bank_id, content="Test content")
assert retain_result.success
# Test retain_batch
batch_result = client.retain_batch(
bank_id=bank_id, items=[{"content": "Item 1"}, {"content": "Item 2"}]
)
assert batch_result.success
assert batch_result.items_count >= 2
# Test recall
recall_result = client.recall(bank_id=bank_id, query="test")
assert hasattr(recall_result, "results")
# Test reflect
reflect_result = client.reflect(bank_id=bank_id, query="What is stored?")
assert hasattr(reflect_result, "text")
# Test list_memories
list_result = client.list_memories(bank_id=bank_id, limit=5)
assert hasattr(list_result, "items")
print("✓ All methods successfully proxied")
finally:
client.close()
def test_embedded_multiple_banks(llm_config):
"""
Test that HindsightEmbedded can work with multiple banks.
"""
profile = f"test_multibank_{uuid.uuid4().hex[:8]}"
bank1_id = f"bank1_{uuid.uuid4().hex[:8]}"
bank2_id = f"bank2_{uuid.uuid4().hex[:8]}"
client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
try:
# Create first bank and store data
client.create_bank(bank_id=bank1_id, name="Bank 1")
client.retain(bank_id=bank1_id, content="Alice prefers Python for data science")
# Create second bank and store data
client.create_bank(bank_id=bank2_id, name="Bank 2")
client.retain(bank_id=bank2_id, content="Bob uses JavaScript for web development")
# Recall from both banks
results1 = client.recall(bank_id=bank1_id, query="programming language")
results2 = client.recall(bank_id=bank2_id, query="programming language")
assert len(results1.results) > 0
assert len(results2.results) > 0
# Verify banks are isolated (each should only see their own content)
# This is a basic check - content isolation is tested more thoroughly in other tests
assert results1.results[0].text != results2.results[0].text or len(results1.results) != len(
results2.results
)
finally:
client.close()
def test_embedded_profile_isolation(llm_config):
"""
Test that different profiles create isolated data stores.
"""
profile1 = f"test_iso1_{uuid.uuid4().hex[:8]}"
profile2 = f"test_iso2_{uuid.uuid4().hex[:8]}"
bank_id = "shared_bank_name" # Same bank_id in both profiles
client1 = HindsightEmbedded(profile=profile1, log_level="info", **llm_config)
client2 = HindsightEmbedded(profile=profile2, log_level="info", **llm_config)
try:
# Store data in profile1
client1.retain(bank_id=bank_id, content="User likes TypeScript for frontend development")
# Store different data in profile2
client2.retain(bank_id=bank_id, content="User prefers Rust for systems programming")
# Each profile should only see its own data
results1 = client1.recall(bank_id=bank_id, query="programming preference")
results2 = client2.recall(bank_id=bank_id, query="programming preference")
# Both should have results
assert len(results1.results) > 0
assert len(results2.results) > 0
# Results should be different (basic isolation check)
# Note: This is a basic sanity check. Full isolation is ensured by pg0's data directory separation
finally:
client1.close()
client2.close()
def test_embedded_error_after_close(llm_config):
"""
Test that using HindsightEmbedded after close() raises an error.
"""
profile = f"test_error_{uuid.uuid4().hex[:8]}"
bank_id = f"bank_{uuid.uuid4().hex[:8]}"
client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
# Use it once to start server
client.retain(bank_id=bank_id, content="Test")
# Close the client
client.close()
assert not client.is_running
# Trying to use it after close should raise an error
with pytest.raises(RuntimeError, match="Cannot use HindsightEmbedded after it has been closed"):
client.retain(bank_id=bank_id, content="This should fail")
@@ -1,170 +0,0 @@
"""Test that API namespaces ensure daemon is started before each call."""
from unittest.mock import Mock, patch
import pytest
from hindsight import HindsightEmbedded
@pytest.fixture
def embedded_client():
"""Create an embedded client for testing."""
return HindsightEmbedded(
profile="test",
llm_provider="openai",
llm_api_key="test-key",
)
def test_banks_create_ensures_daemon_started(embedded_client):
"""Test that banks.create() calls _ensure_started()."""
# Mock _ensure_started to track calls
with patch.object(embedded_client, "_ensure_started") as mock_ensure:
# Mock the underlying client to avoid actual API call
mock_client = Mock()
embedded_client._client = mock_client
# Call namespace method
try:
embedded_client.banks.create(bank_id="test", name="Test Bank")
except Exception:
pass # We don't care if the actual call fails
# Verify _ensure_started was called
mock_ensure.assert_called_once()
def test_mental_models_list_ensures_daemon_started(embedded_client):
"""Test that mental_models.list() calls _ensure_started()."""
with patch.object(embedded_client, "_ensure_started") as mock_ensure:
mock_client = Mock()
embedded_client._client = mock_client
try:
embedded_client.mental_models.list(bank_id="test")
except Exception:
pass
mock_ensure.assert_called_once()
def test_directives_list_ensures_daemon_started(embedded_client):
"""Test that directives.list() calls _ensure_started()."""
with patch.object(embedded_client, "_ensure_started") as mock_ensure:
mock_client = Mock()
embedded_client._client = mock_client
try:
embedded_client.directives.list(bank_id="test")
except Exception:
pass
mock_ensure.assert_called_once()
def test_memories_list_ensures_daemon_started(embedded_client):
"""Test that memories.list() calls _ensure_started()."""
with patch.object(embedded_client, "_ensure_started") as mock_ensure:
mock_client = Mock()
embedded_client._client = mock_client
try:
embedded_client.memories.list(bank_id="test")
except Exception:
pass
mock_ensure.assert_called_once()
def test_multiple_calls_ensure_daemon_each_time(embedded_client):
"""Test that each namespace call ensures daemon is started."""
with patch.object(embedded_client, "_ensure_started") as mock_ensure:
mock_client = Mock()
embedded_client._client = mock_client
# Make multiple calls
try:
embedded_client.banks.create(bank_id="test", name="Test")
except Exception:
pass
try:
embedded_client.mental_models.list(bank_id="test")
except Exception:
pass
try:
embedded_client.directives.list(bank_id="test")
except Exception:
pass
# Should be called 3 times (once per namespace method call)
assert mock_ensure.call_count == 3
def test_daemon_restart_handling(embedded_client):
"""Test that namespace methods can recover from daemon crash."""
call_count = 0
def mock_ensure_started():
"""Mock that simulates daemon restart."""
nonlocal call_count
call_count += 1
# Create a new mock client each time (simulating daemon restart)
embedded_client._client = Mock()
embedded_client._started = True
with patch.object(embedded_client, "_ensure_started", side_effect=mock_ensure_started):
# First call - daemon starts
embedded_client.banks.create(bank_id="test", name="Test")
assert call_count == 1
# Simulate daemon crash by clearing client
embedded_client._client = None
embedded_client._started = False
# Second call - daemon restarts
embedded_client.banks.create(bank_id="test", name="Test")
assert call_count == 2
def test_ensure_started_calls_manager(embedded_client):
"""Test that _ensure_started actually starts the daemon via manager."""
# Mock the manager
mock_manager = Mock()
mock_manager.ensure_running.return_value = True
mock_manager.get_url.return_value = "http://localhost:54321"
embedded_client._manager = mock_manager
# Mock Hindsight client constructor
with patch("hindsight.embedded.Hindsight") as mock_hindsight_class:
mock_client = Mock()
mock_hindsight_class.return_value = mock_client
# Call _ensure_started
embedded_client._ensure_started()
# Verify manager was called
mock_manager.ensure_running.assert_called_once_with(
embedded_client.config, embedded_client.profile
)
mock_manager.get_url.assert_called_once_with(embedded_client.profile)
# Verify Hindsight client was created
mock_hindsight_class.assert_called_once_with(base_url="http://localhost:54321")
def test_namespace_singleton_behavior(embedded_client):
"""Test that namespace properties return the same instance."""
banks1 = embedded_client.banks
banks2 = embedded_client.banks
# Should be the same instance
assert banks1 is banks2
# Same for other namespaces
assert embedded_client.mental_models is embedded_client.mental_models
assert embedded_client.directives is embedded_client.directives
assert embedded_client.memories is embedded_client.memories
-137
View File
@@ -1,137 +0,0 @@
# Hindsight API
**Memory System for AI Agents** — Temporal + Semantic + Entity Memory Architecture using PostgreSQL with pgvector.
Hindsight gives AI agents persistent memory that works like human memory: it stores facts, tracks entities and relationships, handles temporal reasoning ("what happened last spring?"), and forms opinions based on configurable disposition traits.
## Installation
```bash
pip install hindsight-api
```
## Quick Start
### Run the Server
```bash
# Set your LLM provider
export HINDSIGHT_API_LLM_PROVIDER=openai
export HINDSIGHT_API_LLM_API_KEY=sk-xxxxxxxxxxxx
# Start the server (uses embedded PostgreSQL by default)
hindsight-api
```
The server starts at http://localhost:8888 with:
- REST API for memory operations
- MCP server at `/mcp` for tool-use integration
### Use the Python API
```python
from hindsight_api import MemoryEngine
# Create and initialize the memory engine
memory = MemoryEngine()
await memory.initialize()
# Create a memory bank for your agent
bank = await memory.create_memory_bank(
name="my-assistant",
background="A helpful coding assistant"
)
# Store a memory
await memory.retain(
memory_bank_id=bank.id,
content="The user prefers Python for data science projects"
)
# Recall memories
results = await memory.recall(
memory_bank_id=bank.id,
query="What programming language does the user prefer?"
)
# Reflect with reasoning
response = await memory.reflect(
memory_bank_id=bank.id,
query="Should I recommend Python or R for this ML project?"
)
```
## CLI Options
```bash
hindsight-api --help
# Common options
hindsight-api --port 9000 # Custom port (default: 8888)
hindsight-api --host 127.0.0.1 # Bind to localhost only
hindsight-api --workers 4 # Multiple worker processes
hindsight-api --log-level debug # Verbose logging
```
## Configuration
Configure via environment variables:
| Variable | Description | Default |
|----------|-------------|---------|
| `HINDSIGHT_API_DATABASE_URL` | PostgreSQL connection string | `pg0` (embedded) |
| `HINDSIGHT_API_LLM_PROVIDER` | `openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio` | `openai` |
| `HINDSIGHT_API_LLM_API_KEY` | API key for LLM provider | - |
| `HINDSIGHT_API_LLM_MODEL` | Model name | `gpt-4o-mini` |
| `HINDSIGHT_API_HOST` | Server bind address | `0.0.0.0` |
| `HINDSIGHT_API_PORT` | Server port | `8888` |
### Example with External PostgreSQL
```bash
export HINDSIGHT_API_DATABASE_URL=postgresql://user:pass@localhost:5432/hindsight
export HINDSIGHT_API_LLM_PROVIDER=groq
export HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx
hindsight-api
```
## Docker
```bash
docker run --rm -it -p 8888:8888 \
-e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \
-v $HOME/.hindsight-docker:/home/hindsight/.pg0 \
ghcr.io/vectorize-io/hindsight:latest
```
## MCP Server
For local MCP integration without running the full API server:
```bash
hindsight-local-mcp
```
This runs a stdio-based MCP server that can be used directly with MCP-compatible clients.
## Key Features
- **Multi-Strategy Retrieval (TEMPR)** — Semantic, keyword, graph, and temporal search combined with RRF fusion
- **Entity Graph** — Automatic entity extraction and relationship tracking
- **Temporal Reasoning** — Native support for time-based queries
- **Disposition Traits** — Configurable skepticism, literalism, and empathy influence opinion formation
- **Three Memory Types** — World facts, bank actions, and formed opinions with confidence scores
## Documentation
Full documentation: [https://hindsight.vectorize.io](https://hindsight.vectorize.io)
- [Installation Guide](https://hindsight.vectorize.io/developer/installation)
- [Configuration Reference](https://hindsight.vectorize.io/developer/configuration)
- [API Reference](https://hindsight.vectorize.io/api-reference)
- [Python SDK](https://hindsight.vectorize.io/sdks/python)
## License
Apache 2.0
@@ -1 +0,0 @@
# Admin CLI for Hindsight
@@ -1,383 +0,0 @@
"""
Hindsight Admin CLI - backup and restore operations.
"""
import asyncio
import io
import json
import logging
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import asyncpg
import typer
from ..config import DEFAULT_DATABASE_SCHEMA, HindsightConfig
from ..extensions import TenantExtension, load_extension
from ..pg0 import parse_pg0_url, resolve_database_url
def _fq_table(table: str, schema: str) -> str:
"""Get fully-qualified table name with schema prefix."""
return f"{schema}.{table}"
# Setup logging
logging.basicConfig(
level=logging.INFO,
format="%(message)s",
)
logger = logging.getLogger(__name__)
app = typer.Typer(name="hindsight-admin", help="Hindsight administrative commands")
# Tables to backup/restore in dependency order
# Import must happen in this order due to foreign key constraints
BACKUP_TABLES = [
"banks",
"documents",
"entities",
"chunks",
"memory_units",
"unit_entities",
"entity_cooccurrences",
"memory_links",
]
MANIFEST_VERSION = "1"
async def _backup(database_url: str, output_path: Path, schema: str = "public") -> dict[str, Any]:
"""Backup all tables to a zip file using binary COPY protocol."""
conn = await asyncpg.connect(database_url)
try:
tables: dict[str, Any] = {}
manifest: dict[str, Any] = {
"version": MANIFEST_VERSION,
"created_at": datetime.now(timezone.utc).isoformat(),
"schema": schema,
"tables": tables,
}
# Use a transaction with REPEATABLE READ isolation to get a consistent
# snapshot across all tables. This prevents race conditions where
# entity_cooccurrences could reference entities created after the
# entities table was backed up.
async with conn.transaction(isolation="repeatable_read"):
with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zf:
for i, table in enumerate(BACKUP_TABLES, 1):
typer.echo(f" [{i}/{len(BACKUP_TABLES)}] Backing up {table}...", nl=False)
buffer = io.BytesIO()
# Use binary COPY for exact type preservation
# asyncpg requires schema_name as separate parameter
await conn.copy_from_table(table, schema_name=schema, output=buffer, format="binary")
data = buffer.getvalue()
zf.writestr(f"{table}.bin", data)
# Get row count for manifest
qualified_table = _fq_table(table, schema)
row_count = await conn.fetchval(f"SELECT COUNT(*) FROM {qualified_table}")
tables[table] = {
"rows": row_count,
"size_bytes": len(data),
}
typer.echo(f" {row_count} rows")
zf.writestr("manifest.json", json.dumps(manifest, indent=2))
return manifest
finally:
await conn.close()
async def _restore(database_url: str, input_path: Path, schema: str = "public") -> dict[str, Any]:
"""Restore all tables from a zip file using binary COPY protocol."""
conn = await asyncpg.connect(database_url)
try:
with zipfile.ZipFile(input_path, "r") as zf:
# Read and validate manifest
manifest: dict[str, Any] = json.loads(zf.read("manifest.json"))
if manifest.get("version") != MANIFEST_VERSION:
raise ValueError(f"Unsupported backup version: {manifest.get('version')}")
# Use a transaction for atomic restore - either all tables are
# restored or none are, preventing partial/inconsistent state.
async with conn.transaction():
typer.echo(" Clearing existing data...")
# Truncate tables in reverse order (respects FK constraints)
for table in reversed(BACKUP_TABLES):
qualified_table = _fq_table(table, schema)
await conn.execute(f"TRUNCATE TABLE {qualified_table} CASCADE")
# Restore tables in forward order
for i, table in enumerate(BACKUP_TABLES, 1):
filename = f"{table}.bin"
if filename not in zf.namelist():
typer.echo(f" [{i}/{len(BACKUP_TABLES)}] {table}: skipped (not in backup)")
continue
expected_rows = manifest["tables"].get(table, {}).get("rows", "?")
typer.echo(f" [{i}/{len(BACKUP_TABLES)}] Restoring {table}... {expected_rows} rows")
data = zf.read(filename)
buffer = io.BytesIO(data)
# asyncpg requires schema_name as separate parameter
await conn.copy_to_table(table, schema_name=schema, source=buffer, format="binary")
# Refresh materialized view
typer.echo(" Refreshing materialized views...")
await conn.execute(f"REFRESH MATERIALIZED VIEW {_fq_table('memory_units_bm25', schema)}")
return manifest
finally:
await conn.close()
async def _run_backup(db_url: str, output: Path, schema: str = "public") -> dict[str, Any]:
"""Resolve database URL and run backup."""
is_pg0, instance_name, _ = parse_pg0_url(db_url)
if is_pg0:
typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
resolved_url = await resolve_database_url(db_url)
return await _backup(resolved_url, output, schema)
async def _run_restore(db_url: str, input_file: Path, schema: str = "public") -> dict[str, Any]:
"""Resolve database URL and run restore."""
is_pg0, instance_name, _ = parse_pg0_url(db_url)
if is_pg0:
typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
resolved_url = await resolve_database_url(db_url)
return await _restore(resolved_url, input_file, schema)
@app.command()
def backup(
output: Path = typer.Argument(..., help="Output file path (.zip)"),
schema: str = typer.Option("public", "--schema", "-s", help="Database schema to backup"),
):
"""Backup the Hindsight database to a zip file."""
config = HindsightConfig.from_env()
if not config.database_url:
typer.echo("Error: Database URL not configured.", err=True)
typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
raise typer.Exit(1)
if output.suffix != ".zip":
output = output.with_suffix(".zip")
typer.echo(f"Backing up database (schema: {schema}) to {output}...")
manifest = asyncio.run(_run_backup(config.database_url, output, schema))
total_rows = sum(t["rows"] for t in manifest["tables"].values())
typer.echo(f"Backed up {total_rows} rows across {len(BACKUP_TABLES)} tables")
typer.echo(f"Backup saved to {output}")
@app.command()
def restore(
input_file: Path = typer.Argument(..., help="Input backup file (.zip)"),
schema: str = typer.Option("public", "--schema", "-s", help="Database schema to restore to"),
yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"),
):
"""Restore the database from a backup file. WARNING: This deletes all existing data."""
config = HindsightConfig.from_env()
if not config.database_url:
typer.echo("Error: Database URL not configured.", err=True)
typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
raise typer.Exit(1)
if not input_file.exists():
typer.echo(f"Error: File not found: {input_file}", err=True)
raise typer.Exit(1)
if not yes:
typer.confirm(
"This will DELETE all existing data and replace it with the backup. Continue?",
abort=True,
)
typer.echo(f"Restoring database (schema: {schema}) from {input_file}...")
manifest = asyncio.run(_run_restore(config.database_url, input_file, schema))
total_rows = sum(t["rows"] for t in manifest["tables"].values())
typer.echo(f"Restored {total_rows} rows across {len(BACKUP_TABLES)} tables")
typer.echo("Restore complete")
async def _run_migration(
db_url: str,
schema: str | None = None,
base_schema: str = DEFAULT_DATABASE_SCHEMA,
embedding_dimension: int | None = None,
) -> list[str]:
"""Resolve database URL and run migrations for one schema or all discovered schemas."""
from ..migrations import (
ensure_embedding_dimension,
ensure_text_search_extension,
ensure_vector_extension,
run_migrations,
)
is_pg0, instance_name, _ = parse_pg0_url(db_url)
if is_pg0:
typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
resolved_url = await resolve_database_url(db_url)
config = HindsightConfig.from_env()
if schema:
schemas = [schema]
else:
tenant_extension = load_extension("TENANT", TenantExtension)
schemas = [base_schema or DEFAULT_DATABASE_SCHEMA]
if tenant_extension:
tenants = await tenant_extension.list_tenants()
schemas.extend(tenant.schema for tenant in tenants if tenant.schema)
# Preserve order while removing duplicates.
schemas = list(dict.fromkeys(schemas))
for schema in schemas:
run_migrations(resolved_url, schema=schema)
if embedding_dimension is not None:
for schema in schemas:
ensure_embedding_dimension(
resolved_url,
embedding_dimension,
schema=schema,
vector_extension=config.vector_extension,
)
for schema in schemas:
ensure_vector_extension(
resolved_url,
vector_extension=config.vector_extension,
schema=schema,
)
for schema in schemas:
ensure_text_search_extension(
resolved_url,
text_search_extension=config.text_search_extension,
schema=schema,
)
return schemas
@app.command(name="run-db-migration")
def run_db_migration(
schema: str | None = typer.Option(
None,
"--schema",
"-s",
help="Database schema to run migrations on. If omitted, migrate the base schema and all discovered tenant schemas.",
),
embedding_dimension: int | None = typer.Option(
None,
"--embedding-dimension",
help="Expected embedding dimension to enforce after migrations. Omit to skip dimension sync.",
),
):
"""Run database migrations to the latest version."""
config = HindsightConfig.from_env()
if not config.database_url:
typer.echo("Error: Database URL not configured.", err=True)
typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
raise typer.Exit(1)
if schema:
typer.echo(f"Running database migrations for schema: {schema}...")
else:
typer.echo("Running database migrations for base schema and all discovered tenant schemas...")
schemas = asyncio.run(
_run_migration(
config.database_url,
schema=schema,
base_schema=config.database_schema,
embedding_dimension=embedding_dimension,
)
)
typer.echo(f"Database migrations completed successfully for {len(schemas)} schema(s)")
async def _decommission_worker(db_url: str, worker_id: str, schema: str = "public") -> int:
"""Release all tasks owned by a worker, setting them back to pending status."""
is_pg0, instance_name, _ = parse_pg0_url(db_url)
if is_pg0:
typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
resolved_url = await resolve_database_url(db_url)
conn = await asyncpg.connect(resolved_url)
try:
table = _fq_table("async_operations", schema)
result = await conn.fetch(
f"""
UPDATE {table}
SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
WHERE worker_id = $1 AND status = 'processing'
RETURNING operation_id
""",
worker_id,
)
return len(result)
finally:
await conn.close()
@app.command(name="decommission-worker")
def decommission_worker(
worker_id: str = typer.Argument(..., help="Worker ID to decommission"),
schema: str = typer.Option("public", "--schema", "-s", help="Database schema"),
yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"),
):
"""Release all tasks owned by a worker (sets status back to pending).
Use this command when a worker has crashed or been removed without graceful shutdown.
All tasks that were being processed by the worker will be released back to the queue
so other workers can pick them up.
"""
config = HindsightConfig.from_env()
if not config.database_url:
typer.echo("Error: Database URL not configured.", err=True)
typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
raise typer.Exit(1)
if not yes:
typer.confirm(
f"This will release all tasks owned by worker '{worker_id}' back to pending. Continue?",
abort=True,
)
typer.echo(f"Decommissioning worker '{worker_id}' (schema: {schema})...")
count = asyncio.run(_decommission_worker(config.database_url, worker_id, schema))
if count > 0:
typer.echo(f"Released {count} task(s) from worker '{worker_id}'")
else:
typer.echo(f"No tasks found for worker '{worker_id}'")
def main():
app()
if __name__ == "__main__":
main()
@@ -1,70 +0,0 @@
"""Add file_storage table for BYTEA-based file storage
Revision ID: a1b2c3d4e5f6
Revises: y0t1u2v3w4x5
Create Date: 2026-02-16
Creates a dedicated table for storing uploaded files using BYTEA.
This provides zero-config file storage that "just works" for development
and small deployments. For production/scale, use S3-compatible storage.
Files are stored in a separate table to avoid bloating the documents table.
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "a1b2c3d4e5f6"
down_revision: str | Sequence[str] | None = "y0t1u2v3w4x5"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Create file_storage table for BYTEA storage."""
schema = _get_schema_prefix()
# Create file_storage table (minimal: just key + data)
op.execute(
f"""
CREATE TABLE IF NOT EXISTS {schema}file_storage (
storage_key TEXT PRIMARY KEY,
data BYTEA NOT NULL
)
"""
)
# Add file tracking columns to documents table
op.execute(
f"""
ALTER TABLE {schema}documents
ADD COLUMN IF NOT EXISTS file_storage_key TEXT,
ADD COLUMN IF NOT EXISTS file_original_name TEXT,
ADD COLUMN IF NOT EXISTS file_content_type TEXT
"""
)
def downgrade() -> None:
"""Remove file_storage table and related columns."""
schema = _get_schema_prefix()
# Drop columns from documents table
op.execute(
f"""
ALTER TABLE {schema}documents
DROP COLUMN IF EXISTS file_storage_key,
DROP COLUMN IF EXISTS file_original_name,
DROP COLUMN IF EXISTS file_content_type
"""
)
# Drop file_storage table
op.execute(f"DROP TABLE IF EXISTS {schema}file_storage")
@@ -1,88 +0,0 @@
"""Add text_signals column to memory_units for enriched BM25 indexing.
text_signals stores a denormalized space-separated string of entity names
(and future signals) to improve full-text search recall without polluting
the stored fact text.
- vchord: text_signals included in tokenize() at insert time
- native: search_vector GENERATED column regenerated to include text_signals
- pg_textsearch: no change (index only supports a single base column)
Revision ID: a2b3c4d5e6f7
Revises: z1u2v3w4x5y6
Create Date: 2026-02-28
"""
import os
from collections.abc import Sequence
from alembic import context, op
revision: str = "a2b3c4d5e6f7"
down_revision: str | Sequence[str] | None = "aa2b3c4d5e6f"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def _detect_text_search_extension() -> str:
return os.getenv("HINDSIGHT_API_TEXT_SEARCH_EXTENSION", "native").lower()
def upgrade() -> None:
schema = _get_schema_prefix()
table = f"{schema}memory_units"
text_search_ext = _detect_text_search_extension()
# Add text_signals column (nullable TEXT, populated at retain time)
op.execute(f"ALTER TABLE {table} ADD COLUMN IF NOT EXISTS text_signals TEXT")
if text_search_ext == "native":
# Native PostgreSQL: drop and recreate the GENERATED tsvector column to include text_signals
op.execute(f"ALTER TABLE {table} DROP COLUMN IF EXISTS search_vector")
op.execute(f"""
ALTER TABLE {table}
ADD COLUMN search_vector tsvector
GENERATED ALWAYS AS (
to_tsvector('english',
COALESCE(text, '') || ' ' ||
COALESCE(context, '') || ' ' ||
COALESCE(text_signals, '')
)
) STORED
""")
# Recreate GIN index (was dropped with the column)
op.execute(f"""
CREATE INDEX IF NOT EXISTS idx_memory_units_text_search
ON {table} USING gin(search_vector)
""")
# vchord: tokenize() call in fact_storage.py is updated to include text_signals at insert time
# pg_textsearch: no change — index operates on the base `text` column only
def downgrade() -> None:
schema = _get_schema_prefix()
table = f"{schema}memory_units"
text_search_ext = _detect_text_search_extension()
if text_search_ext == "native":
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_text_search")
op.execute(f"ALTER TABLE {table} DROP COLUMN IF EXISTS search_vector")
op.execute(f"""
ALTER TABLE {table}
ADD COLUMN search_vector tsvector
GENERATED ALWAYS AS (
to_tsvector('english', COALESCE(text, '') || ' ' || COALESCE(context, ''))
) STORED
""")
op.execute(f"""
CREATE INDEX idx_memory_units_text_search
ON {table} USING gin(search_vector)
""")
op.execute(f"ALTER TABLE {table} DROP COLUMN IF EXISTS text_signals")
@@ -1,54 +0,0 @@
"""Add GIN index on source_memory_ids for observation lookup performance
Without this index, queries using the array overlap operator (&&) or array
containment (@>) on source_memory_ids require a full sequential scan over all
observation memory_units. At ~77k observations this was measured at 45ms per
query, becoming a bottleneck during consolidation recall (57-64s timeouts) and
user recall (18-27s average).
The GIN index reduces these queries to index scans: 45ms → 0.049ms (927x
speedup). Recall dropped from 18-27s to ~6s, and consolidation recall
stabilised from timeout to ~15s.
Created with CONCURRENTLY so the migration does not block reads or writes.
CONCURRENTLY requires running outside a transaction block, so the migration
emits an explicit COMMIT before the statement and uses IF NOT EXISTS for
idempotency.
Revision ID: a2b3c4d5e6f8
Revises: f7g8h9i0j1k2
Create Date: 2026-03-04
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "a2b3c4d5e6f8"
down_revision: str | Sequence[str] | None = "f7g8h9i0j1k2"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
# CREATE INDEX CONCURRENTLY cannot run inside a transaction block.
# Commit the current Alembic transaction first.
op.execute("COMMIT")
op.execute(
f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
f"ON {schema}memory_units USING GIN (source_memory_ids) "
f"WHERE source_memory_ids IS NOT NULL"
)
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
@@ -1,52 +0,0 @@
"""Add consolidation_failed_at column to memory_units for tracking persistent LLM failures.
When all LLM retries are exhausted on a single-memory batch, the memory is marked
with consolidation_failed_at instead of consolidated_at, so it is not silently lost
and can be retried later via the API.
Revision ID: a3b4c5d6e7f8
Revises: g7h8i9j0k1l2
Create Date: 2026-03-17
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "a3b4c5d6e7f8"
down_revision: str | Sequence[str] | None = "g7h8i9j0k1l2"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(
f"""
ALTER TABLE {schema}memory_units
ADD COLUMN IF NOT EXISTS consolidation_failed_at TIMESTAMPTZ DEFAULT NULL
"""
)
# Index to efficiently query memories that failed consolidation for a given bank
op.execute(
f"""
CREATE INDEX IF NOT EXISTS idx_memory_units_consolidation_failed
ON {schema}memory_units (bank_id, consolidation_failed_at)
WHERE consolidation_failed_at IS NOT NULL AND fact_type IN ('experience', 'world')
"""
)
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_consolidation_failed")
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS consolidation_failed_at")
@@ -1,36 +0,0 @@
"""Make event_date nullable in memory_units to support timestamp-free content
Revision ID: aa2b3c4d5e6f
Revises: z1u2v3w4x5y6
Create Date: 2026-03-02
When callers retain content without a timestamp (e.g. fictional documents, static text),
the event_date column should be allowed to be NULL rather than defaulting to utcnow().
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "aa2b3c4d5e6f"
down_revision: str | Sequence[str] | None = "z1u2v3w4x5y6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}memory_units ALTER COLUMN event_date DROP NOT NULL")
def downgrade() -> None:
schema = _get_schema_prefix()
# Backfill NULLs with now() before restoring the NOT NULL constraint
op.execute(f"UPDATE {schema}memory_units SET event_date = now() WHERE event_date IS NULL")
op.execute(f"ALTER TABLE {schema}memory_units ALTER COLUMN event_date SET NOT NULL")
@@ -1,68 +0,0 @@
"""Add partial indexes on memory_units temporal date fields for fast temporal retrieval
Revision ID: b3c4d5e6f7g8
Revises: c1a2b3d4e5f6
Create Date: 2026-03-02
The temporal retrieval entry-point query filters memory_units by occurred_start,
occurred_end, and mentioned_at using OR conditions. Without dedicated indexes the
planner falls back to a sequential scan of all bank rows after applying the
(bank_id, fact_type) index, then re-checks each date field.
These three partial indexes give the planner bitmap-index scan options for the
three most common date predicates, dramatically reducing the row set before any
embedding computation is required.
All indexes are created CONCURRENTLY so the migration does not block writes on
memory_units during production deployments. CONCURRENTLY requires running outside
a transaction block; see migrations.py for how this is handled safely.
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "b3c4d5e6f7g8"
down_revision: str | Sequence[str] | None = "c1a2b3d4e5f6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
# Partial index on occurred_start (covers "occurred_start BETWEEN $4 AND $5")
op.execute("COMMIT")
op.execute(
f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_occurred_start "
f"ON {schema}memory_units(bank_id, fact_type, occurred_start) "
f"WHERE occurred_start IS NOT NULL"
)
# Partial index on occurred_end (covers "occurred_end BETWEEN $4 AND $5")
op.execute("COMMIT")
op.execute(
f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_occurred_end "
f"ON {schema}memory_units(bank_id, fact_type, occurred_end) "
f"WHERE occurred_end IS NOT NULL"
)
# Partial index on mentioned_at (covers "mentioned_at BETWEEN $4 AND $5")
op.execute("COMMIT")
op.execute(
f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_mentioned_at "
f"ON {schema}memory_units(bank_id, fact_type, mentioned_at) "
f"WHERE mentioned_at IS NOT NULL"
)
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_mentioned_at")
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_occurred_end")
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_occurred_start")
@@ -1,34 +0,0 @@
"""Backfill observation_scopes column if missing.
This migration ensures observation_scopes exists even on databases that had
revision z1u2v3w4x5y6 applied when it referred to the old text_signals migration
(before it was renamed to a2b3c4d5e6f7). The ADD COLUMN IF NOT EXISTS makes this
a no-op on databases that already have the column.
Revision ID: b4c5d6e7f8a9
Revises: a2b3c4d5e6f7
Create Date: 2026-03-02
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "b4c5d6e7f8a9"
down_revision: str | Sequence[str] | None = "a2b3c4d5e6f7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS observation_scopes JSONB")
def downgrade() -> None:
pass # intentionally no-op — safe to leave the column in place
@@ -1,46 +0,0 @@
"""Enable pg_trgm extension and add GIN trigram index on entities.canonical_name
Revision ID: c1a2b3d4e5f6
Revises: b4c5d6e7f8a9
Create Date: 2026-03-02
Index is created CONCURRENTLY so the migration does not block writes on entities
during production deployments. CONCURRENTLY requires running outside a transaction
block; see migrations.py for how this is handled safely.
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "c1a2b3d4e5f6"
down_revision: str | Sequence[str] | None = "b4c5d6e7f8a9"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
# pg_trgm ships with every standard PostgreSQL installation as a contrib module.
# It enables fast similarity lookups via GIN indexes, used for entity name matching.
op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
schema = _get_schema_prefix()
# GIN index on canonical_name enables sub-millisecond trigram similarity queries
# (% operator, similarity()) instead of full-table scans across all bank entities.
op.execute("COMMIT")
op.execute(
f"CREATE INDEX CONCURRENTLY IF NOT EXISTS entities_canonical_name_trgm_idx "
f"ON {schema}entities USING GIN (canonical_name gin_trgm_ops)"
)
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}entities_canonical_name_trgm_idx")
# Note: not dropping pg_trgm extension as other indexes may depend on it
@@ -1,30 +0,0 @@
"""Add history column to mental_models
Revision ID: c3d4e5f6g7h8
Revises: a2b3c4d5e6f7, a2b3c4d5e6f8
Create Date: 2026-03-06
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "c3d4e5f6g7h8"
down_revision: str | Sequence[str] | None = ("a2b3c4d5e6f7", "a2b3c4d5e6f8")
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}mental_models ADD COLUMN IF NOT EXISTS history JSONB DEFAULT '[]'::jsonb")
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS history")
@@ -1,83 +0,0 @@
"""Add covering and composite indexes to speed up link expansion graph retrieval.
Two indexes target the two bottlenecks identified by EXPLAIN ANALYZE on a 17M-row
memory_links table:
1. idx_memory_links_to_type_weight (to_unit_id, link_type, weight DESC)
The semantic incoming direction — finding facts that consider seeds as their
nearest neighbour — currently hits an expensive BitmapAnd of two separate
bitmap scans (to_unit_id bitmap ∩ link_type bitmap). A composite index
on (to_unit_id, link_type) turns this into a single index scan and reduces
latency from ~36 ms to < 5 ms per query.
2. idx_memory_links_entity_covering (from_unit_id) INCLUDE (to_unit_id, entity_id)
WHERE link_type = 'entity'
The entity co-occurrence expansion uses COUNT(DISTINCT ml.entity_id) and
joins on ml.to_unit_id. Without a covering index the planner must read
~2 500 heap pages to fetch entity_id and to_unit_id after the bitmap index
scan, adding ~230 ms of random I/O. INCLUDE adds those two columns to the
index leaf pages so the entire query can be served from the index (index-only
scan), eliminating the heap reads entirely.
Partial index (WHERE link_type = 'entity') keeps index size ~40 % smaller.
Both indexes are created with CONCURRENTLY so the migration does not block
concurrent reads or writes on memory_links. CONCURRENTLY requires running
outside a transaction block, so the migration emits an explicit COMMIT before
each statement and uses IF NOT EXISTS for idempotency.
Revision ID: d2e3f4a5b6c7
Revises: b3c4d5e6f7g8
Create Date: 2026-03-02
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "d2e3f4a5b6c7"
down_revision: str | Sequence[str] | None = "b3c4d5e6f7g8"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
# CREATE INDEX CONCURRENTLY cannot run inside a transaction block.
# Commit the current Alembic transaction, then issue each CONCURRENTLY
# statement in its own implicit autocommit transaction.
# IF NOT EXISTS makes each statement idempotent if the migration is retried.
# Index for the semantic *incoming* direction in link_expansion_retrieval.py.
# Replaces the BitmapAnd of idx_memory_links_to_unit ∩ idx_memory_links_link_type
# with a single composite index scan.
op.execute("COMMIT")
op.execute(
f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_to_type_weight "
f"ON {schema}memory_links(to_unit_id, link_type, weight DESC)"
)
# Covering index for entity co-occurrence expansion.
# Enables an index-only scan: entity_id and to_unit_id are read from the
# index leaf pages instead of the heap, eliminating ~2 500 random heap-page
# reads per expansion query.
op.execute("COMMIT")
op.execute(
f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_entity_covering "
f"ON {schema}memory_links(from_unit_id) "
f"INCLUDE (to_unit_id, entity_id) "
f"WHERE link_type = 'entity'"
)
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_entity_covering")
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_to_type_weight")
@@ -1,53 +0,0 @@
"""Recreate idx_memory_units_source_memory_ids GIN index with fastupdate=off
GIN indexes use a "fastupdate" pending list by default: small writes are
buffered there and flushed to the main GIN tree in bulk. Flushing requires
AccessExclusiveLock on the index. Under high insert concurrency (e.g. 8
parallel pytest-xdist workers all calling retain_async) two transactions can
each trigger a flush simultaneously and deadlock.
Disabling fastupdate makes every insert write directly to the GIN tree
(slightly slower per insert, but no pending-list lock cycles).
Revision ID: d4e5f6g7h8i9
Revises: d5e6f7a8b9c0
Create Date: 2026-03-11
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "d4e5f6g7h8i9"
down_revision: str | Sequence[str] | None = "d5e6f7a8b9c0"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
# DROP + CREATE CONCURRENTLY must run outside a transaction block.
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
op.execute(
f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
f"ON {schema}memory_units USING GIN (source_memory_ids) "
f"WITH (fastupdate=off) "
f"WHERE source_memory_ids IS NOT NULL"
)
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
op.execute(
f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
f"ON {schema}memory_units USING GIN (source_memory_ids) "
f"WHERE source_memory_ids IS NOT NULL"
)
@@ -1,131 +0,0 @@
"""Add internal_id to banks and per-(bank, fact_type) partial HNSW indexes
Revision ID: d5e6f7a8b9c0
Revises: a3b4c5d6e7f8
Create Date: 2026-03-11
This migration:
1. Adds internal_id UUID column to banks (stable identifier for index naming)
2. Drops the global HNSW index (competes with per-bank partial indexes)
3. Creates per-(bank_id, fact_type) partial HNSW indexes for all existing banks
(new banks get indexes created at bank-creation time via bank_utils.create_bank_hnsw_indexes)
Why per-(bank, fact_type) indexes:
- fact_type-only partial indexes are never chosen by the planner when bank_id is in the WHERE
clause, because the idx_memory_units_bank_id B-tree index always wins at planning time.
- Per-(bank, fact_type) partial indexes have both predicates matching → planner selects them.
- The global HNSW index competes for larger partitions (world, observation) and must be dropped.
For large deployments, create indexes CONCURRENTLY before running this migration:
SELECT internal_id, bank_id FROM banks;
-- for each bank and each fact_type in (world, experience, observation):
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mu_emb_{ft}_{uid16}
ON memory_units USING hnsw (embedding vector_cosine_ops)
WHERE fact_type = '{ft}' AND bank_id = '{bank_id}';
DROP INDEX CONCURRENTLY IF EXISTS idx_memory_units_embedding;
"""
from collections.abc import Sequence
from alembic import context, op
from sqlalchemy import text
revision: str = "d5e6f7a8b9c0"
down_revision: str | Sequence[str] | None = "c3d4e5f6g7h8"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
_HNSW_FACT_TYPES: dict[str, str] = {
"world": "worl",
"experience": "expr",
"observation": "obsv",
}
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
# 1. Add internal_id column to banks
op.execute(
f"ALTER TABLE {schema}banks ADD COLUMN IF NOT EXISTS internal_id UUID DEFAULT gen_random_uuid() NOT NULL"
)
op.execute(f"ALTER TABLE {schema}banks ADD CONSTRAINT banks_internal_id_unique UNIQUE (internal_id)")
# 2. Drop any fact_type-only partial HNSW indexes that may exist from prior migrations
# (bank_id B-tree always wins over them when bank_id is in the WHERE clause)
op.execute(f"DROP INDEX IF EXISTS {schema}idx_mu_emb_world")
op.execute(f"DROP INDEX IF EXISTS {schema}idx_mu_emb_observation")
op.execute(f"DROP INDEX IF EXISTS {schema}idx_mu_emb_experience")
# 4. Drop global HNSW index (competes with per-bank partial indexes)
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_embedding")
# 5. Create per-(bank, fact_type) partial HNSW indexes for all existing banks
bind = op.get_bind()
schema_name = context.config.get_main_option("target_schema")
table_ref = f'"{schema_name}".memory_units' if schema_name else "memory_units"
banks_ref = f'"{schema_name}".banks' if schema_name else "banks"
rows = bind.execute(text(f"SELECT bank_id, internal_id FROM {banks_ref}")).fetchall() # noqa: S608
for row in rows:
bank_id = row[0]
internal_id = str(row[1]).replace("-", "")[:16]
escaped_bank_id = bank_id.replace("'", "''")
for ft, ft_short in _HNSW_FACT_TYPES.items():
idx_name = f"idx_mu_emb_{ft_short}_{internal_id}"
# Index name is schema-unqualified (indexes live in the schema of their table)
bind.execute(
text(
f"CREATE INDEX IF NOT EXISTS {idx_name} "
f"ON {table_ref} USING hnsw (embedding vector_cosine_ops) "
f"WHERE fact_type = '{ft}' AND bank_id = '{escaped_bank_id}'"
)
)
def downgrade() -> None:
schema = _get_schema_prefix()
# Drop per-bank HNSW indexes (iterate existing banks)
bind = op.get_bind()
schema_name = context.config.get_main_option("target_schema")
banks_ref = f'"{schema_name}".banks' if schema_name else "banks"
rows = bind.execute(text(f"SELECT internal_id FROM {banks_ref}")).fetchall() # noqa: S608
for row in rows:
internal_id = str(row[0]).replace("-", "")[:16]
for ft_short in _HNSW_FACT_TYPES.values():
idx_name = f"idx_mu_emb_{ft_short}_{internal_id}"
bind.execute(text(f"DROP INDEX IF EXISTS {schema}{idx_name}"))
# Restore the global HNSW index
table_ref = f'"{schema_name}".memory_units' if schema_name else "memory_units"
op.execute(
f"CREATE INDEX IF NOT EXISTS idx_memory_units_embedding ON {table_ref} USING hnsw (embedding vector_cosine_ops)"
)
# Restore old fact_type-only partial indexes
op.execute(
f"CREATE INDEX IF NOT EXISTS idx_mu_emb_world "
f"ON {table_ref} USING hnsw (embedding vector_cosine_ops) "
f"WHERE fact_type = 'world'"
)
op.execute(
f"CREATE INDEX IF NOT EXISTS idx_mu_emb_observation "
f"ON {table_ref} USING hnsw (embedding vector_cosine_ops) "
f"WHERE fact_type = 'observation'"
)
op.execute(
f"CREATE INDEX IF NOT EXISTS idx_mu_emb_experience "
f"ON {table_ref} USING hnsw (embedding vector_cosine_ops) "
f"WHERE fact_type = 'experience'"
)
# Drop internal_id column
op.execute(f"ALTER TABLE {schema}banks DROP CONSTRAINT IF EXISTS banks_internal_id_unique")
op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS internal_id")
@@ -1,62 +0,0 @@
"""Add webhooks table and next_retry_at to async_operations.
Webhook deliveries are handled as async_operations tasks (operation_type='webhook_delivery')
rather than a dedicated webhook_deliveries table.
Revision ID: e4f5a6b7c8d9
Revises: d2e3f4a5b6c7
Create Date: 2026-03-04
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "e4f5a6b7c8d9"
down_revision: str | Sequence[str] | None = "d2e3f4a5b6c7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(
f"""
CREATE TABLE IF NOT EXISTS {schema}webhooks (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
bank_id TEXT,
url TEXT NOT NULL,
secret TEXT,
event_types TEXT[] NOT NULL DEFAULT '{{}}',
enabled BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
"""
)
# Index for bank-scoped webhook lookup
op.execute(f"CREATE INDEX IF NOT EXISTS idx_webhooks_bank_id ON {schema}webhooks(bank_id)")
# Add next_retry_at to async_operations for task-owned retry scheduling
op.execute(f"ALTER TABLE {schema}async_operations ADD COLUMN IF NOT EXISTS next_retry_at TIMESTAMPTZ NULL")
# Index for polling: status + next_retry_at
op.execute(
f"CREATE INDEX IF NOT EXISTS idx_async_operations_status_retry "
f"ON {schema}async_operations(status, next_retry_at)"
)
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_status_retry")
op.execute(f"ALTER TABLE {schema}async_operations DROP COLUMN IF EXISTS next_retry_at")
op.execute(f"DROP INDEX IF EXISTS {schema}idx_webhooks_bank_id")
op.execute(f"DROP TABLE IF EXISTS {schema}webhooks")
@@ -1,73 +0,0 @@
"""Add CASCADE DELETE FK from async_operations and webhooks to banks.
When a bank is deleted, all its async_operations and webhooks rows are
automatically deleted by the database. This ensures that any in-flight
worker tasks detect the deletion via _check_op_alive() and abort early.
Revision ID: e5f6g7h8i9j0
Revises: d4e5f6g7h8i9
Create Date: 2026-03-11
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "e5f6g7h8i9j0"
down_revision: str | Sequence[str] | None = "d4e5f6g7h8i9"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
# Remove orphaned async_operations rows whose bank no longer exists
# (can happen because there was no FK before this migration).
op.execute(
f"""
DELETE FROM {schema}async_operations
WHERE bank_id IS NOT NULL
AND bank_id NOT IN (SELECT bank_id FROM {schema}banks)
"""
)
# Remove orphaned webhooks rows whose bank no longer exists.
op.execute(
f"""
DELETE FROM {schema}webhooks
WHERE bank_id IS NOT NULL
AND bank_id NOT IN (SELECT bank_id FROM {schema}banks)
"""
)
# Add FK with ON DELETE CASCADE so that deleting a bank automatically
# cleans up all its pending/processing operations and webhook configs.
op.execute(
f"""
ALTER TABLE {schema}async_operations
ADD CONSTRAINT fk_async_operations_bank_id
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id)
ON DELETE CASCADE
"""
)
op.execute(
f"""
ALTER TABLE {schema}webhooks
ADD CONSTRAINT fk_webhooks_bank_id
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id)
ON DELETE CASCADE
"""
)
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}async_operations DROP CONSTRAINT IF EXISTS fk_async_operations_bank_id")
op.execute(f"ALTER TABLE {schema}webhooks DROP CONSTRAINT IF EXISTS fk_webhooks_bank_id")
@@ -1,44 +0,0 @@
"""add_memory_links_from_type_weight_index
Revision ID: f1a2b3c4d5e6
Revises: e0a1b2c3d4e5
Create Date: 2025-01-12
Add composite index on memory_links (from_unit_id, link_type, weight DESC)
to optimize MPFP graph traversal queries that need top-k edges per type.
"""
from collections.abc import Sequence
from alembic import context, op
# revision identifiers, used by Alembic.
revision: str = "f1a2b3c4d5e6"
down_revision: str | Sequence[str] | None = "e0a1b2c3d4e5"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (e.g., 'tenant_x.' or '' for public)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Add composite index for efficient MPFP edge loading."""
schema = _get_schema_prefix()
# Create composite index for efficient top-k per (from_node, link_type) queries
# This enables LATERAL joins to use index-only scans with early termination
# Note: Not using CONCURRENTLY here as it requires running outside a transaction
# For production with large tables, consider running this manually with CONCURRENTLY
op.execute(
f"CREATE INDEX IF NOT EXISTS idx_memory_links_from_type_weight "
f"ON {schema}memory_links(from_unit_id, link_type, weight DESC)"
)
def downgrade() -> None:
"""Remove the composite index."""
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_links_from_type_weight")
@@ -1,38 +0,0 @@
"""chunk_fk_cascade_delete
Revision ID: f6g7h8i9j0k1
Revises: e5f6g7h8i9j0
Create Date: 2026-03-16 00:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "f6g7h8i9j0k1"
down_revision: str | Sequence[str] | None = "e5f6g7h8i9j0"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Change memory_units.chunk_id FK from SET NULL to CASCADE.
When a document is deleted the CASCADE reaches chunks first; with SET NULL
the memory_units rows survived with chunk_id = NULL, leaving ghost records.
Switching to CASCADE ensures they are removed together with their chunk.
"""
op.drop_constraint("memory_units_chunk_fkey", "memory_units", type_="foreignkey")
op.create_foreign_key(
"memory_units_chunk_fkey", "memory_units", "chunks", ["chunk_id"], ["chunk_id"], ondelete="CASCADE"
)
def downgrade() -> None:
"""Revert to SET NULL behaviour."""
op.drop_constraint("memory_units_chunk_fkey", "memory_units", type_="foreignkey")
op.create_foreign_key(
"memory_units_chunk_fkey", "memory_units", "chunks", ["chunk_id"], ["chunk_id"], ondelete="SET NULL"
)
@@ -1,33 +0,0 @@
"""Add http_config JSONB column to webhooks table.
Stores HTTP delivery configuration (method, timeout, headers, params) as a
single JSONB column rather than separate columns.
Revision ID: f7g8h9i0j1k2
Revises: e4f5a6b7c8d9
Create Date: 2026-03-04
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "f7g8h9i0j1k2"
down_revision: str | Sequence[str] | None = "e4f5a6b7c8d9"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}webhooks ADD COLUMN IF NOT EXISTS http_config JSONB NOT NULL DEFAULT '{{}}'")
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}webhooks DROP COLUMN IF EXISTS http_config")
@@ -1,48 +0,0 @@
"""add_tags_column
Revision ID: g2a3b4c5d6e7
Revises: f1a2b3c4d5e6
Create Date: 2025-01-13
Add tags column to memory_units and documents tables for visibility scoping.
Tags enable filtering memories by scope (e.g., user IDs, session IDs) during recall/reflect.
"""
from collections.abc import Sequence
from alembic import context, op
# revision identifiers, used by Alembic.
revision: str = "g2a3b4c5d6e7"
down_revision: str | Sequence[str] | None = "f1a2b3c4d5e6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (e.g., 'tenant_x.' or '' for public)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Add tags column to memory_units and documents tables."""
schema = _get_schema_prefix()
# Add tags column to memory_units table
op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS tags VARCHAR[] NOT NULL DEFAULT '{{}}'")
# Create GIN index for efficient array containment queries (tags && ARRAY['x'])
op.execute(f"CREATE INDEX IF NOT EXISTS idx_memory_units_tags ON {schema}memory_units USING GIN (tags)")
# Add tags column to documents table for document-level tags
op.execute(f"ALTER TABLE {schema}documents ADD COLUMN IF NOT EXISTS tags VARCHAR[] NOT NULL DEFAULT '{{}}'")
def downgrade() -> None:
"""Remove tags columns and index."""
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_tags")
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS tags")
op.execute(f"ALTER TABLE {schema}documents DROP COLUMN IF EXISTS tags")
@@ -1,71 +0,0 @@
"""backsweep_orphan_memory_units
Two-pass cleanup of memory_units rows that were never removed by earlier bugs:
Pass 1 — any fact_type, bank gone:
memory_units whose bank_id no longer exists in banks. These accumulate when
a bank is deleted without a proper cascade (no FK from memory_units to banks
exists in the schema).
Pass 2 — observations only, all sources gone:
observation rows whose bank still exists but every source_memory_id points
to a deleted memory unit. These were left behind before PR #580 fixed the
chunk FK cascade and before delete_document() called
_delete_stale_observations_for_memories.
Revision ID: g7h8i9j0k1l2
Revises: f6g7h8i9j0k1
Create Date: 2026-03-16
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "g7h8i9j0k1l2"
down_revision: str | Sequence[str] | None = "f6g7h8i9j0k1"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
mu = f"{schema}memory_units"
banks = f"{schema}banks"
# Pass 1: delete all memory_units (any fact_type) whose bank no longer exists.
# There is no FK from memory_units to banks, so these never cascade away.
op.execute(
f"""
DELETE FROM {mu}
WHERE NOT EXISTS (
SELECT 1 FROM {banks} b WHERE b.bank_id = {mu}.bank_id
)
"""
)
# Pass 2: delete orphaned observations whose bank still exists but every
# source_memory_id refers to a now-deleted memory unit (or the array is
# empty). Observations with at least one surviving source are left alone.
op.execute(
f"""
DELETE FROM {mu} orphan
WHERE orphan.fact_type = 'observation'
AND NOT EXISTS (
SELECT 1
FROM {mu} src
WHERE src.id = ANY(orphan.source_memory_ids)
AND src.bank_id = orphan.bank_id
)
"""
)
def downgrade() -> None:
# Deleted rows cannot be restored.
pass
@@ -1,112 +0,0 @@
"""mental_models_v4
Revision ID: h3c4d5e6f7g8
Revises: g2a3b4c5d6e7
Create Date: 2026-01-08 00:00:00.000000
This migration implements the v4 mental models system:
1. Deletes existing observation memory_units (observations now in mental models)
2. Adds mission column to banks (replacing background)
3. Creates mental_models table with final schema
Mental models can reference entities when an entity is "promoted" to a mental model.
Summary content is stored as JSONB observations with per-observation fact attribution.
"""
from collections.abc import Sequence
from alembic import context, op
# revision identifiers, used by Alembic.
revision: str = "h3c4d5e6f7g8"
down_revision: str | Sequence[str] | None = "g2a3b4c5d6e7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Apply mental models v4 changes."""
schema = _get_schema_prefix()
# Step 1: Delete observation memory_units (cascades to unit_entities links)
# Observations are now handled through mental models, not memory_units
op.execute(f"DELETE FROM {schema}memory_units WHERE fact_type = 'observation'")
# Step 2: Drop observation-specific index (if it exists)
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_observation_date")
# Step 3: Add mission column to banks (replacing background)
op.execute(f"ALTER TABLE {schema}banks ADD COLUMN IF NOT EXISTS mission TEXT")
# Migrate: copy background to mission if background column exists
# Use DO block to check column existence first (idempotent for re-runs)
schema_name = context.config.get_main_option("target_schema") or "public"
op.execute(f"""
DO $$
BEGIN
IF EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_schema = '{schema_name}' AND table_name = 'banks' AND column_name = 'background'
) THEN
UPDATE {schema}banks
SET mission = background
WHERE mission IS NULL;
END IF;
END $$;
""")
# Remove background column (replaced by mission)
op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS background")
# Step 4: Create mental_models table with final v4 schema (if not exists)
op.execute(f"""
CREATE TABLE IF NOT EXISTS {schema}mental_models (
id VARCHAR(64) NOT NULL,
bank_id VARCHAR(64) NOT NULL,
subtype VARCHAR(32) NOT NULL,
name VARCHAR(256) NOT NULL,
description TEXT NOT NULL,
entity_id UUID,
observations JSONB DEFAULT '{{"observations": []}}'::jsonb,
links VARCHAR[],
tags VARCHAR[] DEFAULT '{{}}',
last_updated TIMESTAMP WITH TIME ZONE,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
PRIMARY KEY (id, bank_id),
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE,
FOREIGN KEY (entity_id) REFERENCES {schema}entities(id) ON DELETE SET NULL,
CONSTRAINT ck_mental_models_subtype CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
)
""")
# Step 5: Create indexes for efficient queries (if not exist)
op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_bank_id ON {schema}mental_models(bank_id)")
op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_subtype ON {schema}mental_models(bank_id, subtype)")
op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_entity_id ON {schema}mental_models(entity_id)")
# GIN index for efficient tags array filtering
op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_tags ON {schema}mental_models USING GIN(tags)")
def downgrade() -> None:
"""Revert mental models v4 changes."""
schema = _get_schema_prefix()
# Drop mental_models table (cascades to indexes)
op.execute(f"DROP TABLE IF EXISTS {schema}mental_models CASCADE")
# Add back background column to banks
op.execute(f"ALTER TABLE {schema}banks ADD COLUMN IF NOT EXISTS background TEXT")
# Migrate mission back to background
op.execute(f"UPDATE {schema}banks SET background = mission WHERE background IS NULL")
# Remove mission column
op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS mission")
# Note: Cannot restore deleted observations - they are lost on downgrade
@@ -1,41 +0,0 @@
"""delete_opinions
Revision ID: i4d5e6f7g8h9
Revises: h3c4d5e6f7g8
Create Date: 2026-01-15 00:00:00.000000
This migration removes opinion facts from memory_units.
Opinions are no longer a separate fact type - they are now represented
through mental model observations with confidence scores.
"""
from collections.abc import Sequence
from alembic import context, op
# revision identifiers, used by Alembic.
revision: str = "i4d5e6f7g8h9"
down_revision: str | Sequence[str] | None = "h3c4d5e6f7g8"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Delete opinion memory_units."""
schema = _get_schema_prefix()
# Delete opinion memory_units (cascades to unit_entities links)
# Opinions are now handled through mental model observations
op.execute(f"DELETE FROM {schema}memory_units WHERE fact_type = 'opinion'")
def downgrade() -> None:
"""Cannot restore deleted opinions."""
# Note: Cannot restore deleted opinions - they are lost on downgrade
pass
@@ -1,95 +0,0 @@
"""mental_model_versions
Revision ID: j5e6f7g8h9i0
Revises: i4d5e6f7g8h9
Create Date: 2026-01-16 00:00:00.000000
This migration adds versioning support for mental models:
1. Creates mental_model_versions table to store observation snapshots
2. Adds version column to mental_models for tracking current version
This enables changelog/diff functionality for mental model observations.
"""
from collections.abc import Sequence
from alembic import context, op
# revision identifiers, used by Alembic.
revision: str = "j5e6f7g8h9i0"
down_revision: str | Sequence[str] | None = "i4d5e6f7g8h9"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Create mental_model_versions table and add version tracking."""
schema = _get_schema_prefix()
# Create mental_model_versions table for storing observation snapshots
op.execute(f"""
CREATE TABLE {schema}mental_model_versions (
id SERIAL PRIMARY KEY,
mental_model_id VARCHAR(64) NOT NULL,
bank_id VARCHAR(64) NOT NULL,
version INT NOT NULL,
observations JSONB NOT NULL DEFAULT '{{"observations": []}}'::jsonb,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
FOREIGN KEY (mental_model_id, bank_id)
REFERENCES {schema}mental_models(id, bank_id) ON DELETE CASCADE,
UNIQUE (mental_model_id, bank_id, version)
)
""")
# Index for efficient version queries (get latest, list versions)
op.execute(f"""
CREATE INDEX idx_mental_model_versions_lookup
ON {schema}mental_model_versions(mental_model_id, bank_id, version DESC)
""")
# Add version column to mental_models to track current version
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD COLUMN IF NOT EXISTS version INT NOT NULL DEFAULT 0
""")
# Migrate existing mental models: create version 1 for any that have observations
op.execute(f"""
INSERT INTO {schema}mental_model_versions (mental_model_id, bank_id, version, observations, created_at)
SELECT id, bank_id, 1, observations, COALESCE(last_updated, created_at)
FROM {schema}mental_models
WHERE observations IS NOT NULL
AND observations != '{{"observations": []}}'::jsonb
AND (observations->'observations') IS NOT NULL
AND jsonb_array_length(observations->'observations') > 0
""")
# Update version to 1 for migrated mental models
op.execute(f"""
UPDATE {schema}mental_models
SET version = 1
WHERE observations IS NOT NULL
AND observations != '{{"observations": []}}'::jsonb
AND (observations->'observations') IS NOT NULL
AND jsonb_array_length(observations->'observations') > 0
""")
def downgrade() -> None:
"""Remove mental_model_versions table and version column."""
schema = _get_schema_prefix()
# Drop index
op.execute(f"DROP INDEX IF EXISTS {schema}idx_mental_model_versions_lookup")
# Drop versions table
op.execute(f"DROP TABLE IF EXISTS {schema}mental_model_versions")
# Remove version column from mental_models
op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS version")
@@ -1,58 +0,0 @@
"""add_directive_subtype
Revision ID: k6f7g8h9i0j1
Revises: j5e6f7g8h9i0
Create Date: 2026-01-16 00:00:00.000000
This migration adds 'directive' to the mental_models subtype constraint.
Directives are hard rules with user-provided observations that the reflect agent must follow.
"""
from collections.abc import Sequence
from alembic import context, op
# revision identifiers, used by Alembic.
revision: str = "k6f7g8h9i0j1"
down_revision: str | Sequence[str] | None = "j5e6f7g8h9i0"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Add 'directive' to mental_models subtype constraint."""
schema = _get_schema_prefix()
# Drop existing constraint
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
# Create new constraint with 'directive' added
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD CONSTRAINT ck_mental_models_subtype
CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned', 'directive'))
""")
def downgrade() -> None:
"""Remove 'directive' from mental_models subtype constraint."""
schema = _get_schema_prefix()
# First delete any directives (cannot downgrade if they exist)
op.execute(f"DELETE FROM {schema}mental_models WHERE subtype = 'directive'")
# Drop constraint with directive
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
# Recreate original constraint without directive
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD CONSTRAINT ck_mental_models_subtype
CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
""")
@@ -1,109 +0,0 @@
"""add_worker_columns
Revision ID: l7g8h9i0j1k2
Revises: k6f7g8h9i0j1
Create Date: 2026-01-19 00:00:00.000000
This migration adds columns to async_operations for distributed worker support:
- worker_id: ID of the worker that claimed the task
- claimed_at: When the task was claimed
- retry_count: Number of retry attempts
- task_payload: The serialized task dictionary
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import context, op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = "l7g8h9i0j1k2"
down_revision: str | Sequence[str] | None = "k6f7g8h9i0j1"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Add worker columns to async_operations."""
schema = _get_schema_prefix()
# Add worker_id column (ID of worker that claimed the task)
op.add_column(
"async_operations",
sa.Column("worker_id", sa.Text(), nullable=True),
schema=context.config.get_main_option("target_schema") or None,
)
# Add claimed_at column (when task was claimed by worker)
op.add_column(
"async_operations",
sa.Column("claimed_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
schema=context.config.get_main_option("target_schema") or None,
)
# Add retry_count column (number of retry attempts)
op.add_column(
"async_operations",
sa.Column("retry_count", sa.Integer(), server_default="0", nullable=False),
schema=context.config.get_main_option("target_schema") or None,
)
# Add task_payload column (serialized task dictionary)
op.add_column(
"async_operations",
sa.Column(
"task_payload",
postgresql.JSONB(astext_type=sa.Text()),
nullable=True,
),
schema=context.config.get_main_option("target_schema") or None,
)
# Add index for efficient worker polling (pending tasks ordered by creation time)
op.execute(
f"CREATE INDEX idx_async_operations_pending_claim ON {schema}async_operations (status, created_at) "
f"WHERE status = 'pending' AND task_payload IS NOT NULL"
)
# Add index for finding tasks by worker_id (for decommissioning)
op.execute(
f"CREATE INDEX idx_async_operations_worker_id ON {schema}async_operations (worker_id) WHERE worker_id IS NOT NULL"
)
def downgrade() -> None:
"""Remove worker columns from async_operations."""
schema = _get_schema_prefix()
# Drop indexes
op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_pending_claim")
op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_worker_id")
# Drop columns
op.drop_column(
"async_operations",
"task_payload",
schema=context.config.get_main_option("target_schema") or None,
)
op.drop_column(
"async_operations",
"retry_count",
schema=context.config.get_main_option("target_schema") or None,
)
op.drop_column(
"async_operations",
"claimed_at",
schema=context.config.get_main_option("target_schema") or None,
)
op.drop_column(
"async_operations",
"worker_id",
schema=context.config.get_main_option("target_schema") or None,
)
@@ -1,41 +0,0 @@
"""mental_model_id_to_text
Revision ID: m8h9i0j1k2l3
Revises: l7g8h9i0j1k2
Create Date: 2026-01-19 00:00:00.000000
This migration changes the mental_models.id column from VARCHAR(64) to TEXT
to support longer model IDs (e.g., entity names that exceed 64 characters).
"""
from collections.abc import Sequence
from alembic import context, op
# revision identifiers, used by Alembic.
revision: str = "m8h9i0j1k2l3"
down_revision: str | Sequence[str] | None = "l7g8h9i0j1k2"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Change mental_models.id from VARCHAR(64) to TEXT."""
schema = _get_schema_prefix()
# Alter the id column type from VARCHAR(64) to TEXT
op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE TEXT")
def downgrade() -> None:
"""Revert mental_models.id from TEXT to VARCHAR(64)."""
schema = _get_schema_prefix()
# Note: This may fail if any id values exceed 64 characters
op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE VARCHAR(64)")
@@ -1,317 +0,0 @@
"""learnings_and_pinned_reflections
Revision ID: n9i0j1k2l3m4
Revises: m8h9i0j1k2l3
Create Date: 2026-01-21 00:00:00.000000
This migration:
1. Creates the 'learnings' table for automatic bottom-up consolidation
2. Creates the 'pinned_reflections' table for user-curated living documents
3. Adds consolidation tracking columns to the 'banks' table
"""
import os
from collections.abc import Sequence
from alembic import context, op
from sqlalchemy import text
# revision identifiers, used by Alembic.
revision: str = "n9i0j1k2l3m4"
down_revision: str | Sequence[str] | None = "m8h9i0j1k2l3"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def _detect_vector_extension() -> str:
"""
Detect or validate vector extension: 'pgvector', 'vchord', or 'pgvectorscale'.
Respects HINDSIGHT_API_VECTOR_EXTENSION env var if set.
"""
conn = op.get_bind()
vector_extension = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
# Validate configured extension is installed
if vector_extension == "pgvectorscale":
# pgvectorscale/DiskANN requires pgvector
pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
if not pgvector_check:
raise RuntimeError(
"DiskANN requires pgvector. Install with: CREATE EXTENSION vector; then vectorscale or pg_diskann CASCADE;"
)
# Check for either vectorscale (open source) or pg_diskann (Azure)
vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar()
pg_diskann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_diskann'")).scalar()
if vectorscale_check:
return "pgvectorscale"
elif pg_diskann_check:
return "pg_diskann"
else:
raise RuntimeError(
"Configured vector extension 'pgvectorscale' not found. Install either:\n"
" - pgvectorscale: CREATE EXTENSION vectorscale CASCADE;\n"
" - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
)
elif vector_extension == "vchord":
vchord_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord'")).scalar()
if not vchord_check:
raise RuntimeError(
"Configured vector extension 'vchord' not found. Install it with: CREATE EXTENSION vchord CASCADE;"
)
return "vchord"
elif vector_extension == "pgvector":
pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
if not pgvector_check:
raise RuntimeError(
"Configured vector extension 'pgvector' not found. Install it with: CREATE EXTENSION vector;"
)
return "pgvector"
else:
raise ValueError(
f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {vector_extension}. Must be 'pgvector', 'vchord', or 'pgvectorscale'"
)
def _detect_text_search_extension() -> str:
"""
Detect or validate text search extension: 'native', 'vchord', or 'pg_textsearch'.
Respects HINDSIGHT_API_TEXT_SEARCH_EXTENSION env var.
Creates the extension if needed.
"""
text_search_extension = os.getenv("HINDSIGHT_API_TEXT_SEARCH_EXTENSION", "native").lower()
if text_search_extension == "vchord":
# Create vchord_bm25 extension if not exists
try:
op.execute("CREATE EXTENSION IF NOT EXISTS vchord_bm25 CASCADE")
except Exception:
# Extension might already exist or user lacks permissions - verify it exists
conn = op.get_bind()
result = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord_bm25'")).fetchone()
if not result:
# Extension truly doesn't exist - re-raise the error
raise
return "vchord"
elif text_search_extension == "pg_textsearch":
# Create pg_textsearch extension if not exists
try:
op.execute("CREATE EXTENSION IF NOT EXISTS pg_textsearch CASCADE")
except Exception:
# Extension might already exist or user lacks permissions - verify it exists
conn = op.get_bind()
result = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_textsearch'")).fetchone()
if not result:
# Extension truly doesn't exist - re-raise the error
raise
return "pg_textsearch"
elif text_search_extension == "native":
return "native"
else:
raise ValueError(
f"Invalid HINDSIGHT_API_TEXT_SEARCH_EXTENSION: {text_search_extension}. Must be 'native', 'vchord', or 'pg_textsearch'"
)
def upgrade() -> None:
"""Create learnings and pinned_reflections tables."""
schema = _get_schema_prefix()
# Detect which vector extension is available
vector_ext = _detect_vector_extension()
# Detect which text search extension to use
text_search_ext = _detect_text_search_extension()
# 1. Create learnings table
op.execute(f"""
CREATE TABLE {schema}learnings (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
bank_id VARCHAR(64) NOT NULL,
text TEXT NOT NULL,
proof_count INT NOT NULL DEFAULT 1,
history JSONB DEFAULT '[]'::jsonb,
mission_context VARCHAR(64),
pre_mission_change BOOLEAN DEFAULT FALSE,
embedding vector(384),
tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
)
""")
# Add foreign key constraint
op.execute(f"""
ALTER TABLE {schema}learnings
ADD CONSTRAINT fk_learnings_bank_id
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
""")
# Indexes for learnings
op.execute(f"CREATE INDEX idx_learnings_bank_id ON {schema}learnings(bank_id)")
# Create vector index based on detected extension
if vector_ext == "pgvectorscale":
op.execute(f"""
CREATE INDEX idx_learnings_embedding ON {schema}learnings
USING diskann (embedding vector_cosine_ops)
WITH (num_neighbors = 50)
""")
elif vector_ext == "pg_diskann":
op.execute(f"""
CREATE INDEX idx_learnings_embedding ON {schema}learnings
USING diskann (embedding vector_cosine_ops)
WITH (max_neighbors = 50)
""")
elif vector_ext == "vchord":
op.execute(f"""
CREATE INDEX idx_learnings_embedding ON {schema}learnings
USING vchordrq (embedding vector_l2_ops)
""")
else: # pgvector
op.execute(f"""
CREATE INDEX idx_learnings_embedding ON {schema}learnings
USING hnsw (embedding vector_cosine_ops)
""")
op.execute(f"CREATE INDEX idx_learnings_tags ON {schema}learnings USING GIN(tags)")
# Full-text search for learnings
if text_search_ext == "vchord":
# VectorChord BM25: bm25vector type (no GENERATED - tokenization happens on INSERT)
# Note: vchord_bm25 extension creates types in bm25_catalog schema
op.execute(f"""
ALTER TABLE {schema}learnings ADD COLUMN search_vector bm25_catalog.bm25vector
""")
op.execute(f"""
CREATE INDEX idx_learnings_text_search ON {schema}learnings
USING bm25 (search_vector bm25_catalog.bm25_ops)
""")
elif text_search_ext == "pg_textsearch":
# Timescale pg_textsearch: dummy TEXT column for consistency (indexes operate on base columns directly)
op.execute(f"""
ALTER TABLE {schema}learnings ADD COLUMN search_vector TEXT
""")
op.execute(f"""
CREATE INDEX idx_learnings_text_search ON {schema}learnings
USING bm25(text) WITH (text_config='english')
""")
else: # native
# Native PostgreSQL: tsvector with automatic generation
op.execute(f"""
ALTER TABLE {schema}learnings ADD COLUMN search_vector tsvector
GENERATED ALWAYS AS (to_tsvector('english', text)) STORED
""")
op.execute(f"CREATE INDEX idx_learnings_text_search ON {schema}learnings USING gin(search_vector)")
# 2. Create pinned_reflections table
op.execute(f"""
CREATE TABLE {schema}pinned_reflections (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
bank_id VARCHAR(64) NOT NULL,
name VARCHAR(256) NOT NULL,
source_query TEXT NOT NULL,
content TEXT NOT NULL,
embedding vector(384),
tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
last_refreshed_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
)
""")
# Add foreign key constraint
op.execute(f"""
ALTER TABLE {schema}pinned_reflections
ADD CONSTRAINT fk_pinned_reflections_bank_id
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
""")
# Indexes for pinned_reflections
op.execute(f"CREATE INDEX idx_pinned_reflections_bank_id ON {schema}pinned_reflections(bank_id)")
# Create vector index based on detected extension
if vector_ext == "pgvectorscale":
op.execute(f"""
CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
USING diskann (embedding vector_cosine_ops)
WITH (num_neighbors = 50)
""")
elif vector_ext == "pg_diskann":
op.execute(f"""
CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
USING diskann (embedding vector_cosine_ops)
WITH (max_neighbors = 50)
""")
elif vector_ext == "vchord":
op.execute(f"""
CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
USING vchordrq (embedding vector_l2_ops)
""")
else: # pgvector
op.execute(f"""
CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
USING hnsw (embedding vector_cosine_ops)
""")
op.execute(f"CREATE INDEX idx_pinned_reflections_tags ON {schema}pinned_reflections USING GIN(tags)")
# Full-text search for pinned_reflections
if text_search_ext == "vchord":
# VectorChord BM25: bm25vector type (no GENERATED - tokenization happens on INSERT/UPDATE)
# Note: vchord_bm25 extension creates types in bm25_catalog schema
op.execute(f"""
ALTER TABLE {schema}pinned_reflections ADD COLUMN search_vector bm25_catalog.bm25vector
""")
op.execute(f"""
CREATE INDEX idx_pinned_reflections_text_search ON {schema}pinned_reflections
USING bm25 (search_vector bm25_catalog.bm25_ops)
""")
elif text_search_ext == "pg_textsearch":
# Timescale pg_textsearch: dummy TEXT column for consistency (indexes operate on base columns directly)
op.execute(f"""
ALTER TABLE {schema}pinned_reflections ADD COLUMN search_vector TEXT
""")
op.execute(f"""
CREATE INDEX idx_pinned_reflections_text_search ON {schema}pinned_reflections
USING bm25(content)
WITH (text_config='english')
""")
else: # native
# Native PostgreSQL: tsvector with automatic generation
op.execute(f"""
ALTER TABLE {schema}pinned_reflections ADD COLUMN search_vector tsvector
GENERATED ALWAYS AS (to_tsvector('english', COALESCE(name, '') || ' ' || content)) STORED
""")
op.execute(f"""
CREATE INDEX idx_pinned_reflections_text_search ON {schema}pinned_reflections
USING gin(search_vector)
""")
# 3. Add consolidation tracking columns to banks table
op.execute(f"""
ALTER TABLE {schema}banks
ADD COLUMN IF NOT EXISTS last_consolidated_at TIMESTAMP WITH TIME ZONE
""")
op.execute(f"""
ALTER TABLE {schema}banks
ADD COLUMN IF NOT EXISTS mission_changed_at TIMESTAMP WITH TIME ZONE
""")
def downgrade() -> None:
"""Drop learnings and pinned_reflections tables."""
schema = _get_schema_prefix()
# Drop tables
op.execute(f"DROP TABLE IF EXISTS {schema}learnings CASCADE")
op.execute(f"DROP TABLE IF EXISTS {schema}pinned_reflections CASCADE")
# Remove columns from banks
op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS last_consolidated_at")
op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS mission_changed_at")
@@ -1,113 +0,0 @@
"""migrate_mental_models_data
Revision ID: o0j1k2l3m4n5
Revises: n9i0j1k2l3m4
Create Date: 2026-01-21 00:00:00.000000
This migration:
1. Migrates existing 'pinned' mental models to the new 'pinned_reflections' table
2. Migrates existing 'learned' mental models to the new 'learnings' table
3. Deletes non-directive mental models (structural, emergent, pinned, learned)
4. Drops the mental_model_versions table (no longer used)
5. Adds a CHECK constraint that only 'directive' subtype is allowed
"""
from collections.abc import Sequence
from alembic import context, op
# revision identifiers, used by Alembic.
revision: str = "o0j1k2l3m4n5"
down_revision: str | Sequence[str] | None = "n9i0j1k2l3m4"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Migrate data and clean up old mental models."""
schema = _get_schema_prefix()
# 1. Migrate 'pinned' mental models to pinned_reflections
# For pinned models, the first observation's content becomes the pinned reflection content
op.execute(f"""
INSERT INTO {schema}pinned_reflections (bank_id, name, source_query, content, tags, created_at)
SELECT
bank_id,
name,
description AS source_query,
COALESCE(
observations->'observations'->0->>'content',
description,
''
) AS content,
tags,
created_at
FROM {schema}mental_models
WHERE subtype = 'pinned'
ON CONFLICT DO NOTHING
""")
# 2. Migrate 'learned' mental models to learnings
# Each observation in a learned model becomes a separate learning
op.execute(f"""
INSERT INTO {schema}learnings (bank_id, text, proof_count, tags, created_at)
SELECT
mm.bank_id,
obs->>'content' AS text,
GREATEST(1, COALESCE(jsonb_array_length(obs->'evidence'), 1)) AS proof_count,
mm.tags,
mm.created_at
FROM {schema}mental_models mm,
LATERAL jsonb_array_elements(mm.observations->'observations') AS obs
WHERE mm.subtype = 'learned'
AND obs->>'content' IS NOT NULL
AND obs->>'content' != ''
ON CONFLICT DO NOTHING
""")
# 3. Delete all non-directive mental models (they've been migrated or are obsolete)
op.execute(f"""
DELETE FROM {schema}mental_models
WHERE subtype != 'directive'
""")
# 4. Drop the mental_model_versions table (no longer used)
op.execute(f"DROP TABLE IF EXISTS {schema}mental_model_versions CASCADE")
# 5. Drop old constraints and add new one that only allows 'directive'
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD CONSTRAINT ck_mental_models_subtype CHECK (subtype = 'directive')
""")
def downgrade() -> None:
"""Reverse the migration (data migration is one-way, so this just removes constraints)."""
schema = _get_schema_prefix()
# Remove the directive-only constraint
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
# Re-create mental_model_versions table
op.execute(f"""
CREATE TABLE IF NOT EXISTS {schema}mental_model_versions (
id SERIAL PRIMARY KEY,
bank_id VARCHAR(64) NOT NULL,
model_id VARCHAR(128) NOT NULL,
version INT NOT NULL,
observations JSONB NOT NULL,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
)
""")
op.execute(
f"CREATE INDEX IF NOT EXISTS idx_mm_versions_lookup ON {schema}mental_model_versions(bank_id, model_id, version DESC)"
)
# Note: Data migration cannot be reversed - pinned_reflections and learnings data remains
@@ -1,194 +0,0 @@
"""new_knowledge_architecture
Revision ID: p1k2l3m4n5o6
Revises: o0j1k2l3m4n5
Create Date: 2026-01-21 00:00:00.000000
This migration implements the new knowledge architecture:
1. Drops the 'learnings' table (mental models are now in memory_units)
2. Renames 'pinned_reflections' to 'reflections'
3. Drops the 'mental_models' table completely
4. Creates 'directives' table for hard rules
5. Adds mental model support columns to 'memory_units' (proof_count, source_memory_ids, history)
The new architecture:
- Directives: Hard rules in their own table
- Mental Models: Stored in memory_units with fact_type='mental_model'
- Reflections: User-curated documents (renamed from pinned_reflections)
"""
from collections.abc import Sequence
from alembic import context, op
# revision identifiers, used by Alembic.
revision: str = "p1k2l3m4n5o6"
down_revision: str | Sequence[str] | None = "o0j1k2l3m4n5"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Implement new knowledge architecture."""
schema = _get_schema_prefix()
# 1. Drop the learnings table (mental models will be in memory_units)
op.execute(f"DROP TABLE IF EXISTS {schema}learnings CASCADE")
# 2. Rename pinned_reflections to reflections
op.execute(f"ALTER TABLE IF EXISTS {schema}pinned_reflections RENAME TO reflections")
# Rename indexes for reflections
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_bank_id RENAME TO idx_reflections_bank_id")
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_embedding RENAME TO idx_reflections_embedding")
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_tags RENAME TO idx_reflections_tags")
op.execute(
f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_text_search RENAME TO idx_reflections_text_search"
)
# Rename foreign key constraint
op.execute(f"""
ALTER TABLE {schema}reflections
DROP CONSTRAINT IF EXISTS fk_pinned_reflections_bank_id
""")
op.execute(f"""
ALTER TABLE {schema}reflections
ADD CONSTRAINT fk_reflections_bank_id
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
""")
# 3. Drop the mental_models table completely
op.execute(f"DROP TABLE IF EXISTS {schema}mental_models CASCADE")
# 4. Create directives table
op.execute(f"""
CREATE TABLE {schema}directives (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
bank_id VARCHAR(64) NOT NULL,
name VARCHAR(256) NOT NULL,
content TEXT NOT NULL,
priority INT NOT NULL DEFAULT 0,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
)
""")
# Add foreign key and indexes for directives
op.execute(f"""
ALTER TABLE {schema}directives
ADD CONSTRAINT fk_directives_bank_id
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
""")
op.execute(f"CREATE INDEX idx_directives_bank_id ON {schema}directives(bank_id)")
op.execute(f"CREATE INDEX idx_directives_bank_active ON {schema}directives(bank_id, is_active)")
op.execute(f"CREATE INDEX idx_directives_tags ON {schema}directives USING GIN(tags)")
# 5. Add mental model support columns to memory_units
# proof_count: Number of memories that support this mental model
op.execute(f"""
ALTER TABLE {schema}memory_units
ADD COLUMN IF NOT EXISTS proof_count INT DEFAULT 1
""")
# source_memory_ids: Array of memory IDs that consolidated into this mental model
op.execute(f"""
ALTER TABLE {schema}memory_units
ADD COLUMN IF NOT EXISTS source_memory_ids UUID[] DEFAULT ARRAY[]::UUID[]
""")
# history: JSONB array tracking changes to mental models
op.execute(f"""
ALTER TABLE {schema}memory_units
ADD COLUMN IF NOT EXISTS history JSONB DEFAULT '[]'::jsonb
""")
# Add index for finding mental models
op.execute(f"""
CREATE INDEX IF NOT EXISTS idx_memory_units_mental_models
ON {schema}memory_units(bank_id, fact_type)
WHERE fact_type = 'mental_model'
""")
# 6. Update fact_type check constraint to include 'mental_model'
op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
op.execute(f"""
ALTER TABLE {schema}memory_units
ADD CONSTRAINT memory_units_fact_type_check
CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation', 'mental_model'))
""")
def downgrade() -> None:
"""Reverse the migration."""
schema = _get_schema_prefix()
# Restore original fact_type check constraint (without 'mental_model')
op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
op.execute(f"""
ALTER TABLE {schema}memory_units
ADD CONSTRAINT memory_units_fact_type_check
CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation'))
""")
# Drop mental model columns from memory_units
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS proof_count")
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS source_memory_ids")
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS history")
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_mental_models")
# Drop directives table
op.execute(f"DROP TABLE IF EXISTS {schema}directives CASCADE")
# Rename reflections back to pinned_reflections
op.execute(f"ALTER TABLE IF EXISTS {schema}reflections RENAME TO pinned_reflections")
# Restore indexes
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_bank_id RENAME TO idx_pinned_reflections_bank_id")
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_embedding RENAME TO idx_pinned_reflections_embedding")
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_tags RENAME TO idx_pinned_reflections_tags")
op.execute(
f"ALTER INDEX IF EXISTS {schema}idx_reflections_text_search RENAME TO idx_pinned_reflections_text_search"
)
# Restore foreign key
op.execute(f"""
ALTER TABLE {schema}pinned_reflections
DROP CONSTRAINT IF EXISTS fk_reflections_bank_id
""")
op.execute(f"""
ALTER TABLE {schema}pinned_reflections
ADD CONSTRAINT fk_pinned_reflections_bank_id
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
""")
# Re-create learnings table
op.execute(f"""
CREATE TABLE IF NOT EXISTS {schema}learnings (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
bank_id VARCHAR(64) NOT NULL,
text TEXT NOT NULL,
proof_count INT NOT NULL DEFAULT 1,
history JSONB DEFAULT '[]'::jsonb,
mission_context VARCHAR(64),
pre_mission_change BOOLEAN DEFAULT FALSE,
embedding vector(384),
tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
)
""")
op.execute(f"""
ALTER TABLE {schema}learnings
ADD CONSTRAINT fk_learnings_bank_id
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
""")
# Note: mental_models table recreation is complex and would need separate handling
@@ -1,50 +0,0 @@
"""fix_mental_model_fact_type
Revision ID: q2l3m4n5o6p7
Revises: p1k2l3m4n5o6
Create Date: 2026-01-21 13:30:00.000000
Fix the fact_type check constraint to include 'mental_model'.
This is a fix for p1k2l3m4n5o6 which should have included this change.
"""
from collections.abc import Sequence
from alembic import context, op
# revision identifiers, used by Alembic.
revision: str = "q2l3m4n5o6p7"
down_revision: str | Sequence[str] | None = "p1k2l3m4n5o6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Add 'mental_model' to the fact_type check constraint."""
schema = _get_schema_prefix()
# Drop the old constraint and add the new one with mental_model included
op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
op.execute(f"""
ALTER TABLE {schema}memory_units
ADD CONSTRAINT memory_units_fact_type_check
CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation', 'mental_model'))
""")
def downgrade() -> None:
"""Remove 'mental_model' from the fact_type check constraint."""
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
op.execute(f"""
ALTER TABLE {schema}memory_units
ADD CONSTRAINT memory_units_fact_type_check
CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation'))
""")
@@ -1,47 +0,0 @@
"""Add reflect_response JSONB column to reflections
Revision ID: r3m4n5o6p7q8
Revises: q2l3m4n5o6p7
Create Date: 2026-01-21
This migration adds a reflect_response JSONB column to store the full
reflect API response payload, including based_on facts and trace data.
Note: Table was renamed from pinned_reflections to reflections in p1k2l3m4n5o6.
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "r3m4n5o6p7q8"
down_revision: str | Sequence[str] | None = "q2l3m4n5o6p7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Add reflect_response JSONB column to reflections."""
schema = _get_schema_prefix()
# Add reflect_response column to store the full reflect API response
op.execute(f"""
ALTER TABLE {schema}reflections
ADD COLUMN IF NOT EXISTS reflect_response JSONB
""")
def downgrade() -> None:
"""Remove reflect_response column from reflections."""
schema = _get_schema_prefix()
op.execute(f"""
ALTER TABLE {schema}reflections
DROP COLUMN IF EXISTS reflect_response
""")
@@ -1,53 +0,0 @@
"""Add consolidated_at column to memory_units for incremental consolidation tracking.
This allows consolidation to track progress at the memory level rather than
using a bank-level watermark. If consolidation crashes, already-processed
memories won't be reprocessed.
Revision ID: s4n5o6p7q8r9
Revises: r3m4n5o6p7q8
Create Date: 2025-01-22
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "s4n5o6p7q8r9"
down_revision: str | Sequence[str] | None = "r3m4n5o6p7q8"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
# Add consolidated_at column to memory_units
op.execute(
f"""
ALTER TABLE {schema}memory_units
ADD COLUMN IF NOT EXISTS consolidated_at TIMESTAMPTZ DEFAULT NULL
"""
)
# Create index for efficient querying of unconsolidated memories
op.execute(
f"""
CREATE INDEX IF NOT EXISTS idx_memory_units_unconsolidated
ON {schema}memory_units (bank_id, created_at)
WHERE consolidated_at IS NULL AND fact_type IN ('experience', 'world')
"""
)
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_unconsolidated")
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS consolidated_at")
@@ -1,134 +0,0 @@
"""Rename mental_model fact_type to observation and reflections table to mental_models
Revision ID: t5o6p7q8r9s0
Revises: s4n5o6p7q8r9
Create Date: 2026-01-26
This migration implements the terminology rename:
1. mental_model (fact_type in memory_units) -> observation
2. reflections table -> mental_models table
The new terminology:
- Observations: Consolidated knowledge synthesized from facts (was mental_model)
- Mental Models: Stored reflect responses (was reflections)
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "t5o6p7q8r9s0"
down_revision: str | Sequence[str] | None = "s4n5o6p7q8r9"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Rename mental_model -> observation and reflections -> mental_models."""
schema = _get_schema_prefix()
# 1. Update fact_type values: mental_model -> observation
op.execute(f"""
UPDATE {schema}memory_units
SET fact_type = 'observation'
WHERE fact_type = 'mental_model'
""")
# 2. Update the CHECK constraint - remove mental_model, keep observation
op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
op.execute(f"""
ALTER TABLE {schema}memory_units
ADD CONSTRAINT memory_units_fact_type_check
CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation'))
""")
# 3. Rename the index for observations (was for mental_models)
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_mental_models")
op.execute(f"""
CREATE INDEX IF NOT EXISTS idx_memory_units_observations
ON {schema}memory_units(bank_id, fact_type)
WHERE fact_type = 'observation'
""")
# 4. Update the unconsolidated index to not filter by fact_type since observations
# are now the consolidated type
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_unconsolidated")
op.execute(f"""
CREATE INDEX IF NOT EXISTS idx_memory_units_unconsolidated
ON {schema}memory_units (bank_id, created_at)
WHERE consolidated_at IS NULL AND fact_type IN ('experience', 'world')
""")
# 5. Rename reflections table to mental_models
op.execute(f"ALTER TABLE IF EXISTS {schema}reflections RENAME TO mental_models")
# 6. Rename indexes for mental_models (was reflections)
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_bank_id RENAME TO idx_mental_models_bank_id")
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_embedding RENAME TO idx_mental_models_embedding")
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_tags RENAME TO idx_mental_models_tags")
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_text_search RENAME TO idx_mental_models_text_search")
# 7. Rename foreign key constraint
op.execute(f"""
ALTER TABLE {schema}mental_models
DROP CONSTRAINT IF EXISTS fk_reflections_bank_id
""")
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD CONSTRAINT fk_mental_models_bank_id
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
""")
def downgrade() -> None:
"""Reverse: observation -> mental_model and mental_models -> reflections."""
schema = _get_schema_prefix()
# 1. Rename mental_models table back to reflections
op.execute(f"ALTER TABLE IF EXISTS {schema}mental_models RENAME TO reflections")
# 2. Rename indexes back
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_mental_models_bank_id RENAME TO idx_reflections_bank_id")
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_mental_models_embedding RENAME TO idx_reflections_embedding")
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_mental_models_tags RENAME TO idx_reflections_tags")
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_mental_models_text_search RENAME TO idx_reflections_text_search")
# 3. Rename foreign key back
op.execute(f"""
ALTER TABLE {schema}reflections
DROP CONSTRAINT IF EXISTS fk_mental_models_bank_id
""")
op.execute(f"""
ALTER TABLE {schema}reflections
ADD CONSTRAINT fk_reflections_bank_id
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
""")
# 4. Update fact_type values: observation -> mental_model
op.execute(f"""
UPDATE {schema}memory_units
SET fact_type = 'mental_model'
WHERE fact_type = 'observation'
""")
# 5. Update the CHECK constraint back
op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
op.execute(f"""
ALTER TABLE {schema}memory_units
ADD CONSTRAINT memory_units_fact_type_check
CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation', 'mental_model'))
""")
# 6. Rename index back
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_observations")
op.execute(f"""
CREATE INDEX IF NOT EXISTS idx_memory_units_mental_models
ON {schema}memory_units(bank_id, fact_type)
WHERE fact_type = 'mental_model'
""")
@@ -1,41 +0,0 @@
"""Change mental_models.id from UUID to TEXT
Revision ID: u6p7q8r9s0t1
Revises: t5o6p7q8r9s0
Create Date: 2026-01-27
This migration changes the mental_models.id column from UUID to TEXT
to support user-defined text identifiers like 'team-communication' instead of UUIDs.
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "u6p7q8r9s0t1"
down_revision: str | Sequence[str] | None = "t5o6p7q8r9s0"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Change mental_models.id from UUID to TEXT."""
schema = _get_schema_prefix()
# Change the id column type from UUID to TEXT
# Existing UUIDs will be converted to their string representation
op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE TEXT USING id::TEXT")
def downgrade() -> None:
"""Revert mental_models.id from TEXT to UUID."""
schema = _get_schema_prefix()
# Note: This will fail if any id values are not valid UUIDs
op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE UUID USING id::UUID")
@@ -1,50 +0,0 @@
"""Add max_tokens and trigger columns to mental_models
Revision ID: v7q8r9s0t1u2
Revises: u6p7q8r9s0t1
Create Date: 2026-01-27
This migration adds:
- max_tokens column: token limit for content generation during refresh
- trigger column: JSONB for trigger settings (e.g., refresh_after_consolidation)
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "v7q8r9s0t1u2"
down_revision: str | Sequence[str] | None = "u6p7q8r9s0t1"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Add max_tokens and trigger columns to mental_models."""
schema = _get_schema_prefix()
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD COLUMN IF NOT EXISTS max_tokens INT NOT NULL DEFAULT 2048
""")
# trigger column stores trigger settings as JSONB
# Default: refresh_after_consolidation = false (not "real time")
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD COLUMN IF NOT EXISTS trigger JSONB NOT NULL DEFAULT '{{"refresh_after_consolidation": false}}'::jsonb
""")
def downgrade() -> None:
"""Remove max_tokens and trigger columns from mental_models."""
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS max_tokens")
op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS trigger")
@@ -1,60 +0,0 @@
"""Fix mental_models primary key to be scoped per bank
Revision ID: w8r9s0t1u2v3
Revises: v7q8r9s0t1u2
Create Date: 2026-02-05
This migration fixes a critical bank isolation bug where mental_models.id was
globally unique across all banks instead of being scoped per bank. This caused
conflicts when different banks tried to use the same custom ID.
CRITICAL FIX: Changes primary key from (id) to (bank_id, id) to ensure proper isolation.
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "w8r9s0t1u2v3"
down_revision: str | Sequence[str] | None = "v7q8r9s0t1u2"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Change mental_models primary key from (id) to (bank_id, id) for proper bank isolation."""
schema = _get_schema_prefix()
# Drop the old primary key constraint (just id)
# Note: The constraint might be named differently on different DBs
# Try both old names (pinned_reflections_pkey from original, mental_models_pkey from rename)
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS pinned_reflections_pkey")
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS mental_models_pkey")
# Create the new composite primary key (bank_id, id)
# This ensures IDs are scoped per bank, not globally
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD CONSTRAINT mental_models_pkey PRIMARY KEY (bank_id, id)
""")
def downgrade() -> None:
"""Revert mental_models primary key from (bank_id, id) to (id)."""
schema = _get_schema_prefix()
# Drop the composite primary key
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS mental_models_pkey")
# Restore the old primary key (just id)
# WARNING: This downgrade will fail if there are duplicate IDs across banks
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD CONSTRAINT mental_models_pkey PRIMARY KEY (id)
""")
@@ -1,64 +0,0 @@
"""Add config JSONB column to banks table for hierarchical configuration
Revision ID: x9s0t1u2v3w4
Revises: w8r9s0t1u2v3
Create Date: 2026-02-09
This migration adds a `config` JSONB column to the banks table to support
per-bank configuration overrides. This enables hierarchical configuration where:
- Global config is loaded from environment variables
- Tenant config is provided via TenantExtension
- Bank config overrides are stored in banks.config JSONB column
The config column stores overrides for hierarchical fields (LLM settings,
retention parameters, retrieval settings, etc.) in Python field name format.
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import context, op
from sqlalchemy.dialects.postgresql import JSONB
revision: str = "x9s0t1u2v3w4"
down_revision: str | Sequence[str] | None = "w8r9s0t1u2v3"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Add config JSONB column to banks table with GIN index."""
schema = _get_schema_prefix()
# Add config column to banks table
op.execute(f"""
ALTER TABLE {schema}banks
ADD COLUMN config JSONB NOT NULL DEFAULT '{{}}'::jsonb
""")
# Add GIN index for efficient JSONB queries
op.execute(f"""
CREATE INDEX idx_banks_config
ON {schema}banks
USING gin(config)
""")
def downgrade() -> None:
"""Remove config column and index from banks table."""
schema = _get_schema_prefix()
# Drop index first
op.execute(f"DROP INDEX IF EXISTS {schema}idx_banks_config")
# Drop column
op.execute(f"""
ALTER TABLE {schema}banks
DROP COLUMN IF EXISTS config
""")
@@ -1,49 +0,0 @@
"""Add GIN index on async_operations.result_metadata for parent_operation_id queries
Revision ID: y0t1u2v3w4x5
Revises: x9s0t1u2v3w4
Create Date: 2026-02-13
This migration adds a GIN index on the result_metadata JSONB column in the
async_operations table to support efficient queries for child operations by
parent_operation_id.
The index enables fast lookups when querying for child operations:
SELECT * FROM async_operations
WHERE result_metadata::jsonb @> '{"parent_operation_id": "uuid"}'::jsonb
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "y0t1u2v3w4x5"
down_revision: str | Sequence[str] | None = "x9s0t1u2v3w4"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
"""Add GIN index on result_metadata for efficient parent_operation_id queries."""
schema = _get_schema_prefix()
# Add GIN index for JSONB containment queries (@> operator)
op.execute(f"""
CREATE INDEX IF NOT EXISTS idx_async_operations_result_metadata
ON {schema}async_operations
USING gin(result_metadata)
""")
def downgrade() -> None:
"""Remove GIN index on result_metadata."""
schema = _get_schema_prefix()
# Drop index
op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_result_metadata")
@@ -1,35 +0,0 @@
"""Add observation_scopes column to memory_units table
Revision ID: z1u2v3w4x5y6
Revises: a1b2c3d4e5f6
Create Date: 2026-02-25
Adds observation_scopes JSONB column to memory_units to control how observations
are scoped during consolidation. Accepts "per_tag", "combined", or an explicit
list of tag-set lists for custom multi-pass consolidation.
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "z1u2v3w4x5y6"
down_revision: str | Sequence[str] | None = "a1b2c3d4e5f6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS observation_scopes JSONB")
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS observation_scopes")
File diff suppressed because it is too large Load Diff
-461
View File
@@ -1,461 +0,0 @@
"""Hindsight MCP Server implementation using FastMCP (HTTP transport)."""
import json
import logging
import os
from contextvars import ContextVar
from fastmcp import FastMCP
from hindsight_api import MemoryEngine
from hindsight_api.config import _get_raw_config
from hindsight_api.engine.memory_engine import _current_schema
from hindsight_api.extensions import MCPExtension, load_extension
from hindsight_api.extensions.tenant import AuthenticationError
from hindsight_api.mcp_tools import MCPToolsConfig, register_mcp_tools
from hindsight_api.models import RequestContext
# All tools available in the system (explicit list — no wildcards)
_ALL_TOOLS: frozenset[str] = frozenset(
{
"retain",
"recall",
"reflect",
"list_banks",
"create_bank",
"list_mental_models",
"get_mental_model",
"create_mental_model",
"update_mental_model",
"delete_mental_model",
"refresh_mental_model",
"list_directives",
"create_directive",
"delete_directive",
"list_memories",
"get_memory",
"delete_memory",
"list_documents",
"get_document",
"delete_document",
"list_operations",
"get_operation",
"cancel_operation",
"list_tags",
"get_bank",
"get_bank_stats",
"update_bank",
"delete_bank",
"clear_memories",
}
)
# Configure logging from HINDSIGHT_API_LOG_LEVEL environment variable
_log_level_str = os.environ.get("HINDSIGHT_API_LOG_LEVEL", "info").lower()
_log_level_map = {
"critical": logging.CRITICAL,
"error": logging.ERROR,
"warning": logging.WARNING,
"info": logging.INFO,
"debug": logging.DEBUG,
"trace": logging.DEBUG,
}
logging.basicConfig(
level=_log_level_map.get(_log_level_str, logging.INFO),
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
)
logger = logging.getLogger(__name__)
# Default bank_id from environment variable
DEFAULT_BANK_ID = os.environ.get("HINDSIGHT_MCP_BANK_ID", "default")
# Legacy MCP authentication token (for backwards compatibility)
# If set, this token is checked first before TenantExtension auth
MCP_AUTH_TOKEN = os.environ.get("HINDSIGHT_API_MCP_AUTH_TOKEN")
# Context variable to hold the current bank_id
_current_bank_id: ContextVar[str | None] = ContextVar("current_bank_id", default=None)
# Context variable to hold the current API key (for tenant auth propagation)
_current_api_key: ContextVar[str | None] = ContextVar("current_api_key", default=None)
# Context variables for tenant_id and api_key_id (set by authenticate, used by usage metering)
_current_tenant_id: ContextVar[str | None] = ContextVar("current_tenant_id", default=None)
_current_api_key_id: ContextVar[str | None] = ContextVar("current_api_key_id", default=None)
def get_current_bank_id() -> str | None:
"""Get the current bank_id from context."""
return _current_bank_id.get()
def get_current_api_key() -> str | None:
"""Get the current API key from context."""
return _current_api_key.get()
def get_current_tenant_id() -> str | None:
"""Get the current tenant_id from context."""
return _current_tenant_id.get()
def get_current_api_key_id() -> str | None:
"""Get the current api_key_id from context."""
return _current_api_key_id.get()
def create_mcp_server(memory: MemoryEngine, multi_bank: bool = True) -> FastMCP:
"""
Create and configure the Hindsight MCP server.
Args:
memory: MemoryEngine instance (required)
multi_bank: If True, expose all tools with bank_id parameters (default).
If False, only expose bank-scoped tools without bank_id parameters.
Returns:
Configured FastMCP server instance
"""
mcp = FastMCP("hindsight-mcp-server")
global_config = _get_raw_config()
# Tools available for this mode (multi-bank exposes all tools; single-bank excludes bank-management tools)
_SINGLE_BANK_TOOLS: frozenset[str] = frozenset(
{
"retain",
"recall",
"reflect",
"list_mental_models",
"get_mental_model",
"create_mental_model",
"update_mental_model",
"delete_mental_model",
"refresh_mental_model",
"list_directives",
"create_directive",
"delete_directive",
"list_memories",
"get_memory",
"delete_memory",
"list_documents",
"get_document",
"delete_document",
"list_operations",
"get_operation",
"cancel_operation",
"list_tags",
"get_bank",
"update_bank",
"delete_bank",
"clear_memories",
}
)
base_tools: frozenset[str] | None = None if multi_bank else _SINGLE_BANK_TOOLS
# Apply global mcp_enabled_tools filter (env-level allowlist)
if global_config.mcp_enabled_tools is not None:
allowed = frozenset(global_config.mcp_enabled_tools)
base_tools = (base_tools if base_tools is not None else _ALL_TOOLS) & allowed
# Configure and register tools using shared module
config = MCPToolsConfig(
bank_id_resolver=get_current_bank_id,
api_key_resolver=get_current_api_key, # Propagate API key for tenant auth
tenant_id_resolver=get_current_tenant_id, # Propagate tenant_id for usage metering
api_key_id_resolver=get_current_api_key_id, # Propagate api_key_id for usage metering
include_bank_id_param=multi_bank,
tools=base_tools,
)
register_mcp_tools(mcp, memory, config)
# Load and register additional tools from MCP extension if configured
mcp_extension = load_extension("MCP", MCPExtension)
if mcp_extension:
logger.info(f"Loading MCP extension: {mcp_extension.__class__.__name__}")
mcp_extension.register_tools(mcp, memory)
# Make all tools tolerant of extra arguments from LLMs (e.g., "explanation")
_make_tools_tolerant(mcp)
return mcp
def _make_tools_tolerant(mcp: FastMCP) -> None:
"""Wrap all tool run methods to strip unknown arguments before validation.
LLMs frequently add extra fields like "explanation" or "reasoning" to tool calls.
FastMCP's Pydantic TypeAdapter rejects these with "Unexpected keyword argument".
This wraps each tool's run() to filter arguments to only known parameters.
"""
try:
for name, tool in mcp._tool_manager._tools.items():
if hasattr(tool, "parameters") and tool.parameters:
allowed = set(tool.parameters.get("properties", {}).keys())
original_run = tool.run
async def _tolerant_run(arguments, _allowed=allowed, _orig=original_run):
extra_keys = set(arguments.keys()) - _allowed
if extra_keys:
logger.debug(f"Stripping unknown arguments from tool call: {extra_keys}")
arguments = {k: v for k, v in arguments.items() if k in _allowed}
return await _orig(arguments)
# FunctionTool is a Pydantic model with extra='forbid', so use
# object.__setattr__ to bypass Pydantic's setter validation.
object.__setattr__(tool, "run", _tolerant_run)
except (AttributeError, KeyError) as e:
logger.warning(f"Could not make tools tolerant of extra arguments: {e}")
class MCPMiddleware:
"""ASGI middleware that intercepts MCP requests and routes to appropriate MCP server.
This middleware wraps the main FastAPI app and intercepts requests matching the
configured prefix (default: /mcp). Non-MCP requests pass through to the inner app.
Authentication:
1. If HINDSIGHT_API_MCP_AUTH_TOKEN is set (legacy), validates against that token
2. Otherwise, uses TenantExtension.authenticate_mcp() from the MemoryEngine
- DefaultTenantExtension: no auth required (local dev)
- ApiKeyTenantExtension: validates against env var
Two modes based on URL structure:
1. Multi-bank mode (for /mcp/ root endpoint):
- Exposes all tools: retain, recall, reflect, list_banks, create_bank
- All tools include optional bank_id parameter for cross-bank operations
- Bank ID from: X-Bank-Id header or HINDSIGHT_MCP_BANK_ID env var
2. Single-bank mode (for /mcp/{bank_id}/ endpoints):
- Exposes bank-scoped tools only: retain, recall, reflect
- No bank_id parameter (comes from URL)
- No bank management tools (list_banks, create_bank)
- Recommended for agent isolation
Bank ID resolution priority:
1. URL path (e.g., /mcp/{bank_id}/) → single-bank mode
2. X-Bank-Id header → multi-bank mode
3. HINDSIGHT_MCP_BANK_ID env var → multi-bank mode (default: "default")
Examples:
# Single-bank mode (recommended for agent isolation)
claude mcp add --transport http my-agent http://localhost:8888/mcp/my-agent-bank/ \\
--header "Authorization: Bearer <token>"
# Multi-bank mode (for cross-bank operations)
claude mcp add --transport http hindsight http://localhost:8888/mcp \\
--header "X-Bank-Id: my-bank" --header "Authorization: Bearer <token>"
"""
def __init__(
self,
app,
memory: MemoryEngine,
prefix: str = "/mcp",
multi_bank_app=None,
single_bank_app=None,
multi_bank_server=None,
single_bank_server=None,
):
self.app = app
self.prefix = prefix
self.memory = memory
self.tenant_extension = memory._tenant_extension
if multi_bank_app and single_bank_app:
# Pre-created servers (used when called via add_middleware from create_app)
self.multi_bank_app = multi_bank_app
self.single_bank_app = single_bank_app
self.multi_bank_server = multi_bank_server
self.single_bank_server = single_bank_server
else:
# Create servers internally (for direct construction / tests)
self.multi_bank_server = create_mcp_server(memory, multi_bank=True)
self.multi_bank_app = self.multi_bank_server.http_app(path="/", stateless_http=True)
self.single_bank_server = create_mcp_server(memory, multi_bank=False)
self.single_bank_app = self.single_bank_server.http_app(path="/", stateless_http=True)
def _get_header(self, scope: dict, name: str) -> str | None:
"""Extract a header value from ASGI scope."""
name_lower = name.lower().encode()
for header_name, header_value in scope.get("headers", []):
if header_name.lower() == name_lower:
return header_value.decode()
return None
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
await self.app(scope, receive, send)
return
path = scope.get("path", "")
# Check if this is an MCP request (matches prefix)
if not (path == self.prefix or path.startswith(self.prefix + "/")):
# Not an MCP request — pass through to the inner app
await self.app(scope, receive, send)
return
# Strip prefix from path
path = path[len(self.prefix) :] or "/"
# Extract auth token from header (for tenant auth propagation)
auth_header = self._get_header(scope, "Authorization")
auth_token: str | None = None
if auth_header:
# Support both "Bearer <token>" and direct token
auth_token = auth_header[7:].strip() if auth_header.startswith("Bearer ") else auth_header.strip()
# Authenticate: check legacy MCP_AUTH_TOKEN first, then TenantExtension
tenant_context = None
auth_tenant_id: str | None = None
auth_api_key_id: str | None = None
if MCP_AUTH_TOKEN:
# Legacy authentication mode - validate against static token
if not auth_token:
await self._send_error(send, 401, "Authorization header required")
return
if auth_token != MCP_AUTH_TOKEN:
await self._send_error(send, 401, "Invalid authentication token")
return
# Legacy mode doesn't use tenant schemas
tenant_context = None
else:
# Use TenantExtension.authenticate_mcp() for auth
try:
auth_context = RequestContext(api_key=auth_token)
tenant_context = await self.tenant_extension.authenticate_mcp(auth_context)
# Capture tenant_id and api_key_id set by authenticate() for usage metering
auth_tenant_id = auth_context.tenant_id
auth_api_key_id = auth_context.api_key_id
except AuthenticationError as e:
await self._send_error(send, 401, str(e), extra_headers=e.headers)
return
# Set schema from tenant context so downstream DB queries use the correct schema
schema_token = (
_current_schema.set(tenant_context.schema_name) if tenant_context and tenant_context.schema_name else None
)
# Resolve bank_id: path takes priority over header.
# Path = user's explicit connection endpoint (e.g., /mcp/my-bank/).
# X-Bank-Id header = per-request override for multi-bank mode only.
bank_id = None
bank_id_from_path = False
new_path = path
# First, try to extract from path: /{bank_id}/...
if path.startswith("/") and len(path) > 1:
parts = path[1:].split("/", 1)
if parts[0]:
bank_id = parts[0]
bank_id_from_path = True
new_path = "/" + parts[1] if len(parts) > 1 else "/"
# If no path-based bank_id, try X-Bank-Id header (multi-bank mode)
if not bank_id:
bank_id = self._get_header(scope, "X-Bank-Id")
# Fall back to default bank_id
if not bank_id:
bank_id = DEFAULT_BANK_ID
logger.debug(f"Using default bank_id: {bank_id}")
# Select the appropriate MCP app based on how bank_id was provided:
# - Path-based bank_id → single-bank app (no bank_id param, scoped tools)
# - Header/env bank_id → multi-bank app (bank_id param, all tools)
target_app = self.single_bank_app if bank_id_from_path else self.multi_bank_app
# Set bank_id, api_key, tenant_id, and api_key_id context
bank_id_token = _current_bank_id.set(bank_id)
# Store the auth token for tenant extension to validate
api_key_token = _current_api_key.set(auth_token) if auth_token else None
# Store tenant_id and api_key_id from authentication for usage metering
tenant_id_token = _current_tenant_id.set(auth_tenant_id) if auth_tenant_id else None
api_key_id_token = _current_api_key_id.set(auth_api_key_id) if auth_api_key_id else None
try:
new_scope = scope.copy()
new_scope["path"] = new_path
# Clear root_path since we're passing directly to the app
new_scope["root_path"] = ""
# Ensure Accept header includes required MIME types for MCP SDK.
# Some clients (e.g., Claude Code) don't send Accept, causing
# the SDK to reject with 406 Not Acceptable.
accept_header = self._get_header(new_scope, "accept")
if not accept_header or "text/event-stream" not in accept_header:
headers = [(k, v) for k, v in new_scope.get("headers", []) if k.lower() != b"accept"]
headers.append((b"accept", b"application/json, text/event-stream"))
new_scope["headers"] = headers
# Wrap send to rewrite the SSE endpoint URL to include bank_id if using path-based routing.
# Only rewrite SSE (text/event-stream) responses to avoid corrupting tool results
# that might contain the literal string "data: /messages".
is_sse_response = False
async def send_wrapper(message):
nonlocal is_sse_response
if message["type"] == "http.response.start":
for header_name, header_value in message.get("headers", []):
if header_name == b"content-type" and b"text/event-stream" in header_value:
is_sse_response = True
break
if message["type"] == "http.response.body" and bank_id_from_path and is_sse_response:
body = message.get("body", b"")
if body and b"/messages" in body:
# Rewrite /messages to /{bank_id}/messages in SSE endpoint event
body = body.replace(b"data: /messages", f"data: /{bank_id}/messages".encode())
message = {**message, "body": body}
await send(message)
await target_app(new_scope, receive, send_wrapper)
finally:
_current_bank_id.reset(bank_id_token)
if api_key_token is not None:
_current_api_key.reset(api_key_token)
if tenant_id_token is not None:
_current_tenant_id.reset(tenant_id_token)
if api_key_id_token is not None:
_current_api_key_id.reset(api_key_id_token)
if schema_token is not None:
_current_schema.reset(schema_token)
async def _send_error(self, send, status: int, message: str, extra_headers: dict[str, str] | None = None):
"""Send an error response."""
body = json.dumps({"error": message}).encode()
headers = [(b"content-type", b"application/json")]
for key, value in (extra_headers or {}).items():
headers.append((key.encode(), value.encode()))
await send(
{
"type": "http.response.start",
"status": status,
"headers": headers,
}
)
await send(
{
"type": "http.response.body",
"body": body,
}
)
def create_mcp_servers(memory: MemoryEngine):
"""Create multi-bank and single-bank MCP servers and their Starlette apps.
Returns the servers and apps separately so lifespans can be chained before
the middleware wraps the main app.
Returns:
Tuple of (multi_bank_server, single_bank_server, multi_bank_app, single_bank_app)
"""
multi_bank_server = create_mcp_server(memory, multi_bank=True)
multi_bank_app = multi_bank_server.http_app(path="/", stateless_http=True)
single_bank_server = create_mcp_server(memory, multi_bank=False)
single_bank_app = single_bank_server.http_app(path="/", stateless_http=True)
return multi_bank_server, single_bank_server, multi_bank_app, single_bank_app
File diff suppressed because it is too large Load Diff
@@ -1,315 +0,0 @@
"""
Configuration resolution with hierarchical overrides.
Resolves config values through the hierarchy:
Global (env vars) → Tenant config (via extension) → Bank config (database)
Config values are resolved on every request to ensure consistency across
multiple API servers.
"""
import json
import logging
from dataclasses import asdict, replace
from typing import Any
import asyncpg
from hindsight_api.config import HindsightConfig, _get_raw_config, normalize_config_dict
from hindsight_api.engine.memory_engine import fq_table
from hindsight_api.extensions.tenant import TenantExtension
from hindsight_api.models import RequestContext
logger = logging.getLogger(__name__)
class ConfigResolver:
"""Resolves hierarchical configuration with tenant/bank overrides."""
def __init__(self, pool: asyncpg.Pool, tenant_extension: TenantExtension | None = None):
"""
Initialize config resolver.
Args:
pool: Database connection pool
tenant_extension: Optional tenant extension for tenant-level config and permissions
"""
self.pool = pool
self.tenant_extension = tenant_extension
self._global_config = _get_raw_config()
self._configurable_fields = HindsightConfig.get_configurable_fields()
self._credential_fields = HindsightConfig.get_credential_fields()
async def resolve_full_config(self, bank_id: str, context: RequestContext | None = None) -> HindsightConfig:
"""
Resolve full HindsightConfig for a bank with hierarchical overrides applied.
This is for INTERNAL USE ONLY. Returns the complete config object with all fields
including credentials and static fields. Use get_bank_config() for API responses.
Resolution order:
1. Global config (from environment variables)
2. Tenant config overrides (from TenantExtension.get_tenant_config())
3. Bank config overrides (from banks.config JSONB)
Args:
bank_id: Bank identifier
context: Request context for tenant config resolution
Returns:
Complete HindsightConfig with hierarchical overrides applied
"""
# Start with global config (all fields)
config_dict = asdict(self._global_config)
# Load tenant config overrides (if tenant extension available)
if self.tenant_extension and context:
try:
tenant_overrides = await self.tenant_extension.get_tenant_config(context)
if tenant_overrides:
# Normalize keys and filter to configurable fields only
normalized_tenant = normalize_config_dict(tenant_overrides)
configurable_tenant = {k: v for k, v in normalized_tenant.items() if k in self._configurable_fields}
config_dict.update(configurable_tenant)
logger.debug(
f"Applied tenant config overrides for bank {bank_id}: {list(configurable_tenant.keys())}"
)
except Exception as e:
logger.warning(f"Failed to load tenant config for bank {bank_id}: {e}")
# Load bank config overrides
bank_overrides = await self._load_bank_config(bank_id)
if bank_overrides:
config_dict.update(bank_overrides)
logger.debug(f"Applied bank config overrides for bank {bank_id}: {list(bank_overrides.keys())}")
# Return full config object (dataclass doesn't have __init__ that accepts kwargs, so we update the object)
# Create a new config instance by copying the global config and updating fields
resolved_config = HindsightConfig(**config_dict)
return resolved_config
async def get_bank_config(self, bank_id: str, context: RequestContext | None = None) -> dict[str, Any]:
"""
Get fully resolved config for a bank (filtered by permissions).
Resolution order:
1. Global config (from environment variables)
2. Tenant config overrides (from TenantExtension.get_tenant_config())
3. Bank config overrides (from banks.config JSONB)
Note: Config is resolved on every call (not cached) to ensure consistency
across multiple API servers.
SECURITY:
- Only returns configurable fields (excludes static/infrastructure fields)
- Filters out ALL credential fields (API keys, base URLs, etc.)
- Further filtered by tenant/bank permissions if extension provides them
Args:
bank_id: Bank identifier
context: Request context for tenant config resolution and permissions
Returns:
Dict of allowed configurable fields only (never includes credentials or static fields)
"""
# Resolve full config with all hierarchical overrides
resolved_config = await self.resolve_full_config(bank_id, context)
config_dict = asdict(resolved_config)
# SECURITY: Filter to only configurable fields (exclude static/infrastructure)
filtered = {k: v for k, v in config_dict.items() if k in self._configurable_fields}
# SECURITY: Remove ALL credential fields (API keys, base URLs, etc.)
filtered = {k: v for k, v in filtered.items() if k not in self._credential_fields}
# PERMISSIONS: Further filter based on tenant/bank permissions
if self.tenant_extension and context:
try:
allowed_fields = await self.tenant_extension.get_allowed_config_fields(context, bank_id)
if allowed_fields is not None: # None means "allow all"
filtered = {k: v for k, v in filtered.items() if k in allowed_fields}
logger.debug(
f"Applied permission filter for bank {bank_id}: allowed={len(allowed_fields)} fields, "
f"returned={len(filtered)} fields"
)
except Exception as e:
logger.warning(f"Failed to load permissions for bank {bank_id}: {e}")
return filtered
async def _load_bank_config(self, bank_id: str) -> dict[str, Any]:
"""
Load bank config overrides from banks.config JSONB column.
Args:
bank_id: Bank identifier
Returns:
Dict of config overrides (only configurable fields, normalized keys)
"""
try:
async with self.pool.acquire() as conn:
row = await conn.fetchrow(
f"""
SELECT config FROM {fq_table("banks")} WHERE bank_id = $1
""",
bank_id,
)
if row and row["config"]:
config_data = row["config"]
# Handle case where JSONB is returned as JSON string
if isinstance(config_data, str):
config_data = json.loads(config_data)
# Normalize keys (handle both env var format and Python field format)
normalized = normalize_config_dict(config_data)
# Only return overrides for configurable fields
return {k: v for k, v in normalized.items() if k in self._configurable_fields}
except Exception as e:
logger.error(f"Failed to load bank config for {bank_id}: {e}")
return {}
async def update_bank_config(
self, bank_id: str, updates: dict[str, Any], context: RequestContext | None = None
) -> None:
"""
Update bank configuration overrides (with permission checking).
Args:
bank_id: Bank identifier
updates: Dict of config field names to new values.
Keys can be in env var format (HINDSIGHT_API_LLM_PROVIDER)
or Python field format (llm_provider).
Only configurable fields are allowed.
context: Request context for permission checking
Raises:
ValueError: If attempting to override invalid/disallowed fields
"""
# Normalize keys
normalized_updates = normalize_config_dict(updates)
# SECURITY: Reject credential fields explicitly
credential_attempts = set(normalized_updates.keys()) & self._credential_fields
if credential_attempts:
raise ValueError(
f"Cannot set credential fields via API: {sorted(credential_attempts)}. "
f"Credentials (API keys, base URLs) must be set at server level only."
)
# Validate all fields are configurable
invalid_fields = set(normalized_updates.keys()) - self._configurable_fields
if invalid_fields:
static_fields = HindsightConfig.get_static_fields()
invalid_static = invalid_fields & static_fields
if invalid_static:
raise ValueError(
f"Cannot override static (server-level) fields: {sorted(invalid_static)}. "
f"Only configurable fields can be overridden per-bank. "
f"Configurable fields include: {sorted(list(self._configurable_fields)[:10])}... "
f"(total: {len(self._configurable_fields)} fields)"
)
else:
raise ValueError(
f"Unknown configuration fields: {sorted(invalid_fields)}. "
f"Valid configurable fields: {sorted(list(self._configurable_fields)[:10])}..."
)
# PERMISSIONS: Check tenant/bank permissions
if self.tenant_extension and context:
try:
allowed_fields = await self.tenant_extension.get_allowed_config_fields(context, bank_id)
if allowed_fields is not None: # None means "allow all"
disallowed = set(normalized_updates.keys()) - allowed_fields
if disallowed:
raise ValueError(
f"Not allowed to modify fields: {sorted(disallowed)}. "
f"Your permissions allow: {sorted(list(allowed_fields)[:10])}..."
if allowed_fields
else "Not allowed to modify fields: {sorted(disallowed)}. "
"Your permissions do not allow any config modifications."
)
except ValueError:
raise # Re-raise permission errors
except Exception as e:
logger.warning(f"Failed to check permissions for bank {bank_id}: {e}")
# Continue without permission check (fail open for backward compatibility)
# Validate retain_strategies: reject empty string keys
if "retain_strategies" in normalized_updates and normalized_updates["retain_strategies"]:
empty_keys = [k for k in normalized_updates["retain_strategies"] if not str(k).strip()]
if empty_keys:
raise ValueError(
"Strategy names must not be empty strings. Remove entries with empty names before saving."
)
# Merge with existing config (JSONB || operator)
async with self.pool.acquire() as conn:
await conn.execute(
f"""
UPDATE {fq_table("banks")}
SET config = config || $1::jsonb,
updated_at = now()
WHERE bank_id = $2
""",
json.dumps(normalized_updates),
bank_id,
)
logger.info(f"Updated bank config for {bank_id}: {list(normalized_updates.keys())}")
async def reset_bank_config(self, bank_id: str) -> None:
"""
Reset bank configuration to defaults (remove all overrides).
Args:
bank_id: Bank identifier
"""
async with self.pool.acquire() as conn:
await conn.execute(
f"""
UPDATE {fq_table("banks")}
SET config = '{{}}'::jsonb,
updated_at = now()
WHERE bank_id = $1
""",
bank_id,
)
logger.info(f"Reset bank config for {bank_id} to defaults")
def apply_strategy(config: HindsightConfig, strategy_name: str) -> HindsightConfig:
"""
Apply a named retain strategy's overrides on top of a resolved config.
A strategy is a named set of hierarchical field overrides stored in
config.retain_strategies. Any field in _HIERARCHICAL_FIELDS can be
overridden, including retain_extraction_mode, retain_chunk_size,
entity_labels, entities_allow_free_form, etc.
Unknown strategy names log a warning and return config unchanged.
Unknown or non-hierarchical fields in the strategy are silently ignored.
"""
strategies = config.retain_strategies or {}
if strategy_name not in strategies:
logger.warning(f"Unknown retain strategy '{strategy_name}', using resolved config as-is")
return config
overrides = strategies[strategy_name]
if not isinstance(overrides, dict):
logger.warning(f"Retain strategy '{strategy_name}' is not a dict, skipping")
return config
configurable = HindsightConfig.get_configurable_fields()
filtered = {k: v for k, v in overrides.items() if k in configurable}
if not filtered:
return config
logger.debug(f"Applying retain strategy '{strategy_name}': {list(filtered.keys())}")
return replace(config, **filtered)
-113
View File
@@ -1,113 +0,0 @@
"""
Daemon mode support for Hindsight API.
Provides idle timeout for running as a background daemon.
"""
import asyncio
import logging
import os
import sys
import time
from pathlib import Path
logger = logging.getLogger(__name__)
# Default daemon configuration
DEFAULT_DAEMON_PORT = 8888
DEFAULT_IDLE_TIMEOUT = 0 # 0 = no auto-exit (hindsight-embed passes its own timeout)
# Allow override via environment variable for profile-specific logs
DAEMON_LOG_PATH = Path(os.getenv("HINDSIGHT_API_DAEMON_LOG", str(Path.home() / ".hindsight" / "daemon.log")))
class IdleTimeoutMiddleware:
"""ASGI middleware that tracks activity and exits after idle timeout."""
def __init__(self, app, idle_timeout: int = DEFAULT_IDLE_TIMEOUT):
self.app = app
self.idle_timeout = idle_timeout
self.last_activity = time.time()
self._checker_task = None
async def __call__(self, scope, receive, send):
# Update activity timestamp on each request
self.last_activity = time.time()
await self.app(scope, receive, send)
def start_idle_checker(self):
"""Start the background task that checks for idle timeout."""
self._checker_task = asyncio.create_task(self._check_idle())
async def _check_idle(self):
"""Background task that exits the process after idle timeout."""
# If idle_timeout is 0, don't auto-exit
if self.idle_timeout <= 0:
return
while True:
await asyncio.sleep(30) # Check every 30 seconds
idle_time = time.time() - self.last_activity
if idle_time > self.idle_timeout:
logger.info(f"Idle timeout reached ({self.idle_timeout}s), shutting down daemon")
# Give a moment for any in-flight requests
await asyncio.sleep(1)
# Send SIGTERM to ourselves to trigger graceful shutdown
import signal
os.kill(os.getpid(), signal.SIGTERM)
def daemonize():
"""
Fork the current process into a background daemon.
Uses double-fork technique to properly detach from terminal.
"""
# First fork - detach from parent
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError as e:
sys.stderr.write(f"fork #1 failed: {e}\n")
sys.exit(1)
# Decouple from parent environment
os.chdir("/")
os.setsid()
os.umask(0)
# Second fork - prevent zombie
pid = os.fork()
if pid > 0:
sys.exit(0)
# Redirect standard file descriptors to log file
DAEMON_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
sys.stdout.flush()
sys.stderr.flush()
# Redirect stdin to /dev/null
with open("/dev/null", "r") as devnull:
os.dup2(devnull.fileno(), sys.stdin.fileno())
# Redirect stdout/stderr to log file
log_fd = open(DAEMON_LOG_PATH, "a")
os.dup2(log_fd.fileno(), sys.stdout.fileno())
os.dup2(log_fd.fileno(), sys.stderr.fileno())
def check_daemon_running(port: int = DEFAULT_DAEMON_PORT) -> bool:
"""Check if a daemon is running and responsive on the given port."""
import socket
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(1)
result = sock.connect_ex(("127.0.0.1", port))
sock.close()
return result == 0
except Exception:
return False
@@ -1,5 +0,0 @@
"""Consolidation engine for automatic learning creation from memories."""
from .consolidator import run_consolidation_job
__all__ = ["run_consolidation_job"]

Some files were not shown because too many files have changed in this diff Show More