Compare commits

..
Author SHA1 Message Date
DK09876andClaude Opus 4.6 b22f4bf258 fix(opencode): fix message parsing, shared state, and post-compaction retain
Three bugs fixed:
1. msg.role → msg.info.role: OpenCode SDK wraps role inside info, so all
   messages were silently filtered out, breaking retain and recall (#941)
2. Move PluginState to module level so it persists across sessions instead
   of being recreated per plugin instantiation
3. Reset lastRetainedTurn after compaction so idle-retain resumes when the
   message list shrinks

Co-Authored-By: Claude Opus 4.6 <[email protected]>
2026-04-13 09:04:11 -07:00
1388 changed files with 28927 additions and 137178 deletions
+2 -38
View File
@@ -2,7 +2,7 @@
# Copy this file to .env and fill in your values
# LLM Configuration (Required)
# Supported providers: openai, groq, ollama, gemini, anthropic, lmstudio, vertexai, minimax, deepseek, zai, volcano
# Supported providers: openai, groq, ollama, gemini, anthropic, lmstudio, vertexai, minimax, volcano
HINDSIGHT_API_LLM_PROVIDER=openai
HINDSIGHT_API_LLM_API_KEY=your-api-key-here
HINDSIGHT_API_LLM_MODEL=gpt-4o-mini
@@ -25,16 +25,6 @@ HINDSIGHT_API_LLM_BASE_URL=https://api.openai.com/v1
# HINDSIGHT_API_LLM_API_KEY=your-minimax-api-key
# HINDSIGHT_API_LLM_MODEL=MiniMax-M2.7
# Example: DeepSeek configuration (https://api.deepseek.com)
# HINDSIGHT_API_LLM_PROVIDER=deepseek
# HINDSIGHT_API_LLM_API_KEY=your-deepseek-api-key
# HINDSIGHT_API_LLM_MODEL=deepseek-v4-flash # or deepseek-v4-pro / deepseek-chat / deepseek-reasoner
# Example: z.ai configuration (Zhipu GLM series, https://z.ai)
# HINDSIGHT_API_LLM_PROVIDER=zai
# HINDSIGHT_API_LLM_API_KEY=your-zai-api-key
# HINDSIGHT_API_LLM_MODEL=glm-4.5-flash # or glm-4.5-air for the paid tier
# Example: LM Studio local configuration (Qwen 2.5 32B recommended)
# HINDSIGHT_API_LLM_PROVIDER=lmstudio
# HINDSIGHT_API_LLM_API_KEY=lmstudio
@@ -54,7 +44,6 @@ HINDSIGHT_API_LOG_LEVEL=info
# Database (Optional - uses embedded pg0 by default)
# HINDSIGHT_API_DATABASE_URL=postgresql://user:pass@host:5432/db
# HINDSIGHT_API_READ_DATABASE_URL= # Optional read-replica URL. When set, recall queries (semantic, BM25, graph, temporal) flow through a separate pool against this URL, offloading the primary. Typically points to a read-only endpoint (CNPG's <cluster>-ro service or Aurora reader endpoint).
# HINDSIGHT_API_MIGRATION_DATABASE_URL= # Direct PostgreSQL URL for migrations (bypasses PgBouncer). Falls back to DATABASE_URL.
# HINDSIGHT_API_DATABASE_SCHEMA=public # PostgreSQL schema name (default: public)
@@ -65,25 +54,12 @@ HINDSIGHT_API_LOG_LEVEL=info
# HINDSIGHT_API_VECTOR_EXTENSION=pgvectorscale # Auto-detects pg_diskann on Azure
# Embeddings Configuration (Optional - uses local by default)
# Provider: "local" (default), "tei", "openai", "cohere", "google", "openrouter", "litellm", or "litellm-sdk"
# Provider: "local" (default) or "tei" (HuggingFace Text Embeddings Inference)
# HINDSIGHT_API_EMBEDDINGS_PROVIDER=local
# For local provider:
# HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL=BAAI/bge-small-en-v1.5
# Optional for China network / restricted HF access:
# HF_ENDPOINT=https://hf-mirror.com
# For TEI provider:
# HINDSIGHT_API_EMBEDDINGS_TEI_URL=http://localhost:8080
# For OpenAI-compatible embeddings:
# HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=sk-xxxx
# HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL=text-embedding-3-small
# HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL=https://api.openai.com/v1
#
# IMPORTANT: Embedding keys require provider-specific names:
# HINDSIGHT_API_EMBEDDINGS_{PROVIDER}_{PARAMETER}
# (for example, HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL).
#
# DeepSeek note: DeepSeek is supported for LLM calls, but not for embeddings.
# If using DeepSeek as LLM provider, keep embeddings on local/openai/cohere/google/etc.
# Reranker Configuration (Optional - uses local by default)
# Provider: "local" (default) or "tei" (HuggingFace Text Embeddings Inference)
@@ -107,15 +83,3 @@ HINDSIGHT_API_LOG_LEVEL=info
# Custom service name and environment (optional, defaults: hindsight-api, development)
# HINDSIGHT_API_OTEL_SERVICE_NAME=hindsight-production
# HINDSIGHT_API_OTEL_DEPLOYMENT_ENVIRONMENT=production
# -----------------------------------------------------------------------------
# Control Plane (Optional)
# -----------------------------------------------------------------------------
# Dataplane API URL - where the CP proxies requests to
# HINDSIGHT_CP_DATAPLANE_API_URL=http://localhost:8888
# Optional: Require a shared access key to view the Control Plane UI.
# When set, visitors see a login page and must enter the key before
# accessing the dashboard or any /api/* routes (except /api/health).
# HINDSIGHT_CP_ACCESS_KEY=your-shared-secret-key
+1 -1
View File
@@ -34,7 +34,7 @@ jobs:
env:
UMAMI_URL: https://analytics.hindsight.vectorize.io
UMAMI_WEBSITE_ID: ${{ secrets.UMAMI_WEBSITE_ID }}
- uses: actions/upload-pages-artifact@v5
- uses: actions/upload-pages-artifact@v4
with:
path: hindsight-docs/build
deploy:
-200
View File
@@ -1,200 +0,0 @@
name: Performance Tests
on:
schedule:
# Run daily at 06:00 UTC
- cron: "0 6 * * *"
workflow_dispatch:
inputs:
scale:
description: "Test scale (perf-test)"
type: choice
options:
- tiny
- small
- medium
- large
default: large
suite:
description: "Perf-test suite to run (blank = all)"
type: choice
options:
- ""
- retain
- recall
- recall-with-observations
- consolidation
default: ""
locomo_conversations:
description: "LoComo conversation IDs (space-separated). Blank = curated set (conv-26 conv-30 conv-43)."
type: string
default: ""
locomo_skip:
description: "Skip LoComo job"
type: boolean
default: false
ref:
description: "Git ref to test (branch, tag, or SHA). Defaults to main."
type: string
default: ""
concurrency:
group: perf-test
cancel-in-progress: true
jobs:
perf-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.ref }}
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
enable-cache: true
prune-cache: false
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version-file: ".python-version"
- name: Cache HuggingFace models
uses: actions/cache@v5
with:
path: ~/.cache/huggingface
key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
restore-keys: |
${{ runner.os }}-huggingface-
- name: Pre-download models
working-directory: ./hindsight-api-slim
run: |
uv run --frozen --all-extras --index-strategy unsafe-best-match python -c "
from sentence_transformers import SentenceTransformer
print('Downloading embedding model...')
SentenceTransformer('BAAI/bge-small-en-v1.5')
print('Model downloaded successfully')
"
- name: Install hindsight-dev dependencies
run: |
cd hindsight-dev && uv sync --frozen --all-extras --index-strategy unsafe-best-match
- name: Run perf-test
run: |
SUITE_ARG=""
if [ -n "${{ inputs.suite }}" ]; then
SUITE_ARG="--suite ${{ inputs.suite }}"
fi
./scripts/benchmarks/run-perf-test.sh \
--scale ${{ inputs.scale || 'large' }} \
$SUITE_ARG \
--output perf-results.json
- name: Upload perf results
if: always()
uses: actions/upload-artifact@v7
with:
name: perf-results-${{ github.sha }}
path: hindsight-dev/perf-results.json
retention-days: 90
# Publish enriched results (perf JSON + commit metadata) to the dashboard
# repo's gh-pages branch. The static site at
# https://vectorize-io.github.io/hindsight-continuous-performance-monitor/
# reads data/index.json + data/<run>.json and renders charts client-side.
- name: Publish to dashboard
if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
env:
PERF_DASHBOARD_TOKEN: ${{ secrets.PERF_DASHBOARD_TOKEN }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: ./scripts/benchmarks/publish-perf-results.sh hindsight-dev/perf-results.json
locomo:
if: inputs.locomo_skip != true
runs-on: ubuntu-latest
env:
HINDSIGHT_API_LLM_PROVIDER: vertexai
HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY: /tmp/gcp-credentials.json
HINDSIGHT_API_LLM_MODEL: google/gemini-2.5-flash-lite
HINDSIGHT_API_JUDGE_LLM_PROVIDER: vertexai
HINDSIGHT_API_JUDGE_LLM_MODEL: google/gemini-2.5-flash-lite
HINDSIGHT_API_ANSWER_LLM_PROVIDER: vertexai
HINDSIGHT_API_ANSWER_LLM_MODEL: google/gemini-2.5-flash
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.ref }}
- name: Setup GCP credentials
run: |
printf '%s' '${{ secrets.GCP_VERTEXAI_CREDENTIALS }}' > /tmp/gcp-credentials.json
PROJECT_ID=$(jq -r '.project_id' /tmp/gcp-credentials.json)
echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
enable-cache: true
prune-cache: false
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version-file: ".python-version"
- name: Cache HuggingFace models
uses: actions/cache@v5
with:
path: ~/.cache/huggingface
key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
restore-keys: |
${{ runner.os }}-huggingface-
- name: Pre-download models
working-directory: ./hindsight-api-slim
run: |
uv run --frozen --all-extras --index-strategy unsafe-best-match python -c "
from sentence_transformers import SentenceTransformer
print('Downloading embedding model...')
SentenceTransformer('BAAI/bge-small-en-v1.5')
print('Model downloaded successfully')
"
- name: Install hindsight-dev dependencies
run: |
cd hindsight-dev && uv sync --frozen --all-extras --index-strategy unsafe-best-match
- name: Run LoComo benchmark
# Curated 3-conversation subset (best/middle/worst by accuracy on the
# last successful full run): conv-26 (best), conv-30 (middle), conv-43
# (worst). Excludes conv-44, the bank with the largest unconsolidated
# set that has been pushing scheduled runs over the per-bank
# _wait_for_consolidation timeout. Override via workflow_dispatch with
# the locomo_conversations input.
run: |
CONVERSATIONS="${{ inputs.locomo_conversations }}"
if [ -z "$CONVERSATIONS" ]; then
CONVERSATIONS="conv-26 conv-30 conv-43"
fi
uv run python hindsight-dev/benchmarks/locomo/locomo_benchmark.py \
--wait-consolidation \
--conversation $CONVERSATIONS
- name: Upload LoComo results
if: always()
uses: actions/upload-artifact@v7
with:
name: locomo-results-${{ github.sha }}
path: hindsight-dev/benchmarks/locomo/results/
retention-days: 90
- name: Publish LoComo to dashboard
if: success() && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch')
env:
PERF_DASHBOARD_TOKEN: ${{ secrets.PERF_DASHBOARD_TOKEN }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: ./scripts/benchmarks/publish-locomo-results.sh hindsight-dev/benchmarks/locomo/results/benchmark_results.json
+7 -18
View File
@@ -81,28 +81,17 @@ jobs:
with:
node-version: '22'
registry-url: 'https://registry.npmjs.org'
cache: 'npm'
cache-dependency-path: package-lock.json
# Guard: fail fast if the integration's lockfile resolves any dep from a
# monorepo workspace (link=true) or a relative file path. The release
# runner has no pre-built workspace `dist/` so `npm run build` would
# later fail at tsc with "Cannot find module". See:
# https://github.com/vectorize-io/hindsight/issues/… (0.6.0 openclaw retry)
- name: Check integration lockfile
if: steps.type.outputs.type == 'typescript'
run: ./scripts/check-integration-lockfiles.sh
# Some integrations depend on workspace packages (hindsight-client,
# hindsight-all, hindsight-agent-sdk) via file: refs. Install from root
# so npm resolves them, then build the workspace deps before the integration.
- name: Install root workspace dependencies
if: steps.type.outputs.type == 'typescript'
run: npm ci
- name: Build workspace deps (hindsight-client, hindsight-all, hindsight-agent-sdk)
if: steps.type.outputs.type == 'typescript'
run: |
npm run build --workspace=hindsight-clients/typescript
npm run build --workspace=hindsight-all-npm
npm run build --workspace=hindsight-tools/hindsight-agent-sdk
- name: Install integration dependencies
- name: Install dependencies
if: steps.type.outputs.type == 'typescript'
working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
run: npm ci
@@ -117,7 +106,7 @@ jobs:
working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}
run: |
set +e
OUTPUT=$(npm publish --access public --provenance 2>&1)
OUTPUT=$(npm publish --access public 2>&1)
EXIT_CODE=$?
echo "$OUTPUT"
if [ $EXIT_CODE -ne 0 ]; then
-65
View File
@@ -1,65 +0,0 @@
name: Release Tool
on:
push:
tags:
- 'tools/**'
jobs:
publish:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Extract tool info
id: info
run: |
# refs/tags/tools/self-driving-agents/v0.0.1 → tool=self-driving-agents, version=0.0.1
TAG="${GITHUB_REF#refs/tags/}"
TOOL=$(echo "$TAG" | cut -d'/' -f2)
VERSION=$(echo "$TAG" | cut -d'/' -f3 | sed 's/^v//')
echo "tool=$TOOL" >> $GITHUB_OUTPUT
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "tag=$TAG" >> $GITHUB_OUTPUT
echo "Tool: $TOOL, Version: $VERSION"
- name: Set up Node.js
uses: actions/setup-node@v6
with:
node-version: '22'
registry-url: 'https://registry.npmjs.org'
cache: 'npm'
cache-dependency-path: package-lock.json
# Tools live under hindsight-tools/ and may depend on workspace packages
# (e.g. @vectorize-io/hindsight-client). Install from root so npm resolves
# workspace deps, then build any required workspace packages first.
- name: Install root workspace dependencies
run: npm ci
- name: Build hindsight-client (workspace dep)
run: npm run build --workspace=hindsight-clients/typescript
- name: Build hindsight-agent-sdk (workspace dep)
run: npm run build --workspace=hindsight-tools/hindsight-agent-sdk
- name: Build tool
run: npm run build --workspace=hindsight-tools/${{ steps.info.outputs.tool }}
- name: Publish to npm
working-directory: ./hindsight-tools/${{ steps.info.outputs.tool }}
run: |
set +e
OUTPUT=$(npm publish --access public 2>&1)
EXIT_CODE=$?
echo "$OUTPUT"
if [ $EXIT_CODE -ne 0 ]; then
if echo "$OUTPUT" | grep -q "cannot publish over"; then
echo "Package version already published, skipping..."
exit 0
fi
exit $EXIT_CODE
fi
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-34
View File
@@ -314,7 +314,6 @@ jobs:
permissions:
contents: read
packages: write
id-token: write
strategy:
matrix:
include:
@@ -411,7 +410,6 @@ jobs:
# Build multi-platform and push to release tags
- name: Build and push release images
id: build
uses: docker/build-push-action@v7
with:
context: .
@@ -423,31 +421,6 @@ jobs:
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
- name: Install cosign
uses: sigstore/cosign-installer@v3
- name: Sign published images
env:
TAGS: ${{ steps.meta.outputs.tags }}
DIGEST: ${{ steps.build.outputs.digest }}
run: |
set -euo pipefail
refs=()
while IFS= read -r tag; do
[[ -z "${tag}" ]] && continue
refs+=("${tag}@${DIGEST}")
done <<< "${TAGS}"
cosign sign --yes "${refs[@]}"
- name: Verify signature on primary tag
env:
IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}
DIGEST: ${{ steps.build.outputs.digest }}
run: |
cosign verify "${IMAGE}@${DIGEST}" \
--certificate-identity-regexp "^https://github\.com/${{ github.repository }}/\.github/workflows/release\.yml@.*" \
--certificate-oidc-issuer https://token.actions.githubusercontent.com
release-helm-chart:
runs-on: ubuntu-latest
permissions:
@@ -524,12 +497,6 @@ jobs:
name: rust-cli-hindsight-linux-amd64
path: ./artifacts/rust-cli-linux
- name: Download Rust CLI (Linux ARM)
uses: actions/download-artifact@v8
with:
name: rust-cli-hindsight-linux-arm64
path: ./artifacts/rust-cli-linux-arm64
- name: Download Rust CLI (macOS Intel)
uses: actions/download-artifact@v8
with:
@@ -566,7 +533,6 @@ jobs:
cp artifacts/control-plane/*.tgz release-assets/ || true
# Rust CLI binaries
cp artifacts/rust-cli-linux/hindsight-linux-amd64 release-assets/ || true
cp artifacts/rust-cli-linux-arm64/hindsight-linux-arm64 release-assets/ || true
cp artifacts/rust-cli-darwin-amd64/hindsight-darwin-amd64 release-assets/ || true
cp artifacts/rust-cli-darwin-arm64/hindsight-darwin-arm64 release-assets/ || true
# Helm chart
-71
View File
@@ -1,71 +0,0 @@
name: Sign published images
on:
workflow_dispatch:
inputs:
version:
description: 'Version to sign (without leading v, e.g. 0.6.0)'
required: true
type: string
default: '0.6.0'
permissions:
contents: read
packages: write
id-token: write
jobs:
sign:
name: Sign ${{ matrix.image }}:${{ inputs.version }}${{ matrix.suffix }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- { image: hindsight-api, suffix: '' }
- { image: hindsight-api, suffix: '-slim' }
- { image: hindsight-control-plane, suffix: '' }
- { image: hindsight, suffix: '' }
- { image: hindsight, suffix: '-slim' }
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Log in to GHCR
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Install cosign
uses: sigstore/cosign-installer@v3
- name: Resolve image digest
id: resolve
env:
IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}
TAG: ${{ inputs.version }}${{ matrix.suffix }}
run: |
set -euo pipefail
DIGEST=$(docker buildx imagetools inspect "${IMAGE}:${TAG}" --format '{{json .Manifest.Digest}}' | tr -d '"')
if [[ -z "${DIGEST}" || "${DIGEST}" != sha256:* ]]; then
echo "Failed to resolve digest for ${IMAGE}:${TAG} (got: ${DIGEST})" >&2
exit 1
fi
echo "Resolved ${IMAGE}:${TAG} -> ${DIGEST}"
echo "ref=${IMAGE}@${DIGEST}" >> "$GITHUB_OUTPUT"
- name: Sign image
env:
REF: ${{ steps.resolve.outputs.ref }}
run: cosign sign --yes "${REF}"
- name: Verify signature
env:
REF: ${{ steps.resolve.outputs.ref }}
run: |
cosign verify "${REF}" \
--certificate-identity-regexp "^https://github\.com/${{ github.repository }}/\.github/workflows/sign-images\.yml@.*" \
--certificate-oidc-issuer https://token.actions.githubusercontent.com
+71 -1045
View File
File diff suppressed because it is too large Load Diff
-7
View File
@@ -1,7 +0,0 @@
{
"semi": true,
"singleQuote": false,
"tabWidth": 2,
"trailingComma": "es5",
"printWidth": 100
}
+8 -47
View File
@@ -68,9 +68,8 @@ cd hindsight-control-plane && npm run dev
./scripts/benchmarks/run-locomo.sh
# Performance benchmarks
./scripts/benchmarks/run-perf-test.sh # System perf (mock LLM + pg0)
./scripts/benchmarks/run-perf-test.sh --scale tiny # Quick smoke test
./scripts/benchmarks/run-consolidation.sh
./scripts/benchmarks/run-retain-perf.sh --document <path> # Requires API server running
# Results viewer
./scripts/benchmarks/start-visualizer.sh # View results at localhost:8001
@@ -123,17 +122,12 @@ Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
### Adding Database Migrations
Hindsight runs the same Alembic tree against PostgreSQL and Oracle 23ai. Each
migration file dispatches through `run_for_dialect`, which calls either
`_pg_upgrade` or `_oracle_upgrade` based on the live connection. A pytest lint
(`tests/test_migration_shape.py`) fails CI if a migration omits the dispatcher.
1. **Create a new migration file** in `hindsight-api-slim/hindsight_api/alembic/versions/`:
- File name format: `<revision_id>_<description>.py` (e.g., `f1a2b3c4d5e6_add_new_index.py`)
- Use a unique hex revision ID (12 chars)
- Set `down_revision` to the previous migration's revision ID
2. **Migration template** (the `script.py.mako` template scaffolds this; fill in the bodies):
2. **Migration template**:
```python
"""Description of the migration
@@ -144,58 +138,25 @@ migration file dispatches through `run_for_dialect`, which calls either
from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "f1a2b3c4d5e6"
down_revision: str | Sequence[str] | None = "<previous_revision_id>"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _pg_schema_prefix() -> str:
"""Schema-qualifier for raw SQL on PG (multi-tenant search_path)."""
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
schema = _pg_schema_prefix()
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"CREATE INDEX ... ON {schema}table_name(...)")
def _pg_downgrade() -> None:
schema = _pg_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}index_name")
def _oracle_upgrade() -> None:
# Oracle 23ai equivalent. Use op.get_bind().exec_driver_sql for forms
# that Alembic core does not model (vector/text indexes, partitions).
op.execute("CREATE INDEX ... ON table_name(...)")
def _oracle_downgrade() -> None:
op.execute("DROP INDEX IF EXISTS index_name")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade, oracle=_oracle_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade, oracle=_oracle_downgrade)
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}index_name")
```
**Dialect-only migrations.** If a change genuinely doesn't apply to one
dialect (e.g. enabling `pg_trgm` is PG-only), omit the unused slot:
```python
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade) # oracle slot intentionally absent → no-op
```
Make the asymmetry deliberate. Don't leave an Oracle slot empty just because
you didn't think about it — copy-pasting a PG migration without the Oracle
half is exactly how schemas drift.
3. **Run migrations locally**:
```bash
# Set database URL and run migrations for the base schema plus all tenants
+1 -3
View File
@@ -30,7 +30,7 @@ It eliminates the shortcomings of alternative techniques such as RAG and knowled
Hindsight is the most accurate agent memory system ever tested according to benchmark performance. It has achieved state-of-the-art performance on the LongMemEval benchmark, widely used to assess memory system performance across a variety of conversational AI scenarios. The current reported performance of Hindsight and other agent memory solutions as of January 2026 is shown here:
![Overview](./hindsight-docs/static/img/hindsight-benchmarks.png)
![Overview](./hindsight-docs/static/img/hindsight-bench.jpg)
The benchmark performance data for Hindsight has been independently reproduced by research collaborators at the Virginia Tech [Sanghani Center for Artificial Intelligence and Data Analytics](https://sanghani.cs.vt.edu/) and The Washington Post. Other scores are self-reported by software vendors.
@@ -84,8 +84,6 @@ cd docker/docker-compose
docker compose up
```
> Oracle AI Database is also supported for enterprise deployments with full feature parity. See the [storage documentation](https://hindsight.vectorize.io/developer/storage) for details.
>API: http://localhost:8888
>UI: http://localhost:9999
Generated
+1 -33
View File
@@ -42,16 +42,6 @@
},
"workspace": {
"members": {
"hindsight-all-npm": {
"packageJson": {
"dependencies": [
"npm:@types/node@22",
"npm:tsup@^8.5.1",
"npm:typescript@^5.7.0",
"npm:vitest@^4.1.2"
]
}
},
"hindsight-clients/typescript": {
"packageJson": {
"dependencies": [
@@ -68,7 +58,6 @@
"hindsight-control-plane": {
"packageJson": {
"dependencies": [
"npm:@chenglou/pretext@^0.0.3",
"npm:@eslint/eslintrc@^3.3.3",
"npm:@eslint/js@^9.39.2",
"npm:@radix-ui/react-alert-dialog@^1.1.15",
@@ -102,12 +91,11 @@
"npm:eslint@^9.39.1",
"npm:[email protected]",
"npm:next-themes@~0.4.6",
"npm:next@^16.1.7",
"npm:next@^16.1.6",
"npm:postcss@^8.5.6",
"npm:prettier@^3.7.4",
"npm:react-chrono@^2.9.1",
"npm:react-dom@^19.2.0",
"npm:react-is@^19.2.4",
"npm:react-markdown@^10.1.0",
"npm:react18-json-view@~0.2.9",
"npm:react@^19.2.0",
@@ -145,26 +133,6 @@
"npm:typescript@~5.6.2"
]
}
},
"hindsight-tools/hindsight-agent-sdk": {
"packageJson": {
"dependencies": [
"npm:@vectorize-io/hindsight-client@~0.5.6",
"npm:typescript@^5.4.0",
"npm:vitest@^4.1.2"
]
}
},
"hindsight-tools/self-driving-agents": {
"packageJson": {
"dependencies": [
"npm:@clack/prompts@^1.2.0",
"npm:@vectorize-io/hindsight-client@~0.5.6",
"npm:picocolors@^1.1.0",
"npm:typescript@^5.4.0",
"npm:vitest@^4.1.2"
]
}
}
}
}
@@ -1,90 +0,0 @@
name: hindsight
# Docker Compose file for Hindsight with AlloyDB Omni and ScaNN
# Uses Google's free AlloyDB Omni container image: https://hub.docker.com/r/google/alloydbomni
#
# Usage:
# docker compose -f docker/docker-compose/alloydb/docker-compose.yaml up -d
#
# Make sure to set the required environment variables before running:
# - HINDSIGHT_DB_PASSWORD: password for the AlloyDB Omni/PostgreSQL user
# - Configure LLM provider variables as needed (see the hindsight service below)
#
# Optional environment variables with defaults:
# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
# - HINDSIGHT_DB_VERSION: AlloyDB Omni image tag (default: 17)
# - HINDSIGHT_DB_USER: database user (default: hindsight_user)
# - HINDSIGHT_DB_NAME: database name (default: hindsight_db)
services:
db:
image: google/alloydbomni:${HINDSIGHT_DB_VERSION:-17}
container_name: hindsight-db-alloydb
restart: always
ports:
- "5438:5432"
environment:
POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:-hindsight_password}
POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
volumes:
- alloydb_data:/var/lib/postgresql/data
networks:
- hindsight-net
alloydb-init:
image: google/alloydbomni:${HINDSIGHT_DB_VERSION:-17}
depends_on:
- db
environment:
- PGPASSWORD=${HINDSIGHT_DB_PASSWORD:-hindsight_password}
command:
- bash
- -c
- |
echo 'Waiting for AlloyDB Omni to be ready...'
until pg_isready -h hindsight-db-alloydb -p 5432 -U ${HINDSIGHT_DB_USER:-hindsight_user}; do
echo 'AlloyDB Omni is unavailable - sleeping'
sleep 2
done
echo 'AlloyDB Omni is ready - creating ${HINDSIGHT_DB_NAME:-hindsight_db} database'
psql -h hindsight-db-alloydb -p 5432 -U ${HINDSIGHT_DB_USER:-hindsight_user} -c 'CREATE DATABASE ${HINDSIGHT_DB_NAME:-hindsight_db};' 2>/dev/null || echo 'Database already exists'
echo 'Creating vector and alloydb_scann extensions'
psql -h hindsight-db-alloydb -p 5432 -U ${HINDSIGHT_DB_USER:-hindsight_user} -d ${HINDSIGHT_DB_NAME:-hindsight_db} -c 'CREATE EXTENSION IF NOT EXISTS vector;'
psql -h hindsight-db-alloydb -p 5432 -U ${HINDSIGHT_DB_USER:-hindsight_user} -d ${HINDSIGHT_DB_NAME:-hindsight_db} -c 'CREATE EXTENSION IF NOT EXISTS alloydb_scann CASCADE;'
echo 'Database and extensions created successfully'
restart: "no"
networks:
- hindsight-net
hindsight:
image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
container_name: hindsight-app
ports:
- "8888:8888"
- "9999:9999"
environment:
# LLM Configuration
HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai}
HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key}
# Database Configuration
HINDSIGHT_API_DATABASE_URL: postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:-hindsight_password}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
# Vector and Text Search Extensions
HINDSIGHT_API_VECTOR_EXTENSION: scann
HINDSIGHT_API_TEXT_SEARCH_EXTENSION: native
depends_on:
db:
condition: service_started
alloydb-init:
condition: service_completed_successfully
networks:
- hindsight-net
networks:
hindsight-net:
driver: bridge
volumes:
alloydb_data:
@@ -1,34 +0,0 @@
# Example: custom Hindsight image with non-default local models baked in.
#
# Use this pattern in production when you run a non-default embedder or
# reranker. Baking models into the image removes the runtime dependency on
# HuggingFace and lets the container registry handle caching per node, so
# you don't need a model-cache PVC.
#
# Built on top of the slim image so only the deps and models you actually
# use end up in the final image.
FROM ghcr.io/vectorize-io/hindsight:latest-slim
# Install the local-ml deps required to load sentence-transformers /
# cross-encoder models at runtime. Pinned ranges mirror hindsight-api-slim's
# `local-ml` extra in hindsight-api-slim/pyproject.toml. Use `uv pip
# install` against the image's venv explicitly: the slim image's venv was
# created by `uv sync` and does not ship its own `pip`, so a bare
# `pip install` would fall back to user site-packages and not be visible
# to the runtime python.
RUN uv pip install --python /app/api/.venv/bin/python --no-cache \
'sentence-transformers>=3.3.0' \
'transformers>=4.53.0' \
'torch>=2.6.0'
# Pre-download the models you want to use. Replace these with your own.
# The defaults bundled in the full image are BAAI/bge-small-en-v1.5 and
# cross-encoder/ms-marco-MiniLM-L-6-v2; here we pick multilingual variants
# as a concrete non-default example.
ARG EMBEDDER=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
ARG RERANKER=cross-encoder/mmarco-mMiniLMv2-L12-H384-v1
ENV HF_HUB_DOWNLOAD_TIMEOUT=600
RUN python -c "\
from sentence_transformers import SentenceTransformer, CrossEncoder; \
SentenceTransformer('${EMBEDDER}'); \
CrossEncoder('${RERANKER}')"
@@ -1,81 +0,0 @@
# Hindsight with Custom Local Models
Example Docker Compose setup that builds a Hindsight image with **non-default
local embedder and reranker models baked in at build time**.
This is the recommended pattern for production when you use a non-default
local model: the container registry caches model layers per node, pod
startup is deterministic, and you don't need a model-cache PVC (or any
runtime dependency on HuggingFace).
## When to use this
- You override `HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL` or
`HINDSIGHT_API_RERANKER_LOCAL_MODEL` to a non-default model.
- You want pod startup to be deterministic and offline-capable.
- You'd otherwise reach for a Helm `modelCache` PVC just to avoid
re-downloading models.
If you're using the **default** local models, the published full image
(`ghcr.io/vectorize-io/hindsight:latest`) already bakes them in — you don't
need this example.
If you're using **external** providers (TEI, OpenAI, Cohere, ...) for
embeddings and reranking, use the slim image directly — no models are
needed in the image.
## Quick start
```bash
export OPENAI_API_KEY=sk-xxx
docker compose -f docker/docker-compose/custom-models/docker-compose.yaml up --build
```
- API: http://localhost:8888
- Control Plane: http://localhost:9999
## Using your own models
Override the build args to bake different models:
```bash
docker compose -f docker/docker-compose/custom-models/docker-compose.yaml build \
--build-arg EMBEDDER=your-org/your-embedder \
--build-arg RERANKER=your-org/your-reranker
```
Then update the matching `HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL` and
`HINDSIGHT_API_RERANKER_LOCAL_MODEL` values in `docker-compose.yaml` so the
runtime points at the same model IDs.
## Verifying the models are baked in
`docker-compose.yaml` sets `HF_HUB_OFFLINE=1` and `TRANSFORMERS_OFFLINE=1`
so that any attempt to download a model at runtime fails loudly instead of
silently re-downloading. If the container starts and serves recall queries
with these set, the models are correctly baked in.
You can also inspect the image directly:
```bash
docker run --rm --entrypoint sh hindsight-custom-models-hindsight \
-c 'ls ~/.cache/huggingface/hub/'
```
## Why not a model-cache PVC?
The Helm chart exposes an optional `api.persistence.modelCache` PVC for
caching downloaded models across pod restarts. Compared to baking models
into the image:
- A PVC adds storage cost — one PVC per worker replica with
`volumeClaimTemplates`.
- `ReadWriteOnce` (the default) pins pods to a node.
- The PVC needs lifecycle management on `helm uninstall` / `helm upgrade`
— without `helm.sh/resource-policy: keep` it is deleted on uninstall;
with it, storage keeps billing forever until manually cleaned up.
- Pod startup still depends on HuggingFace being reachable on first run.
Image layers, by contrast, are pulled once per node and cached for free by
the container runtime, with no orphaned-storage cleanup story.
@@ -1,44 +0,0 @@
name: hindsight-custom-models
# Example: run a custom Hindsight image with non-default local models baked
# in at build time, so pod startup does not depend on HuggingFace at runtime.
#
# Quick start:
# export OPENAI_API_KEY=sk-xxx
# docker compose -f docker/docker-compose/custom-models/docker-compose.yaml up --build
#
# Required environment variables:
# - OPENAI_API_KEY (or configure another LLM provider via HINDSIGHT_API_LLM_*)
services:
hindsight:
build:
context: .
dockerfile: Dockerfile
# Override at build time to bake different models:
# docker compose build --build-arg EMBEDDER=your-org/your-embedder
args:
EMBEDDER: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
RERANKER: cross-encoder/mmarco-mMiniLMv2-L12-H384-v1
container_name: hindsight-custom-models
ports:
- "8888:8888"
- "9999:9999"
environment:
HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai}
HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key}
# Point Hindsight at the models baked into the image above.
HINDSIGHT_API_EMBEDDINGS_PROVIDER: local
HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
HINDSIGHT_API_RERANKER_PROVIDER: local
HINDSIGHT_API_RERANKER_LOCAL_MODEL: cross-encoder/mmarco-mMiniLMv2-L12-H384-v1
# Fail fast if a model is missing from the image instead of silently
# falling back to a HuggingFace download at runtime.
HF_HUB_OFFLINE: "1"
TRANSFORMERS_OFFLINE: "1"
volumes:
- pg_data:/home/hindsight/.pg0
volumes:
pg_data:
@@ -51,8 +51,6 @@ services:
# Control Plane config
HINDSIGHT_CP_DATAPLANE_API_URL: http://localhost:8888
# Optional: Require a shared access key for Control Plane UI access
# HINDSIGHT_CP_ACCESS_KEY: your-secret-key
volumes:
# Persist embedded pg0 database
- hindsight_data:/app/data
-8
View File
@@ -172,10 +172,6 @@ USER hindsight
# "Permission denied" error when mounting a fresh root-owned volume.
RUN mkdir -p /home/hindsight/.pg0
# Make /home/hindsight traversable when running with --user UID:GID overrides
# (default 0700 blocks traversal by non-owner UIDs needed for bind-mount ownership matching)
RUN chmod 755 /home/hindsight
ENV PATH="/app/api/.venv/bin:${PATH}"
# Pre-download tiktoken encoding (ALWAYS - required for token counting even in air-gapped envs)
@@ -332,10 +328,6 @@ USER hindsight
# "Permission denied" error when mounting a fresh root-owned volume.
RUN mkdir -p /home/hindsight/.pg0
# Make /home/hindsight traversable when running with --user UID:GID overrides
# (default 0700 blocks traversal by non-owner UIDs needed for bind-mount ownership matching)
RUN chmod 755 /home/hindsight
ENV PATH="/app/api/.venv/bin:${PATH}"
# Pre-download tiktoken encoding (ALWAYS - required for token counting even in air-gapped envs)
+2 -2
View File
@@ -2,8 +2,8 @@ apiVersion: v2
name: hindsight
description: Hindsight helm chart
type: application
version: 0.6.2
appVersion: "0.6.2"
version: 0.5.1
appVersion: "0.5.1"
keywords:
- ai
- memory
+1 -10
View File
@@ -70,12 +70,6 @@ api:
# Persistent volume for local model cache (reranker, embeddings)
# Models are downloaded to /home/hindsight/.cache on first use.
# Without persistence, models are re-downloaded on every pod restart.
#
# For production, prefer baking models into a custom image instead of
# enabling this PVC: image layers are pulled once per node and cached
# for free, while a PVC adds storage cost, pins pods to a node
# (ReadWriteOnce), and needs lifecycle management on uninstall/upgrade.
# See docs: developer/installation#bundling-custom-models-in-a-custom-image
persistence:
modelCache:
enabled: false
@@ -174,10 +168,7 @@ worker:
# affinity: {}
# Persistent volume for local model cache (reranker, embeddings)
# Uses volumeClaimTemplates since worker is a StatefulSet — one PVC per
# replica. For production, prefer baking models into a custom image; see
# api.persistence.modelCache above and docs:
# developer/installation#bundling-custom-models-in-a-custom-image
# Uses volumeClaimTemplates since worker is a StatefulSet.
persistence:
modelCache:
enabled: false
+10 -10
View File
@@ -18,17 +18,17 @@ npm install @vectorize-io/hindsight-all @vectorize-io/hindsight-client
## Example
```ts
import { HindsightServer, consoleLogger } from "@vectorize-io/hindsight-all";
import { HindsightClient } from "@vectorize-io/hindsight-client";
import { HindsightServer, consoleLogger } from '@vectorize-io/hindsight-all';
import { HindsightClient } from '@vectorize-io/hindsight-client';
const server = new HindsightServer({
profile: "my-app",
profile: 'my-app',
port: 9077,
env: {
HINDSIGHT_API_LLM_PROVIDER: "anthropic",
HINDSIGHT_API_LLM_PROVIDER: 'anthropic',
HINDSIGHT_API_LLM_API_KEY: process.env.ANTHROPIC_API_KEY,
HINDSIGHT_API_LLM_MODEL: "claude-sonnet-4-20250514",
HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT: "0",
HINDSIGHT_API_LLM_MODEL: 'claude-sonnet-4-20250514',
HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT: '0',
},
logger: consoleLogger,
});
@@ -37,11 +37,11 @@ await server.start();
const client = new HindsightClient({ baseUrl: server.getBaseUrl() });
await client.retain("user-123", "User prefers dark mode and concise answers.", {
documentId: "pref-2026-04-01",
await client.retain('user-123', 'User prefers dark mode and concise answers.', {
documentId: 'pref-2026-04-01',
});
const recall = await client.recall("user-123", "what are the user preferences?");
const recall = await client.recall('user-123', 'what are the user preferences?');
console.log(recall.results);
await server.stop();
@@ -62,7 +62,7 @@ If you're hacking on the Python `hindsight-embed` package in the same monorepo,
```ts
new HindsightServer({
embedPackagePath: "/path/to/hindsight-embed",
embedPackagePath: '/path/to/hindsight-embed',
// ...
});
```
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@vectorize-io/hindsight-all",
"version": "0.6.2",
"version": "0.5.1",
"description": "Node.js programmatic lifecycle manager for Hindsight — embeds a local hindsight daemon in a Node application. Pair with @vectorize-io/hindsight-client for memory operations.",
"main": "dist/index.js",
"types": "dist/index.d.ts",
+20 -24
View File
@@ -1,36 +1,32 @@
import { describe, it, expect } from "vitest";
import { getEmbedCommand } from "./command.js";
import { describe, it, expect } from 'vitest';
import { getEmbedCommand } from './command.js';
describe("getEmbedCommand", () => {
it("defaults to uvx hindsight-embed@latest", () => {
expect(getEmbedCommand()).toEqual(["uvx", "hindsight-embed@latest"]);
describe('getEmbedCommand', () => {
it('defaults to uvx hindsight-embed@latest', () => {
expect(getEmbedCommand()).toEqual(['uvx', 'hindsight-embed@latest']);
});
it("honours an explicit version", () => {
expect(getEmbedCommand({ embedVersion: "0.5.0" })).toEqual(["uvx", "[email protected]"]);
it('honours an explicit version', () => {
expect(getEmbedCommand({ embedVersion: '0.5.0' })).toEqual(['uvx', '[email protected]']);
});
it("treats an empty version as latest", () => {
expect(getEmbedCommand({ embedVersion: "" })).toEqual(["uvx", "hindsight-embed@latest"]);
it('treats an empty version as latest', () => {
expect(getEmbedCommand({ embedVersion: '' })).toEqual(['uvx', 'hindsight-embed@latest']);
});
it("uses uv run --directory when a local path is given", () => {
expect(getEmbedCommand({ embedPackagePath: "/abs/path" })).toEqual([
"uv",
"run",
"--directory",
"/abs/path",
"hindsight-embed",
it('uses uv run --directory when a local path is given', () => {
expect(getEmbedCommand({ embedPackagePath: '/abs/path' })).toEqual([
'uv',
'run',
'--directory',
'/abs/path',
'hindsight-embed',
]);
});
it("local path takes precedence over version", () => {
expect(getEmbedCommand({ embedPackagePath: "/abs/path", embedVersion: "0.5.0" })).toEqual([
"uv",
"run",
"--directory",
"/abs/path",
"hindsight-embed",
]);
it('local path takes precedence over version', () => {
expect(
getEmbedCommand({ embedPackagePath: '/abs/path', embedVersion: '0.5.0' }),
).toEqual(['uv', 'run', '--directory', '/abs/path', 'hindsight-embed']);
});
});
+3 -3
View File
@@ -18,8 +18,8 @@ export interface EmbedCommandOptions {
export function getEmbedCommand(opts: EmbedCommandOptions = {}): string[] {
if (opts.embedPackagePath) {
return ["uv", "run", "--directory", opts.embedPackagePath, "hindsight-embed"];
return ['uv', 'run', '--directory', opts.embedPackagePath, 'hindsight-embed'];
}
const version = opts.embedVersion && opts.embedVersion.length > 0 ? opts.embedVersion : "latest";
return ["uvx", `hindsight-embed@${version}`];
const version = opts.embedVersion && opts.embedVersion.length > 0 ? opts.embedVersion : 'latest';
return ['uvx', `hindsight-embed@${version}`];
}
+6 -6
View File
@@ -1,7 +1,7 @@
export { HindsightServer } from "./server.js";
export { getEmbedCommand } from "./command.js";
export { silentLogger, consoleLogger } from "./logger.js";
export { HindsightServer } from './server.js';
export { getEmbedCommand } from './command.js';
export { silentLogger, consoleLogger } from './logger.js';
export type { Logger } from "./logger.js";
export type { EmbedCommandOptions } from "./command.js";
export type { HindsightServerOptions } from "./types.js";
export type { Logger } from './logger.js';
export type { EmbedCommandOptions } from './command.js';
export type { HindsightServerOptions } from './types.js';
+15 -15
View File
@@ -1,32 +1,32 @@
import { describe, it, expect } from "vitest";
import { HindsightServer } from "./server.js";
import { describe, it, expect } from 'vitest';
import { HindsightServer } from './server.js';
describe("HindsightServer construction", () => {
it("defaults base URL to http://127.0.0.1:8888", () => {
describe('HindsightServer construction', () => {
it('defaults base URL to http://127.0.0.1:8888', () => {
const server = new HindsightServer();
expect(server.getBaseUrl()).toBe("http://127.0.0.1:8888");
expect(server.getProfile()).toBe("default");
expect(server.getBaseUrl()).toBe('http://127.0.0.1:8888');
expect(server.getProfile()).toBe('default');
});
it("honours custom profile, port, and host", () => {
const server = new HindsightServer({ profile: "app", port: 9077, host: "0.0.0.0" });
expect(server.getProfile()).toBe("app");
expect(server.getBaseUrl()).toBe("http://0.0.0.0:9077");
it('honours custom profile, port, and host', () => {
const server = new HindsightServer({ profile: 'app', port: 9077, host: '0.0.0.0' });
expect(server.getProfile()).toBe('app');
expect(server.getBaseUrl()).toBe('http://0.0.0.0:9077');
});
it("accepts open env pass-through without complaining about unknown keys", () => {
it('accepts open env pass-through without complaining about unknown keys', () => {
const server = new HindsightServer({
env: {
HINDSIGHT_API_LLM_PROVIDER: "openai",
HINDSIGHT_API_LLM_MODEL: "gpt-4o-mini",
HINDSIGHT_API_LLM_PROVIDER: 'openai',
HINDSIGHT_API_LLM_MODEL: 'gpt-4o-mini',
// A field that does not exist today — should still be accepted
HINDSIGHT_FUTURE_FLAG: "enabled",
HINDSIGHT_FUTURE_FLAG: 'enabled',
},
});
expect(server).toBeInstanceOf(HindsightServer);
});
it("exposes checkHealth that returns false when no daemon is running", async () => {
it('exposes checkHealth that returns false when no daemon is running', async () => {
// Random high port that nothing is listening on.
const server = new HindsightServer({ port: 1, readyTimeoutMs: 100 });
const healthy = await server.checkHealth();
+43 -43
View File
@@ -1,12 +1,12 @@
import { spawn } from "child_process";
import { getEmbedCommand } from "./command.js";
import { silentLogger } from "./logger.js";
import type { Logger } from "./logger.js";
import type { HindsightServerOptions } from "./types.js";
import { spawn } from 'child_process';
import { getEmbedCommand } from './command.js';
import { silentLogger } from './logger.js';
import type { Logger } from './logger.js';
import type { HindsightServerOptions } from './types.js';
const DEFAULT_PORT = 8888;
const DEFAULT_HOST = "127.0.0.1";
const DEFAULT_PROFILE = "default";
const DEFAULT_HOST = '127.0.0.1';
const DEFAULT_PROFILE = 'default';
const DEFAULT_READY_TIMEOUT_MS = 30_000;
const DEFAULT_READY_POLL_INTERVAL_MS = 1_000;
@@ -61,7 +61,7 @@ export class HindsightServer {
this.userEnv = opts.env ?? {};
this.extraProfileCreateArgs = opts.extraProfileCreateArgs ?? [];
this.extraDaemonStartArgs = opts.extraDaemonStartArgs ?? [];
this.platformCpuWorkaround = opts.platformCpuWorkaround ?? process.platform === "darwin";
this.platformCpuWorkaround = opts.platformCpuWorkaround ?? (process.platform === 'darwin');
this.readyTimeoutMs = opts.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS;
this.readyPollIntervalMs = opts.readyPollIntervalMs ?? DEFAULT_READY_POLL_INTERVAL_MS;
this.logger = opts.logger ?? silentLogger;
@@ -100,22 +100,22 @@ export class HindsightServer {
embedVersion: this.embedVersion,
embedPackagePath: this.embedPackagePath,
});
const args = [...baseArgs, "daemon", "--profile", this.profile, "stop"];
const args = [...baseArgs, 'daemon', '--profile', this.profile, 'stop'];
const child = spawn(cmd, args, { stdio: "pipe" });
this.pipeOutput(child, "daemon.stop");
const child = spawn(cmd, args, { stdio: 'pipe' });
this.pipeOutput(child, 'daemon.stop');
await new Promise<void>((resolve) => {
const timeout = setTimeout(() => {
this.logger.warn(`[hindsight] daemon stop timed out after 5s`);
resolve();
}, 5_000);
child.on("exit", () => {
child.on('exit', () => {
clearTimeout(timeout);
this.logger.info(`[hindsight] daemon stopped`);
resolve();
});
child.on("error", (err) => {
child.on('error', (err) => {
clearTimeout(timeout);
this.logger.warn(`[hindsight] error stopping daemon: ${err.message}`);
resolve();
@@ -147,9 +147,9 @@ export class HindsightServer {
private buildEnv(): NodeJS.ProcessEnv {
const merged: NodeJS.ProcessEnv = { ...process.env };
if (this.platformCpuWorkaround && process.platform === "darwin") {
merged["HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU"] = "1";
merged["HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU"] = "1";
if (this.platformCpuWorkaround && process.platform === 'darwin') {
merged['HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU'] = '1';
merged['HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU'] = '1';
}
for (const [key, value] of Object.entries(this.userEnv)) {
@@ -175,11 +175,11 @@ export class HindsightServer {
});
const createArgs = [
...baseArgs,
"profile",
"create",
'profile',
'create',
this.profile,
"--merge",
"--port",
'--merge',
'--port',
String(this.port),
];
@@ -189,12 +189,12 @@ export class HindsightServer {
// host state into profile config.
const envForProfile = this.collectProfileEnv(env);
for (const [key, value] of Object.entries(envForProfile)) {
createArgs.push("--env", `${key}=${value}`);
createArgs.push('--env', `${key}=${value}`);
}
createArgs.push(...this.extraProfileCreateArgs);
await this.runCommand(cmd, createArgs, env, "profile.create");
await this.runCommand(cmd, createArgs, env, 'profile.create');
}
/** Collect only the env vars that should be written into the profile file. */
@@ -209,10 +209,10 @@ export class HindsightServer {
}
// 2. CPU workaround — only if auto-applied and not already overridden.
if (this.platformCpuWorkaround && process.platform === "darwin") {
if (this.platformCpuWorkaround && process.platform === 'darwin') {
const cpuKeys = [
"HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU",
"HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU",
'HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU',
'HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU',
];
for (const key of cpuKeys) {
if (!(key in out) && env[key] !== undefined) {
@@ -231,14 +231,14 @@ export class HindsightServer {
});
const args = [
...baseArgs,
"daemon",
"--profile",
'daemon',
'--profile',
this.profile,
"start",
'start',
...this.extraDaemonStartArgs,
];
await this.runCommand(cmd, args, env, "daemon.start");
await this.runCommand(cmd, args, env, 'daemon.start');
}
/**
@@ -249,34 +249,34 @@ export class HindsightServer {
cmd: string,
args: string[],
env: NodeJS.ProcessEnv,
label: string
label: string,
): Promise<void> {
const child = spawn(cmd, args, { stdio: "pipe", env });
let output = "";
child.stdout?.on("data", (data: Buffer) => {
const child = spawn(cmd, args, { stdio: 'pipe', env });
let output = '';
child.stdout?.on('data', (data: Buffer) => {
const text = data.toString();
output += text;
for (const line of text.trimEnd().split("\n")) {
for (const line of text.trimEnd().split('\n')) {
if (line) this.logger.info(`[hindsight:${label}] ${line}`);
}
});
child.stderr?.on("data", (data: Buffer) => {
child.stderr?.on('data', (data: Buffer) => {
const text = data.toString();
output += text;
for (const line of text.trimEnd().split("\n")) {
for (const line of text.trimEnd().split('\n')) {
if (line) this.logger.warn(`[hindsight:${label}] ${line}`);
}
});
await new Promise<void>((resolve, reject) => {
child.on("exit", (code) => {
child.on('exit', (code) => {
if (code === 0) {
resolve();
} else {
reject(new Error(`${label} failed with code ${code}: ${output.trim()}`));
}
});
child.on("error", (err) => {
child.on('error', (err) => {
reject(new Error(`${label} failed to spawn: ${err.message}`, { cause: err }));
});
});
@@ -284,13 +284,13 @@ export class HindsightServer {
/** Stream a spawned child's stdout/stderr through the logger without blocking. */
private pipeOutput(child: ReturnType<typeof spawn>, label: string): void {
child.stdout?.on("data", (data: Buffer) => {
for (const line of data.toString().trimEnd().split("\n")) {
child.stdout?.on('data', (data: Buffer) => {
for (const line of data.toString().trimEnd().split('\n')) {
if (line) this.logger.info(`[hindsight:${label}] ${line}`);
}
});
child.stderr?.on("data", (data: Buffer) => {
for (const line of data.toString().trimEnd().split("\n")) {
child.stderr?.on('data', (data: Buffer) => {
for (const line of data.toString().trimEnd().split('\n')) {
if (line) this.logger.warn(`[hindsight:${label}] ${line}`);
}
});
@@ -316,7 +316,7 @@ export class HindsightServer {
await new Promise((resolve) => setTimeout(resolve, this.readyPollIntervalMs));
}
throw new Error(
`Hindsight daemon did not become ready within ${this.readyTimeoutMs}ms at ${this.baseUrl}`
`Hindsight daemon did not become ready within ${this.readyTimeoutMs}ms at ${this.baseUrl}`,
);
}
}
+1 -1
View File
@@ -1,4 +1,4 @@
import type { Logger } from "./logger.js";
import type { Logger } from './logger.js';
/**
* Options for {@link HindsightServer}.
+4 -4
View File
@@ -1,10 +1,10 @@
import { defineConfig } from "tsup";
import { defineConfig } from 'tsup';
export default defineConfig({
entry: ["src/index.ts"],
format: ["esm"],
entry: ['src/index.ts'],
format: ['esm'],
dts: true,
outDir: "dist",
outDir: 'dist',
clean: true,
sourcemap: true,
bundle: true,
+3 -3
View File
@@ -1,8 +1,8 @@
import { defineConfig } from "vitest/config";
import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
include: ["src/**/*.test.ts"],
environment: "node",
include: ['src/**/*.test.ts'],
environment: 'node',
},
});
+2 -2
View File
@@ -4,12 +4,12 @@ build-backend = "setuptools.build_meta"
[project]
name = "hindsight-all-slim"
version = "0.6.2"
version = "0.5.1"
description = "Hindsight: Agent Memory That Works Like Human Memory - Slim All-in-One Bundle"
readme = "README.md"
requires-python = ">=3.11"
dependencies = [
"hindsight-api-slim==0.6.2",
"hindsight-api-slim>=0.4.17",
"hindsight-client>=0.0.7",
"hindsight-embed>=0.1.0",
]
+30 -41
View File
@@ -71,7 +71,7 @@ class HindsightEmbedded:
llm_model: Model name to use
llm_base_url: Optional custom base URL for LLM API
database_url: Optional database URL override (default: profile-specific pg0)
idle_timeout: Seconds before daemon auto-exits when idle (default: 0, disabled)
idle_timeout: Seconds before daemon auto-exits when idle (default: 300)
log_level: Daemon log level (default: "info")
ui: Whether to start the control plane web UI alongside the daemon (default: False)
ui_port: Port for the UI. Defaults to daemon_port + 10000.
@@ -86,7 +86,7 @@ class HindsightEmbedded:
llm_model: str = "openai/gpt-oss-120b",
llm_base_url: Optional[str] = None,
database_url: Optional[str] = None,
idle_timeout: int = 0,
idle_timeout: int = 300,
log_level: str = "info",
ui: bool = False,
ui_port: Optional[int] = None,
@@ -102,7 +102,7 @@ class HindsightEmbedded:
llm_model: Model name to use
llm_base_url: Optional custom base URL for LLM API
database_url: Optional database URL override
idle_timeout: Seconds before daemon auto-exits when idle (0 = disabled)
idle_timeout: Seconds before daemon auto-exits when idle
log_level: Daemon log level
ui: Whether to start the control plane web UI alongside the daemon
ui_port: Port for the UI (defaults to daemon_port + 10000)
@@ -142,37 +142,14 @@ class HindsightEmbedded:
self._memories_api: Optional[MemoriesAPI] = None
def _ensure_started(self):
"""Ensure daemon is running (thread-safe), restarting if crashed."""
"""Ensure daemon is running (thread-safe)."""
if self._started and self._client is not None:
if self._manager.is_running(self.profile):
return
# Daemon crashed — reset state and fall through to restart
logger.warning(
"Daemon for profile '%s' is no longer responsive, restarting...",
self.profile,
)
try:
self._client.close()
except Exception:
logger.debug("Error closing stale client", exc_info=True)
self._client = None
self._started = False
return
with self._lock:
# Double-check after acquiring lock
if self._started and self._client is not None:
if self._manager.is_running(self.profile):
return
logger.warning(
"Daemon for profile '%s' is no longer responsive (lock path), restarting...",
self.profile,
)
try:
self._client.close()
except Exception:
logger.debug("Error closing stale client", exc_info=True)
self._client = None
self._started = False
return
if self._closed:
raise RuntimeError(
@@ -276,10 +253,23 @@ class HindsightEmbedded:
This allows HindsightEmbedded to expose all HindsightClient methods
without manually wrapping each one.
"""
# Ensure server is started (and restart if crashed) before proxying
# Ensure server is started before proxying
self._ensure_started()
return getattr(self._client, name)
# Get the attribute from the underlying client
attr = getattr(self._client, name)
# If it's a callable, wrap it to ensure server is started
# (shouldn't be needed since _ensure_started already called, but defensive)
if callable(attr):
def wrapper(*args, **kwargs):
self._ensure_started()
return attr(*args, **kwargs)
return wrapper
return attr
def __enter__(self):
"""Context manager entry - ensures server is started."""
@@ -404,8 +394,11 @@ class HindsightEmbedded:
"""
Get the underlying Hindsight client for direct access.
Ensures daemon is started (and restarts it if it has crashed) before
returning the client.
WARNING: Using this property directly means daemon restarts won't be
handled automatically. Prefer using the API namespaces (banks, mental_models,
directives, memories) or direct method calls on HindsightEmbedded instead.
Ensures daemon is started before returning the client.
Returns:
Hindsight: The underlying client instance
@@ -416,8 +409,9 @@ class HindsightEmbedded:
embedded = HindsightEmbedded(profile="myapp", ...)
# Direct access (not recommended - daemon crashes won't be handled)
client = embedded.client
banks = client.list_banks()
banks = client.list_banks() # If daemon crashes, this will fail
```
"""
self._ensure_started()
@@ -431,13 +425,8 @@ class HindsightEmbedded:
@property
def is_running(self) -> bool:
"""Check if the client is initialized and the daemon is responsive."""
return (
self._started
and not self._closed
and self._client is not None
and self._manager.is_running(self.profile)
)
"""Check if the client is initialized."""
return self._started and not self._closed and self._client is not None
@property
def ui_url(self) -> str:
+3 -3
View File
@@ -4,12 +4,12 @@ build-backend = "hatchling.build"
[project]
name = "hindsight-all"
version = "0.6.2"
version = "0.5.1"
description = "Hindsight: Agent Memory That Works Like Human Memory - All-in-One Bundle"
readme = "README.md"
requires-python = ">=3.11"
dependencies = [
"hindsight-api-slim[all]==0.6.2",
"hindsight-api-slim[all]>=0.4.17",
"hindsight-client>=0.0.7",
"hindsight-embed>=0.1.0",
]
@@ -21,7 +21,7 @@ hindsight-embed = { workspace = true }
[project.optional-dependencies]
local-llm = [
"hindsight-api-slim[local-llm]==0.6.2",
"hindsight-api-slim[local-llm]>=0.4.17",
]
test = [
"pytest>=7.0.0",
-39
View File
@@ -401,42 +401,3 @@ def test_embedded_ui_flag(llm_config):
finally:
client.close()
def test_embedded_daemon_crash_recovery(llm_config):
"""
Test that HindsightEmbedded recovers when the daemon crashes.
Simulates a crash by stopping the daemon, then verifies
that the next operation transparently restarts it.
"""
profile = f"test_crash_{uuid.uuid4().hex[:8]}"
bank_id = f"bank_{uuid.uuid4().hex[:8]}"
client = HindsightEmbedded(profile=profile, log_level="info", **llm_config)
try:
# Start daemon and store a memory
result = client.retain(bank_id=bank_id, content="Before crash")
assert result.success, "Initial retain should succeed"
assert client.is_running, "Daemon should be running"
original_url = client.url
# Simulate daemon crash by stopping it
client._manager.stop(client.profile)
assert not client._manager.is_running(client.profile), (
"Daemon should be stopped after simulated crash"
)
# Next operation should transparently restart the daemon
result2 = client.retain(bank_id=bank_id, content="After crash recovery")
assert result2.success, "Retain after crash recovery should succeed"
assert client.is_running, "Daemon should be running again after recovery"
# Verify recall still works
recall_result = client.recall(bank_id=bank_id, query="crash")
assert isinstance(recall_result.results, list), "Recall should return results"
finally:
client.close()
+1 -1
View File
@@ -46,4 +46,4 @@ __all__ = [
"RemoteTEICrossEncoder",
"LLMConfig",
]
__version__ = "0.6.2"
__version__ = "0.5.1"
@@ -1,170 +0,0 @@
"""Shared PostgreSQL vector-extension dispatch helpers."""
from __future__ import annotations
import logging
from sqlalchemy import text
from sqlalchemy.engine import Connection
logger = logging.getLogger(__name__)
# Extensions a user can set via HINDSIGHT_API_VECTOR_EXTENSION.
CONFIGURABLE_EXTENSIONS = ("pgvector", "pgvectorscale", "vchord", "scann")
# Extensions detect_vector_extension() can return. pg_diskann is a runtime-only
# resolution from a configured "pgvectorscale" backend on Azure (uses a different
# WITH clause), never a value the user sets directly.
RESOLVED_EXTENSIONS = (*CONFIGURABLE_EXTENSIONS, "pg_diskann")
# Backwards-compatible alias for older imports.
VALID_EXTENSIONS = CONFIGURABLE_EXTENSIONS
SCANN_MIN_ROWS_FOR_AUTO_INDEX = 10_000
_EXTENSION_NAMES = {
"pgvector": "vector",
"pgvectorscale": "vectorscale",
"vchord": "vchord",
"scann": "alloydb_scann",
}
_INDEX_USING_CLAUSES = {
"pgvector": "USING hnsw (embedding vector_cosine_ops)",
"pgvectorscale": "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)",
"pg_diskann": "USING diskann (embedding vector_cosine_ops) WITH (max_neighbors = 50)",
"vchord": "USING vchordrq (embedding vector_l2_ops)",
"scann": "USING scann (embedding cosine) WITH (mode = 'AUTO')",
}
_INDEX_TYPE_KEYWORDS = {
"pgvector": "hnsw",
"pgvectorscale": "diskann",
"pg_diskann": "diskann",
"vchord": "vchordrq",
"scann": "scann",
}
_EXTENSION_INSTALL_SQL = {
"pgvector": ("CREATE EXTENSION IF NOT EXISTS vector",),
"pgvectorscale": (
"CREATE EXTENSION IF NOT EXISTS vector",
"CREATE EXTENSION IF NOT EXISTS vectorscale CASCADE",
),
"vchord": ("CREATE EXTENSION IF NOT EXISTS vchord CASCADE",),
"scann": (
"CREATE EXTENSION IF NOT EXISTS vector",
"CREATE EXTENSION IF NOT EXISTS alloydb_scann CASCADE",
),
}
_INSTALL_HINTS = {
"pgvector": "CREATE EXTENSION vector;",
"pgvectorscale": "CREATE EXTENSION vector; then CREATE EXTENSION vectorscale CASCADE; (or pg_diskann on Azure)",
"vchord": "CREATE EXTENSION vchord CASCADE;",
"scann": "CREATE EXTENSION vector; then CREATE EXTENSION alloydb_scann CASCADE;",
}
def validate_extension(name: str) -> str:
"""Return a normalized configurable vector extension name or raise.
Used at the user-facing config boundary; pg_diskann is rejected here because
it is a detection-time alias, never a value the user sets directly.
"""
ext = name.lower()
if ext not in CONFIGURABLE_EXTENSIONS:
valid = ", ".join(CONFIGURABLE_EXTENSIONS)
raise ValueError(f"Invalid vector_extension: {name}. Must be one of: {valid}")
return ext
def _normalize_resolved(name: str) -> str:
"""Normalize either a user-configurable or detect-time extension name."""
ext = name.lower()
if ext not in RESOLVED_EXTENSIONS:
valid = ", ".join(RESOLVED_EXTENSIONS)
raise ValueError(f"Unknown vector extension: {name}. Must be one of: {valid}")
return ext
def pg_extension_name(ext: str) -> str:
"""Return the PostgreSQL extension name for a configured vector backend."""
return _EXTENSION_NAMES[validate_extension(ext)]
def index_using_clause(ext: str) -> str:
"""Return the CREATE INDEX USING clause for the vector backend."""
return _INDEX_USING_CLAUSES[_normalize_resolved(ext)]
def index_type_keyword(ext: str) -> str:
"""Return the keyword that identifies this index type in pg_indexes.indexdef."""
return _INDEX_TYPE_KEYWORDS[_normalize_resolved(ext)]
def minimum_rows_for_index(ext: str) -> int:
"""Return the minimum populated embedding rows before creating this index type."""
return SCANN_MIN_ROWS_FOR_AUTO_INDEX if _normalize_resolved(ext) == "scann" else 0
def should_defer_index_creation(ext: str, row_count: int) -> bool:
"""Return True when index creation should wait for more embeddings."""
minimum_rows = minimum_rows_for_index(ext)
return minimum_rows > 0 and row_count < minimum_rows
def uses_per_bank_vector_indexes(ext: str) -> bool:
"""Return whether the backend should create per-bank partial vector indexes."""
return _normalize_resolved(ext) != "scann"
def bootstrap_extension(conn: Connection, ext: str) -> None:
"""Install the configured vector extension and any prerequisites if possible."""
normalized = validate_extension(ext)
for statement in _EXTENSION_INSTALL_SQL[normalized]:
conn.execute(text(statement))
def detect_vector_extension(conn: Connection, vector_extension: str = "pgvector") -> str:
"""Validate the configured vector extension exists and return the index backend."""
configured_ext = validate_extension(vector_extension)
if configured_ext == "pgvectorscale":
pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
if not pgvector_check:
raise RuntimeError(
"DiskANN (pgvectorscale/pg_diskann) requires pgvector to be installed. "
f"Install it with: {_INSTALL_HINTS['pgvectorscale']}"
)
vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar()
pg_diskann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_diskann'")).scalar()
if vectorscale_check:
logger.debug("Using vector extension: pgvectorscale (DiskANN)")
return "pgvectorscale"
if pg_diskann_check:
logger.debug("Using vector extension: pg_diskann (Azure DiskANN)")
return "pg_diskann"
raise RuntimeError(
"Configured vector extension 'pgvectorscale' not found. Install either:\n"
" - pgvectorscale (open source): CREATE EXTENSION vectorscale CASCADE;\n"
" - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
)
extension_name = pg_extension_name(configured_ext)
extension_check = conn.execute(
text("SELECT 1 FROM pg_extension WHERE extname = :extension_name"),
{"extension_name": extension_name},
).scalar()
if not extension_check:
raise RuntimeError(
f"Configured vector extension '{configured_ext}' not found. "
f"Install it with: {_INSTALL_HINTS[configured_ext]}"
)
logger.debug("Using configured vector extension: %s", configured_ext)
return configured_ext
+8 -139
View File
@@ -1,7 +1,5 @@
"""PostgreSQL-only admin utilities (backup, restore, migration, worker management).
Not supported on Oracle backends. Uses asyncpg.connect() directly, binary COPY,
TRUNCATE CASCADE, and REFRESH MATERIALIZED VIEW — all inherently PG-specific.
"""
Hindsight Admin CLI - backup and restore operations.
"""
import asyncio
@@ -17,10 +15,15 @@ import asyncpg
import typer
from ..config import DEFAULT_DATABASE_SCHEMA, HindsightConfig
from ..engine.schema import fq_table_explicit as _fq_table
from ..extensions import TenantExtension, load_extension
from ..pg0 import parse_pg0_url, resolve_database_url
def _fq_table(table: str, schema: str) -> str:
"""Get fully-qualified table name with schema prefix."""
return f"{schema}.{table}"
# Setup logging
logging.basicConfig(
level=logging.INFO,
@@ -372,140 +375,6 @@ def decommission_worker(
typer.echo(f"No tasks found for worker '{worker_id}'")
async def _decommission_all_workers(db_url: str, schema: str = "public") -> list[dict[str, Any]]:
"""Release all processing tasks from all workers, setting them back to pending status."""
is_pg0, instance_name, _ = parse_pg0_url(db_url)
if is_pg0:
typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
resolved_url = await resolve_database_url(db_url)
conn = await asyncpg.connect(resolved_url)
try:
table = _fq_table("async_operations", schema)
rows = await conn.fetch(
f"""
UPDATE {table}
SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
WHERE status = 'processing'
RETURNING operation_id, worker_id, operation_type
""",
)
return [dict(r) for r in rows]
finally:
await conn.close()
@app.command(name="decommission-workers")
def decommission_workers(
schema: str = typer.Option("public", "--schema", "-s", help="Database schema"),
yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"),
):
"""Release all processing tasks from all workers (sets status back to pending).
Use this command to recover from situations where one or more workers have crashed
or been removed without graceful shutdown. All tasks currently in 'processing' status
will be released back to the queue regardless of which worker owns them.
"""
config = HindsightConfig.from_env()
if not config.database_url:
typer.echo("Error: Database URL not configured.", err=True)
typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
raise typer.Exit(1)
if not yes:
typer.confirm(
"This will release ALL processing tasks from ALL workers back to pending. Continue?",
abort=True,
)
typer.echo(f"Decommissioning all workers (schema: {schema})...")
released = asyncio.run(_decommission_all_workers(config.database_url, schema))
if released:
# Group by worker_id for summary
by_worker: dict[str, int] = {}
for row in released:
wid = row["worker_id"] or "unknown"
by_worker[wid] = by_worker.get(wid, 0) + 1
typer.echo(f"Released {len(released)} task(s):")
for wid, count in by_worker.items():
typer.echo(f" {wid}: {count} task(s)")
else:
typer.echo("No processing tasks found")
async def _worker_status(db_url: str, schema: str = "public") -> list[dict[str, Any]]:
"""Get all processing tasks grouped by worker with their last update time."""
is_pg0, instance_name, _ = parse_pg0_url(db_url)
if is_pg0:
typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
resolved_url = await resolve_database_url(db_url)
conn = await asyncpg.connect(resolved_url)
try:
table = _fq_table("async_operations", schema)
rows = await conn.fetch(
f"""
SELECT worker_id, operation_id, operation_type, bank_id,
claimed_at, updated_at,
now() - claimed_at AS running_for,
now() - updated_at AS last_update_ago
FROM {table}
WHERE status = 'processing'
ORDER BY worker_id, claimed_at
""",
)
return [dict(r) for r in rows]
finally:
await conn.close()
@app.command(name="worker-status")
def worker_status(
schema: str = typer.Option("public", "--schema", "-s", help="Database schema"),
):
"""Show all currently processing tasks grouped by worker.
Displays each worker's active tasks with operation type, bank, how long
the task has been running, and when it was last updated. Useful for
identifying dead workers with orphaned tasks.
"""
config = HindsightConfig.from_env()
if not config.database_url:
typer.echo("Error: Database URL not configured.", err=True)
typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
raise typer.Exit(1)
rows = asyncio.run(_worker_status(config.database_url, schema))
if not rows:
typer.echo("No processing tasks found")
return
# Group by worker_id
by_worker: dict[str, list[dict[str, Any]]] = {}
for row in rows:
wid = row["worker_id"] or "unknown"
by_worker.setdefault(wid, []).append(row)
typer.echo(f"Processing tasks across {len(by_worker)} worker(s):\n")
for wid, tasks in by_worker.items():
typer.echo(f"Worker: {wid} ({len(tasks)} task(s))")
for task in tasks:
op_id = str(task["operation_id"])[:8]
running_for = task["running_for"]
last_update = task["last_update_ago"]
typer.echo(
f" {op_id} {task['operation_type']:<20s} bank={task['bank_id']}"
f" running={running_for} last_update={last_update} ago"
)
typer.echo("")
def main():
app()
@@ -1,42 +0,0 @@
"""Dialect dispatcher for Alembic migrations.
Each migration file declares a ``_pg_upgrade``/``_oracle_upgrade`` (and matching
downgrades) function and routes ``upgrade()``/``downgrade()`` through
``run_for_dialect``. The helper inspects the live connection's dialect name and
runs the matching function — or no-ops if the migration doesn't apply to the
current backend.
Use ``None`` (or omit the kwarg) when a migration intentionally has no effect
on a dialect; the helper treats it as a no-op.
"""
from __future__ import annotations
from collections.abc import Callable
from alembic import op
DialectFn = Callable[[], None]
_SUPPORTED = ("postgresql", "oracle")
def run_for_dialect(
*,
pg: DialectFn | None = None,
oracle: DialectFn | None = None,
) -> None:
"""Dispatch to the function matching the current bind's dialect.
Args:
pg: Function to run when the active bind is PostgreSQL.
oracle: Function to run when the active bind is Oracle.
Unrecognized dialects raise; an explicit ``None`` for the active dialect
is a no-op (the migration deliberately does nothing here).
"""
name = op.get_bind().dialect.name
if name not in _SUPPORTED:
raise RuntimeError(f"Unsupported dialect for migration dispatch: {name!r}. Expected one of {_SUPPORTED}.")
fn = {"postgresql": pg, "oracle": oracle}[name]
if fn is not None:
fn()
+76 -101
View File
@@ -1,37 +1,28 @@
"""
Alembic environment for Hindsight.
Supports two dialects:
* PostgreSQL (sync psycopg2 driver) — default; uses ``search_path`` for
multi-tenant schema isolation and forces read-write transactions to work
around Supabase's read-only-by-default sessions.
* Oracle 23ai (``oracledb`` driver) — uses ``CURRENT_SCHEMA`` for tenant
isolation; no equivalent of ``search_path`` or read-only session quirks.
Each migration file dispatches its DDL through ``alembic._dialect.run_for_dialect``
so a single revision tree serves both backends.
Alembic environment configuration for SQLAlchemy with pgvector.
Uses synchronous psycopg2 driver for migrations to avoid pgbouncer issues.
"""
import logging
import os
from pathlib import Path
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from alembic import context
from dotenv import load_dotenv
from sqlalchemy import Connection, engine_from_config, pool
from sqlalchemy.engine import Engine
from sqlalchemy import engine_from_config, pool
from hindsight_api.db_url import is_oracle_url, to_libpq_url
# Import your models here
from hindsight_api.models import Base
def load_env() -> None:
"""Load environment variables from .env (skipped if already configured)."""
# Load environment variables based on HINDSIGHT_API_DATABASE_URL env var or default to local
def load_env():
"""Load environment variables from .env"""
# Check if HINDSIGHT_API_DATABASE_URL is already set (e.g., by CI/CD)
if os.getenv("HINDSIGHT_API_DATABASE_URL"):
return
# Look for .env file in the parent directory (root of the workspace)
root_dir = Path(__file__).parent.parent.parent
env_file = root_dir / ".env"
@@ -41,45 +32,30 @@ def load_env() -> None:
load_env()
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Note: We don't call fileConfig() here to avoid overriding the application's logging configuration.
# Alembic will use the existing logging configuration from the application.
# add your model's MetaData object here
# for 'autogenerate' support
target_metadata = Base.metadata
def _normalize_oracle_url(url: str) -> str:
"""Coerce an Oracle URL into the SQLAlchemy form the oracledb dialect expects.
Two issues to handle:
1. Force the ``oracle+oracledb`` driver — bare ``oracle://`` defaults to
cx_Oracle.
2. Map a path-style service to ``?service_name=...``. SQLAlchemy's oracledb
dialect treats the URL path as a *SID* (legacy), but Oracle Free /
Autonomous DB only register a service name. Without this rewrite we get
``DPY-6003: SID "FREEPDB1" is not registered`` even though the listener
is happy to accept the same name as a service.
"""
parts = urlsplit(url)
if not parts.scheme.startswith("oracle"):
return url
new_scheme = "oracle+oracledb" if parts.scheme == "oracle" else parts.scheme
service = parts.path.lstrip("/")
new_query = parts.query
new_path = parts.path
# Promote /SERVICE to ?service_name=SERVICE unless the caller already
# supplied an explicit ?sid= or ?service_name=.
if service and "service_name=" not in new_query and "sid=" not in new_query:
params = [(k, v) for k, v in parse_qsl(new_query, keep_blank_values=True)]
params.append(("service_name", service))
new_query = urlencode(params)
new_path = ""
return urlunsplit((new_scheme, parts.netloc, new_path, new_query, parts.fragment))
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def get_database_url() -> str:
"""Resolve the migration URL from Alembic config or env, normalizing per-dialect."""
"""
Get and process the database URL from config or environment.
Returns the URL with the correct driver (psycopg2) for migrations.
"""
# Get database URL from config (set programmatically) or environment
database_url = config.get_main_option("sqlalchemy.url")
if not database_url:
database_url = os.getenv("HINDSIGHT_API_DATABASE_URL")
@@ -89,18 +65,30 @@ def get_database_url() -> str:
"Set HINDSIGHT_API_DATABASE_URL environment variable or pass database_url to run_migrations()."
)
if is_oracle_url(database_url):
database_url = _normalize_oracle_url(database_url)
else:
# PG: convert SQLAlchemy-style asyncpg URLs and ?ssl= params to libpq form
# for the sync engine used during migrations.
database_url = to_libpq_url(database_url)
# For migrations, use psycopg2 (sync driver) to avoid pgbouncer prepared statement issues
if database_url.startswith("postgresql+asyncpg://"):
database_url = database_url.replace("postgresql+asyncpg://", "postgresql://", 1)
elif database_url.startswith("postgres+asyncpg://"):
database_url = database_url.replace("postgres+asyncpg://", "postgresql://", 1)
# Update config with processed URL for engine_from_config to use
config.set_main_option("sqlalchemy.url", database_url)
return database_url
def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
logging.info("running offline")
database_url = get_database_url()
@@ -115,40 +103,14 @@ def run_migrations_offline() -> None:
context.run_migrations()
def _configure_pg_session(engine: Engine, connection: Connection, target_schema: str | None) -> None:
"""PG-only: ensure the session is RW (Supabase) and bind ``search_path``."""
def run_migrations_online() -> None:
"""Run migrations in 'online' mode with synchronous engine."""
from sqlalchemy import event, text
@event.listens_for(engine, "connect")
def set_read_write_mode(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
cursor.execute("SET SESSION CHARACTERISTICS AS TRANSACTION READ WRITE")
if target_schema:
cursor.execute(f'CREATE SCHEMA IF NOT EXISTS "{target_schema}"')
cursor.execute(f'SET search_path TO "{target_schema}", public')
cursor.close()
get_database_url() # Process and set the database URL in config
connection.execute(text("SET SESSION CHARACTERISTICS AS TRANSACTION READ WRITE"))
if target_schema:
connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{target_schema}"'))
connection.execute(text(f'SET search_path TO "{target_schema}", public'))
connection.commit()
def _configure_oracle_session(connection: Connection, target_schema: str | None) -> None:
"""Oracle: switch the session's default schema; tolerate DDL contention."""
from sqlalchemy import text
# Wait up to 30s for DDL locks instead of failing immediately (ORA-00054).
connection.execute(text("ALTER SESSION SET DDL_LOCK_TIMEOUT = 30"))
if target_schema:
connection.execute(text(f'ALTER SESSION SET CURRENT_SCHEMA = "{target_schema}"'))
def run_migrations_online() -> None:
database_url = get_database_url()
# Check if we're targeting a specific schema (for multi-tenant isolation)
target_schema = config.get_main_option("target_schema")
is_oracle = is_oracle_url(database_url)
connectable = engine_from_config(
config.get_section(config.config_ini_section, {}),
@@ -156,19 +118,37 @@ def run_migrations_online() -> None:
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
if is_oracle:
_configure_oracle_session(connection, target_schema)
else:
_configure_pg_session(connectable, connection, target_schema)
# Add event listener to ensure connection is in read-write mode
# This is needed for Supabase which may start connections in read-only mode
@event.listens_for(connectable, "connect")
def set_read_write_mode(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
cursor.execute("SET SESSION CHARACTERISTICS AS TRANSACTION READ WRITE")
# If targeting a specific schema, set search_path
# Include public in search_path for access to shared extensions (pgvector)
if target_schema:
cursor.execute(f'CREATE SCHEMA IF NOT EXISTS "{target_schema}"')
cursor.execute(f'SET search_path TO "{target_schema}", public')
cursor.close()
with connectable.connect() as connection:
# Also explicitly set read-write mode on this connection
connection.execute(text("SET SESSION CHARACTERISTICS AS TRANSACTION READ WRITE"))
# If targeting a specific schema, set search_path
# Include public in search_path for access to shared extensions (pgvector)
if target_schema:
connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{target_schema}"'))
connection.execute(text(f'SET search_path TO "{target_schema}", public'))
connection.commit() # Commit the SET command
# Configure context with version_table_schema if using a specific schema
context_opts = {
"connection": connection,
"target_metadata": target_metadata,
}
if target_schema and not is_oracle:
# Oracle has no equivalent of PG's per-schema version table; the
# ``alembic_version`` table lives in CURRENT_SCHEMA implicitly.
if target_schema:
context_opts["version_table_schema"] = target_schema
context.configure(**context_opts)
@@ -176,12 +156,7 @@ def run_migrations_online() -> None:
with context.begin_transaction():
context.run_migrations()
# Always commit. PG needs it for the explicit RW-mode SET to persist;
# Oracle needs it because each DDL auto-commits but the trailing
# ``UPDATE alembic_version`` is plain DML that would otherwise stay in
# an open transaction and roll back when the connection closes —
# producing the "schema is created but the version row is one revision
# behind" failure mode.
# Explicit commit to ensure changes are persisted (especially for Supabase)
connection.commit()
@@ -11,8 +11,6 @@ from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
@@ -20,27 +18,11 @@ branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def _pg_upgrade() -> None:
"""PostgreSQL upgrade. Set to ``None`` below if this migration is Oracle-only."""
def upgrade() -> None:
"""Upgrade schema."""
${upgrades if upgrades else "pass"}
def _pg_downgrade() -> None:
${downgrades if downgrades else "pass"}
def _oracle_upgrade() -> None:
"""Oracle upgrade. Set to ``None`` below if this migration is Postgres-only."""
pass
def _oracle_downgrade() -> None:
pass
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade, oracle=_oracle_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade, oracle=_oracle_downgrade)
"""Downgrade schema."""
${downgrades if downgrades else "pass"}
@@ -4,8 +4,8 @@ The previous GIN trigram index on canonical_name was case-sensitive, causing
"Alice" and "alice" to have different trigram sets. This recreates it on
LOWER(canonical_name) so the % operator matches case-insensitively.
Revision ID: 2eee35aa3cfc
Revises: d6e7f8a9b0c1
Revision ID: d6e7f8a9b0c1
Revises: c5d6e7f8a9b0
Create Date: 2026-03-31
"""
@@ -13,10 +13,8 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "2eee35aa3cfc"
down_revision: str | Sequence[str] | None = "d6e7f8a9b0c1"
revision: str = "d6e7f8a9b0c1"
down_revision: str | Sequence[str] | None = "c5d6e7f8a9b0"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
@@ -26,7 +24,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
# Drop the old case-sensitive trigram index
op.execute("DROP INDEX IF EXISTS entities_canonical_name_trgm_idx")
@@ -37,7 +35,7 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS entities_canonical_name_lower_trgm_idx")
schema = _get_schema_prefix()
# Restore original case-sensitive index
@@ -45,11 +43,3 @@ def _pg_downgrade() -> None:
f"CREATE INDEX IF NOT EXISTS entities_canonical_name_trgm_idx "
f"ON {schema}entities USING GIN (canonical_name gin_trgm_ops)"
)
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -15,8 +15,6 @@ from pgvector.sqlalchemy import Vector
from sqlalchemy import text
from sqlalchemy.dialects import postgresql
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "5a366d414dce"
down_revision: str | Sequence[str] | None = None
@@ -26,67 +24,52 @@ depends_on: str | Sequence[str] | None = None
def _detect_vector_extension() -> str:
"""
Detect or validate vector extension for this immutable migration revision.
Detect or validate vector extension: 'pgvector', 'vchord', or 'pgvectorscale'.
Respects HINDSIGHT_API_VECTOR_EXTENSION env var if set.
"""
conn = op.get_bind()
vector_extension = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
# Validate configured extension is installed
if vector_extension == "pgvectorscale":
# pgvectorscale/DiskANN requires pgvector
pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
if not pgvector_check:
raise RuntimeError(
"DiskANN requires pgvector. Install with: CREATE EXTENSION vector; then vectorscale or pg_diskann CASCADE;"
)
# Check for either vectorscale (open source) or pg_diskann (Azure)
vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar()
pg_diskann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_diskann'")).scalar()
if vectorscale_check:
return "pgvectorscale"
if pg_diskann_check:
elif pg_diskann_check:
return "pg_diskann"
raise RuntimeError(
"Configured vector extension 'pgvectorscale' not found. Install either:\n"
" - pgvectorscale: CREATE EXTENSION vectorscale CASCADE;\n"
" - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
)
if vector_extension == "vchord":
else:
raise RuntimeError(
"Configured vector extension 'pgvectorscale' not found. Install either:\n"
" - pgvectorscale: CREATE EXTENSION vectorscale CASCADE;\n"
" - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
)
elif vector_extension == "vchord":
vchord_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord'")).scalar()
if not vchord_check:
raise RuntimeError(
"Configured vector extension 'vchord' not found. Install it with: CREATE EXTENSION vchord CASCADE;"
)
return "vchord"
if vector_extension == "scann":
scann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'alloydb_scann'")).scalar()
if not scann_check:
raise RuntimeError(
"Configured vector extension 'scann' not found. Install it with: CREATE EXTENSION alloydb_scann CASCADE;"
)
return "scann"
if vector_extension == "pgvector":
elif vector_extension == "pgvector":
pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
if not pgvector_check:
raise RuntimeError(
"Configured vector extension 'pgvector' not found. Install it with: CREATE EXTENSION vector;"
)
return "pgvector"
raise ValueError(
"Invalid HINDSIGHT_API_VECTOR_EXTENSION: "
f"{vector_extension}. Must be 'pgvector', 'vchord', 'pgvectorscale', or 'scann'"
)
def _vector_index_using_clause(ext: str) -> str:
if ext == "pgvectorscale":
return "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)"
if ext == "pg_diskann":
return "USING diskann (embedding vector_cosine_ops) WITH (max_neighbors = 50)"
if ext == "vchord":
return "USING vchordrq (embedding vector_l2_ops)"
if ext == "scann":
return "USING scann (embedding cosine) WITH (mode = 'AUTO')"
return "USING hnsw (embedding vector_cosine_ops)"
else:
raise ValueError(
f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {vector_extension}. Must be 'pgvector', 'vchord', or 'pgvectorscale'"
)
def _detect_text_search_extension() -> str:
@@ -129,7 +112,7 @@ def _detect_text_search_extension() -> str:
)
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Upgrade schema - create all tables from scratch."""
# Note: pgvector extension is installed globally BEFORE migrations run
@@ -328,11 +311,36 @@ def _pg_upgrade() -> None:
)
# Create vector index - conditional based on available extension
vector_ext = _detect_vector_extension()
if vector_ext != "scann":
op.execute(f"""
if vector_ext == "pgvectorscale":
# Use DiskANN index for pgvectorscale (disk-based, scalable)
op.execute("""
CREATE INDEX idx_memory_units_embedding ON memory_units
{_vector_index_using_clause(vector_ext)}
USING diskann (embedding vector_cosine_ops)
WITH (num_neighbors = 50)
""")
elif vector_ext == "pg_diskann":
# Use DiskANN index for pg_diskann (Azure)
op.execute("""
CREATE INDEX idx_memory_units_embedding ON memory_units
USING diskann (embedding vector_cosine_ops)
WITH (max_neighbors = 50)
""")
elif vector_ext == "vchord":
# Use vchordrq index for vchord (supports high-dimensional embeddings)
op.execute("""
CREATE INDEX idx_memory_units_embedding ON memory_units
USING vchordrq (embedding vector_l2_ops)
""")
else: # pgvector
# Use HNSW index for pgvector
op.create_index(
"idx_memory_units_embedding",
"memory_units",
["embedding"],
postgresql_using="hnsw",
postgresql_ops={"embedding": "vector_cosine_ops"},
)
# Create full-text search index on search_vector
# Index type depends on text search backend
@@ -455,7 +463,7 @@ def _pg_upgrade() -> None:
op.create_index("idx_unit_entities_entity", "unit_entities", ["entity_id"])
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Downgrade schema - drop all tables."""
# Drop tables in reverse dependency order
@@ -515,11 +523,3 @@ def _pg_downgrade() -> None:
# Drop extensions (optional - comment out if you want to keep them)
# op.execute('DROP EXTENSION IF EXISTS vector')
# op.execute('DROP EXTENSION IF EXISTS "uuid-ossp"')
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,117 +0,0 @@
"""Repair mental_models.subtype on databases stuck at m3rg3h3ad5f6
Three production deployments reported `column "subtype" of relation
"mental_models" does not exist` on `create_mental_model` even after their
container reported `Database migrations completed successfully` and
`alembic_version` advanced to `m3rg3h3ad5f6` (see issue #1553, #1553#1
confirmations from @4Lienau and @khanhduyvt0101).
Both `h3c4d5e6f7g8_mental_models_v4` and `d5y6z7a8b9c0_backfill_mental_models_subtype`
were meant to ensure `subtype` exists, but on databases that came through the
`reflections -> mental_models` rename chain *and* whose alembic_version
advanced past `d5y6z7a8b9c0` along an alternate path during the divergent-heads
reorganization, neither column-add actually fired. The result is a head-tagged
database with a v3-shaped `mental_models` table missing six columns:
``subtype``, ``description``, ``entity_id``, ``observations``, ``links``,
``last_updated``.
This migration sits at the current head (`m3rg3h3ad5f6`) so every affected
deployment will pick it up on next container start. It mirrors the column-add
block from `d5y6z7a8b9c0_backfill_mental_models_subtype` using
``ADD COLUMN IF NOT EXISTS`` so it is a no-op on databases where the columns
are already present.
Revision ID: 86f7a033d372
Revises: m3rg3h3ad5f6
Create Date: 2026-05-14
"""
from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "86f7a033d372"
down_revision: str | Sequence[str] | None = "m3rg3h3ad5f6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _pg_schema_prefix() -> str:
"""Schema-qualifier for raw SQL on PG (multi-tenant search_path)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
"""Idempotently ensure mental_models has the v4 column set.
Safe to re-apply on databases that already received the columns via
`h3c4d5e6f7g8_mental_models_v4` or `d5y6z7a8b9c0_backfill_mental_models_subtype` —
every column-add uses ``IF NOT EXISTS`` and the constraint is recreated
from scratch with the canonical v4 allowlist.
"""
schema = _pg_schema_prefix()
bare_schema = schema.strip(".").strip('"') if schema else ""
schema_clause = f"AND table_schema = '{bare_schema}'" if bare_schema else ""
# Wrapped in a DO block so the existence check skips databases that
# predate the reflections -> mental_models rename chain (no table to
# repair). On those, every ALTER below would error.
op.execute(
f"""
DO $$
BEGIN
IF EXISTS (
SELECT 1 FROM information_schema.tables
WHERE table_name = 'mental_models'
{schema_clause}
) THEN
-- Add the six v4 columns idempotently.
ALTER TABLE {schema}mental_models
ADD COLUMN IF NOT EXISTS subtype VARCHAR(32) NOT NULL DEFAULT 'structural';
ALTER TABLE {schema}mental_models
ADD COLUMN IF NOT EXISTS description TEXT NOT NULL DEFAULT '';
ALTER TABLE {schema}mental_models
ADD COLUMN IF NOT EXISTS entity_id UUID;
ALTER TABLE {schema}mental_models
ADD COLUMN IF NOT EXISTS observations JSONB DEFAULT '{{"observations": []}}'::jsonb;
ALTER TABLE {schema}mental_models
ADD COLUMN IF NOT EXISTS links VARCHAR[];
ALTER TABLE {schema}mental_models
ADD COLUMN IF NOT EXISTS last_updated TIMESTAMP WITH TIME ZONE;
-- Recreate the CHECK constraint with the canonical v4 allowlist.
-- Existing rows with subtype = 'directive' (possible on databases
-- that ran the o0j1k2l3m4n5 directive-only path) are rewritten to
-- 'structural' first so the constraint add succeeds.
UPDATE {schema}mental_models SET subtype = 'structural' WHERE subtype = 'directive';
ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype;
ALTER TABLE {schema}mental_models
ADD CONSTRAINT ck_mental_models_subtype
CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'));
CREATE INDEX IF NOT EXISTS idx_mental_models_subtype
ON {schema}mental_models(bank_id, subtype);
END IF;
END$$;
"""
)
def _pg_downgrade() -> None:
"""No-op: dropping these columns would corrupt v4 application code."""
pass
def upgrade() -> None:
# PG-only: Oracle's baseline (o1a2b3c4d5e6) creates mental_models with its
# own subtype shape (chk_mm_subtype IN ('directive', 'pinned')) and a
# different table topology, so this PG-shaped repair does not apply.
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,50 +0,0 @@
"""Merge divergent migration heads for v0.5.3
v0.5.3 shipped with two migration heads that were never unified:
* ``c4x5y6z7a8b9`` — delta-refresh chain
(``add_last_refreshed_source_query`` ->
``add_structured_content_to_mental_models`` ->
``backsweep_orphan_observations_v2``)
* ``h3i4j5k6l7m8`` — per-bank vector indexes / audit log chain
(the ``merge_heads_and_add_unit_entities_index`` subtree)
Both fork from ``z1u2v3w4x5y6``. Upgrades from v0.5.2 still succeed — the
walker applies the three c4x5 revisions and leaves the database stamped at
both heads — but the result is a split DAG: ``alembic upgrade head``
(singular) is ambiguous, and any future migration has to pick one head as
its parent, orphaning the other.
This revision linearises the DAG into a single head. It has no schema
effect.
Revision ID: 8c6fa6f7230b
Revises: c4x5y6z7a8b9, h3i4j5k6l7m8
Create Date: 2026-04-18
"""
from collections.abc import Sequence
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "8c6fa6f7230b"
down_revision: str | Sequence[str] | None = ("c4x5y6z7a8b9", "h3i4j5k6l7m8")
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _pg_upgrade() -> None:
pass
def _pg_downgrade() -> None:
pass
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,128 +0,0 @@
"""Make memory_links.from_unit_id and memory_links.to_unit_id FKs deferrable.
Revision ID: 9f8e7d6c5b4a
Revises: o1a2b3c4d5e6
Create Date: 2026-05-03
Background
----------
Concurrent retain (which INSERTs into ``memory_links``) and any code path
that DELETEs a row whose deletion cascades into ``memory_links`` (e.g.
delta-retain superseding chunks, which CASCADEs chunks → memory_units →
memory_links) can deadlock under sustained single-tenant write load.
The deadlock cycle:
* Tx A: ``DELETE FROM chunks WHERE chunk_id = ANY(...)``
→ CASCADE acquires row locks on memory_units, then on memory_links rows
where ``to_unit_id`` matches the deleted units.
* Tx B: ``INSERT INTO memory_links (...)`` referencing one of the same
memory_units rows.
→ The immediate FK check takes ``FOR KEY SHARE`` on those memory_units
rows.
The two transactions take row locks on the same memory_units rows in
opposite orders depending on which side started first. PostgreSQL detects
the cycle and aborts one transaction; the loser is killed mid-batch, the
winner continues. Workers then retry, but under sustained write load the
pattern repeats.
Fix
---
Make both ``memory_links → memory_units`` FKs (``from_unit_id`` and
``to_unit_id``) ``DEFERRABLE INITIALLY DEFERRED``. This pushes the FK
check from INSERT time to COMMIT time:
* INSERT no longer takes ``FOR KEY SHARE`` on the memory_units row → no
contention with the cascading DELETE's row lock.
* At COMMIT the engine validates referential integrity in one shot. If a
cascade-DELETE has since removed the referenced unit, the INSERT
transaction commits OR fails with a clean FK violation (sqlstate
23503) instead of a deadlock (sqlstate 40P01).
The ``WHERE EXISTS`` filter already in ``_bulk_insert_links`` continues to
filter out the typical "stale unit_id" case at INSERT time; the deferred
FK is only the backstop for the narrow race window between the EXISTS
probe and COMMIT. ``ON DELETE CASCADE`` semantics are unchanged — only
the *timing* of the constraint check moves.
The ``entity_id`` FK on ``memory_links`` is not changed; entities are not
involved in the observed deadlock cycle and leaving the constraint
immediate keeps the error message specific when an entity row is missing.
"""
from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "9f8e7d6c5b4a"
down_revision: str | Sequence[str] | None = "o1a2b3c4d5e6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Schema-qualifier for raw SQL on PG (multi-tenant search_path)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
# The two FK constraints installed by the initial schema migration
# (5a366d414dce_initial_schema), mapped to the column they constrain.
# They reference memory_units(id) with ON DELETE CASCADE — that
# semantics is preserved; only the deferral attribute changes.
_FK_COLUMNS: dict[str, str] = {
"fk_memory_links_from_unit_id_memory_units": "from_unit_id",
"fk_memory_links_to_unit_id_memory_units": "to_unit_id",
}
def _pg_upgrade() -> None:
schema = _get_schema_prefix()
# PostgreSQL doesn't allow altering the deferrability of an existing
# constraint with ALTER CONSTRAINT — the constraint must be dropped
# and recreated. DROP IF EXISTS makes the migration safe to re-run
# on schemas where the constraint was already recreated.
for fk_name, column in _FK_COLUMNS.items():
op.execute(f"ALTER TABLE {schema}memory_links DROP CONSTRAINT IF EXISTS {fk_name}")
op.execute(
f"""
ALTER TABLE {schema}memory_links
ADD CONSTRAINT {fk_name}
FOREIGN KEY ({column})
REFERENCES {schema}memory_units (id)
ON DELETE CASCADE
DEFERRABLE INITIALLY DEFERRED
"""
)
def _pg_downgrade() -> None:
schema = _get_schema_prefix()
# Revert to the default (NOT DEFERRABLE) form so a downgrade actually
# restores the prior schema state, even though that re-introduces the
# deadlock window.
for fk_name, column in _FK_COLUMNS.items():
op.execute(f"ALTER TABLE {schema}memory_links DROP CONSTRAINT IF EXISTS {fk_name}")
op.execute(
f"""
ALTER TABLE {schema}memory_links
ADD CONSTRAINT {fk_name}
FOREIGN KEY ({column})
REFERENCES {schema}memory_units (id)
ON DELETE CASCADE
"""
)
def upgrade() -> None:
# PG-only: Oracle's deferrable-FK semantics differ and the deadlock
# cycle was only observed on PostgreSQL. Oracle slot intentionally
# absent → no-op there.
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -15,8 +15,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "a1b2c3d4e5f6"
down_revision: str | Sequence[str] | None = "y0t1u2v3w4x5"
branch_labels: str | Sequence[str] | None = None
@@ -29,7 +27,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Create file_storage table for BYTEA storage."""
schema = _get_schema_prefix()
@@ -54,7 +52,7 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Remove file_storage table and related columns."""
schema = _get_schema_prefix()
@@ -70,11 +68,3 @@ def _pg_downgrade() -> None:
# Drop file_storage table
op.execute(f"DROP TABLE IF EXISTS {schema}file_storage")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -18,8 +18,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "a2b3c4d5e6f7"
down_revision: str | Sequence[str] | None = "aa2b3c4d5e6f"
branch_labels: str | Sequence[str] | None = None
@@ -35,7 +33,7 @@ def _detect_text_search_extension() -> str:
return os.getenv("HINDSIGHT_API_TEXT_SEARCH_EXTENSION", "native").lower()
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
table = f"{schema}memory_units"
text_search_ext = _detect_text_search_extension()
@@ -67,7 +65,7 @@ def _pg_upgrade() -> None:
# pg_textsearch: no change — index operates on the base `text` column only
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
table = f"{schema}memory_units"
text_search_ext = _detect_text_search_extension()
@@ -88,11 +86,3 @@ def _pg_downgrade() -> None:
""")
op.execute(f"ALTER TABLE {table} DROP COLUMN IF EXISTS text_signals")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -24,8 +24,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "a2b3c4d5e6f8"
down_revision: str | Sequence[str] | None = "f7g8h9i0j1k2"
branch_labels: str | Sequence[str] | None = None
@@ -37,7 +35,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
# CREATE INDEX CONCURRENTLY cannot run inside a transaction block.
@@ -50,15 +48,7 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,48 +0,0 @@
"""Add last_refreshed_source_query column to mental_models
Revision ID: a2v3w4x5y6z7
Revises: z1u2v3w4x5y6
Create Date: 2026-04-15
Tracks the source_query that was used during the most recent refresh.
Used by delta-mode refresh to detect when the query has changed: if it has,
delta mode falls back to a full regeneration because the surgical-edit
assumption (same topic, new facts) no longer holds.
"""
from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "a2v3w4x5y6z7"
down_revision: str | Sequence[str] | None = "z1u2v3w4x5y6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD COLUMN IF NOT EXISTS last_refreshed_source_query TEXT
""")
def _pg_downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS last_refreshed_source_query")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -13,8 +13,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "a3b4c5d6e7f8"
down_revision: str | Sequence[str] | None = "g7h8i9j0k1l2"
branch_labels: str | Sequence[str] | None = None
@@ -27,7 +25,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(
@@ -47,16 +45,8 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_consolidation_failed")
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS consolidation_failed_at")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,7 +1,7 @@
"""Fix per-bank vector indexes to match configured extension
Revision ID: a4b5c6d7e8f9
Revises: 2eee35aa3cfc
Revises: d6e7f8a9b0c1
Create Date: 2026-04-01
Migration d5e6f7a8b9c0 hardcoded HNSW when creating per-bank partial vector
@@ -11,8 +11,7 @@ was configured.
This migration detects the mismatch and recreates the affected indexes with
the correct type. Skipped entirely when the configured extension is pgvector
(the default) or scann. ScaNN uses global vector indexes because empty or tiny
per-bank indexes cannot be built safely on AlloyDB.
(the default), since those indexes are already correct.
"""
import os
@@ -21,10 +20,8 @@ from collections.abc import Sequence
from alembic import context, op
from sqlalchemy import text
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "a4b5c6d7e8f9"
down_revision: str | Sequence[str] | None = "2eee35aa3cfc"
down_revision: str | Sequence[str] | None = "d6e7f8a9b0c1"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
@@ -40,47 +37,39 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _validate_extension(name: str) -> str:
ext = name.lower()
if ext not in {"pgvector", "pgvectorscale", "vchord", "scann"}:
raise ValueError(
f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {ext}. Must be 'pgvector', 'vchord', 'pgvectorscale', or 'scann'"
)
return ext
def _index_type_keyword(ext: str) -> str:
def _target_index_type() -> str | None:
"""Return the target index type, or None if pgvector (no fix needed)."""
ext = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
if ext == "pgvectorscale":
return "diskann"
if ext == "vchord":
elif ext == "vchord":
return "vchordrq"
if ext == "scann":
return "scann"
return "hnsw"
return None
def _vector_index_using_clause(ext: str) -> str:
def _vector_index_using_clause() -> str:
"""Return the USING clause based on the configured vector extension."""
ext = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
if ext == "pgvectorscale":
return "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)"
if ext == "vchord":
elif ext == "vchord":
return "USING vchordrq (embedding vector_l2_ops)"
if ext == "scann":
return "USING scann (embedding cosine) WITH (mode = 'AUTO')"
return "USING hnsw (embedding vector_cosine_ops)"
else:
return "USING hnsw (embedding vector_cosine_ops)"
def _pg_upgrade() -> None:
ext = _validate_extension(os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector"))
if ext in {"pgvector", "scann"}:
def upgrade() -> None:
target = _target_index_type()
if target is None:
# pgvector — indexes are already HNSW, nothing to fix
return
target = _index_type_keyword(ext)
bind = op.get_bind()
schema_name = context.config.get_main_option("target_schema")
schema = _get_schema_prefix()
table_ref = f'"{schema_name}".memory_units' if schema_name else "memory_units"
banks_ref = f'"{schema_name}".banks' if schema_name else "banks"
using_clause = _vector_index_using_clause(ext)
using_clause = _vector_index_using_clause()
pg_schema = schema_name or "public"
rows = bind.execute(text(f"SELECT bank_id, internal_id FROM {banks_ref}")).fetchall() # noqa: S608
@@ -124,10 +113,10 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
# Downgrade recreates indexes as HNSW (the original hardcoded behavior)
ext = _validate_extension(os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector"))
if ext in {"pgvector", "scann"}:
target = _target_index_type()
if target is None:
return
bind = op.get_bind()
@@ -151,11 +140,3 @@ def _pg_downgrade() -> None:
f"WHERE fact_type = '{ft}' AND bank_id = '{escaped_bank_id}'"
)
)
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -12,8 +12,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "aa2b3c4d5e6f"
down_revision: str | Sequence[str] | None = "z1u2v3w4x5y6"
branch_labels: str | Sequence[str] | None = None
@@ -26,21 +24,13 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}memory_units ALTER COLUMN event_date DROP NOT NULL")
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
# Backfill NULLs with now() before restoring the NOT NULL constraint
op.execute(f"UPDATE {schema}memory_units SET event_date = now() WHERE event_date IS NULL")
op.execute(f"ALTER TABLE {schema}memory_units ALTER COLUMN event_date SET NOT NULL")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -9,8 +9,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "b3c4d5e6f7a8"
down_revision: str | Sequence[str] | None = "a3b4c5d6e7f8"
branch_labels: str | Sequence[str] | None = None
@@ -23,20 +21,12 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
# Add content_hash column to chunks table for delta comparison
op.execute(f"ALTER TABLE {schema}chunks ADD COLUMN IF NOT EXISTS content_hash TEXT")
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}chunks DROP COLUMN IF EXISTS content_hash")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -22,8 +22,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "b3c4d5e6f7g8"
down_revision: str | Sequence[str] | None = "c1a2b3d4e5f6"
branch_labels: str | Sequence[str] | None = None
@@ -35,7 +33,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
# Partial index on occurred_start (covers "occurred_start BETWEEN $4 AND $5")
op.execute("COMMIT")
@@ -60,7 +58,7 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_mentioned_at")
@@ -68,11 +66,3 @@ def _pg_downgrade() -> None:
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_occurred_end")
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_occurred_start")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,54 +0,0 @@
"""Add structured_content JSONB column to mental_models
Revision ID: b3w4x5y6z7a8
Revises: a2v3w4x5y6z7
Create Date: 2026-04-16
Stores the structured representation of a mental model document (sections,
blocks). The plain ``content`` column remains the rendered markdown shown to
users. ``structured_content`` is the source of truth for delta-mode refreshes:
each refresh applies a list of typed operations to the structured doc, then
re-renders to markdown — so unchanged sections come through byte-identical
without an LLM round-trip.
Nullable: existing markdown-only mental models continue to work in full mode;
the column is populated lazily the first time a model is refreshed in delta
mode.
"""
from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "b3w4x5y6z7a8"
down_revision: str | Sequence[str] | None = "a2v3w4x5y6z7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD COLUMN IF NOT EXISTS structured_content JSONB
""")
def _pg_downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS structured_content")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -14,8 +14,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "b4c5d6e7f8a9"
down_revision: str | Sequence[str] | None = "a2b3c4d5e6f7"
branch_labels: str | Sequence[str] | None = None
@@ -27,18 +25,10 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS observation_scopes JSONB")
def _pg_downgrade() -> None:
pass # intentionally no-op — safe to leave the column in place
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
pass # intentionally no-op — safe to leave the column in place
@@ -1,91 +0,0 @@
"""Backfill entity_cooccurrences.last_cooccurred from memory_units event time
Revision ID: b5d4e3f2a1c9
Revises: o1a2b3c4d5e6
Create Date: 2026-04-24
The writer path in `entity_resolver.link_units_to_entities_batch` historically
stamped `entity_cooccurrences.last_cooccurred` with `datetime.now(UTC)` at
flush time, ignoring the source memory unit's event date. For normal online
retains that's fine (now ≈ event time), but for any corpus that was
backfilled in a single session — migrating from another memory system, for
example — every co-occurrence collapsed to the import moment, which hid the
underlying knowledge timeline from the dashboard's entity graph recency heat
and from any downstream consumer of the column.
The writer is fixed in the same change set to propagate the unit's event_date;
this migration repairs historical rows by reading the true event time off
`unit_entities × memory_units` (falling back to `created_at` when
`mentioned_at` / `occurred_start` are NULL, so rows never regress).
Oracle slot is intentionally absent: the Oracle baseline (`o1a2b3c4d5e6`)
landed days before this fix, so any Oracle deployment runs the corrected
writer against an effectively empty `entity_cooccurrences` — there is no
historical residue on Oracle to repair. PG-only matches the asymmetry of
the data, not negligence.
"""
from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "b5d4e3f2a1c9"
down_revision: str | Sequence[str] | None = "o1a2b3c4d5e6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
schema = _get_schema_prefix()
# Recompute last_cooccurred from the true event time per entity pair.
# COALESCE picks the first non-null of mentioned_at / occurred_start /
# created_at so banks without event-time metadata still see a sane value
# (equivalent to the pre-fix behaviour) instead of NULL.
#
# The self-join on `unit_entities` is O(k²) per memory_unit in the number
# of distinct entities mentioned (k). For typical units k is small (single
# digits), but a bank with units containing hundreds of entities and tens
# of millions of co-occurrence rows may want to run this off-hours — the
# whole UPDATE is one statement, so it locks every targeted ec row for
# the duration. The migration is one-time; subsequent online writes
# already carry event time via the writer fix.
op.execute(
f"""
UPDATE {schema}entity_cooccurrences ec
SET last_cooccurred = sub.event_time
FROM (
SELECT
LEAST(ue1.entity_id, ue2.entity_id) AS e1,
GREATEST(ue1.entity_id, ue2.entity_id) AS e2,
MAX(COALESCE(mu.mentioned_at, mu.occurred_start, mu.created_at)) AS event_time
FROM {schema}memory_units mu
JOIN {schema}unit_entities ue1 ON ue1.unit_id = mu.id
JOIN {schema}unit_entities ue2 ON ue2.unit_id = mu.id AND ue1.entity_id <> ue2.entity_id
GROUP BY 1, 2
) sub
WHERE ec.entity_id_1 = sub.e1 AND ec.entity_id_2 = sub.e2
"""
)
def _pg_downgrade() -> None:
# No-op: the previous column value was `now()` at the time of write and
# isn't recoverable. Rolling back the code is sufficient — new writes will
# revert to the old behaviour for subsequent retains.
pass
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade) # oracle slot intentionally absent — see header
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -12,8 +12,6 @@ import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "b7c4d8e9f1a2"
down_revision: str | Sequence[str] | None = "5a366d414dce"
@@ -21,7 +19,7 @@ branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Add chunks table and link memory_units to chunks."""
# Create chunks table with single text PK (bank_id_document_id_chunk_index)
@@ -58,7 +56,7 @@ def _pg_upgrade() -> None:
op.create_index("idx_memory_units_chunk_id", "memory_units", ["chunk_id"])
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Remove chunks table and chunk_id from memory_units."""
# Drop index and foreign key from memory_units
@@ -70,11 +68,3 @@ def _pg_downgrade() -> None:
op.drop_index("idx_chunks_bank_id", table_name="chunks")
op.drop_index("idx_chunks_document_id", table_name="chunks")
op.drop_table("chunks")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -14,8 +14,6 @@ from collections.abc import Sequence
import sqlalchemy as sa
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "c1a2b3d4e5f6"
down_revision: str | Sequence[str] | None = "b4c5d6e7f8a9"
branch_labels: str | Sequence[str] | None = None
@@ -27,7 +25,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
# pg_trgm ships with most PostgreSQL installations as a contrib module.
# It enables fast similarity lookups via GIN indexes, used for entity name matching.
# On managed services (e.g. Azure Flexible Server), the extension may not be
@@ -54,16 +52,8 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}entities_canonical_name_trgm_idx")
# Note: not dropping pg_trgm extension as other indexes may depend on it
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -14,8 +14,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "c2d3e4f5g6h7"
down_revision: str | Sequence[str] | None = ("a3b4c5d6e7f8", "c8e5f2a3b4d1")
branch_labels: str | Sequence[str] | None = None
@@ -28,7 +26,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(
@@ -54,18 +52,10 @@ def _pg_upgrade() -> None:
op.execute(f"CREATE INDEX IF NOT EXISTS idx_audit_log_started ON {schema}audit_log (started_at DESC)")
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}idx_audit_log_started")
op.execute(f"DROP INDEX IF EXISTS {schema}idx_audit_log_bank_started")
op.execute(f"DROP INDEX IF EXISTS {schema}idx_audit_log_action_started")
op.execute(f"DROP TABLE IF EXISTS {schema}audit_log")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -9,8 +9,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "c3d4e5f6g7h8"
down_revision: str | Sequence[str] | None = ("a2b3c4d5e6f7", "a2b3c4d5e6f8")
branch_labels: str | Sequence[str] | None = None
@@ -22,19 +20,11 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}mental_models ADD COLUMN IF NOT EXISTS history JSONB DEFAULT '[]'::jsonb")
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS history")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,76 +0,0 @@
"""backsweep_orphan_observations_v2
Re-run of Pass 2 from migration ``g7h8i9j0k1l2_backsweep_orphan_observations``
to sweep observations that became orphaned between then and now.
Why we need it again:
``fact_storage.handle_document_tracking`` (the retain/upsert path) deleted
the existing document via the FK cascade — which removes the source
``memory_units`` — but never invalidated the observations derived from
them. Only the explicit ``MemoryEngine.delete_document`` API called
``_delete_stale_observations_for_memories``. Every document re-ingest
therefore left orphan observations whose ``source_memory_ids`` arrays
pointed at IDs that no longer existed in ``memory_units``.
``handle_document_tracking`` now calls the same cleanup helper before the
cascade, so no new orphans will accumulate going forward. This migration
cleans up the historical residue.
Identical to Pass 2 of g7h8i9j0k1l2. Pass 1 (memory_units whose bank is
gone) is intentionally not re-run; that scenario has no fresh source.
Revision ID: c4x5y6z7a8b9
Revises: b3w4x5y6z7a8
Create Date: 2026-04-16
"""
from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "c4x5y6z7a8b9"
down_revision: str | Sequence[str] | None = "b3w4x5y6z7a8"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
schema = _get_schema_prefix()
mu = f"{schema}memory_units"
# Delete observations whose every source_memory_id refers to a now-deleted
# memory_unit (or the array is empty). Observations with at least one
# surviving source are left alone — the consolidation engine will refresh
# their text on the next pass.
op.execute(
f"""
DELETE FROM {mu} orphan
WHERE orphan.fact_type = 'observation'
AND NOT EXISTS (
SELECT 1
FROM {mu} src
WHERE src.id = ANY(orphan.source_memory_ids)
AND src.bank_id = orphan.bank_id
)
"""
)
def _pg_downgrade() -> None:
# Deleted rows cannot be restored.
pass
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -13,8 +13,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "c5d6e7f8a9b0"
down_revision: str | Sequence[str] | None = "b3c4d5e6f7a8"
branch_labels: str | Sequence[str] | None = None
@@ -26,7 +24,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
# 1. Add nullable column
@@ -45,14 +43,6 @@ def _pg_upgrade() -> None:
op.execute(f"ALTER TABLE {schema}memory_links ALTER COLUMN bank_id SET NOT NULL")
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}memory_links DROP COLUMN IF EXISTS bank_id")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -12,8 +12,6 @@ import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "c8e5f2a3b4d1"
down_revision: str | Sequence[str] | None = "b7c4d8e9f1a2"
@@ -21,7 +19,7 @@ branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Add retain_params JSONB column to documents table."""
# Add retain_params column to store parameters passed during retain
@@ -31,7 +29,7 @@ def _pg_upgrade() -> None:
op.create_index("idx_documents_retain_params", "documents", ["retain_params"], postgresql_using="gin")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Remove retain_params column from documents table."""
# Drop index
@@ -39,11 +37,3 @@ def _pg_downgrade() -> None:
# Drop column
op.drop_column("documents", "retain_params")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -34,8 +34,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "d2e3f4a5b6c7"
down_revision: str | Sequence[str] | None = "b3c4d5e6f7g8"
branch_labels: str | Sequence[str] | None = None
@@ -47,7 +45,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
# CREATE INDEX CONCURRENTLY cannot run inside a transaction block.
@@ -77,17 +75,9 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_entity_covering")
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_to_type_weight")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -18,8 +18,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "d4e5f6g7h8i9"
down_revision: str | Sequence[str] | None = "d5e6f7a8b9c0"
branch_labels: str | Sequence[str] | None = None
@@ -31,7 +29,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
# DROP + CREATE CONCURRENTLY must run outside a transaction block.
op.execute("COMMIT")
@@ -44,7 +42,7 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute("COMMIT")
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
@@ -53,11 +51,3 @@ def _pg_downgrade() -> None:
f"ON {schema}memory_units USING GIN (source_memory_ids) "
f"WHERE source_memory_ids IS NOT NULL"
)
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -6,11 +6,10 @@ Create Date: 2026-03-11
This migration:
1. Adds internal_id UUID column to banks (stable identifier for index naming)
2. For non-ScaNN backends, drops the global vector index (competes with
per-bank partial indexes)
3. For non-ScaNN backends, creates per-(bank_id, fact_type) partial vector
indexes for all existing banks using the configured vector extension
(HNSW for pgvector, DiskANN for pgvectorscale, vchordrq for vchord).
2. Drops the global vector index (competes with per-bank partial indexes)
3. Creates per-(bank_id, fact_type) partial vector indexes for all existing banks
using the configured vector extension (HNSW for pgvector, DiskANN for
pgvectorscale, vchordrq for vchord).
(new banks get indexes created at bank-creation time via bank_utils.create_bank_vector_indexes)
Why per-(bank, fact_type) indexes:
@@ -18,8 +17,6 @@ Why per-(bank, fact_type) indexes:
clause, because the idx_memory_units_bank_id B-tree index always wins at planning time.
- Per-(bank, fact_type) partial indexes have both predicates matching → planner selects them.
- The global vector index competes for larger partitions (world, observation) and must be dropped.
- AlloyDB ScaNN uses global vector indexes with filtered vector search instead
because empty or tiny per-bank indexes cannot be built safely.
"""
import os
@@ -28,8 +25,6 @@ from collections.abc import Sequence
from alembic import context, op
from sqlalchemy import text
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "d5e6f7a8b9c0"
down_revision: str | Sequence[str] | None = "c3d4e5f6g7h8"
branch_labels: str | Sequence[str] | None = None
@@ -42,31 +37,23 @@ _FACT_TYPES: dict[str, str] = {
}
def _configured_vector_extension() -> str:
ext = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
if ext not in {"pgvector", "pgvectorscale", "vchord", "scann"}:
raise ValueError(
f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {ext}. Must be 'pgvector', 'vchord', 'pgvectorscale', or 'scann'"
)
return ext
def _vector_index_using_clause(ext: str) -> str:
if ext == "pgvectorscale":
return "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)"
if ext == "vchord":
return "USING vchordrq (embedding vector_l2_ops)"
if ext == "scann":
return "USING scann (embedding cosine) WITH (mode = 'AUTO')"
return "USING hnsw (embedding vector_cosine_ops)"
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def _vector_index_using_clause() -> str:
"""Return the USING clause based on the configured vector extension."""
ext = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
if ext == "pgvectorscale":
return "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)"
elif ext == "vchord":
return "USING vchordrq (embedding vector_l2_ops)"
else:
return "USING hnsw (embedding vector_cosine_ops)"
def upgrade() -> None:
schema = _get_schema_prefix()
# 1. Add internal_id column to banks
@@ -75,14 +62,6 @@ def _pg_upgrade() -> None:
)
op.execute(f"ALTER TABLE {schema}banks ADD CONSTRAINT banks_internal_id_unique UNIQUE (internal_id)")
ext = _configured_vector_extension()
if ext == "scann":
# ScaNN should keep/use a global vector index. Per-bank partial indexes
# are created while banks are empty and can fail AlloyDB's ScaNN build
# requirements, so this migration leaves vector index reconciliation to
# runtime ensure_vector_extension once enough rows exist.
return
# 2. Drop any fact_type-only partial indexes that may exist from prior migrations
# (bank_id B-tree always wins over them when bank_id is in the WHERE clause)
op.execute(f"DROP INDEX IF EXISTS {schema}idx_mu_emb_world")
@@ -98,7 +77,7 @@ def _pg_upgrade() -> None:
schema_name = context.config.get_main_option("target_schema")
table_ref = f'"{schema_name}".memory_units' if schema_name else "memory_units"
banks_ref = f'"{schema_name}".banks' if schema_name else "banks"
using_clause = _vector_index_using_clause(ext)
using_clause = _vector_index_using_clause()
rows = bind.execute(text(f"SELECT bank_id, internal_id FROM {banks_ref}")).fetchall() # noqa: S608
for row in rows:
@@ -117,7 +96,7 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
# Drop per-bank HNSW indexes (iterate existing banks)
@@ -128,7 +107,7 @@ def _pg_downgrade() -> None:
rows = bind.execute(text(f"SELECT internal_id FROM {banks_ref}")).fetchall() # noqa: S608
for row in rows:
internal_id = str(row[0]).replace("-", "")[:16]
for ft_short in _FACT_TYPES.values():
for ft_short in _HNSW_FACT_TYPES.values():
idx_name = f"idx_mu_emb_{ft_short}_{internal_id}"
bind.execute(text(f"DROP INDEX IF EXISTS {schema}{idx_name}"))
@@ -158,11 +137,3 @@ def _pg_downgrade() -> None:
# Drop internal_id column
op.execute(f"ALTER TABLE {schema}banks DROP CONSTRAINT IF EXISTS banks_internal_id_unique")
op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS internal_id")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,67 +0,0 @@
"""Backfill mental_models.subtype for databases that ran h3c4d5e6f7g8 before the fix
Migration h3c4d5e6f7g8 used CREATE TABLE IF NOT EXISTS to create the
mental_models table with a subtype column. But on databases where the table
already existed (from the reflections -> mental_models rename chain), the
CREATE was a no-op and subtype was never added. A fix was later added to
h3c4d5e6f7g8 (Step 4b), but databases that had already run the migration
never re-execute it. This migration adds the missing columns idempotently.
Revision ID: d5y6z7a8b9c0
Revises: 8c6fa6f7230b
Create Date: 2026-04-18
"""
from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "d5y6z7a8b9c0"
down_revision: str | Sequence[str] | None = "8c6fa6f7230b"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
schema = _get_schema_prefix()
# Add columns that h3c4d5e6f7g8 intended to create but missed when
# the table already existed from the reflections rename chain.
for col_ddl in [
"subtype VARCHAR(32) NOT NULL DEFAULT 'structural'",
"description TEXT NOT NULL DEFAULT ''",
"entity_id UUID",
"observations JSONB DEFAULT '{\"observations\": []}'::jsonb",
"links VARCHAR[]",
"last_updated TIMESTAMP WITH TIME ZONE",
]:
op.execute(f"ALTER TABLE {schema}mental_models ADD COLUMN IF NOT EXISTS {col_ddl}")
# Ensure the CHECK constraint exists
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD CONSTRAINT ck_mental_models_subtype CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
""")
op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_subtype ON {schema}mental_models(bank_id, subtype)")
def _pg_downgrade() -> None:
# No-op: these columns are part of the intended schema
pass
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,49 +0,0 @@
"""Drop unused metadata column from documents table
Revision ID: d6e7f8a9b0c1
Revises: c2d3e4f5g6h7, c5d6e7f8a9b0
Create Date: 2026-03-30
The metadata column on documents was always stored as an empty dict {}.
Actual document metadata is stored inside retain_params.metadata.
This migration was originally shipped in v0.4.22, then its file was deleted
in v0.5.0 (and its revision ID accidentally reused by 2eee35aa3cfc).
Restoring the file so that databases stamped at this revision can upgrade
cleanly to v0.5.x+.
"""
from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "d6e7f8a9b0c1"
down_revision: str | Sequence[str] | None = ("c2d3e4f5g6h7", "c5d6e7f8a9b0")
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}documents DROP COLUMN IF EXISTS metadata")
def _pg_downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}documents ADD COLUMN IF NOT EXISTS metadata jsonb DEFAULT '{{}}'")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -8,8 +8,6 @@ Create Date: 2024-12-04 15:00:00.000000
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision = "d9f6a3b4c5e2"
down_revision = "c8e5f2a3b4d1"
@@ -23,7 +21,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade():
def upgrade():
schema = _get_schema_prefix()
# Drop old check constraint FIRST (before updating data)
@@ -40,7 +38,7 @@ def _pg_upgrade():
)
def _pg_downgrade():
def downgrade():
schema = _get_schema_prefix()
# Drop new check constraint FIRST
@@ -53,11 +51,3 @@ def _pg_downgrade():
op.create_check_constraint(
"memory_units_fact_type_check", "memory_units", "fact_type IN ('world', 'bank', 'opinion', 'observation')"
)
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -14,8 +14,6 @@ from collections.abc import Sequence
import sqlalchemy as sa
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "e0a1b2c3d4e5"
down_revision: str | Sequence[str] | None = "rename_personality"
@@ -35,7 +33,7 @@ def _get_target_schema() -> str:
return schema if schema else "public"
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Convert Big Five disposition to 3-trait disposition."""
conn = op.get_bind()
schema = _get_schema_prefix()
@@ -77,7 +75,7 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Convert back to Big Five disposition."""
conn = op.get_bind()
schema = _get_schema_prefix()
@@ -111,11 +109,3 @@ def _pg_downgrade() -> None:
ALTER COLUMN disposition SET DEFAULT '{{"openness": 0.5, "conscientiousness": 0.5, "extraversion": 0.5, "agreeableness": 0.5, "neuroticism": 0.5, "bias_strength": 0.5}}'::jsonb
""")
)
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -12,8 +12,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "e4f5a6b7c8d9"
down_revision: str | Sequence[str] | None = "d2e3f4a5b6c7"
branch_labels: str | Sequence[str] | None = None
@@ -25,7 +23,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(
@@ -56,17 +54,9 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_status_retry")
op.execute(f"ALTER TABLE {schema}async_operations DROP COLUMN IF EXISTS next_retry_at")
op.execute(f"DROP INDEX IF EXISTS {schema}idx_webhooks_bank_id")
op.execute(f"DROP TABLE IF EXISTS {schema}webhooks")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -13,8 +13,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "e5f6g7h8i9j0"
down_revision: str | Sequence[str] | None = "d4e5f6g7h8i9"
branch_labels: str | Sequence[str] | None = None
@@ -26,7 +24,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
# Remove orphaned async_operations rows whose bank no longer exists
@@ -69,15 +67,7 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}async_operations DROP CONSTRAINT IF EXISTS fk_async_operations_bank_id")
op.execute(f"ALTER TABLE {schema}webhooks DROP CONSTRAINT IF EXISTS fk_webhooks_bank_id")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -12,8 +12,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "f1a2b3c4d5e6"
down_revision: str | Sequence[str] | None = "e0a1b2c3d4e5"
@@ -27,7 +25,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Add composite index for efficient graph retrieval edge loading."""
schema = _get_schema_prefix()
# Create composite index for efficient top-k per (from_node, link_type) queries
@@ -40,15 +38,7 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Remove the composite index."""
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_links_from_type_weight")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -10,8 +10,6 @@ from collections.abc import Sequence
from alembic import op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "f6g7h8i9j0k1"
down_revision: str | Sequence[str] | None = "e5f6g7h8i9j0"
@@ -19,7 +17,7 @@ branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Change memory_units.chunk_id FK from SET NULL to CASCADE.
When a document is deleted the CASCADE reaches chunks first; with SET NULL
@@ -51,17 +49,9 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Revert to SET NULL behaviour."""
op.drop_constraint("memory_units_chunk_fkey", "memory_units", type_="foreignkey")
op.create_foreign_key(
"memory_units_chunk_fkey", "memory_units", "chunks", ["chunk_id"], ["chunk_id"], ondelete="SET NULL"
)
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -12,8 +12,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "f7g8h9i0j1k2"
down_revision: str | Sequence[str] | None = "e4f5a6b7c8d9"
branch_labels: str | Sequence[str] | None = None
@@ -25,19 +23,11 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}webhooks ADD COLUMN IF NOT EXISTS http_config JSONB NOT NULL DEFAULT '{{}}'")
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}webhooks DROP COLUMN IF EXISTS http_config")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -12,8 +12,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "g2a3b4c5d6e7"
down_revision: str | Sequence[str] | None = "f1a2b3c4d5e6"
@@ -27,7 +25,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Add tags column to memory_units and documents tables."""
schema = _get_schema_prefix()
@@ -41,18 +39,10 @@ def _pg_upgrade() -> None:
op.execute(f"ALTER TABLE {schema}documents ADD COLUMN IF NOT EXISTS tags VARCHAR[] NOT NULL DEFAULT '{{}}'")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Remove tags columns and index."""
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_tags")
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS tags")
op.execute(f"ALTER TABLE {schema}documents DROP COLUMN IF EXISTS tags")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -13,8 +13,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "g2h3i4j5k6l7"
down_revision: str | Sequence[str] | None = "f1a2b3c4d5e6"
@@ -28,7 +26,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
# 1. Delete any remaining opinion rows
@@ -51,7 +49,7 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
# Restore confidence_score column
@@ -83,11 +81,3 @@ def _pg_downgrade() -> None:
f"CREATE INDEX idx_memory_units_opinion_date ON {schema}memory_units "
f"(bank_id, event_date DESC) WHERE fact_type = 'opinion'"
)
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -22,8 +22,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "g7h8i9j0k1l2"
down_revision: str | Sequence[str] | None = "f6g7h8i9j0k1"
branch_labels: str | Sequence[str] | None = None
@@ -35,7 +33,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
mu = f"{schema}memory_units"
banks = f"{schema}banks"
@@ -68,14 +66,6 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
# Deleted rows cannot be restored.
pass
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -17,8 +17,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "h3c4d5e6f7g8"
down_revision: str | Sequence[str] | None = "g2a3b4c5d6e7"
@@ -32,7 +30,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Apply mental models v4 changes."""
schema = _get_schema_prefix()
@@ -87,26 +85,6 @@ def _pg_upgrade() -> None:
)
""")
# Step 4b: If the table already existed (from reflections rename chain),
# it won't have the v4 columns. Add them idempotently so the migration
# works regardless of whether CREATE TABLE above was a no-op.
for col_ddl in [
"subtype VARCHAR(32) NOT NULL DEFAULT 'directive'",
"description TEXT NOT NULL DEFAULT ''",
"entity_id UUID",
"observations JSONB DEFAULT '{\"observations\": []}'::jsonb",
"links VARCHAR[]",
"last_updated TIMESTAMP WITH TIME ZONE",
]:
op.execute(f"ALTER TABLE {schema}mental_models ADD COLUMN IF NOT EXISTS {col_ddl}")
# Ensure the subtype CHECK constraint exists (may not if table was renamed)
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD CONSTRAINT ck_mental_models_subtype CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
""")
# Step 5: Create indexes for efficient queries (if not exist)
op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_bank_id ON {schema}mental_models(bank_id)")
op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_subtype ON {schema}mental_models(bank_id, subtype)")
@@ -115,7 +93,7 @@ def _pg_upgrade() -> None:
op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_tags ON {schema}mental_models USING GIN(tags)")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Revert mental models v4 changes."""
schema = _get_schema_prefix()
@@ -132,11 +110,3 @@ def _pg_downgrade() -> None:
op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS mission")
# Note: Cannot restore deleted observations - they are lost on downgrade
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,7 +1,7 @@
"""Merge 3 migration heads and add unit_entities composite index
Revision ID: h3i4j5k6l7m8
Revises: a4b5c6d7e8f9, g2h3i4j5k6l7
Revises: a4b5c6d7e8f9, c2d3e4f5g6h7, g2h3i4j5k6l7
Create Date: 2026-04-07
Merges three unmerged migration heads into one, and adds a composite index
@@ -13,10 +13,8 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "h3i4j5k6l7m8"
down_revision: str | Sequence[str] | None = ("a4b5c6d7e8f9", "g2h3i4j5k6l7")
down_revision: str | Sequence[str] | None = ("a4b5c6d7e8f9", "c2d3e4f5g6h7", "g2h3i4j5k6l7")
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
@@ -27,7 +25,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
# Composite index enables index-only scans for entity_id -> unit_id lookups
op.execute(
@@ -37,16 +35,8 @@ def _pg_upgrade() -> None:
op.execute(f"DROP INDEX IF EXISTS {schema}idx_unit_entities_entity")
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}idx_unit_entities_entity_unit")
# Restore the single-column index
op.execute(f"CREATE INDEX IF NOT EXISTS idx_unit_entities_entity ON {schema}unit_entities (entity_id)")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -13,8 +13,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "i4d5e6f7g8h9"
down_revision: str | Sequence[str] | None = "h3c4d5e6f7g8"
@@ -28,7 +26,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Delete opinion memory_units."""
schema = _get_schema_prefix()
@@ -37,15 +35,7 @@ def _pg_upgrade() -> None:
op.execute(f"DELETE FROM {schema}memory_units WHERE fact_type = 'opinion'")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Cannot restore deleted opinions."""
# Note: Cannot restore deleted opinions - they are lost on downgrade
pass
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,49 +0,0 @@
"""Add 'cancelled' to async_operations status check constraint
Revision ID: i4j5k6l7m8n9
Revises: d5y6z7a8b9c0
Create Date: 2026-04-23
"""
from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "i4j5k6l7m8n9"
down_revision: str | Sequence[str] | None = "d5y6z7a8b9c0"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}async_operations DROP CONSTRAINT IF EXISTS async_operations_status_check")
op.execute(
f"ALTER TABLE {schema}async_operations ADD CONSTRAINT async_operations_status_check "
f"CHECK (status IN ('pending', 'processing', 'completed', 'failed', 'cancelled'))"
)
def _pg_downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}async_operations DROP CONSTRAINT IF EXISTS async_operations_status_check")
op.execute(
f"ALTER TABLE {schema}async_operations ADD CONSTRAINT async_operations_status_check "
f"CHECK (status IN ('pending', 'processing', 'completed', 'failed'))"
)
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -15,8 +15,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "j5e6f7g8h9i0"
down_revision: str | Sequence[str] | None = "i4d5e6f7g8h9"
@@ -30,7 +28,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Create mental_model_versions table and add version tracking."""
schema = _get_schema_prefix()
@@ -83,7 +81,7 @@ def _pg_upgrade() -> None:
""")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Remove mental_model_versions table and version column."""
schema = _get_schema_prefix()
@@ -95,11 +93,3 @@ def _pg_downgrade() -> None:
# Remove version column from mental_models
op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS version")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -12,8 +12,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "k6f7g8h9i0j1"
down_revision: str | Sequence[str] | None = "j5e6f7g8h9i0"
@@ -27,7 +25,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Add 'directive' to mental_models subtype constraint."""
schema = _get_schema_prefix()
@@ -42,7 +40,7 @@ def _pg_upgrade() -> None:
""")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Remove 'directive' from mental_models subtype constraint."""
schema = _get_schema_prefix()
@@ -58,11 +56,3 @@ def _pg_downgrade() -> None:
ADD CONSTRAINT ck_mental_models_subtype
CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
""")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,38 +0,0 @@
"""No-op: observation_sources table is Oracle-only
Originally created the observation_sources junction table for all backends,
but PG uses native array ops on the source_memory_ids column (faster at scale).
Oracle creates this table in the o1a2b3c4d5e6 baseline migration instead.
Kept as a no-op to preserve the Alembic revision chain.
Revision ID: k6l7m8n9o0p1
Revises: i4j5k6l7m8n9
Create Date: 2026-04-24
"""
from collections.abc import Sequence
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "k6l7m8n9o0p1"
down_revision: str | Sequence[str] | None = "i4j5k6l7m8n9"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _pg_upgrade() -> None:
# PG uses source_memory_ids[] array on memory_units — no junction table.
pass
def _pg_downgrade() -> None:
pass
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -17,8 +17,6 @@ import sqlalchemy as sa
from alembic import context, op
from sqlalchemy.dialects import postgresql
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "l7g8h9i0j1k2"
down_revision: str | Sequence[str] | None = "k6f7g8h9i0j1"
@@ -32,7 +30,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Add worker columns to async_operations."""
schema = _get_schema_prefix()
@@ -80,7 +78,7 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Remove worker columns from async_operations."""
schema = _get_schema_prefix()
@@ -109,11 +107,3 @@ def _pg_downgrade() -> None:
"worker_id",
schema=context.config.get_main_option("target_schema") or None,
)
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,31 +0,0 @@
"""Merge divergent heads from deferrable FK and cooccurrence backfill
Revision ID: m3rg3h3ad5f6
Revises: 9f8e7d6c5b4a, b5d4e3f2a1c9
Create Date: 2026-05-04
"""
from collections.abc import Sequence
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "m3rg3h3ad5f6"
down_revision: tuple[str, ...] = ("9f8e7d6c5b4a", "b5d4e3f2a1c9")
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _pg_upgrade() -> None:
pass
def _pg_downgrade() -> None:
pass
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -12,8 +12,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "m8h9i0j1k2l3"
down_revision: str | Sequence[str] | None = "l7g8h9i0j1k2"
@@ -27,7 +25,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Change mental_models.id from VARCHAR(64) to TEXT."""
schema = _get_schema_prefix()
@@ -35,17 +33,9 @@ def _pg_upgrade() -> None:
op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE TEXT")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Revert mental_models.id from TEXT to VARCHAR(64)."""
schema = _get_schema_prefix()
# Note: This may fail if any id values exceed 64 characters
op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE VARCHAR(64)")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -16,8 +16,6 @@ from collections.abc import Sequence
from alembic import context, op
from sqlalchemy import text
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "n9i0j1k2l3m4"
down_revision: str | Sequence[str] | None = "m8h9i0j1k2l3"
@@ -32,65 +30,53 @@ def _get_schema_prefix() -> str:
def _detect_vector_extension() -> str:
"""Detect or validate vector extension for this immutable migration revision."""
"""
Detect or validate vector extension: 'pgvector', 'vchord', or 'pgvectorscale'.
Respects HINDSIGHT_API_VECTOR_EXTENSION env var if set.
"""
conn = op.get_bind()
vector_extension = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower()
# Validate configured extension is installed
if vector_extension == "pgvectorscale":
# pgvectorscale/DiskANN requires pgvector
pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
if not pgvector_check:
raise RuntimeError(
"DiskANN requires pgvector. Install with: CREATE EXTENSION vector; then vectorscale or pg_diskann CASCADE;"
)
# Check for either vectorscale (open source) or pg_diskann (Azure)
vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar()
pg_diskann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_diskann'")).scalar()
if vectorscale_check:
return "pgvectorscale"
if pg_diskann_check:
elif pg_diskann_check:
return "pg_diskann"
raise RuntimeError(
"Configured vector extension 'pgvectorscale' not found. Install either:\n"
" - pgvectorscale: CREATE EXTENSION vectorscale CASCADE;\n"
" - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
)
if vector_extension == "vchord":
else:
raise RuntimeError(
"Configured vector extension 'pgvectorscale' not found. Install either:\n"
" - pgvectorscale: CREATE EXTENSION vectorscale CASCADE;\n"
" - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
)
elif vector_extension == "vchord":
vchord_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord'")).scalar()
if not vchord_check:
raise RuntimeError(
"Configured vector extension 'vchord' not found. Install it with: CREATE EXTENSION vchord CASCADE;"
)
return "vchord"
if vector_extension == "scann":
scann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'alloydb_scann'")).scalar()
if not scann_check:
raise RuntimeError(
"Configured vector extension 'scann' not found. Install it with: CREATE EXTENSION alloydb_scann CASCADE;"
)
return "scann"
if vector_extension == "pgvector":
elif vector_extension == "pgvector":
pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
if not pgvector_check:
raise RuntimeError(
"Configured vector extension 'pgvector' not found. Install it with: CREATE EXTENSION vector;"
)
return "pgvector"
raise ValueError(
"Invalid HINDSIGHT_API_VECTOR_EXTENSION: "
f"{vector_extension}. Must be 'pgvector', 'vchord', 'pgvectorscale', or 'scann'"
)
def _vector_index_using_clause(ext: str) -> str:
if ext == "pgvectorscale":
return "USING diskann (embedding vector_cosine_ops) WITH (num_neighbors = 50)"
if ext == "pg_diskann":
return "USING diskann (embedding vector_cosine_ops) WITH (max_neighbors = 50)"
if ext == "vchord":
return "USING vchordrq (embedding vector_l2_ops)"
if ext == "scann":
return "USING scann (embedding cosine) WITH (mode = 'AUTO')"
return "USING hnsw (embedding vector_cosine_ops)"
else:
raise ValueError(
f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {vector_extension}. Must be 'pgvector', 'vchord', or 'pgvectorscale'"
)
def _detect_text_search_extension() -> str:
@@ -133,7 +119,7 @@ def _detect_text_search_extension() -> str:
)
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Create learnings and pinned_reflections tables."""
schema = _get_schema_prefix()
@@ -170,12 +156,28 @@ def _pg_upgrade() -> None:
# Indexes for learnings
op.execute(f"CREATE INDEX idx_learnings_bank_id ON {schema}learnings(bank_id)")
# Create vector index based on detected extension. ScaNN is deferred because
# this table is empty during migration and AlloyDB rejects empty ScaNN builds.
if vector_ext != "scann":
# Create vector index based on detected extension
if vector_ext == "pgvectorscale":
op.execute(f"""
CREATE INDEX idx_learnings_embedding ON {schema}learnings
{_vector_index_using_clause(vector_ext)}
USING diskann (embedding vector_cosine_ops)
WITH (num_neighbors = 50)
""")
elif vector_ext == "pg_diskann":
op.execute(f"""
CREATE INDEX idx_learnings_embedding ON {schema}learnings
USING diskann (embedding vector_cosine_ops)
WITH (max_neighbors = 50)
""")
elif vector_ext == "vchord":
op.execute(f"""
CREATE INDEX idx_learnings_embedding ON {schema}learnings
USING vchordrq (embedding vector_l2_ops)
""")
else: # pgvector
op.execute(f"""
CREATE INDEX idx_learnings_embedding ON {schema}learnings
USING hnsw (embedding vector_cosine_ops)
""")
op.execute(f"CREATE INDEX idx_learnings_tags ON {schema}learnings USING GIN(tags)")
@@ -233,12 +235,28 @@ def _pg_upgrade() -> None:
# Indexes for pinned_reflections
op.execute(f"CREATE INDEX idx_pinned_reflections_bank_id ON {schema}pinned_reflections(bank_id)")
# Create vector index based on detected extension. ScaNN is deferred because
# this table is empty during migration and AlloyDB rejects empty ScaNN builds.
if vector_ext != "scann":
# Create vector index based on detected extension
if vector_ext == "pgvectorscale":
op.execute(f"""
CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
{_vector_index_using_clause(vector_ext)}
USING diskann (embedding vector_cosine_ops)
WITH (num_neighbors = 50)
""")
elif vector_ext == "pg_diskann":
op.execute(f"""
CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
USING diskann (embedding vector_cosine_ops)
WITH (max_neighbors = 50)
""")
elif vector_ext == "vchord":
op.execute(f"""
CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
USING vchordrq (embedding vector_l2_ops)
""")
else: # pgvector
op.execute(f"""
CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
USING hnsw (embedding vector_cosine_ops)
""")
op.execute(f"CREATE INDEX idx_pinned_reflections_tags ON {schema}pinned_reflections USING GIN(tags)")
@@ -286,7 +304,7 @@ def _pg_upgrade() -> None:
""")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Drop learnings and pinned_reflections tables."""
schema = _get_schema_prefix()
@@ -297,11 +315,3 @@ def _pg_downgrade() -> None:
# Remove columns from banks
op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS last_consolidated_at")
op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS mission_changed_at")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -16,8 +16,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "o0j1k2l3m4n5"
down_revision: str | Sequence[str] | None = "n9i0j1k2l3m4"
@@ -31,7 +29,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Migrate data and clean up old mental models."""
schema = _get_schema_prefix()
@@ -82,17 +80,15 @@ def _pg_upgrade() -> None:
# 4. Drop the mental_model_versions table (no longer used)
op.execute(f"DROP TABLE IF EXISTS {schema}mental_model_versions CASCADE")
# 5. Drop old constraints and add new one that allows current subtypes.
# 'pinned' is still used by the code for user-created mental models;
# 'directive' is used for system directives.
# 5. Drop old constraints and add new one that only allows 'directive'
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
op.execute(f"""
ALTER TABLE {schema}mental_models
ADD CONSTRAINT ck_mental_models_subtype CHECK (subtype IN ('directive', 'pinned'))
ADD CONSTRAINT ck_mental_models_subtype CHECK (subtype = 'directive')
""")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Reverse the migration (data migration is one-way, so this just removes constraints)."""
schema = _get_schema_prefix()
@@ -115,11 +111,3 @@ def _pg_downgrade() -> None:
)
# Note: Data migration cannot be reversed - pinned_reflections and learnings data remains
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -1,445 +0,0 @@
"""oracle_baseline
Brings a fresh Oracle 23ai database up to the current schema in a single step.
PostgreSQL is a no-op here — the prior 59 revisions already build the PG schema
incrementally; this revision just closes the loop so both dialects share a
single head from this point on.
After this migration ships, *every* new revision must fill both the ``_pg_*``
and ``_oracle_*`` slots (or explicitly leave one ``None``); a CI check enforces
that.
Tables mirror the PostgreSQL schema but use Oracle-native types:
- UUID -> RAW(16) DEFAULT SYS_GUID()
- TEXT / large VARCHAR -> CLOB
- JSONB -> CLOB with IS JSON CHECK
- BOOLEAN -> NUMBER(1)
- FLOAT -> BINARY_DOUBLE
- VARCHAR[] -> CLOB (JSON array stored as string)
- BYTEA -> BLOB
- vector(384) -> VECTOR(384, FLOAT32) (Oracle 23ai native)
Revision ID: o1a2b3c4d5e6
Revises: k6l7m8n9o0p1
Create Date: 2026-04-29
"""
from collections.abc import Sequence
from alembic import op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "o1a2b3c4d5e6"
down_revision: str | Sequence[str] | None = "k6l7m8n9o0p1"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# ---------------------------------------------------------------------------
# Tables — created in dependency order
# ---------------------------------------------------------------------------
_TABLES: tuple[str, ...] = (
"""
CREATE TABLE IF NOT EXISTS banks (
bank_id VARCHAR2(256) NOT NULL,
internal_id RAW(16) DEFAULT SYS_GUID() NOT NULL,
name VARCHAR2(512),
disposition CLOB DEFAULT '{"skepticism":3,"literalism":3,"empathy":3}' NOT NULL
CONSTRAINT banks_disposition_json CHECK (disposition IS JSON),
mission CLOB,
personality CLOB DEFAULT '{}' NOT NULL
CONSTRAINT banks_personality_json CHECK (personality IS JSON),
config CLOB DEFAULT '{}' NOT NULL
CONSTRAINT banks_config_json CHECK (config IS JSON),
created_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
CONSTRAINT pk_banks PRIMARY KEY (bank_id),
CONSTRAINT banks_internal_id_unique UNIQUE (internal_id)
)
""",
"""
CREATE TABLE IF NOT EXISTS documents (
id VARCHAR2(512) NOT NULL,
bank_id VARCHAR2(256) NOT NULL,
original_text CLOB,
content_hash VARCHAR2(128),
metadata CLOB DEFAULT '{}' NOT NULL
CONSTRAINT docs_metadata_json CHECK (metadata IS JSON),
retain_params CLOB CONSTRAINT docs_retain_params_json CHECK (retain_params IS JSON OR retain_params IS NULL),
file_storage_key VARCHAR2(512),
file_original_name VARCHAR2(512),
file_content_type VARCHAR2(256),
tags CLOB DEFAULT '[]' NOT NULL,
created_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
CONSTRAINT pk_documents PRIMARY KEY (id, bank_id),
CONSTRAINT fk_documents_bank FOREIGN KEY (bank_id) REFERENCES banks(bank_id) ON DELETE CASCADE
)
""",
"""
CREATE TABLE IF NOT EXISTS chunks (
chunk_id VARCHAR2(512) NOT NULL,
document_id VARCHAR2(512) NOT NULL,
bank_id VARCHAR2(256) NOT NULL,
chunk_index NUMBER(10) NOT NULL,
chunk_text CLOB NOT NULL,
content_hash VARCHAR2(128),
created_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
CONSTRAINT pk_chunks PRIMARY KEY (chunk_id),
CONSTRAINT fk_chunks_document FOREIGN KEY (document_id, bank_id)
REFERENCES documents(id, bank_id) ON DELETE CASCADE
)
""",
# memory_units uses automatic list partitioning on bank_id at create time —
# no post-create ALTER required (we used to do that for legacy installs).
"""
CREATE TABLE IF NOT EXISTS memory_units (
id RAW(16) DEFAULT SYS_GUID() NOT NULL,
bank_id VARCHAR2(256) NOT NULL,
document_id VARCHAR2(512),
chunk_id VARCHAR2(512),
text CLOB NOT NULL,
embedding VECTOR(384, FLOAT32),
context CLOB,
event_date TIMESTAMP WITH TIME ZONE NOT NULL,
occurred_start TIMESTAMP WITH TIME ZONE,
occurred_end TIMESTAMP WITH TIME ZONE,
mentioned_at TIMESTAMP WITH TIME ZONE,
fact_type VARCHAR2(64) DEFAULT 'world' NOT NULL,
confidence_score BINARY_DOUBLE,
access_count NUMBER(10) DEFAULT 0 NOT NULL,
consolidated_at TIMESTAMP WITH TIME ZONE,
observation_scopes CLOB CONSTRAINT mu_obs_scopes_json CHECK (observation_scopes IS JSON OR observation_scopes IS NULL),
tags CLOB DEFAULT '[]' NOT NULL,
metadata CLOB DEFAULT '{}' NOT NULL
CONSTRAINT mu_metadata_json CHECK (metadata IS JSON),
proof_count NUMBER(10) DEFAULT 1,
source_memory_ids CLOB,
history CLOB DEFAULT '[]'
CONSTRAINT mu_history_json CHECK (history IS JSON OR history IS NULL),
text_signals CLOB,
consolidation_failed_at TIMESTAMP WITH TIME ZONE,
search_vector CLOB,
created_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
CONSTRAINT pk_memory_units PRIMARY KEY (id),
CONSTRAINT fk_mu_document FOREIGN KEY (document_id, bank_id)
REFERENCES documents(id, bank_id) ON DELETE CASCADE,
CONSTRAINT fk_mu_chunk FOREIGN KEY (chunk_id)
REFERENCES chunks(chunk_id) ON DELETE SET NULL,
CONSTRAINT chk_mu_fact_type CHECK (fact_type IN ('world', 'experience', 'observation')),
CONSTRAINT chk_mu_confidence CHECK (
confidence_score IS NULL
OR (confidence_score >= 0.0 AND confidence_score <= 1.0)
)
)
PARTITION BY LIST (bank_id) AUTOMATIC
(PARTITION p_default VALUES ('__default__'))
""",
"""
CREATE TABLE IF NOT EXISTS entities (
id RAW(16) DEFAULT SYS_GUID() NOT NULL,
bank_id VARCHAR2(256) NOT NULL,
canonical_name VARCHAR2(512) NOT NULL,
metadata CLOB DEFAULT '{}' NOT NULL
CONSTRAINT ent_metadata_json CHECK (metadata IS JSON),
first_seen TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
last_seen TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
mention_count NUMBER(10) DEFAULT 1 NOT NULL,
CONSTRAINT pk_entities PRIMARY KEY (id)
)
""",
"""
CREATE TABLE IF NOT EXISTS unit_entities (
unit_id RAW(16) NOT NULL,
entity_id RAW(16) NOT NULL,
CONSTRAINT pk_unit_entities PRIMARY KEY (unit_id, entity_id),
CONSTRAINT fk_ue_unit FOREIGN KEY (unit_id) REFERENCES memory_units(id) ON DELETE CASCADE,
CONSTRAINT fk_ue_entity FOREIGN KEY (entity_id) REFERENCES entities(id) ON DELETE CASCADE
)
""",
"""
CREATE TABLE IF NOT EXISTS entity_cooccurrences (
entity_id_1 RAW(16) NOT NULL,
entity_id_2 RAW(16) NOT NULL,
cooccurrence_count NUMBER(10) DEFAULT 1 NOT NULL,
last_cooccurred TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
CONSTRAINT pk_entity_cooccurrences PRIMARY KEY (entity_id_1, entity_id_2),
CONSTRAINT fk_ec_entity1 FOREIGN KEY (entity_id_1) REFERENCES entities(id) ON DELETE CASCADE,
CONSTRAINT fk_ec_entity2 FOREIGN KEY (entity_id_2) REFERENCES entities(id) ON DELETE CASCADE
)
""",
"""
CREATE TABLE IF NOT EXISTS memory_links (
from_unit_id RAW(16) NOT NULL,
to_unit_id RAW(16) NOT NULL,
link_type VARCHAR2(64) NOT NULL,
entity_id RAW(16),
bank_id VARCHAR2(256),
weight BINARY_DOUBLE DEFAULT 1.0 NOT NULL,
source_memory_ids CLOB,
created_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
CONSTRAINT fk_ml_from FOREIGN KEY (from_unit_id) REFERENCES memory_units(id) ON DELETE CASCADE,
CONSTRAINT fk_ml_to FOREIGN KEY (to_unit_id) REFERENCES memory_units(id) ON DELETE CASCADE,
CONSTRAINT fk_ml_entity FOREIGN KEY (entity_id) REFERENCES entities(id) ON DELETE CASCADE,
CONSTRAINT chk_ml_link_type CHECK (
link_type IN ('temporal', 'semantic', 'entity', 'causes', 'caused_by', 'enables', 'prevents')
),
CONSTRAINT chk_ml_weight CHECK (weight >= 0.0 AND weight <= 1.0)
)
""",
"""
CREATE TABLE IF NOT EXISTS mental_models (
id VARCHAR2(256) NOT NULL,
bank_id VARCHAR2(256) NOT NULL,
subtype VARCHAR2(32) NOT NULL,
name VARCHAR2(256) NOT NULL,
description CLOB NOT NULL,
source_query CLOB,
content CLOB,
embedding VECTOR(384, FLOAT32),
entity_id RAW(16),
observations CLOB DEFAULT '{"observations":[]}' NOT NULL
CONSTRAINT mm_obs_json CHECK (observations IS JSON),
links CLOB,
tags CLOB DEFAULT '[]' NOT NULL,
max_tokens NUMBER(10) DEFAULT 2048 NOT NULL,
"trigger" CLOB DEFAULT '{"refresh_after_consolidation":false}' NOT NULL
CONSTRAINT mm_trigger_json CHECK ("trigger" IS JSON),
structured_content CLOB CONSTRAINT mm_sc_json CHECK (structured_content IS JSON OR structured_content IS NULL),
last_refreshed_source_query CLOB,
reflect_response CLOB CONSTRAINT mm_reflect_resp_json CHECK (reflect_response IS JSON OR reflect_response IS NULL),
history CLOB DEFAULT '[]' NOT NULL
CONSTRAINT mm_history_json CHECK (history IS JSON),
last_refreshed_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
last_updated TIMESTAMP WITH TIME ZONE,
created_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
CONSTRAINT pk_mental_models PRIMARY KEY (id, bank_id),
CONSTRAINT fk_mm_bank FOREIGN KEY (bank_id) REFERENCES banks(bank_id) ON DELETE CASCADE,
CONSTRAINT fk_mm_entity FOREIGN KEY (entity_id) REFERENCES entities(id) ON DELETE SET NULL,
CONSTRAINT chk_mm_subtype CHECK (subtype IN ('directive', 'pinned'))
)
""",
"""
CREATE TABLE IF NOT EXISTS directives (
id RAW(16) DEFAULT SYS_GUID() NOT NULL,
bank_id VARCHAR2(256) NOT NULL,
name VARCHAR2(256) NOT NULL,
content CLOB NOT NULL,
priority NUMBER(10) DEFAULT 0 NOT NULL,
is_active NUMBER(1) DEFAULT 1 NOT NULL,
tags CLOB DEFAULT '[]' NOT NULL,
created_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
CONSTRAINT pk_directives PRIMARY KEY (id),
CONSTRAINT fk_dir_bank FOREIGN KEY (bank_id) REFERENCES banks(bank_id) ON DELETE CASCADE
)
""",
"""
CREATE TABLE IF NOT EXISTS async_operations (
operation_id RAW(16) DEFAULT SYS_GUID() NOT NULL,
bank_id VARCHAR2(256) NOT NULL,
operation_type VARCHAR2(128) NOT NULL,
status VARCHAR2(32) DEFAULT 'pending' NOT NULL,
worker_id VARCHAR2(256),
claimed_at TIMESTAMP WITH TIME ZONE,
retry_count NUMBER(10) DEFAULT 0 NOT NULL,
next_retry_at TIMESTAMP WITH TIME ZONE,
task_payload CLOB CONSTRAINT ao_payload_json CHECK (task_payload IS JSON OR task_payload IS NULL),
result_metadata CLOB DEFAULT '{}' NOT NULL
CONSTRAINT ao_result_json CHECK (result_metadata IS JSON),
error_message CLOB,
created_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
completed_at TIMESTAMP WITH TIME ZONE,
CONSTRAINT pk_async_operations PRIMARY KEY (operation_id),
CONSTRAINT fk_ao_bank FOREIGN KEY (bank_id) REFERENCES banks(bank_id) ON DELETE CASCADE,
CONSTRAINT chk_ao_status CHECK (status IN ('pending', 'processing', 'completed', 'failed', 'cancelled'))
)
""",
"""
CREATE TABLE IF NOT EXISTS webhooks (
id RAW(16) DEFAULT SYS_GUID() NOT NULL,
bank_id VARCHAR2(256) NOT NULL,
url VARCHAR2(2048) NOT NULL,
secret VARCHAR2(512),
event_types CLOB DEFAULT '[]' NOT NULL,
http_config CLOB DEFAULT '{}' NOT NULL
CONSTRAINT wh_http_config_json CHECK (http_config IS JSON),
enabled NUMBER(1) DEFAULT 1 NOT NULL,
created_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
CONSTRAINT pk_webhooks PRIMARY KEY (id),
CONSTRAINT fk_wh_bank FOREIGN KEY (bank_id) REFERENCES banks(bank_id) ON DELETE CASCADE
)
""",
"""
CREATE TABLE IF NOT EXISTS file_storage (
storage_key VARCHAR2(512) NOT NULL,
data BLOB NOT NULL,
CONSTRAINT pk_file_storage PRIMARY KEY (storage_key)
)
""",
"""
CREATE TABLE IF NOT EXISTS audit_log (
id RAW(16) DEFAULT SYS_GUID() NOT NULL,
action VARCHAR2(128) NOT NULL,
transport VARCHAR2(64) NOT NULL,
bank_id VARCHAR2(256),
started_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
ended_at TIMESTAMP WITH TIME ZONE,
request CLOB CONSTRAINT al_request_json CHECK (request IS JSON OR request IS NULL),
response CLOB CONSTRAINT al_response_json CHECK (response IS JSON OR response IS NULL),
metadata CLOB DEFAULT '{}' NOT NULL
CONSTRAINT al_metadata_json CHECK (metadata IS JSON),
CONSTRAINT pk_audit_log PRIMARY KEY (id)
)
""",
"""
CREATE TABLE IF NOT EXISTS observation_sources (
observation_id RAW(16) NOT NULL,
source_id RAW(16) NOT NULL,
CONSTRAINT pk_observation_sources PRIMARY KEY (observation_id, source_id),
CONSTRAINT fk_obs_src_observation FOREIGN KEY (observation_id)
REFERENCES memory_units(id) ON DELETE CASCADE
)
""",
)
# ---------------------------------------------------------------------------
# B-tree indexes
# ---------------------------------------------------------------------------
_INDEXES: tuple[str, ...] = (
# documents
"CREATE INDEX idx_docs_bank_id ON documents(bank_id)",
"CREATE INDEX idx_docs_content_hash ON documents(content_hash)",
# chunks
"CREATE INDEX idx_chunks_document_id ON chunks(document_id)",
"CREATE INDEX idx_chunks_bank_id ON chunks(bank_id)",
# memory_units
"CREATE INDEX idx_mu_bank_id ON memory_units(bank_id)",
"CREATE INDEX idx_mu_document_id ON memory_units(document_id)",
"CREATE INDEX idx_mu_chunk_id ON memory_units(chunk_id)",
"CREATE INDEX idx_mu_event_date ON memory_units(event_date DESC)",
"CREATE INDEX idx_mu_bank_date ON memory_units(bank_id, event_date DESC)",
"CREATE INDEX idx_mu_access_count ON memory_units(access_count DESC)",
"CREATE INDEX idx_mu_fact_type ON memory_units(fact_type)",
"CREATE INDEX idx_mu_bank_fact_type ON memory_units(bank_id, fact_type)",
"CREATE INDEX idx_mu_bank_type_date ON memory_units(bank_id, fact_type, event_date DESC)",
# entities
"CREATE INDEX idx_ent_bank_id ON entities(bank_id)",
"CREATE INDEX idx_ent_canonical_name ON entities(canonical_name)",
"CREATE INDEX idx_ent_bank_name ON entities(bank_id, canonical_name)",
"CREATE UNIQUE INDEX idx_ent_bank_lower_name ON entities(bank_id, LOWER(canonical_name))",
# unit_entities
"CREATE INDEX idx_ue_unit ON unit_entities(unit_id)",
"CREATE INDEX idx_ue_entity ON unit_entities(entity_id)",
# entity_cooccurrences
"CREATE INDEX idx_ec_entity1 ON entity_cooccurrences(entity_id_1)",
"CREATE INDEX idx_ec_entity2 ON entity_cooccurrences(entity_id_2)",
"CREATE INDEX idx_ec_count ON entity_cooccurrences(cooccurrence_count DESC)",
# memory_links — function-based unique index uses NVL with the nil UUID raw
# to handle nullable entity_id (matches PG idx_memory_links_unique).
"CREATE UNIQUE INDEX idx_memory_links_unique ON memory_links("
"from_unit_id, to_unit_id, link_type, "
"NVL(entity_id, HEXTORAW('00000000000000000000000000000000')))",
"CREATE INDEX idx_ml_from_unit ON memory_links(from_unit_id)",
"CREATE INDEX idx_ml_to_unit ON memory_links(to_unit_id)",
"CREATE INDEX idx_ml_entity ON memory_links(entity_id)",
"CREATE INDEX idx_ml_link_type ON memory_links(link_type)",
"CREATE INDEX idx_ml_bank_id ON memory_links(bank_id)",
# directives
"CREATE INDEX idx_dir_bank_id ON directives(bank_id)",
"CREATE INDEX idx_dir_bank_active ON directives(bank_id, is_active)",
# mental_models
"CREATE INDEX idx_mm_bank_id ON mental_models(bank_id)",
"CREATE INDEX idx_mm_subtype ON mental_models(bank_id, subtype)",
"CREATE INDEX idx_mm_entity_id ON mental_models(entity_id)",
# async_operations
"CREATE INDEX idx_ao_bank_id ON async_operations(bank_id)",
"CREATE INDEX idx_ao_status ON async_operations(status)",
"CREATE INDEX idx_ao_bank_status ON async_operations(bank_id, status)",
"CREATE INDEX idx_ao_status_retry ON async_operations(status, next_retry_at)",
# webhooks
"CREATE INDEX idx_wh_bank_id ON webhooks(bank_id)",
# audit_log
"CREATE INDEX idx_al_action_started ON audit_log(action, started_at DESC)",
"CREATE INDEX idx_al_bank_started ON audit_log(bank_id, started_at DESC)",
"CREATE INDEX idx_al_started ON audit_log(started_at DESC)",
# observation_sources
"CREATE INDEX idx_obs_sources_source_id ON observation_sources(source_id, observation_id)",
)
_VECTOR_INDEX = (
"CREATE VECTOR INDEX idx_mu_embedding_hnsw ON memory_units(embedding) "
"ORGANIZATION NEIGHBOR PARTITIONS "
"DISTANCE COSINE "
"WITH TARGET ACCURACY 95"
)
# Oracle Text (CTXSYS.CONTEXT) — ``SYNC (ON COMMIT)`` makes it auto-update
# without a maintenance job. Doubled single quotes for the embedded literal.
_TEXT_INDEX = (
"BEGIN "
"EXECUTE IMMEDIATE '"
"CREATE INDEX idx_mu_content_text ON memory_units(text) "
"INDEXTYPE IS CTXSYS.CONTEXT "
"PARAMETERS (''SYNC (ON COMMIT)'')"
"'; "
"EXCEPTION WHEN OTHERS THEN "
"IF SQLCODE = -955 THEN NULL; ELSE RAISE; END IF; "
"END;"
)
def _execute_ignoring_955(sql: str) -> None:
"""Run a CREATE statement and swallow ORA-00955 (object already exists).
Wraps the statement in PL/SQL so the exception handler runs server-side —
no round-trip cost for the common case.
"""
block = (
"BEGIN "
"EXECUTE IMMEDIATE :stmt; "
"EXCEPTION WHEN OTHERS THEN "
"IF SQLCODE = -955 THEN NULL; ELSE RAISE; END IF; "
"END;"
)
op.get_bind().exec_driver_sql(block, {"stmt": sql.strip()})
def _oracle_upgrade() -> None:
bind = op.get_bind()
# Tolerate concurrent DDL instead of failing immediately (ORA-00054).
bind.exec_driver_sql("ALTER SESSION SET DDL_LOCK_TIMEOUT = 30")
for ddl in _TABLES:
_execute_ignoring_955(ddl)
for idx in _INDEXES:
_execute_ignoring_955(idx)
# Hindsight on Oracle requires 23ai with VECTOR support (ASSM tablespace)
# and the CTXSYS package for full-text. Both index creations must succeed
# — the migration fails hard if either feature is unavailable, by design.
# We only swallow ORA-00955 (object already exists) so reruns are safe.
_execute_ignoring_955(_VECTOR_INDEX)
bind.exec_driver_sql(_TEXT_INDEX)
def _oracle_downgrade() -> None:
# Baseline downgrades aren't supported — dropping every table here would
# destroy customer data. Use point-in-time recovery instead.
raise NotImplementedError("Cannot downgrade past the Oracle baseline.")
def upgrade() -> None:
run_for_dialect(oracle=_oracle_upgrade)
def downgrade() -> None:
run_for_dialect(oracle=_oracle_downgrade)
@@ -21,8 +21,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "p1k2l3m4n5o6"
down_revision: str | Sequence[str] | None = "o0j1k2l3m4n5"
@@ -36,7 +34,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Implement new knowledge architecture."""
schema = _get_schema_prefix()
@@ -128,7 +126,7 @@ def _pg_upgrade() -> None:
""")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Reverse the migration."""
schema = _get_schema_prefix()
@@ -194,11 +192,3 @@ def _pg_downgrade() -> None:
""")
# Note: mental_models table recreation is complex and would need separate handling
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -12,8 +12,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "q2l3m4n5o6p7"
down_revision: str | Sequence[str] | None = "p1k2l3m4n5o6"
@@ -27,7 +25,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Add 'mental_model' to the fact_type check constraint."""
schema = _get_schema_prefix()
@@ -40,7 +38,7 @@ def _pg_upgrade() -> None:
""")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Remove 'mental_model' from the fact_type check constraint."""
schema = _get_schema_prefix()
@@ -50,11 +48,3 @@ def _pg_downgrade() -> None:
ADD CONSTRAINT memory_units_fact_type_check
CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation'))
""")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -14,8 +14,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "r3m4n5o6p7q8"
down_revision: str | Sequence[str] | None = "q2l3m4n5o6p7"
branch_labels: str | Sequence[str] | None = None
@@ -28,7 +26,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Add reflect_response JSONB column to reflections."""
schema = _get_schema_prefix()
@@ -39,7 +37,7 @@ def _pg_upgrade() -> None:
""")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Remove reflect_response column from reflections."""
schema = _get_schema_prefix()
@@ -47,11 +45,3 @@ def _pg_downgrade() -> None:
ALTER TABLE {schema}reflections
DROP COLUMN IF EXISTS reflect_response
""")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -12,8 +12,6 @@ import sqlalchemy as sa
from alembic import context, op
from sqlalchemy.dialects import postgresql
from hindsight_api.alembic._dialect import run_for_dialect
# revision identifiers, used by Alembic.
revision: str = "rename_personality"
down_revision: str | Sequence[str] | None = "d9f6a3b4c5e2"
@@ -27,7 +25,7 @@ def _get_target_schema() -> str:
return schema if schema else "public"
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Rename personality column to disposition in banks table (if it exists)."""
conn = op.get_bind()
target_schema = _get_target_schema()
@@ -71,7 +69,7 @@ def _pg_upgrade() -> None:
# else: disposition already exists, nothing to do
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Revert disposition column back to personality."""
conn = op.get_bind()
target_schema = _get_target_schema()
@@ -85,11 +83,3 @@ def _pg_downgrade() -> None:
)
if result.fetchone():
op.alter_column("banks", "disposition", new_column_name="personality")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -13,8 +13,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "s4n5o6p7q8r9"
down_revision: str | Sequence[str] | None = "r3m4n5o6p7q8"
branch_labels: str | Sequence[str] | None = None
@@ -27,7 +25,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
schema = _get_schema_prefix()
# Add consolidated_at column to memory_units
@@ -48,16 +46,8 @@ def _pg_upgrade() -> None:
)
def _pg_downgrade() -> None:
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_unconsolidated")
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS consolidated_at")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -17,8 +17,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "t5o6p7q8r9s0"
down_revision: str | Sequence[str] | None = "s4n5o6p7q8r9"
branch_labels: str | Sequence[str] | None = None
@@ -31,7 +29,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Rename mental_model -> observation and reflections -> mental_models."""
schema = _get_schema_prefix()
@@ -88,7 +86,7 @@ def _pg_upgrade() -> None:
""")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Reverse: observation -> mental_model and mental_models -> reflections."""
schema = _get_schema_prefix()
@@ -134,11 +132,3 @@ def _pg_downgrade() -> None:
ON {schema}memory_units(bank_id, fact_type)
WHERE fact_type = 'mental_model'
""")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -12,8 +12,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "u6p7q8r9s0t1"
down_revision: str | Sequence[str] | None = "t5o6p7q8r9s0"
branch_labels: str | Sequence[str] | None = None
@@ -26,7 +24,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Change mental_models.id from UUID to TEXT."""
schema = _get_schema_prefix()
@@ -35,17 +33,9 @@ def _pg_upgrade() -> None:
op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE TEXT USING id::TEXT")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Revert mental_models.id from TEXT to UUID."""
schema = _get_schema_prefix()
# Note: This will fail if any id values are not valid UUIDs
op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE UUID USING id::UUID")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)
@@ -13,8 +13,6 @@ from collections.abc import Sequence
from alembic import context, op
from hindsight_api.alembic._dialect import run_for_dialect
revision: str = "v7q8r9s0t1u2"
down_revision: str | Sequence[str] | None = "u6p7q8r9s0t1"
branch_labels: str | Sequence[str] | None = None
@@ -27,7 +25,7 @@ def _get_schema_prefix() -> str:
return f'"{schema}".' if schema else ""
def _pg_upgrade() -> None:
def upgrade() -> None:
"""Add max_tokens and trigger columns to mental_models."""
schema = _get_schema_prefix()
@@ -44,17 +42,9 @@ def _pg_upgrade() -> None:
""")
def _pg_downgrade() -> None:
def downgrade() -> None:
"""Remove max_tokens and trigger columns from mental_models."""
schema = _get_schema_prefix()
op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS max_tokens")
op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS trigger")
def upgrade() -> None:
run_for_dialect(pg=_pg_upgrade)
def downgrade() -> None:
run_for_dialect(pg=_pg_downgrade)

Some files were not shown because too many files have changed in this diff Show More