Compare commits
38
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
39df7eab99 | ||
|
|
68957c428f | ||
|
|
034d604e72 | ||
|
|
308e79551a | ||
|
|
92fde4abf2 | ||
|
|
32bd8796e9 | ||
|
|
948d8291a0 | ||
|
|
de943e88f2 | ||
|
|
64baad5e6c | ||
|
|
05562d3472 | ||
|
|
ef483e39a2 | ||
|
|
48483221ee | ||
|
|
8d8a2453c8 | ||
|
|
a7aae18721 | ||
|
|
06f71f869d | ||
|
|
16e5bcfbea | ||
|
|
1ebb182fa0 | ||
|
|
70df6d313c | ||
|
|
f413175799 | ||
|
|
4ce7af0cd4 | ||
|
|
d13bb728f8 | ||
|
|
bca8dd7c94 | ||
|
|
69b1af26ac | ||
|
|
e7ccf0b70c | ||
|
|
0b044845f2 | ||
|
|
787449620b | ||
|
|
a32949a342 | ||
|
|
1cef364719 | ||
|
|
dff293ca8c | ||
|
|
f4bc8443b3 | ||
|
|
ae26a8603b | ||
|
|
183b9dacb4 | ||
|
|
8a7c6e4e91 | ||
|
|
dfccbf29f1 | ||
|
|
dfea4dbe15 | ||
|
|
fcea8afa6c | ||
|
|
94c2b85c81 | ||
|
|
160c5581ec |
@@ -20,18 +20,15 @@ concurrency:
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: hindsight-docs
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: npm
|
||||
cache-dependency-path: hindsight-docs/package-lock.json
|
||||
- run: npm ci
|
||||
- run: npm run build
|
||||
cache-dependency-path: package-lock.json
|
||||
- run: npm ci --workspace=hindsight-docs
|
||||
- run: npm run build --workspace=hindsight-docs
|
||||
- uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: hindsight-docs/build
|
||||
|
||||
@@ -38,6 +38,10 @@ jobs:
|
||||
working-directory: ./hindsight
|
||||
run: uv build --out-dir dist
|
||||
|
||||
- name: Build hindsight-litellm
|
||||
working-directory: ./hindsight-integrations/litellm
|
||||
run: uv build --out-dir dist
|
||||
|
||||
# Publish in order (client and api first, then hindsight-all which depends on them)
|
||||
- name: Publish hindsight-client to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
@@ -57,6 +61,12 @@ jobs:
|
||||
packages-dir: ./hindsight/dist
|
||||
skip-existing: true
|
||||
|
||||
- name: Publish hindsight-litellm to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
packages-dir: ./hindsight-integrations/litellm/dist
|
||||
skip-existing: true
|
||||
|
||||
# Upload artifacts for GitHub release
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
@@ -66,6 +76,7 @@ jobs:
|
||||
hindsight-clients/python/dist/*
|
||||
hindsight-api/dist/*
|
||||
hindsight/dist/*
|
||||
hindsight-integrations/litellm/dist/*
|
||||
retention-days: 1
|
||||
|
||||
release-typescript-client:
|
||||
@@ -80,14 +91,14 @@ jobs:
|
||||
with:
|
||||
node-version: '20'
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: ./hindsight-clients/typescript
|
||||
run: npm ci
|
||||
run: npm ci --workspace=hindsight-clients/typescript
|
||||
|
||||
- name: Build
|
||||
working-directory: ./hindsight-clients/typescript
|
||||
run: npm run build
|
||||
run: npm run build --workspace=hindsight-clients/typescript
|
||||
|
||||
- name: Publish to npm
|
||||
working-directory: ./hindsight-clients/typescript
|
||||
@@ -306,6 +317,7 @@ jobs:
|
||||
cp artifacts/python-packages/hindsight-clients/python/dist/* release-assets/ || true
|
||||
cp artifacts/python-packages/hindsight-api/dist/* release-assets/ || true
|
||||
cp artifacts/python-packages/hindsight/dist/* release-assets/ || true
|
||||
cp artifacts/python-packages/hindsight-integrations/litellm/dist/* release-assets/ || true
|
||||
# TypeScript client
|
||||
cp artifacts/typescript-client/*.tgz release-assets/ || true
|
||||
# Rust CLI binaries
|
||||
@@ -316,54 +328,11 @@ jobs:
|
||||
cp artifacts/helm-chart/*.tgz release-assets/ || true
|
||||
ls -la release-assets/
|
||||
|
||||
- name: Generate release notes
|
||||
run: |
|
||||
cat << 'EOF' > release-notes.md
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Install the CLI
|
||||
curl -fsSL https://raw.githubusercontent.com/vectorize-io/hindsight/refs/heads/main/hindsight-cli/install.sh | bash
|
||||
|
||||
# Start the server
|
||||
docker run -p 8888:8888 -p 9999:9999 \
|
||||
-e HINDSIGHT_API_LLM_PROVIDER=openai \
|
||||
-e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \
|
||||
-e HINDSIGHT_API_LLM_MODEL=gpt-4o-mini \
|
||||
ghcr.io/${{ github.repository_owner }}/hindsight:${{ steps.get_version.outputs.VERSION }}
|
||||
```
|
||||
|
||||
## Docker Images
|
||||
- `ghcr.io/${{ github.repository_owner }}/hindsight:${{ steps.get_version.outputs.VERSION }}` - Standalone (recommended)
|
||||
- `ghcr.io/${{ github.repository_owner }}/hindsight-api:${{ steps.get_version.outputs.VERSION }}` - API only
|
||||
- `ghcr.io/${{ github.repository_owner }}/hindsight-control-plane:${{ steps.get_version.outputs.VERSION }}` - Web UI only
|
||||
|
||||
## CLI
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/vectorize-io/hindsight/refs/heads/main/hindsight-cli/install.sh | bash
|
||||
```
|
||||
|
||||
## Python
|
||||
```bash
|
||||
pip install hindsight-all # or hindsight-api, hindsight-client
|
||||
```
|
||||
|
||||
## TypeScript/JavaScript
|
||||
```bash
|
||||
npm install @vectorize-io/hindsight-client
|
||||
```
|
||||
|
||||
## Helm
|
||||
```bash
|
||||
helm install hindsight oci://ghcr.io/${{ github.repository_owner }}/charts/hindsight --version ${{ steps.get_version.outputs.VERSION }}
|
||||
```
|
||||
EOF
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: release-assets/*
|
||||
body_path: release-notes.md
|
||||
generate_release_notes: true
|
||||
draft: false
|
||||
prerelease: false
|
||||
env:
|
||||
|
||||
+245
-4
@@ -9,6 +9,54 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-python-packages:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- name: hindsight-all
|
||||
path: hindsight
|
||||
- name: hindsight-api
|
||||
path: hindsight-api
|
||||
- name: hindsight-client
|
||||
path: hindsight-clients/python
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version-file: ".python-version"
|
||||
|
||||
- name: Build ${{ matrix.name }}
|
||||
working-directory: ./${{ matrix.path }}
|
||||
run: uv build
|
||||
|
||||
build-typescript-client:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci --workspace=hindsight-clients/typescript
|
||||
|
||||
- name: Build TypeScript client
|
||||
run: npm run build --workspace=hindsight-clients/typescript
|
||||
|
||||
build-docs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -19,14 +67,14 @@ jobs:
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: ./hindsight-docs
|
||||
run: npm ci
|
||||
run: npm ci --workspace=hindsight-docs
|
||||
|
||||
- name: Build docs
|
||||
working-directory: ./hindsight-docs
|
||||
run: npm run build
|
||||
run: npm run build --workspace=hindsight-docs
|
||||
|
||||
build-rust-cli:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -106,6 +154,9 @@ jobs:
|
||||
env:
|
||||
HINDSIGHT_API_LLM_PROVIDER: groq
|
||||
HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# Prefer CPU-only PyTorch in CI (but keep PyPI for everything else)
|
||||
@@ -393,3 +444,193 @@ jobs:
|
||||
run: |
|
||||
echo "=== API Server Logs ==="
|
||||
cat /tmp/api-server.log || echo "No API server log found"
|
||||
|
||||
test-litellm-integration:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
prune-cache: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version-file: ".python-version"
|
||||
|
||||
- name: Build litellm integration
|
||||
working-directory: ./hindsight-integrations/litellm
|
||||
run: uv build
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: ./hindsight-integrations/litellm
|
||||
run: uv sync --extra dev
|
||||
|
||||
- name: Run tests
|
||||
working-directory: ./hindsight-integrations/litellm
|
||||
run: uv run pytest tests -v
|
||||
|
||||
test-doc-examples:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
HINDSIGHT_API_LLM_PROVIDER: groq
|
||||
HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
|
||||
HINDSIGHT_API_URL: http://localhost:8888
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# Model for test generation and analysis (options: gpt-4o, o3-mini, o1, etc.)
|
||||
DOC_TEST_MODEL: o3-mini
|
||||
# Prefer CPU-only PyTorch in CI (but keep PyPI for everything else)
|
||||
UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
prune-cache: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version-file: ".python-version"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: package-lock.json
|
||||
|
||||
- name: Build API
|
||||
working-directory: ./hindsight-api
|
||||
run: uv build
|
||||
|
||||
- name: Build Python client
|
||||
working-directory: ./hindsight-clients/python
|
||||
run: uv build
|
||||
|
||||
- name: Install Python client
|
||||
working-directory: ./hindsight-clients/python
|
||||
run: uv sync --index-strategy unsafe-best-match
|
||||
|
||||
- name: Install API dependencies
|
||||
working-directory: ./hindsight-api
|
||||
run: uv sync --no-install-project --index-strategy unsafe-best-match
|
||||
|
||||
- name: Install test dependencies in API venv
|
||||
working-directory: ./hindsight-api
|
||||
run: |
|
||||
uv pip install ../hindsight-clients/python requests anthropic
|
||||
uv pip install ../hindsight-integrations/litellm
|
||||
uv pip install ../hindsight-integrations/openai
|
||||
|
||||
- name: Verify Python dependencies
|
||||
working-directory: ./hindsight-api
|
||||
run: |
|
||||
echo "=== Verifying Python dependencies ==="
|
||||
uv run python -c "
|
||||
import sys
|
||||
print(f'Python: {sys.executable}')
|
||||
print(f'Prefix: {sys.prefix}')
|
||||
|
||||
# Check required packages
|
||||
packages = [
|
||||
'hindsight_client',
|
||||
'hindsight_litellm',
|
||||
'hindsight_openai',
|
||||
'anthropic',
|
||||
'openai',
|
||||
]
|
||||
|
||||
missing = []
|
||||
for pkg in packages:
|
||||
try:
|
||||
__import__(pkg)
|
||||
print(f' ✓ {pkg}')
|
||||
except ImportError as e:
|
||||
print(f' ✗ {pkg}: {e}')
|
||||
missing.append(pkg)
|
||||
|
||||
if missing:
|
||||
print(f'\nERROR: Missing packages: {missing}')
|
||||
sys.exit(1)
|
||||
print('\nAll Python dependencies verified!')
|
||||
"
|
||||
|
||||
- name: Install TypeScript client dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build TypeScript client
|
||||
run: npm run build --workspace=hindsight-clients/typescript
|
||||
|
||||
- name: Install TypeScript client globally
|
||||
working-directory: ./hindsight-clients/typescript
|
||||
run: npm install -g .
|
||||
|
||||
- name: Make TypeScript client available for temp files
|
||||
run: |
|
||||
# ESM modules don't use NODE_PATH, so create node_modules in /tmp
|
||||
# where test scripts are written
|
||||
mkdir -p /tmp/node_modules/@vectorize-io
|
||||
ln -s ${{ github.workspace }}/hindsight-clients/typescript /tmp/node_modules/@vectorize-io/hindsight-client
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Build and install hindsight CLI
|
||||
working-directory: ./hindsight-cli
|
||||
run: |
|
||||
cargo build --release
|
||||
sudo cp target/release/hindsight /usr/local/bin/
|
||||
|
||||
- name: Create .env file
|
||||
run: |
|
||||
cat > .env << EOF
|
||||
HINDSIGHT_API_LLM_PROVIDER=${{ env.HINDSIGHT_API_LLM_PROVIDER }}
|
||||
HINDSIGHT_API_LLM_API_KEY=${{ env.HINDSIGHT_API_LLM_API_KEY }}
|
||||
HINDSIGHT_API_LLM_MODEL=${{ env.HINDSIGHT_API_LLM_MODEL }}
|
||||
EOF
|
||||
|
||||
- name: Start API server
|
||||
run: |
|
||||
./scripts/dev/start-api.sh > /tmp/api-server.log 2>&1 &
|
||||
echo "Waiting for API server to be ready..."
|
||||
for i in {1..60}; do
|
||||
if curl -sf http://localhost:8888/health > /dev/null 2>&1; then
|
||||
echo "API server is ready after ${i}s"
|
||||
break
|
||||
fi
|
||||
if [ $i -eq 60 ]; then
|
||||
echo "API server failed to start after 60s"
|
||||
cat /tmp/api-server.log
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
- name: Test documentation examples
|
||||
working-directory: ./hindsight-api
|
||||
env:
|
||||
REPO_ROOT: ${{ github.workspace }}
|
||||
run: uv run python ../scripts/test-doc-examples.py
|
||||
|
||||
- name: Write test summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "=== Documentation Test Summary ==="
|
||||
cat /tmp/doc-test-summary.md
|
||||
cat /tmp/doc-test-summary.md >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
- name: Show API server logs
|
||||
if: always()
|
||||
run: |
|
||||
echo "=== API Server Logs ==="
|
||||
cat /tmp/api-server.log || echo "No API server log found"
|
||||
@@ -9,6 +9,9 @@ wheels/
|
||||
# Virtual environments
|
||||
.venv
|
||||
|
||||
# Node
|
||||
node_modules/
|
||||
|
||||
# Environment variables
|
||||
.env
|
||||
|
||||
|
||||
+14
-4
@@ -5,13 +5,23 @@ Thanks for your interest in contributing to Hindsight!
|
||||
## Getting Started
|
||||
|
||||
1. Fork and clone the repository
|
||||
2. Install dependencies:
|
||||
```bash
|
||||
cd hindsight-api && uv sync
|
||||
git clone [email protected]:vectorize-io/hindsight.git
|
||||
cd hindsight
|
||||
```
|
||||
3. Set up your environment:
|
||||
2. Set up your environment:
|
||||
```bash
|
||||
export OPENAI_API_KEY=your-key
|
||||
cp .env.example .env
|
||||
```
|
||||
Edit the .env to add LLM API key and config as required
|
||||
|
||||
3. Install dependencies:
|
||||
```bash
|
||||
# Python dependencies
|
||||
uv sync --directory hindsight-api/
|
||||
|
||||
# Node dependencies (uses npm workspaces)
|
||||
npm install
|
||||
```
|
||||
|
||||
## Development
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
## What is Hindsight?
|
||||
|
||||
Hindsight is an agent memory system built to create smarter agents that learn over time. It eliminates the shortcomings of alternative techniques such as RAG and knowledge graph.
|
||||
Hindsight™ is an agent memory system built to create smarter agents that learn over time. It eliminates the shortcomings of alternative techniques such as RAG and knowledge graph.
|
||||
|
||||
Hindsight addresses common challenges that have frustrated AI engineers building agents to automate tasks and assist users with conversational interfaces. Many of these challenges stem directly from a lack of memory.
|
||||
|
||||
|
||||
@@ -54,13 +54,15 @@ FROM node:20-slim AS sdk-builder
|
||||
ARG INCLUDE_CP
|
||||
RUN if [ "$INCLUDE_CP" != "true" ]; then echo "Skipping SDK build" && exit 0; fi
|
||||
|
||||
WORKDIR /app/sdk
|
||||
WORKDIR /app
|
||||
|
||||
COPY hindsight-clients/typescript/package*.json ./
|
||||
RUN npm ci
|
||||
# Copy root package files for npm workspaces
|
||||
COPY package.json package-lock.json ./
|
||||
COPY hindsight-clients/typescript/ ./hindsight-clients/typescript/
|
||||
|
||||
COPY hindsight-clients/typescript/ ./
|
||||
RUN npm run build
|
||||
# Install and build SDK using workspace
|
||||
RUN npm ci -w @vectorize-io/hindsight-client
|
||||
RUN npm run build -w @vectorize-io/hindsight-client
|
||||
|
||||
# =============================================================================
|
||||
# Stage: Control Plane Builder
|
||||
@@ -73,7 +75,7 @@ RUN if [ "$INCLUDE_CP" != "true" ]; then echo "Skipping CP build" && exit 0; fi
|
||||
WORKDIR /app
|
||||
|
||||
# Copy built SDK
|
||||
COPY --from=sdk-builder /app/sdk /app/sdk
|
||||
COPY --from=sdk-builder /app/hindsight-clients/typescript /app/sdk
|
||||
|
||||
# Install Control Plane dependencies
|
||||
# Only copy package.json (not package-lock.json) to ensure npm installs
|
||||
@@ -165,7 +167,7 @@ FROM node:20-alpine AS cp-only
|
||||
WORKDIR /app
|
||||
|
||||
# Copy built SDK
|
||||
COPY --from=sdk-builder /app/sdk /app/sdk
|
||||
COPY --from=sdk-builder /app/hindsight-clients/typescript /app/sdk
|
||||
|
||||
# Copy Control Plane standalone build
|
||||
WORKDIR /app/control-plane
|
||||
@@ -218,7 +220,7 @@ RUN useradd -m -s /bin/bash hindsight
|
||||
COPY --from=api-builder /app/api /app/api
|
||||
|
||||
# Copy built SDK
|
||||
COPY --from=sdk-builder /app/sdk /app/sdk
|
||||
COPY --from=sdk-builder /app/hindsight-clients/typescript /app/sdk
|
||||
|
||||
# Copy Control Plane standalone build
|
||||
WORKDIR /app/control-plane
|
||||
|
||||
@@ -2,8 +2,8 @@ apiVersion: v2
|
||||
name: hindsight
|
||||
description: Hindsight helm chart
|
||||
type: application
|
||||
version: 0.1.4
|
||||
appVersion: "0.1.4"
|
||||
version: 0.1.5
|
||||
appVersion: "0.1.5"
|
||||
keywords:
|
||||
- ai
|
||||
- memory
|
||||
|
||||
@@ -175,9 +175,13 @@ class LLMProvider:
|
||||
is_reasoning_model = any(x in model_lower for x in ["gpt-5", "o1", "o3"])
|
||||
|
||||
# For GPT-4 and GPT-4.1 models, cap max_completion_tokens to 32000
|
||||
# For GPT-4o models, cap to 16384
|
||||
is_gpt4_model = any(x in model_lower for x in ["gpt-4.1", "gpt-4-"])
|
||||
is_gpt4o_model = "gpt-4o" in model_lower
|
||||
if max_completion_tokens is not None:
|
||||
if is_gpt4_model and max_completion_tokens > 32000:
|
||||
if is_gpt4o_model and max_completion_tokens > 16384:
|
||||
max_completion_tokens = 16384
|
||||
elif is_gpt4_model and max_completion_tokens > 32000:
|
||||
max_completion_tokens = 32000
|
||||
# For reasoning models, max_completion_tokens includes reasoning + output tokens
|
||||
# Enforce minimum of 16000 to ensure enough space for both
|
||||
|
||||
@@ -54,7 +54,9 @@ class EmbeddedPostgres:
|
||||
loop = asyncio.get_event_loop()
|
||||
info = await loop.run_in_executor(None, pg0.start)
|
||||
logger.info(f"PostgreSQL started on port {self.port}")
|
||||
return info.uri
|
||||
# Construct URI manually since pg0-embedded may return None
|
||||
uri = info.uri if info and info.uri else f"postgresql://{self.username}:{self.password}@localhost:{self.port}/{self.database}"
|
||||
return uri
|
||||
except Exception as e:
|
||||
last_error = str(e)
|
||||
if attempt < max_retries:
|
||||
@@ -89,9 +91,9 @@ class EmbeddedPostgres:
|
||||
pg0 = self._get_pg0()
|
||||
loop = asyncio.get_event_loop()
|
||||
info = await loop.run_in_executor(None, pg0.info)
|
||||
if info is None or not info.running:
|
||||
raise RuntimeError("PostgreSQL server is not running or URI not available")
|
||||
return info.uri
|
||||
# Construct URI manually since pg0-embedded may return None
|
||||
uri = info.uri if info and info.uri else f"postgresql://{self.username}:{self.password}@localhost:{self.port}/{self.database}"
|
||||
return uri
|
||||
|
||||
async def is_running(self) -> bool:
|
||||
"""Check if the PostgreSQL server is currently running."""
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "hindsight-api"
|
||||
version = "0.1.4"
|
||||
version = "0.1.5"
|
||||
description = "Temporal + Semantic + Entity Memory System for AI agents using PostgreSQL"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "hindsight-cli"
|
||||
version = "0.1.4"
|
||||
version = "0.1.5"
|
||||
edition = "2021"
|
||||
authors = ["Hindsight Team"]
|
||||
description = "A beautiful CLI for Hindsight - semantic memory system"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "hindsight-client"
|
||||
version = "0.1.4"
|
||||
version = "0.1.5"
|
||||
description = "Python client for Hindsight - Semantic memory system with personality-driven thinking"
|
||||
authors = [
|
||||
{name = "Hindsight Team"}
|
||||
|
||||
-4409
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@vectorize-io/hindsight-client",
|
||||
"version": "0.1.4",
|
||||
"version": "0.1.5",
|
||||
"description": "TypeScript client for Hindsight - Semantic memory system with personality-driven thinking",
|
||||
"main": "./dist/src/index.js",
|
||||
"types": "./dist/src/index.d.ts",
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"module": "commonjs",
|
||||
"lib": ["ES2020", "DOM"],
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable"],
|
||||
"declaration": true,
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./",
|
||||
|
||||
-9845
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "hindsight-control-plane",
|
||||
"version": "0.1.4",
|
||||
"version": "0.1.5",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
|
||||
@@ -180,4 +180,13 @@ code, pre {
|
||||
-webkit-background-clip: text;
|
||||
-webkit-text-fill-color: transparent;
|
||||
background-clip: text;
|
||||
}
|
||||
|
||||
/* Fix datetime-local calendar icon visibility in both light and dark modes */
|
||||
input[type="datetime-local"]::-webkit-calendar-picker-indicator {
|
||||
filter: invert(0.5);
|
||||
}
|
||||
|
||||
.dark input[type="datetime-local"]::-webkit-calendar-picker-indicator {
|
||||
filter: invert(1);
|
||||
}
|
||||
@@ -239,7 +239,7 @@ export function BankProfileView() {
|
||||
<div className="flex gap-2">
|
||||
{editMode ? (
|
||||
<>
|
||||
<Button onClick={handleCancel} variant="outline" disabled={saving}>
|
||||
<Button onClick={handleCancel} variant="secondary" disabled={saving}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button onClick={handleSave} disabled={saving}>
|
||||
@@ -258,7 +258,7 @@ export function BankProfileView() {
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Button onClick={loadData} variant="outline" size="sm">
|
||||
<Button onClick={loadData} variant="secondary" size="sm">
|
||||
<RefreshCw className="w-4 h-4 mr-2" />
|
||||
Refresh
|
||||
</Button>
|
||||
|
||||
@@ -266,7 +266,7 @@ function BankSelectorInner() {
|
||||
</div>
|
||||
<DialogFooter>
|
||||
<Button
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
onClick={() => {
|
||||
setCreateDialogOpen(false);
|
||||
setNewBankId('');
|
||||
@@ -295,7 +295,7 @@ function BankSelectorInner() {
|
||||
</DialogHeader>
|
||||
<div className="py-4 space-y-4">
|
||||
<div>
|
||||
<label className="font-bold block mb-1 text-sm">Content *</label>
|
||||
<label className="font-bold block mb-1 text-sm text-foreground">Content *</label>
|
||||
<Textarea
|
||||
value={docContent}
|
||||
onChange={(e) => setDocContent(e.target.value)}
|
||||
@@ -306,7 +306,7 @@ function BankSelectorInner() {
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="font-bold block mb-1 text-sm">Context</label>
|
||||
<label className="font-bold block mb-1 text-sm text-foreground">Context</label>
|
||||
<Input
|
||||
type="text"
|
||||
value={docContext}
|
||||
@@ -317,16 +317,17 @@ function BankSelectorInner() {
|
||||
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label className="font-bold block mb-1 text-sm">Event Date</label>
|
||||
<label className="font-bold block mb-1 text-sm text-foreground">Event Date</label>
|
||||
<Input
|
||||
type="datetime-local"
|
||||
value={docEventDate}
|
||||
onChange={(e) => setDocEventDate(e.target.value)}
|
||||
className="text-foreground"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="font-bold block mb-1 text-sm">Document ID</label>
|
||||
<label className="font-bold block mb-1 text-sm text-foreground">Document ID</label>
|
||||
<Input
|
||||
type="text"
|
||||
value={docDocumentId}
|
||||
@@ -342,7 +343,7 @@ function BankSelectorInner() {
|
||||
checked={docAsync}
|
||||
onCheckedChange={(checked) => setDocAsync(checked as boolean)}
|
||||
/>
|
||||
<label htmlFor="async-doc" className="text-sm cursor-pointer">
|
||||
<label htmlFor="async-doc" className="text-sm cursor-pointer text-foreground">
|
||||
Process in background (async)
|
||||
</label>
|
||||
</div>
|
||||
@@ -353,7 +354,7 @@ function BankSelectorInner() {
|
||||
</div>
|
||||
<DialogFooter>
|
||||
<Button
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
onClick={() => {
|
||||
setDocDialogOpen(false);
|
||||
setDocContent('');
|
||||
|
||||
@@ -480,7 +480,7 @@ export function DataView({ factType }: DataViewProps) {
|
||||
}`}
|
||||
>
|
||||
<TableCell className="py-2">
|
||||
<div className="line-clamp-2 text-sm leading-snug">{row.text}</div>
|
||||
<div className="line-clamp-2 text-sm leading-snug text-foreground">{row.text}</div>
|
||||
{row.context && (
|
||||
<div className="text-xs text-muted-foreground mt-0.5 truncate">{row.context}</div>
|
||||
)}
|
||||
@@ -506,10 +506,10 @@ export function DataView({ factType }: DataViewProps) {
|
||||
<span className="text-xs text-muted-foreground">-</span>
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell className="text-xs py-2">
|
||||
<TableCell className="text-xs py-2 text-foreground">
|
||||
{occurredDisplay || <span className="text-muted-foreground">-</span>}
|
||||
</TableCell>
|
||||
<TableCell className="text-xs py-2">
|
||||
<TableCell className="text-xs py-2 text-foreground">
|
||||
{mentionedDisplay || <span className="text-muted-foreground">-</span>}
|
||||
</TableCell>
|
||||
<TableCell className="py-2">
|
||||
@@ -519,7 +519,7 @@ export function DataView({ factType }: DataViewProps) {
|
||||
copyToClipboard(row.id);
|
||||
}}
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
className="h-6 w-6 p-0"
|
||||
title="Copy ID"
|
||||
>
|
||||
@@ -799,7 +799,7 @@ function TimelineView({ data, filteredRows }: { data: any; filteredRows: any[] }
|
||||
{/* Zoom controls */}
|
||||
<div className="flex items-center border border-border rounded mr-2">
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={zoomOut}
|
||||
disabled={granularity === 'year'}
|
||||
@@ -808,11 +808,11 @@ function TimelineView({ data, filteredRows }: { data: any; filteredRows: any[] }
|
||||
>
|
||||
<ZoomOut className="h-3 w-3" />
|
||||
</Button>
|
||||
<span className="text-[10px] px-2 min-w-[50px] text-center border-x border-border">
|
||||
<span className="text-[10px] px-2 min-w-[50px] text-center border-x border-border text-foreground">
|
||||
{granularityLabels[granularity]}
|
||||
</span>
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={zoomIn}
|
||||
disabled={granularity === 'day'}
|
||||
@@ -826,7 +826,7 @@ function TimelineView({ data, filteredRows }: { data: any; filteredRows: any[] }
|
||||
{/* Navigation controls */}
|
||||
<div className="flex items-center border border-border rounded">
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={() => scrollToGroup(0)}
|
||||
disabled={timelineGroups.length <= 1}
|
||||
@@ -836,7 +836,7 @@ function TimelineView({ data, filteredRows }: { data: any; filteredRows: any[] }
|
||||
<ChevronsLeft className="h-3 w-3" />
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={() => scrollToGroup(currentIndex - 1)}
|
||||
disabled={currentIndex === 0}
|
||||
@@ -845,11 +845,11 @@ function TimelineView({ data, filteredRows }: { data: any; filteredRows: any[] }
|
||||
>
|
||||
<ChevronLeft className="h-3 w-3" />
|
||||
</Button>
|
||||
<span className="text-[10px] px-2 min-w-[60px] text-center border-x border-border">
|
||||
<span className="text-[10px] px-2 min-w-[60px] text-center border-x border-border text-foreground">
|
||||
{currentIndex + 1} / {timelineGroups.length}
|
||||
</span>
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={() => scrollToGroup(currentIndex + 1)}
|
||||
disabled={currentIndex >= timelineGroups.length - 1}
|
||||
@@ -859,7 +859,7 @@ function TimelineView({ data, filteredRows }: { data: any; filteredRows: any[] }
|
||||
<ChevronRight className="h-3 w-3" />
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={() => scrollToGroup(timelineGroups.length - 1)}
|
||||
disabled={timelineGroups.length <= 1}
|
||||
|
||||
@@ -94,7 +94,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Document ID
|
||||
</div>
|
||||
<div className="text-sm font-mono break-all">{data.id}</div>
|
||||
<div className="text-sm font-mono break-all text-foreground">{data.id}</div>
|
||||
</div>
|
||||
{data.created_at && (
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
@@ -102,7 +102,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Created
|
||||
</div>
|
||||
<div className="text-sm">
|
||||
<div className="text-sm text-foreground">
|
||||
{new Date(data.created_at).toLocaleString()}
|
||||
</div>
|
||||
</div>
|
||||
@@ -110,7 +110,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Memory Units
|
||||
</div>
|
||||
<div className="text-sm">{data.memory_unit_count}</div>
|
||||
<div className="text-sm text-foreground">{data.memory_unit_count}</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
@@ -119,7 +119,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Text Length
|
||||
</div>
|
||||
<div className="text-sm">
|
||||
<div className="text-sm text-foreground">
|
||||
{data.original_text.length.toLocaleString()} characters
|
||||
</div>
|
||||
</div>
|
||||
@@ -132,7 +132,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
||||
Original Text
|
||||
</div>
|
||||
<div className="p-4 bg-muted rounded-lg border border-border max-h-[300px] overflow-y-auto">
|
||||
<pre className="text-sm whitespace-pre-wrap font-mono">
|
||||
<pre className="text-sm whitespace-pre-wrap font-mono text-foreground">
|
||||
{data.original_text}
|
||||
</pre>
|
||||
</div>
|
||||
@@ -146,7 +146,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Chunk ID
|
||||
</div>
|
||||
<div className="text-sm font-mono break-all">
|
||||
<div className="text-sm font-mono break-all text-foreground">
|
||||
{data.chunk_id}
|
||||
</div>
|
||||
</div>
|
||||
@@ -155,7 +155,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Document ID
|
||||
</div>
|
||||
<div className="text-sm font-mono break-all">
|
||||
<div className="text-sm font-mono break-all text-foreground">
|
||||
{data.document_id}
|
||||
</div>
|
||||
</div>
|
||||
@@ -163,7 +163,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Chunk Index
|
||||
</div>
|
||||
<div className="text-sm">{data.chunk_index}</div>
|
||||
<div className="text-sm text-foreground">{data.chunk_index}</div>
|
||||
</div>
|
||||
</div>
|
||||
{data.created_at && (
|
||||
@@ -171,7 +171,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Created
|
||||
</div>
|
||||
<div className="text-sm">
|
||||
<div className="text-sm text-foreground">
|
||||
{new Date(data.created_at).toLocaleString()}
|
||||
</div>
|
||||
</div>
|
||||
@@ -181,7 +181,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Text Length
|
||||
</div>
|
||||
<div className="text-sm">
|
||||
<div className="text-sm text-foreground">
|
||||
{data.chunk_text.length.toLocaleString()} characters
|
||||
</div>
|
||||
</div>
|
||||
@@ -194,7 +194,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
||||
Chunk Text
|
||||
</div>
|
||||
<div className="p-4 bg-muted rounded-lg border border-border max-h-[300px] overflow-y-auto">
|
||||
<pre className="text-sm whitespace-pre-wrap font-mono">
|
||||
<pre className="text-sm whitespace-pre-wrap font-mono text-foreground">
|
||||
{data.chunk_text}
|
||||
</pre>
|
||||
</div>
|
||||
|
||||
@@ -122,17 +122,17 @@ export function DocumentsView() {
|
||||
className={`cursor-pointer hover:bg-muted/50 ${selectedDocument?.id === doc.id ? 'bg-primary/10' : ''}`}
|
||||
onClick={() => viewDocumentText(doc.id)}
|
||||
>
|
||||
<TableCell title={doc.id}>
|
||||
<TableCell title={doc.id} className="text-card-foreground">
|
||||
{doc.id.length > 30 ? doc.id.substring(0, 30) + '...' : doc.id}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<TableCell className="text-card-foreground">
|
||||
{doc.created_at ? new Date(doc.created_at).toLocaleString() : 'N/A'}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<TableCell className="text-card-foreground">
|
||||
{doc.retain_params?.context || '-'}
|
||||
</TableCell>
|
||||
<TableCell>{doc.text_length?.toLocaleString()} chars</TableCell>
|
||||
<TableCell>{doc.memory_unit_count}</TableCell>
|
||||
<TableCell className="text-card-foreground">{doc.text_length?.toLocaleString()} chars</TableCell>
|
||||
<TableCell className="text-card-foreground">{doc.memory_unit_count}</TableCell>
|
||||
<TableCell>
|
||||
<Button
|
||||
onClick={(e) => {
|
||||
@@ -140,7 +140,7 @@ export function DocumentsView() {
|
||||
viewDocumentText(doc.id);
|
||||
}}
|
||||
size="sm"
|
||||
variant={selectedDocument?.id === doc.id ? 'default' : 'outline'}
|
||||
variant={selectedDocument?.id === doc.id ? 'default' : 'secondary'}
|
||||
title="View original text"
|
||||
>
|
||||
View Text
|
||||
@@ -171,7 +171,7 @@ export function DocumentsView() {
|
||||
<p className="text-sm text-muted-foreground mt-1">Original document text and metadata</p>
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={() => setSelectedDocument(null)}
|
||||
className="h-9 px-3 gap-2"
|
||||
@@ -193,7 +193,7 @@ export function DocumentsView() {
|
||||
{/* Document ID */}
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Document ID</div>
|
||||
<code className="text-sm font-mono break-all text-foreground">{selectedDocument.id}</code>
|
||||
<div className="text-sm font-mono break-all text-card-foreground">{selectedDocument.id}</div>
|
||||
</div>
|
||||
|
||||
{/* Created & Memory Units */}
|
||||
@@ -201,11 +201,11 @@ export function DocumentsView() {
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Created</div>
|
||||
<div className="text-sm font-medium text-foreground">{new Date(selectedDocument.created_at).toLocaleString()}</div>
|
||||
<div className="text-sm font-medium text-card-foreground">{new Date(selectedDocument.created_at).toLocaleString()}</div>
|
||||
</div>
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Memory Units</div>
|
||||
<div className="text-sm font-medium text-foreground">{selectedDocument.memory_unit_count}</div>
|
||||
<div className="text-sm font-medium text-card-foreground">{selectedDocument.memory_unit_count}</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
@@ -214,7 +214,7 @@ export function DocumentsView() {
|
||||
{selectedDocument.original_text && (
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Text Length</div>
|
||||
<div className="text-sm font-medium text-foreground">{selectedDocument.original_text.length.toLocaleString()} characters</div>
|
||||
<div className="text-sm font-medium text-card-foreground">{selectedDocument.original_text.length.toLocaleString()} characters</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -222,7 +222,7 @@ export function DocumentsView() {
|
||||
{selectedDocument.retain_params && (
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Retain Parameters</div>
|
||||
<div className="text-sm space-y-2">
|
||||
<div className="text-sm space-y-2 text-card-foreground">
|
||||
{selectedDocument.retain_params.context && (
|
||||
<div><span className="font-semibold">Context:</span> {selectedDocument.retain_params.context}</div>
|
||||
)}
|
||||
@@ -232,7 +232,7 @@ export function DocumentsView() {
|
||||
{selectedDocument.retain_params.metadata && (
|
||||
<div className="mt-2">
|
||||
<span className="font-semibold">Metadata:</span>
|
||||
<pre className="mt-1 text-xs bg-background p-2 rounded">{JSON.stringify(selectedDocument.retain_params.metadata, null, 2)}</pre>
|
||||
<pre className="mt-1 text-xs bg-background p-2 rounded text-card-foreground">{JSON.stringify(selectedDocument.retain_params.metadata, null, 2)}</pre>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
@@ -244,7 +244,7 @@ export function DocumentsView() {
|
||||
<div>
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Original Text</div>
|
||||
<div className="p-4 bg-muted/50 rounded-lg border border-border max-h-[400px] overflow-y-auto">
|
||||
<pre className="text-sm whitespace-pre-wrap font-mono leading-relaxed text-foreground">{selectedDocument.original_text}</pre>
|
||||
<pre className="text-sm whitespace-pre-wrap font-mono leading-relaxed text-card-foreground">{selectedDocument.original_text}</pre>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -126,10 +126,10 @@ export function EntitiesView() {
|
||||
selectedEntity?.id === entity.id ? 'bg-primary/10' : ''
|
||||
}`}
|
||||
>
|
||||
<TableCell className="font-medium">{entity.canonical_name}</TableCell>
|
||||
<TableCell>{entity.mention_count}</TableCell>
|
||||
<TableCell>{formatDate(entity.first_seen)}</TableCell>
|
||||
<TableCell>{formatDate(entity.last_seen)}</TableCell>
|
||||
<TableCell className="font-medium text-card-foreground">{entity.canonical_name}</TableCell>
|
||||
<TableCell className="text-card-foreground">{entity.mention_count}</TableCell>
|
||||
<TableCell className="text-card-foreground">{formatDate(entity.first_seen)}</TableCell>
|
||||
<TableCell className="text-card-foreground">{formatDate(entity.last_seen)}</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
@@ -154,7 +154,7 @@ export function EntitiesView() {
|
||||
{/* Header */}
|
||||
<div className="flex justify-between items-center mb-6 pb-4 border-b border-border">
|
||||
<div>
|
||||
<h3 className="text-xl font-bold text-foreground">{selectedEntity.canonical_name}</h3>
|
||||
<h3 className="text-xl font-bold text-card-foreground">{selectedEntity.canonical_name}</h3>
|
||||
<p className="text-sm text-muted-foreground mt-1">Entity details</p>
|
||||
</div>
|
||||
<Button
|
||||
@@ -172,11 +172,11 @@ export function EntitiesView() {
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Mentions</div>
|
||||
<div className="text-lg font-semibold text-foreground">{selectedEntity.mention_count}</div>
|
||||
<div className="text-lg font-semibold text-card-foreground">{selectedEntity.mention_count}</div>
|
||||
</div>
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">First Seen</div>
|
||||
<div className="text-sm font-medium text-foreground">{formatDate(selectedEntity.first_seen)}</div>
|
||||
<div className="text-sm font-medium text-card-foreground">{formatDate(selectedEntity.first_seen)}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -206,7 +206,7 @@ export function EntitiesView() {
|
||||
<ul className="space-y-2">
|
||||
{selectedEntity.observations.map((obs, idx) => (
|
||||
<li key={idx} className="p-3 bg-muted/50 rounded-lg">
|
||||
<div className="text-sm text-foreground">{obs.text}</div>
|
||||
<div className="text-sm text-card-foreground">{obs.text}</div>
|
||||
{obs.mentioned_at && (
|
||||
<div className="text-xs text-muted-foreground mt-2">
|
||||
{formatDate(obs.mentioned_at)}
|
||||
|
||||
@@ -67,7 +67,7 @@ export function MemoryDetailPanel({
|
||||
<p className="text-sm text-muted-foreground mt-1">Full memory content and metadata</p>
|
||||
</div>
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={onClose}
|
||||
className="h-8 w-8 p-0"
|
||||
@@ -80,14 +80,14 @@ export function MemoryDetailPanel({
|
||||
{/* Full Text */}
|
||||
<div>
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Full Text</div>
|
||||
<div className="text-sm whitespace-pre-wrap leading-relaxed">{memory.text}</div>
|
||||
<div className="text-sm whitespace-pre-wrap leading-relaxed text-foreground">{memory.text}</div>
|
||||
</div>
|
||||
|
||||
{/* Context */}
|
||||
{memory.context && (
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Context</div>
|
||||
<div className="text-sm">{memory.context}</div>
|
||||
<div className="text-sm text-foreground">{memory.context}</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -95,7 +95,7 @@ export function MemoryDetailPanel({
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Occurred</div>
|
||||
<div className="text-sm font-medium">
|
||||
<div className="text-sm font-medium text-foreground">
|
||||
{memory.occurred_start
|
||||
? new Date(memory.occurred_start).toLocaleString()
|
||||
: 'N/A'}
|
||||
@@ -103,7 +103,7 @@ export function MemoryDetailPanel({
|
||||
</div>
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Mentioned</div>
|
||||
<div className="text-sm font-medium">
|
||||
<div className="text-sm font-medium text-foreground">
|
||||
{memory.mentioned_at
|
||||
? new Date(memory.mentioned_at).toLocaleString()
|
||||
: 'N/A'}
|
||||
@@ -159,7 +159,7 @@ export function MemoryDetailPanel({
|
||||
{memory.document_id && (
|
||||
<Button
|
||||
onClick={() => openDocumentModal(memory.document_id)}
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
className="flex-1"
|
||||
>
|
||||
View Document
|
||||
@@ -168,7 +168,7 @@ export function MemoryDetailPanel({
|
||||
{memory.chunk_id && (
|
||||
<Button
|
||||
onClick={() => openChunkModal(memory.chunk_id)}
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
className="flex-1"
|
||||
>
|
||||
View Chunk
|
||||
@@ -300,7 +300,7 @@ export function MemoryDetailPanel({
|
||||
<Button
|
||||
onClick={() => openDocumentModal(memory.document_id)}
|
||||
size="sm"
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
className={`flex-1 ${compact ? 'h-7 text-xs' : ''}`}
|
||||
>
|
||||
View Document
|
||||
@@ -310,7 +310,7 @@ export function MemoryDetailPanel({
|
||||
<Button
|
||||
onClick={() => openChunkModal(memory.chunk_id)}
|
||||
size="sm"
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
className={`flex-1 ${compact ? 'h-7 text-xs' : ''}`}
|
||||
>
|
||||
View Chunk
|
||||
|
||||
@@ -44,7 +44,7 @@ const DialogContent = React.forwardRef<
|
||||
{...props}
|
||||
>
|
||||
{children}
|
||||
<DialogPrimitive.Close className="absolute right-4 top-4 rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground">
|
||||
<DialogPrimitive.Close className="absolute right-4 top-4 rounded-sm ring-offset-background transition-opacity hover:opacity-80 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground text-foreground">
|
||||
<X className="h-4 w-4" />
|
||||
<span className="sr-only">Close</span>
|
||||
</DialogPrimitive.Close>
|
||||
@@ -88,7 +88,7 @@ const DialogTitle = React.forwardRef<
|
||||
<DialogPrimitive.Title
|
||||
ref={ref}
|
||||
className={cn(
|
||||
"text-lg font-semibold leading-none tracking-tight",
|
||||
"text-lg font-semibold leading-none tracking-tight text-foreground",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
|
||||
@@ -0,0 +1,359 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate changelog entry for a new release.
|
||||
|
||||
This script fetches the commit diff between releases, uses an LLM to summarize,
|
||||
and prepends the entry to the changelog page.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from openai import OpenAI
|
||||
from pydantic import BaseModel
|
||||
from rich.console import Console
|
||||
|
||||
console = Console()
|
||||
|
||||
GITHUB_REPO = "vectorize-io/hindsight"
|
||||
GITHUB_RELEASES_URL = f"https://github.com/{GITHUB_REPO}/releases"
|
||||
GITHUB_COMMIT_URL = f"https://github.com/{GITHUB_REPO}/commit"
|
||||
REPO_PATH = Path(__file__).parent.parent.parent
|
||||
CHANGELOG_PATH = REPO_PATH / "hindsight-docs" / "docs" / "changelog" / "index.md"
|
||||
|
||||
|
||||
class ChangelogEntry(BaseModel):
|
||||
"""A single changelog entry."""
|
||||
category: str # "feature", "improvement", "bugfix", "breaking", "other"
|
||||
summary: str # Brief description of the change
|
||||
commit_id: str # Short commit hash
|
||||
|
||||
|
||||
class ChangelogResponse(BaseModel):
|
||||
"""Structured response from LLM."""
|
||||
entries: list[ChangelogEntry]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Commit:
|
||||
"""Parsed commit from git log."""
|
||||
hash: str
|
||||
message: str
|
||||
|
||||
|
||||
def parse_semver(version: str) -> tuple[int, int, int]:
|
||||
"""Parse a semver string into (major, minor, patch)."""
|
||||
version = version.lstrip("v")
|
||||
match = re.match(r"^(\d+)\.(\d+)\.(\d+)", version)
|
||||
if not match:
|
||||
raise ValueError(f"Invalid semver: {version}")
|
||||
return int(match.group(1)), int(match.group(2)), int(match.group(3))
|
||||
|
||||
|
||||
def get_git_tags() -> list[str]:
|
||||
"""Get all git tags sorted by semver (newest first)."""
|
||||
result = subprocess.run(
|
||||
["git", "tag"],
|
||||
cwd=REPO_PATH,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
tags = [t.strip() for t in result.stdout.strip().split("\n") if t.strip()]
|
||||
|
||||
valid_tags = []
|
||||
for tag in tags:
|
||||
try:
|
||||
parse_semver(tag)
|
||||
valid_tags.append(tag)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
valid_tags.sort(key=lambda t: parse_semver(t), reverse=True)
|
||||
return valid_tags
|
||||
|
||||
|
||||
def find_previous_version(new_version: str, existing_tags: list[str]) -> str | None:
|
||||
"""Find the previous version based on semver rules."""
|
||||
new_major, new_minor, new_patch = parse_semver(new_version)
|
||||
|
||||
candidates = []
|
||||
for tag in existing_tags:
|
||||
try:
|
||||
major, minor, patch = parse_semver(tag)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if (major, minor, patch) >= (new_major, new_minor, new_patch):
|
||||
continue
|
||||
|
||||
candidates.append((tag, (major, minor, patch)))
|
||||
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
candidates.sort(key=lambda x: x[1], reverse=True)
|
||||
return candidates[0][0]
|
||||
|
||||
|
||||
def get_commits(from_ref: str | None, to_ref: str) -> list[Commit]:
|
||||
"""Get commits between two refs as structured data."""
|
||||
if from_ref:
|
||||
cmd = ["git", "log", "--format=%h|%s", "--no-merges", f"{from_ref}..{to_ref}"]
|
||||
else:
|
||||
cmd = ["git", "log", "--format=%h|%s", "--no-merges", to_ref]
|
||||
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=REPO_PATH,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
commits = []
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split("|", 1)
|
||||
if len(parts) == 2:
|
||||
commits.append(Commit(hash=parts[0], message=parts[1]))
|
||||
|
||||
return commits
|
||||
|
||||
|
||||
def get_detailed_diff(from_ref: str | None, to_ref: str) -> str:
|
||||
"""Get file change stats between two refs."""
|
||||
if from_ref:
|
||||
cmd = ["git", "diff", "--stat", f"{from_ref}..{to_ref}"]
|
||||
else:
|
||||
cmd = ["git", "diff", "--stat", f"{to_ref}^..{to_ref}"]
|
||||
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=REPO_PATH,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def analyze_commits_with_llm(
|
||||
client: OpenAI,
|
||||
model: str,
|
||||
version: str,
|
||||
commits: list[Commit],
|
||||
file_diff: str,
|
||||
) -> list[ChangelogEntry]:
|
||||
"""Use LLM to analyze commits and return structured changelog entries."""
|
||||
commits_json = json.dumps(
|
||||
[{"commit_id": c.hash, "message": c.message} for c in commits],
|
||||
indent=2
|
||||
)
|
||||
|
||||
prompt = f"""Analyze the following git commits for release {version} of Hindsight (an AI memory system).
|
||||
|
||||
For each meaningful change, create a changelog entry with:
|
||||
- category: one of "feature", "improvement", "bugfix", "breaking", "other"
|
||||
- summary: brief one-line description of the change (user-facing, not technical)
|
||||
- commit_id: the commit hash from the input
|
||||
|
||||
Rules:
|
||||
- Group related commits into a single entry if they're part of the same change
|
||||
- Skip trivial changes (typo fixes, formatting, internal refactoring)
|
||||
- Skip repository-only changes: README updates, CI/GitHub Actions, release scripts, changelog updates, version bumps
|
||||
- Focus on user-facing changes that affect the product functionality
|
||||
- Use the exact commit_id from the input (pick the most relevant one if grouping)
|
||||
- If no meaningful changes remain after filtering, return an empty list
|
||||
|
||||
Commits:
|
||||
{commits_json}
|
||||
|
||||
Files changed summary:
|
||||
{file_diff[:4000]}"""
|
||||
|
||||
response = client.beta.chat.completions.parse(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
response_format=ChangelogResponse,
|
||||
max_completion_tokens=16000,
|
||||
)
|
||||
|
||||
return response.choices[0].message.parsed.entries
|
||||
|
||||
|
||||
def build_changelog_markdown(
|
||||
version: str,
|
||||
tag: str,
|
||||
entries: list[ChangelogEntry],
|
||||
) -> str:
|
||||
"""Build markdown changelog from structured entries."""
|
||||
release_url = f"{GITHUB_RELEASES_URL}/tag/{tag}"
|
||||
|
||||
# Group entries by category
|
||||
categories = {
|
||||
"breaking": ("Breaking Changes", []),
|
||||
"feature": ("Features", []),
|
||||
"improvement": ("Improvements", []),
|
||||
"bugfix": ("Bug Fixes", []),
|
||||
"other": ("Other", []),
|
||||
}
|
||||
|
||||
for entry in entries:
|
||||
cat = entry.category.lower()
|
||||
if cat in categories:
|
||||
categories[cat][1].append(entry)
|
||||
else:
|
||||
categories["other"][1].append(entry)
|
||||
|
||||
# Build markdown
|
||||
lines = [f"## [{version}]({release_url})", ""]
|
||||
|
||||
for cat_key in ["breaking", "feature", "improvement", "bugfix", "other"]:
|
||||
cat_name, cat_entries = categories[cat_key]
|
||||
if cat_entries:
|
||||
lines.append(f"**{cat_name}**")
|
||||
lines.append("")
|
||||
for entry in cat_entries:
|
||||
commit_url = f"{GITHUB_COMMIT_URL}/{entry.commit_id}"
|
||||
lines.append(f"- {entry.summary} ([`{entry.commit_id}`]({commit_url}))")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def read_existing_changelog() -> tuple[str, str]:
|
||||
"""Read existing changelog and split into header and content."""
|
||||
if not CHANGELOG_PATH.exists():
|
||||
header = """---
|
||||
sidebar_position: 1
|
||||
---
|
||||
|
||||
# Changelog
|
||||
|
||||
For full release details, see [GitHub Releases](https://github.com/vectorize-io/hindsight/releases).
|
||||
"""
|
||||
return header, ""
|
||||
|
||||
content = CHANGELOG_PATH.read_text()
|
||||
|
||||
match = re.search(r"^## ", content, re.MULTILINE)
|
||||
if match:
|
||||
header = content[:match.start()].rstrip() + "\n\n"
|
||||
releases = content[match.start():]
|
||||
else:
|
||||
header = content.rstrip() + "\n\n"
|
||||
releases = ""
|
||||
|
||||
return header, releases
|
||||
|
||||
|
||||
def write_changelog(header: str, new_entry: str, existing_releases: str) -> None:
|
||||
"""Write changelog with new entry prepended."""
|
||||
content = header + new_entry + "\n" + existing_releases
|
||||
CHANGELOG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
CHANGELOG_PATH.write_text(content.rstrip() + "\n")
|
||||
|
||||
|
||||
def generate_changelog_entry(
|
||||
version: str,
|
||||
llm_model: str = "gpt-5.2",
|
||||
) -> None:
|
||||
"""Generate changelog entry for a specific version."""
|
||||
api_key = os.environ.get("OPENAI_API_KEY")
|
||||
if not api_key:
|
||||
console.print("[red]Error: OPENAI_API_KEY environment variable not set[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
client = OpenAI(api_key=api_key)
|
||||
|
||||
tag = version if version.startswith("v") else f"v{version}"
|
||||
display_version = version.lstrip("v")
|
||||
|
||||
console.print(f"[blue]Fetching tags from repository...[/blue]")
|
||||
existing_tags = get_git_tags()
|
||||
|
||||
if tag not in existing_tags and display_version not in existing_tags:
|
||||
console.print(f"[red]Error: Tag {tag} not found in repository[/red]")
|
||||
console.print("[red]Create the tag first before generating changelog[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
actual_tag = tag if tag in existing_tags else display_version
|
||||
|
||||
previous_tag = find_previous_version(display_version, existing_tags)
|
||||
|
||||
if previous_tag:
|
||||
console.print(f"[green]Found previous version: {previous_tag}[/green]")
|
||||
else:
|
||||
console.print("[yellow]No previous version found, will include all commits[/yellow]")
|
||||
|
||||
console.print(f"[blue]Getting commits...[/blue]")
|
||||
commits = get_commits(previous_tag, actual_tag)
|
||||
file_diff = get_detailed_diff(previous_tag, actual_tag)
|
||||
|
||||
if not commits:
|
||||
console.print("[red]Error: No commits found for this release[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
console.print(f"[blue]Found {len(commits)} commits[/blue]")
|
||||
|
||||
# Log commits
|
||||
console.print("\n[bold]Commits:[/bold]")
|
||||
for c in commits:
|
||||
console.print(f" {c.hash} {c.message}")
|
||||
|
||||
console.print("\n[bold]Files changed:[/bold]")
|
||||
console.print(file_diff[:4000] if len(file_diff) > 4000 else file_diff)
|
||||
console.print("")
|
||||
|
||||
console.print(f"[blue]Analyzing commits with LLM ({llm_model})...[/blue]")
|
||||
entries = analyze_commits_with_llm(client, llm_model, display_version, commits, file_diff)
|
||||
|
||||
console.print(f"\n[bold]LLM identified {len(entries)} changelog entries:[/bold]")
|
||||
for entry in entries:
|
||||
console.print(f" [{entry.category}] {entry.summary} ({entry.commit_id})")
|
||||
|
||||
new_entry = build_changelog_markdown(display_version, tag, entries)
|
||||
|
||||
header, existing_releases = read_existing_changelog()
|
||||
|
||||
if f"## [{display_version}]" in existing_releases:
|
||||
console.print(f"[red]Error: Version {display_version} already exists in changelog[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
write_changelog(header, new_entry, existing_releases)
|
||||
|
||||
console.print(f"\n[green]Changelog updated: {CHANGELOG_PATH}[/green]")
|
||||
console.print(f"\n[bold]New entry:[/bold]\n{new_entry}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate changelog entry for a release",
|
||||
usage="generate-changelog VERSION [--model MODEL]",
|
||||
)
|
||||
parser.add_argument(
|
||||
"version",
|
||||
help="Version to generate changelog for (e.g., 1.0.5, v1.0.5)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="gpt-5.2",
|
||||
help="OpenAI model to use (default: gpt-5.2)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
generate_changelog_entry(
|
||||
version=args.version,
|
||||
llm_model=args.model,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "hindsight-dev"
|
||||
version = "0.1.4"
|
||||
version = "0.1.5"
|
||||
description = "Development utilities for Hindsight"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
@@ -24,3 +24,4 @@ hindsight-api = { workspace = true }
|
||||
|
||||
[project.scripts]
|
||||
generate-openapi = "hindsight_dev.generate_openapi:generate_openapi_spec"
|
||||
generate-changelog = "hindsight_dev.generate_changelog:main"
|
||||
|
||||
@@ -4,4 +4,35 @@ sidebar_position: 1
|
||||
|
||||
# Changelog
|
||||
|
||||
Coming soon.
|
||||
For full release details, see [GitHub Releases](https://github.com/vectorize-io/hindsight/releases).
|
||||
|
||||
## [0.1.5](https://github.com/vectorize-io/hindsight/releases/tag/v0.1.5)
|
||||
|
||||
**Features**
|
||||
|
||||
- Added LiteLLM integration so Hindsight can capture and manage memories from LiteLLM-based LLM calls. ([`dfccbf2`](https://github.com/vectorize-io/hindsight/commit/dfccbf2))
|
||||
- Added an optional graph-based retriever (MPFP) to improve recall by leveraging relationships between memories. ([`7445cef`](https://github.com/vectorize-io/hindsight/commit/7445cef))
|
||||
|
||||
**Improvements**
|
||||
|
||||
- Switched the embedded Postgres layer to pg0-embedded for a smoother local/standalone experience. ([`94c2b85`](https://github.com/vectorize-io/hindsight/commit/94c2b85))
|
||||
|
||||
**Bug Fixes**
|
||||
|
||||
- Fixed repeated retries on 400 errors from the LLM, preventing unnecessary request loops and failures. ([`70983f5`](https://github.com/vectorize-io/hindsight/commit/70983f5))
|
||||
- Fixed recall trace visualization in the control plane so search/recall debugging displays correctly. ([`922164e`](https://github.com/vectorize-io/hindsight/commit/922164e))
|
||||
- Fixed the CLI installer to make installation more reliable. ([`158a6aa`](https://github.com/vectorize-io/hindsight/commit/158a6aa))
|
||||
- Updated Next.js to patch security vulnerabilities (CVE-2025-55184, CVE-2025-55183). ([`f018cc5`](https://github.com/vectorize-io/hindsight/commit/f018cc5))
|
||||
|
||||
## [0.1.3](https://github.com/vectorize-io/hindsight/releases/tag/v0.1.3)
|
||||
|
||||
**Improvements**
|
||||
|
||||
- Improved CLI and UI branding/polish, including new banner/logo assets and updated interface styling. ([`fa554b8`](https://github.com/vectorize-io/hindsight/commit/fa554b8))
|
||||
|
||||
|
||||
## [0.1.2](https://github.com/vectorize-io/hindsight/releases/tag/v0.1.2)
|
||||
|
||||
**Bug Fixes**
|
||||
|
||||
- Fixed the standalone Docker image so it builds/runs correctly. ([`1056a20`](https://github.com/vectorize-io/hindsight/commit/1056a20))
|
||||
|
||||
@@ -6,14 +6,70 @@ Hindsight uses several machine learning models for different tasks.
|
||||
|
||||
| Model Type | Purpose | Default | Configurable |
|
||||
|------------|---------|---------|--------------|
|
||||
| **LLM** | Fact extraction, reasoning, generation | Provider-specific | Yes |
|
||||
| **Embedding** | Vector representations for semantic search | `BAAI/bge-small-en-v1.5` | Yes |
|
||||
| **Cross-Encoder** | Reranking search results | `cross-encoder/ms-marco-MiniLM-L-6-v2` | Yes |
|
||||
| **LLM** | Fact extraction, reasoning, generation | Provider-specific | Yes |
|
||||
|
||||
All local models (embedding, cross-encoder) are automatically downloaded from HuggingFace on first run.
|
||||
|
||||
---
|
||||
|
||||
## LLM
|
||||
|
||||
Used for fact extraction, entity resolution, opinion generation, and answer synthesis.
|
||||
|
||||
**Supported providers:** OpenAI, Gemini, Groq, Ollama
|
||||
|
||||
### Tested Models
|
||||
|
||||
The following models have been tested and verified to work correctly with Hindsight:
|
||||
|
||||
| Provider | Model |
|
||||
|----------|-------|
|
||||
| **OpenAI** | `gpt-5` |
|
||||
| **OpenAI** | `gpt-5-mini` |
|
||||
| **OpenAI** | `gpt-5-nano` |
|
||||
| **OpenAI** | `gpt-4.1-mini` |
|
||||
| **OpenAI** | `gpt-4.1-nano` |
|
||||
| **OpenAI** | `gpt-4o-mini` |
|
||||
| **Gemini** | `gemini-2.5-flash` |
|
||||
| **Gemini** | `gemini-2.5-flash-lite` |
|
||||
| **Groq** | `openai/gpt-oss-120b` |
|
||||
| **Groq** | `openai/gpt-oss-20b` |
|
||||
| **Groq** | `llama-3.3-70b-versatile` |
|
||||
|
||||
### Using Other Models
|
||||
|
||||
Other LLM models not listed above may work with Hindsight, but they must support **at least 65,000 output tokens** to ensure reliable fact extraction. If you need support for a specific model that doesn't meet this requirement, please [open an issue](https://github.com/hindsight-ai/hindsight/issues) to request an exception.
|
||||
|
||||
### Configuration
|
||||
|
||||
```bash
|
||||
# Groq (recommended)
|
||||
export HINDSIGHT_API_LLM_PROVIDER=groq
|
||||
export HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx
|
||||
export HINDSIGHT_API_LLM_MODEL=openai/gpt-oss-20b
|
||||
|
||||
# OpenAI
|
||||
export HINDSIGHT_API_LLM_PROVIDER=openai
|
||||
export HINDSIGHT_API_LLM_API_KEY=sk-xxxxxxxxxxxx
|
||||
export HINDSIGHT_API_LLM_MODEL=gpt-4o
|
||||
|
||||
# Gemini
|
||||
export HINDSIGHT_API_LLM_PROVIDER=gemini
|
||||
export HINDSIGHT_API_LLM_API_KEY=xxxxxxxxxxxx
|
||||
export HINDSIGHT_API_LLM_MODEL=gemini-2.0-flash
|
||||
|
||||
# Ollama (local)
|
||||
export HINDSIGHT_API_LLM_PROVIDER=ollama
|
||||
export HINDSIGHT_API_LLM_BASE_URL=http://localhost:11434/v1
|
||||
export HINDSIGHT_API_LLM_MODEL=llama3.1
|
||||
```
|
||||
|
||||
**Note:** The LLM is the primary bottleneck for retain operations. See [Performance](./performance) for optimization strategies.
|
||||
|
||||
---
|
||||
|
||||
## Embedding Model
|
||||
|
||||
Converts text into dense vector representations for semantic similarity search.
|
||||
@@ -22,14 +78,13 @@ Converts text into dense vector representations for semantic similarity search.
|
||||
|
||||
**Alternatives:**
|
||||
|
||||
| Model | Dimensions | Use Case |
|
||||
|-------|------------|----------|
|
||||
| `BAAI/bge-small-en-v1.5` | 384 | Default, fast, good quality |
|
||||
| `BAAI/bge-base-en-v1.5` | 768 | Higher accuracy, slower |
|
||||
| `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2` | 384 | Multilingual (50+ languages) |
|
||||
| Model | Use Case |
|
||||
|-------|----------|
|
||||
| `BAAI/bge-small-en-v1.5` | Default, fast, good quality |
|
||||
| `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2` | Multilingual (50+ languages) |
|
||||
|
||||
:::warning
|
||||
All embedding models must produce 384-dimensional vectors to match the database schema.
|
||||
All embedding models must produce **384-dimensional vectors** to match the database schema.
|
||||
:::
|
||||
|
||||
**Configuration:**
|
||||
@@ -71,44 +126,3 @@ export HINDSIGHT_API_RERANKER_LOCAL_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2
|
||||
export HINDSIGHT_API_RERANKER_PROVIDER=tei
|
||||
export HINDSIGHT_API_RERANKER_TEI_URL=http://localhost:8081
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## LLM
|
||||
|
||||
Used for fact extraction, entity resolution, opinion generation, and answer synthesis.
|
||||
|
||||
**Supported providers:** Groq, OpenAI, Gemini, Ollama
|
||||
|
||||
| Provider | Recommended Model | Best For |
|
||||
|----------|------------------|----------|
|
||||
| **Groq** | `openai/gpt-oss-20b` | Fast inference, high throughput (recommended) |
|
||||
| **OpenAI** | `gpt-4o` | Good quality |
|
||||
| **Gemini** | `gemini-2.0-flash` | Good quality, cost effective |
|
||||
| **Ollama** | `llama3.1` | Local deployment, privacy |
|
||||
|
||||
**Configuration:**
|
||||
|
||||
```bash
|
||||
# Groq (recommended)
|
||||
export HINDSIGHT_API_LLM_PROVIDER=groq
|
||||
export HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx
|
||||
export HINDSIGHT_API_LLM_MODEL=openai/gpt-oss-20b
|
||||
|
||||
# OpenAI
|
||||
export HINDSIGHT_API_LLM_PROVIDER=openai
|
||||
export HINDSIGHT_API_LLM_API_KEY=sk-xxxxxxxxxxxx
|
||||
export HINDSIGHT_API_LLM_MODEL=gpt-4o
|
||||
|
||||
# Gemini
|
||||
export HINDSIGHT_API_LLM_PROVIDER=gemini
|
||||
export HINDSIGHT_API_LLM_API_KEY=xxxxxxxxxxxx
|
||||
export HINDSIGHT_API_LLM_MODEL=gemini-2.0-flash
|
||||
|
||||
# Ollama (local)
|
||||
export HINDSIGHT_API_LLM_PROVIDER=ollama
|
||||
export HINDSIGHT_API_LLM_BASE_URL=http://localhost:11434/v1
|
||||
export HINDSIGHT_API_LLM_MODEL=llama3.1
|
||||
```
|
||||
|
||||
**Note:** The LLM is the primary bottleneck for retain operations. See [Performance](./performance) for optimization strategies.
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
---
|
||||
|
||||
# LiteLLM
|
||||
|
||||
Universal LLM memory integration via [LiteLLM](https://github.com/BerriAI/litellm). Add persistent memory to any LLM application with just a few lines of code.
|
||||
|
||||
## Features
|
||||
|
||||
- **Universal LLM Support** - Works with 100+ LLM providers via LiteLLM (OpenAI, Anthropic, Groq, Azure, AWS Bedrock, Google Vertex AI, and more)
|
||||
- **Simple Integration** - Just configure, enable, and use `hindsight_litellm.completion()`
|
||||
- **Automatic Memory Injection** - Relevant memories are injected into prompts before LLM calls
|
||||
- **Automatic Conversation Storage** - Conversations are stored to Hindsight for future recall
|
||||
- **Two Memory Modes** - Choose between `reflect` (synthesized context) or `recall` (raw memory retrieval)
|
||||
- **Direct Memory APIs** - Query, synthesize, and store memories manually
|
||||
- **Native Client Wrappers** - Alternative wrappers for OpenAI and Anthropic SDKs
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install hindsight-litellm
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```python
|
||||
import hindsight_litellm
|
||||
|
||||
# Configure and enable memory integration
|
||||
hindsight_litellm.configure(
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
bank_id="my-agent",
|
||||
)
|
||||
hindsight_litellm.enable()
|
||||
|
||||
# Use the convenience wrapper - memory is automatically injected and stored
|
||||
response = hindsight_litellm.completion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "What did we discuss about AI?"}]
|
||||
)
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
When you call `completion()`, the following happens automatically:
|
||||
|
||||
1. **Memory Retrieval** - Hindsight is queried for relevant memories based on the conversation
|
||||
2. **Prompt Injection** - Memories are injected into the system message
|
||||
3. **LLM Call** - The enriched prompt is sent to the LLM
|
||||
4. **Conversation Storage** - The conversation is stored to Hindsight for future recall
|
||||
5. **Response Returned** - You receive the response as normal
|
||||
|
||||
## Configuration Options
|
||||
|
||||
```python
|
||||
hindsight_litellm.configure(
|
||||
# Required
|
||||
hindsight_api_url="http://localhost:8888", # Hindsight API server URL
|
||||
bank_id="my-agent", # Memory bank ID
|
||||
|
||||
api_key="your-api-key", # Optional API key for authentication
|
||||
|
||||
# Optional - Memory behavior
|
||||
store_conversations=True, # Store conversations after LLM calls
|
||||
inject_memories=True, # Inject relevant memories into prompts
|
||||
use_reflect=False, # Use reflect API (synthesized) vs recall (raw memories)
|
||||
reflect_include_facts=False, # Include source facts with reflect responses
|
||||
max_memories=None, # Maximum memories to inject (None = unlimited)
|
||||
max_memory_tokens=4096, # Maximum tokens for memory context
|
||||
recall_budget="mid", # Recall budget: "low", "mid", "high"
|
||||
fact_types=["world", "agent"], # Filter fact types to inject
|
||||
|
||||
# Optional - Bank Configuration
|
||||
bank_name="My Agent", # Human-readable display name for the memory bank
|
||||
background="This agent...", # Instructions guiding what Hindsight should remember
|
||||
|
||||
# Optional - Advanced
|
||||
injection_mode="system_message", # or "prepend_user"
|
||||
excluded_models=["gpt-3.5*"], # Exclude certain models
|
||||
verbose=True, # Enable verbose logging and debug info
|
||||
)
|
||||
```
|
||||
|
||||
### Bank Configuration
|
||||
|
||||
The `background` and `bank_name` parameters configure the memory bank itself. When provided, `configure()` will automatically create or update the bank with these settings.
|
||||
|
||||
```python
|
||||
hindsight_litellm.configure(
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
bank_id="support-router",
|
||||
bank_name="Customer Support Router",
|
||||
background="""This agent routes customer support requests to the appropriate team.
|
||||
Remember which types of issues should go to which teams (billing, technical, sales).
|
||||
Track customer preferences for communication channels and past issue resolutions.""",
|
||||
)
|
||||
```
|
||||
|
||||
### Memory Modes: Reflect vs Recall
|
||||
|
||||
- **Recall mode** (`use_reflect=False`, default): Retrieves raw memory facts and injects them as a numbered list. Best when you need precise, individual memories.
|
||||
- **Reflect mode** (`use_reflect=True`): Synthesizes memories into a coherent context paragraph. Best for natural, conversational memory context.
|
||||
|
||||
```python
|
||||
# Recall mode - raw memories
|
||||
hindsight_litellm.configure(
|
||||
bank_id="my-agent",
|
||||
use_reflect=False, # Default
|
||||
)
|
||||
# Injects: "1. [WORLD] User prefers Python\n2. [OPINION] User dislikes Java..."
|
||||
|
||||
# Reflect mode - synthesized context
|
||||
hindsight_litellm.configure(
|
||||
bank_id="my-agent",
|
||||
use_reflect=True,
|
||||
)
|
||||
# Injects: "Based on previous conversations, the user is a Python developer who..."
|
||||
```
|
||||
|
||||
## Multi-Provider Support
|
||||
|
||||
Works with any LiteLLM-supported provider:
|
||||
|
||||
```python
|
||||
import hindsight_litellm
|
||||
|
||||
hindsight_litellm.configure(
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
bank_id="my-agent",
|
||||
)
|
||||
hindsight_litellm.enable()
|
||||
|
||||
# OpenAI
|
||||
hindsight_litellm.completion(model="gpt-4o", messages=[...])
|
||||
|
||||
# Anthropic
|
||||
hindsight_litellm.completion(model="claude-3-5-sonnet-20241022", messages=[...])
|
||||
|
||||
# Groq
|
||||
hindsight_litellm.completion(model="groq/llama-3.1-70b-versatile", messages=[...])
|
||||
|
||||
# Azure OpenAI
|
||||
hindsight_litellm.completion(model="azure/gpt-4", messages=[...])
|
||||
|
||||
# AWS Bedrock
|
||||
hindsight_litellm.completion(model="bedrock/anthropic.claude-3", messages=[...])
|
||||
|
||||
# Google Vertex AI
|
||||
hindsight_litellm.completion(model="vertex_ai/gemini-pro", messages=[...])
|
||||
```
|
||||
|
||||
## Direct Memory APIs
|
||||
|
||||
### Recall - Query raw memories
|
||||
|
||||
```python
|
||||
from hindsight_litellm import configure, recall
|
||||
|
||||
configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
|
||||
memories = recall("what projects am I working on?", budget="mid")
|
||||
for m in memories:
|
||||
print(f"- [{m.fact_type}] {m.text}")
|
||||
```
|
||||
|
||||
### Reflect - Get synthesized context
|
||||
|
||||
```python
|
||||
from hindsight_litellm import configure, reflect
|
||||
|
||||
configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
|
||||
result = reflect("what do you know about the user's preferences?")
|
||||
print(result.text)
|
||||
```
|
||||
|
||||
### Retain - Store memories
|
||||
|
||||
```python
|
||||
from hindsight_litellm import configure, retain
|
||||
|
||||
configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
|
||||
result = retain(
|
||||
content="User mentioned they're working on a machine learning project",
|
||||
context="Discussion about current projects",
|
||||
)
|
||||
```
|
||||
|
||||
### Async APIs
|
||||
|
||||
```python
|
||||
from hindsight_litellm import arecall, areflect, aretain
|
||||
|
||||
# Async versions of all memory APIs
|
||||
memories = await arecall("what do you know about me?")
|
||||
context = await areflect("summarize user preferences")
|
||||
result = await aretain(content="New information to remember")
|
||||
```
|
||||
|
||||
## Native Client Wrappers
|
||||
|
||||
Alternative to LiteLLM callbacks for direct SDK integration.
|
||||
|
||||
### OpenAI Wrapper
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
from hindsight_litellm import wrap_openai
|
||||
|
||||
client = OpenAI()
|
||||
wrapped = wrap_openai(
|
||||
client,
|
||||
bank_id="my-agent",
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
)
|
||||
|
||||
response = wrapped.chat.completions.create(
|
||||
model="gpt-4",
|
||||
messages=[{"role": "user", "content": "What do you know about me?"}]
|
||||
)
|
||||
```
|
||||
|
||||
### Anthropic Wrapper
|
||||
|
||||
```python
|
||||
from anthropic import Anthropic
|
||||
from hindsight_litellm import wrap_anthropic
|
||||
|
||||
client = Anthropic()
|
||||
wrapped = wrap_anthropic(
|
||||
client,
|
||||
bank_id="my-agent",
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
)
|
||||
|
||||
response = wrapped.messages.create(
|
||||
model="claude-3-5-sonnet-20241022",
|
||||
max_tokens=1024,
|
||||
messages=[{"role": "user", "content": "Hello!"}]
|
||||
)
|
||||
```
|
||||
|
||||
## Debug Mode
|
||||
|
||||
When `verbose=True`, you can inspect exactly what memories are being injected:
|
||||
|
||||
```python
|
||||
from hindsight_litellm import configure, enable, completion, get_last_injection_debug
|
||||
|
||||
configure(
|
||||
bank_id="my-agent",
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
verbose=True,
|
||||
)
|
||||
enable()
|
||||
|
||||
response = completion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "What's my favorite color?"}]
|
||||
)
|
||||
|
||||
# Inspect what was injected
|
||||
debug = get_last_injection_debug()
|
||||
if debug:
|
||||
print(f"Mode: {debug.mode}") # "reflect" or "recall"
|
||||
print(f"Injected: {debug.injected}") # True/False
|
||||
print(f"Results: {debug.results_count}")
|
||||
print(f"Memory context:\n{debug.memory_context}")
|
||||
```
|
||||
|
||||
## Context Manager
|
||||
|
||||
```python
|
||||
from hindsight_litellm import hindsight_memory
|
||||
import litellm
|
||||
|
||||
with hindsight_memory(bank_id="user-123"):
|
||||
response = litellm.completion(model="gpt-4", messages=[...])
|
||||
# Memory integration automatically disabled after context
|
||||
```
|
||||
|
||||
## Disabling and Cleanup
|
||||
|
||||
```python
|
||||
from hindsight_litellm import disable, cleanup
|
||||
|
||||
# Temporarily disable memory integration
|
||||
disable()
|
||||
|
||||
# Clean up all resources (call when shutting down)
|
||||
cleanup()
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### Main Functions
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `configure(...)` | Configure global Hindsight settings |
|
||||
| `enable()` | Enable memory integration with LiteLLM |
|
||||
| `disable()` | Disable memory integration |
|
||||
| `is_enabled()` | Check if memory integration is enabled |
|
||||
| `cleanup()` | Clean up all resources |
|
||||
|
||||
### Configuration Functions
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `get_config()` | Get current configuration |
|
||||
| `is_configured()` | Check if Hindsight is configured |
|
||||
| `reset_config()` | Reset configuration to defaults |
|
||||
|
||||
### Memory Functions
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `recall(query, ...)` | Synchronously query raw memories |
|
||||
| `arecall(query, ...)` | Asynchronously query raw memories |
|
||||
| `reflect(query, ...)` | Synchronously get synthesized memory context |
|
||||
| `areflect(query, ...)` | Asynchronously get synthesized memory context |
|
||||
| `retain(content, ...)` | Synchronously store a memory |
|
||||
| `aretain(content, ...)` | Asynchronously store a memory |
|
||||
|
||||
### Debug Functions
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `get_last_injection_debug()` | Get debug info from last memory injection |
|
||||
| `clear_injection_debug()` | Clear stored debug info |
|
||||
|
||||
### Client Wrappers
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `wrap_openai(client, ...)` | Wrap OpenAI client with memory |
|
||||
| `wrap_anthropic(client, ...)` | Wrap Anthropic client with memory |
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python >= 3.10
|
||||
- litellm >= 1.40.0
|
||||
- A running Hindsight API server
|
||||
@@ -147,6 +147,18 @@ const sidebars: SidebarsConfig = {
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'category',
|
||||
label: 'Integrations',
|
||||
collapsible: false,
|
||||
items: [
|
||||
{
|
||||
type: 'doc',
|
||||
id: 'sdks/integrations/litellm',
|
||||
label: 'LiteLLM',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
cookbookSidebar: [
|
||||
{
|
||||
|
||||
@@ -514,6 +514,27 @@ article a:not(.button):not([class*="hash-link"]):hover {
|
||||
text-decoration-color: var(--hindsight-gradient-start);
|
||||
}
|
||||
|
||||
/* Links inside code blocks - use solid color instead of gradient */
|
||||
code a,
|
||||
pre a,
|
||||
article code a,
|
||||
article pre a {
|
||||
background: none !important;
|
||||
-webkit-background-clip: unset !important;
|
||||
-webkit-text-fill-color: var(--ifm-color-primary) !important;
|
||||
background-clip: unset !important;
|
||||
color: var(--ifm-color-primary) !important;
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
code a:hover,
|
||||
pre a:hover,
|
||||
article code a:hover,
|
||||
article pre a:hover {
|
||||
color: var(--ifm-color-primary-dark) !important;
|
||||
-webkit-text-fill-color: var(--ifm-color-primary-dark) !important;
|
||||
}
|
||||
|
||||
/* Admonitions - gradient themed */
|
||||
.theme-admonition,
|
||||
[class*="admonition_"] {
|
||||
|
||||
+481
-371
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,437 @@
|
||||
# hindsight-litellm
|
||||
|
||||
Universal LLM memory integration via LiteLLM. Add persistent memory to any LLM application with just a few lines of code.
|
||||
|
||||
## Features
|
||||
|
||||
- **Universal LLM Support** - Works with 100+ LLM providers via LiteLLM (OpenAI, Anthropic, Groq, Azure, AWS Bedrock, Google Vertex AI, and more)
|
||||
- **Simple Integration** - Just configure, enable, and use `hindsight_litellm.completion()`
|
||||
- **Automatic Memory Injection** - Relevant memories are injected into prompts before LLM calls
|
||||
- **Automatic Conversation Storage** - Conversations are stored to Hindsight for future recall
|
||||
- **Two Memory Modes** - Choose between `reflect` (synthesized context) or `recall` (raw memory retrieval)
|
||||
- **Direct Memory APIs** - Query, synthesize, and store memories manually
|
||||
- **Native Client Wrappers** - Alternative wrappers for OpenAI and Anthropic SDKs
|
||||
- **Debug Mode** - Inspect exactly what memories are being injected
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install hindsight-litellm
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```python
|
||||
import hindsight_litellm
|
||||
|
||||
# Configure and enable memory integration
|
||||
hindsight_litellm.configure(
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
bank_id="my-agent",
|
||||
)
|
||||
hindsight_litellm.enable()
|
||||
|
||||
# Use the convenience wrapper - memory is automatically injected and stored
|
||||
response = hindsight_litellm.completion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "What did we discuss about AI?"}]
|
||||
)
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
Here's what happens under the hood when you call `completion()`:
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 1. YOUR CODE │
|
||||
│ ───────────────────────────────────────────────────────────────────────── │
|
||||
│ response = hindsight_litellm.completion( │
|
||||
│ model="gpt-4o-mini", │
|
||||
│ messages=[{"role": "user", "content": "Help me with my Python project"}]│
|
||||
│ ) │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 2. MEMORY RETRIEVAL (before LLM call) │
|
||||
│ ───────────────────────────────────────────────────────────────────────── │
|
||||
│ # hindsight_litellm queries Hindsight for relevant memories │
|
||||
│ │
|
||||
│ # If use_reflect=False (default) - raw memories: │
|
||||
│ memories = hindsight.recall(query="Help me with my Python project") │
|
||||
│ # Returns: ["User prefers pytest", "User is building a FastAPI app", ...] │
|
||||
│ │
|
||||
│ # If use_reflect=True - synthesized context: │
|
||||
│ context = hindsight.reflect(query="Help me with my Python project") │
|
||||
│ # Returns: "The user is an experienced Python developer working on..." │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 3. PROMPT INJECTION │
|
||||
│ ───────────────────────────────────────────────────────────────────────── │
|
||||
│ # Memories are injected into the system message: │
|
||||
│ │
|
||||
│ messages = [ │
|
||||
│ {"role": "system", "content": """ │
|
||||
│ # Relevant Memories │
|
||||
│ 1. [WORLD] User prefers pytest for testing │
|
||||
│ 2. [WORLD] User is building a FastAPI app │
|
||||
│ 3. [OPINION] User likes type hints │
|
||||
│ """}, │
|
||||
│ {"role": "user", "content": "Help me with my Python project"} │
|
||||
│ ] │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 4. LLM CALL │
|
||||
│ ───────────────────────────────────────────────────────────────────────── │
|
||||
│ # The enriched prompt is sent to the LLM │
|
||||
│ response = litellm.completion(model="gpt-4o-mini", messages=messages) │
|
||||
│ │
|
||||
│ # LLM now has context and can give personalized responses like: │
|
||||
│ # "Since you're working on your FastAPI app, here's how to add tests │
|
||||
│ # with pytest..." │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 5. CONVERSATION STORAGE (after LLM call) │
|
||||
│ ───────────────────────────────────────────────────────────────────────── │
|
||||
│ # The conversation is stored to Hindsight for future recall │
|
||||
│ hindsight.retain( │
|
||||
│ content="User: Help me with my Python project\n" │
|
||||
│ "Assistant: Since you're working on FastAPI..." │
|
||||
│ ) │
|
||||
│ # Hindsight extracts facts: "User asked about Python project help" │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 6. RESPONSE RETURNED │
|
||||
│ ───────────────────────────────────────────────────────────────────────── │
|
||||
│ # You receive the response as normal │
|
||||
│ print(response.choices[0].message.content) │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
The memory injection and storage happen automatically - you just use `completion()` as normal.
|
||||
|
||||
## Configuration Options
|
||||
|
||||
```python
|
||||
hindsight_litellm.configure(
|
||||
# Required
|
||||
hindsight_api_url="http://localhost:8888", # Hindsight API server URL
|
||||
bank_id="my-agent", # Memory bank ID
|
||||
|
||||
api_key="your-api-key", # Optional API key for authentication
|
||||
|
||||
# Optional - Memory behavior
|
||||
store_conversations=True, # Store conversations after LLM calls
|
||||
inject_memories=True, # Inject relevant memories into prompts
|
||||
use_reflect=False, # Use reflect API (synthesized) vs recall (raw memories)
|
||||
reflect_include_facts=False, # Include source facts with reflect responses
|
||||
max_memories=None, # Maximum memories to inject (None = unlimited)
|
||||
max_memory_tokens=4096, # Maximum tokens for memory context
|
||||
recall_budget="mid", # Recall budget: "low", "mid", "high"
|
||||
fact_types=["world", "agent"], # Filter fact types to inject
|
||||
|
||||
# Optional - Bank Configuration
|
||||
bank_name="My Agent", # Human-readable display name for the memory bank
|
||||
background="This agent...", # Instructions guiding what Hindsight should remember (see below)
|
||||
|
||||
# Optional - Advanced
|
||||
injection_mode="system_message", # or "prepend_user"
|
||||
excluded_models=["gpt-3.5*"], # Exclude certain models
|
||||
verbose=True, # Enable verbose logging and debug info
|
||||
)
|
||||
```
|
||||
|
||||
### Bank Configuration: background and bank_name
|
||||
|
||||
The `background` and `bank_name` parameters configure the memory bank itself. When provided, `configure()` will automatically create or update the bank with these settings.
|
||||
|
||||
- **bank_name**: A human-readable display name for the memory bank. Useful for identifying banks in the Hindsight UI or when managing multiple banks.
|
||||
|
||||
- **background**: Instructions that guide Hindsight on what information is important to extract and remember from conversations. This influences memory extraction during the `retain` operation and can affect how the bank's "disposition" (skepticism, literalism, empathy) is calibrated.
|
||||
|
||||
```python
|
||||
# Example: Customer support routing agent
|
||||
hindsight_litellm.configure(
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
bank_id="support-router",
|
||||
bank_name="Customer Support Router",
|
||||
background="""This agent routes customer support requests to the appropriate team.
|
||||
Remember which types of issues should go to which teams (billing, technical, sales).
|
||||
Track customer preferences for communication channels and past issue resolutions.
|
||||
Note any escalation patterns or VIP customers who need special handling.""",
|
||||
)
|
||||
```
|
||||
|
||||
### Memory Modes: Reflect vs Recall
|
||||
|
||||
- **Recall mode** (`use_reflect=False`, default): Retrieves raw memory facts and injects them as a numbered list. Best when you need precise, individual memories.
|
||||
- **Reflect mode** (`use_reflect=True`): Synthesizes memories into a coherent context paragraph. Best for natural, conversational memory context.
|
||||
|
||||
```python
|
||||
# Recall mode - raw memories
|
||||
hindsight_litellm.configure(
|
||||
bank_id="my-agent",
|
||||
use_reflect=False, # Default
|
||||
)
|
||||
# Injects: "1. [WORLD] User prefers Python\n2. [OPINION] User dislikes Java..."
|
||||
|
||||
# Reflect mode - synthesized context
|
||||
hindsight_litellm.configure(
|
||||
bank_id="my-agent",
|
||||
use_reflect=True,
|
||||
)
|
||||
# Injects: "Based on previous conversations, the user is a Python developer who..."
|
||||
```
|
||||
|
||||
## Multi-Provider Support
|
||||
|
||||
Works with any LiteLLM-supported provider:
|
||||
|
||||
```python
|
||||
import hindsight_litellm
|
||||
|
||||
hindsight_litellm.configure(
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
bank_id="my-agent",
|
||||
)
|
||||
hindsight_litellm.enable()
|
||||
|
||||
# OpenAI
|
||||
hindsight_litellm.completion(model="gpt-4o", messages=[...])
|
||||
|
||||
# Anthropic
|
||||
hindsight_litellm.completion(model="claude-3-5-sonnet-20241022", messages=[...])
|
||||
|
||||
# Groq
|
||||
hindsight_litellm.completion(model="groq/llama-3.1-70b-versatile", messages=[...])
|
||||
|
||||
# Azure OpenAI
|
||||
hindsight_litellm.completion(model="azure/gpt-4", messages=[...])
|
||||
|
||||
# AWS Bedrock
|
||||
hindsight_litellm.completion(model="bedrock/anthropic.claude-3", messages=[...])
|
||||
|
||||
# Google Vertex AI
|
||||
hindsight_litellm.completion(model="vertex_ai/gemini-pro", messages=[...])
|
||||
```
|
||||
|
||||
## Direct Memory APIs
|
||||
|
||||
### Recall - Query raw memories
|
||||
|
||||
```python
|
||||
from hindsight_litellm import configure, recall
|
||||
|
||||
configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
|
||||
# Query memories
|
||||
memories = recall("what projects am I working on?", budget="mid")
|
||||
for m in memories:
|
||||
print(f"- [{m.fact_type}] {m.text}")
|
||||
|
||||
# Output:
|
||||
# - [world] User is building a FastAPI project
|
||||
# - [opinion] User prefers Python over JavaScript
|
||||
```
|
||||
|
||||
### Reflect - Get synthesized context
|
||||
|
||||
```python
|
||||
from hindsight_litellm import configure, reflect
|
||||
|
||||
configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
|
||||
# Get synthesized memory context
|
||||
result = reflect("what do you know about the user's preferences?")
|
||||
print(result.text)
|
||||
|
||||
# Output:
|
||||
# "Based on our conversations, the user prefers Python for backend development..."
|
||||
```
|
||||
|
||||
### Retain - Store memories
|
||||
|
||||
```python
|
||||
from hindsight_litellm import configure, retain
|
||||
|
||||
configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
|
||||
# Store a memory
|
||||
result = retain(
|
||||
content="User mentioned they're working on a machine learning project",
|
||||
context="Discussion about current projects",
|
||||
)
|
||||
print(f"Retained successfully: {result.success}, items: {result.items_count}")
|
||||
```
|
||||
|
||||
### Async APIs
|
||||
|
||||
```python
|
||||
from hindsight_litellm import arecall, areflect, aretain
|
||||
|
||||
# Async versions of all memory APIs
|
||||
memories = await arecall("what do you know about me?")
|
||||
context = await areflect("summarize user preferences")
|
||||
result = await aretain(content="New information to remember")
|
||||
```
|
||||
|
||||
## Native Client Wrappers
|
||||
|
||||
Alternative to LiteLLM callbacks for direct SDK integration:
|
||||
|
||||
### OpenAI Wrapper
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
from hindsight_litellm import wrap_openai
|
||||
|
||||
client = OpenAI()
|
||||
wrapped = wrap_openai(
|
||||
client,
|
||||
bank_id="my-agent",
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
)
|
||||
|
||||
response = wrapped.chat.completions.create(
|
||||
model="gpt-4",
|
||||
messages=[{"role": "user", "content": "What do you know about me?"}]
|
||||
)
|
||||
```
|
||||
|
||||
### Anthropic Wrapper
|
||||
|
||||
```python
|
||||
from anthropic import Anthropic
|
||||
from hindsight_litellm import wrap_anthropic
|
||||
|
||||
client = Anthropic()
|
||||
wrapped = wrap_anthropic(
|
||||
client,
|
||||
bank_id="my-agent",
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
)
|
||||
|
||||
response = wrapped.messages.create(
|
||||
model="claude-3-5-sonnet-20241022",
|
||||
max_tokens=1024,
|
||||
messages=[{"role": "user", "content": "Hello!"}]
|
||||
)
|
||||
```
|
||||
|
||||
## Debug Mode
|
||||
|
||||
When `verbose=True`, you can inspect exactly what memories are being injected:
|
||||
|
||||
```python
|
||||
from hindsight_litellm import configure, enable, completion, get_last_injection_debug
|
||||
|
||||
configure(
|
||||
bank_id="my-agent",
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
verbose=True,
|
||||
use_reflect=True,
|
||||
)
|
||||
enable()
|
||||
|
||||
response = completion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "What's my favorite color?"}]
|
||||
)
|
||||
|
||||
# Inspect what was injected
|
||||
debug = get_last_injection_debug()
|
||||
if debug:
|
||||
print(f"Mode: {debug.mode}") # "reflect" or "recall"
|
||||
print(f"Injected: {debug.injected}") # True/False
|
||||
print(f"Results: {debug.results_count}")
|
||||
print(f"Memory context:\n{debug.memory_context}")
|
||||
if debug.error:
|
||||
print(f"Error: {debug.error}")
|
||||
```
|
||||
|
||||
## Context Manager
|
||||
|
||||
```python
|
||||
from hindsight_litellm import hindsight_memory
|
||||
import litellm
|
||||
|
||||
with hindsight_memory(bank_id="user-123"):
|
||||
response = litellm.completion(model="gpt-4", messages=[...])
|
||||
# Memory integration automatically disabled after context
|
||||
```
|
||||
|
||||
## Disabling and Cleanup
|
||||
|
||||
```python
|
||||
from hindsight_litellm import disable, cleanup
|
||||
|
||||
# Temporarily disable memory integration
|
||||
disable()
|
||||
|
||||
# Clean up all resources (call when shutting down)
|
||||
cleanup()
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### Main Functions
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `configure(...)` | Configure global Hindsight settings |
|
||||
| `enable()` | Enable memory integration with LiteLLM |
|
||||
| `disable()` | Disable memory integration |
|
||||
| `is_enabled()` | Check if memory integration is enabled |
|
||||
| `cleanup()` | Clean up all resources |
|
||||
|
||||
### Configuration Functions
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `get_config()` | Get current configuration |
|
||||
| `is_configured()` | Check if Hindsight is configured |
|
||||
| `reset_config()` | Reset configuration to defaults |
|
||||
|
||||
### Memory Functions
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `recall(query, ...)` | Synchronously query raw memories |
|
||||
| `arecall(query, ...)` | Asynchronously query raw memories |
|
||||
| `reflect(query, ...)` | Synchronously get synthesized memory context |
|
||||
| `areflect(query, ...)` | Asynchronously get synthesized memory context |
|
||||
| `retain(content, ...)` | Synchronously store a memory |
|
||||
| `aretain(content, ...)` | Asynchronously store a memory |
|
||||
|
||||
### Debug Functions
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `get_last_injection_debug()` | Get debug info from last memory injection |
|
||||
| `clear_injection_debug()` | Clear stored debug info |
|
||||
|
||||
### Client Wrappers
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `wrap_openai(client, ...)` | Wrap OpenAI client with memory |
|
||||
| `wrap_anthropic(client, ...)` | Wrap Anthropic client with memory |
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python >= 3.10
|
||||
- litellm >= 1.40.0
|
||||
- A running Hindsight API server
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
@@ -0,0 +1,817 @@
|
||||
"""Hindsight-LiteLLM: Universal LLM memory integration via LiteLLM.
|
||||
|
||||
This package provides automatic memory integration for any LLM provider
|
||||
supported by LiteLLM (100+ providers including OpenAI, Anthropic, Groq,
|
||||
Azure, AWS Bedrock, Google Vertex AI, and more).
|
||||
|
||||
Features:
|
||||
- Automatic memory injection before LLM calls
|
||||
- Automatic conversation storage after LLM calls
|
||||
- Works with any LiteLLM-supported provider
|
||||
- Zero code changes to existing LiteLLM usage
|
||||
- Multi-user support via separate bank_ids
|
||||
- Document grouping for conversation threading
|
||||
- Direct recall API for manual memory queries
|
||||
- Native client wrappers for OpenAI and Anthropic
|
||||
|
||||
Basic usage:
|
||||
>>> from hindsight_litellm import configure, enable
|
||||
>>>
|
||||
>>> # Configure Hindsight integration
|
||||
>>> configure(
|
||||
... hindsight_api_url="http://localhost:8888",
|
||||
... bank_id="user-123", # Use separate bank_ids for multi-user support
|
||||
... store_conversations=True,
|
||||
... inject_memories=True,
|
||||
... )
|
||||
>>>
|
||||
>>> # Enable memory integration
|
||||
>>> enable()
|
||||
>>>
|
||||
>>> # Now use LiteLLM as normal - memory integration is automatic
|
||||
>>> import litellm
|
||||
>>> response = litellm.completion(
|
||||
... model="gpt-4",
|
||||
... messages=[{"role": "user", "content": "What did we discuss about AI?"}]
|
||||
... )
|
||||
|
||||
Direct recall API:
|
||||
>>> from hindsight_litellm import configure, recall
|
||||
>>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
>>>
|
||||
>>> # Query memories directly
|
||||
>>> memories = recall("what projects am I working on?")
|
||||
>>> for m in memories:
|
||||
... print(f"- [{m.fact_type}] {m.text}")
|
||||
|
||||
Native client wrappers:
|
||||
>>> from openai import OpenAI
|
||||
>>> from hindsight_litellm import wrap_openai
|
||||
>>>
|
||||
>>> client = OpenAI()
|
||||
>>> wrapped = wrap_openai(client, bank_id="user-123")
|
||||
>>>
|
||||
>>> response = wrapped.chat.completions.create(
|
||||
... model="gpt-4",
|
||||
... messages=[{"role": "user", "content": "Hello!"}]
|
||||
... )
|
||||
|
||||
Works with any LiteLLM-supported provider:
|
||||
>>> # OpenAI
|
||||
>>> litellm.completion(model="gpt-4", messages=[...])
|
||||
>>>
|
||||
>>> # Anthropic
|
||||
>>> litellm.completion(model="claude-3-opus-20240229", messages=[...])
|
||||
>>>
|
||||
>>> # Groq
|
||||
>>> litellm.completion(model="groq/llama-3.1-70b-versatile", messages=[...])
|
||||
>>>
|
||||
>>> # Azure OpenAI
|
||||
>>> litellm.completion(model="azure/gpt-4", messages=[...])
|
||||
>>>
|
||||
>>> # AWS Bedrock
|
||||
>>> litellm.completion(model="bedrock/anthropic.claude-3", messages=[...])
|
||||
>>>
|
||||
>>> # Google Vertex AI
|
||||
>>> litellm.completion(model="vertex_ai/gemini-pro", messages=[...])
|
||||
|
||||
Context manager usage:
|
||||
>>> from hindsight_litellm import hindsight_memory
|
||||
>>>
|
||||
>>> with hindsight_memory(bank_id="user-123"):
|
||||
... response = litellm.completion(model="gpt-4", messages=[...])
|
||||
>>> # Memory integration automatically disabled after context
|
||||
|
||||
Configuration options:
|
||||
- hindsight_api_url: URL of your Hindsight API server
|
||||
- bank_id: Memory bank ID for memory operations (required). For multi-user
|
||||
support, use different bank_ids per user (e.g., f"user-{user_id}")
|
||||
- api_key: Optional API key for Hindsight authentication
|
||||
- store_conversations: Whether to store conversations (default: True)
|
||||
- inject_memories: Whether to inject relevant memories (default: True)
|
||||
- injection_mode: How to inject memories (system_message or prepend_user)
|
||||
- max_memories: Maximum number of memories to inject (None = unlimited)
|
||||
- recall_budget: Budget for memory recall (low, mid, high)
|
||||
- excluded_models: List of model patterns to exclude from interception
|
||||
- verbose: Enable verbose logging
|
||||
- bank_name: Display name for the memory bank
|
||||
- background: Instructions that help Hindsight understand what to remember
|
||||
|
||||
Background example:
|
||||
>>> configure(
|
||||
... bank_id="routing-agent",
|
||||
... background="This agent routes customer requests to support channels. "
|
||||
... "Remember which types of issues should go to which channels.",
|
||||
... )
|
||||
"""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, List, Any
|
||||
|
||||
import litellm
|
||||
|
||||
from .config import (
|
||||
configure,
|
||||
get_config,
|
||||
is_configured,
|
||||
reset_config,
|
||||
HindsightConfig,
|
||||
MemoryInjectionMode,
|
||||
)
|
||||
from .callbacks import (
|
||||
HindsightCallback,
|
||||
get_callback,
|
||||
cleanup_callback,
|
||||
)
|
||||
from .wrappers import (
|
||||
recall,
|
||||
arecall,
|
||||
RecallResult,
|
||||
RecallResponse,
|
||||
RecallDebugInfo,
|
||||
reflect,
|
||||
areflect,
|
||||
ReflectResult,
|
||||
ReflectDebugInfo,
|
||||
retain,
|
||||
aretain,
|
||||
RetainResult,
|
||||
RetainDebugInfo,
|
||||
wrap_openai,
|
||||
wrap_anthropic,
|
||||
HindsightOpenAI,
|
||||
HindsightAnthropic,
|
||||
)
|
||||
|
||||
|
||||
__version__ = "0.1.0"
|
||||
|
||||
# Track whether we've registered with LiteLLM
|
||||
_enabled = False
|
||||
|
||||
# Store original functions for restoration
|
||||
_original_completion = None
|
||||
_original_acompletion = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class InjectionDebugInfo:
|
||||
"""Debug information from a memory injection operation.
|
||||
|
||||
This is populated when verbose=True in the config and can be retrieved
|
||||
via get_last_injection_debug() after a completion() call.
|
||||
|
||||
Attributes:
|
||||
mode: The injection mode used ("reflect" or "recall")
|
||||
query: The user query used for memory lookup
|
||||
bank_id: The bank ID used
|
||||
memory_context: The formatted memory context that was injected
|
||||
reflect_text: The raw reflect text (when mode="reflect")
|
||||
reflect_facts: The facts used to generate the reflect response (when reflect_include_facts=True)
|
||||
recall_results: The raw recall results (when mode="recall")
|
||||
results_count: Number of memories/results found
|
||||
injected: Whether memories were actually injected into the prompt
|
||||
error: Error message if injection failed (None on success)
|
||||
"""
|
||||
mode: str # "reflect" or "recall"
|
||||
query: str
|
||||
bank_id: str
|
||||
memory_context: str # The formatted context that was injected
|
||||
reflect_text: Optional[str] = None # Raw reflect response text
|
||||
reflect_facts: Optional[List[dict]] = None # Facts used by reflect (when reflect_include_facts=True)
|
||||
recall_results: Optional[List[dict]] = None # Raw recall results
|
||||
results_count: int = 0
|
||||
injected: bool = False
|
||||
error: Optional[str] = None # Error message if injection failed
|
||||
|
||||
|
||||
# Store the last injection debug info (populated when verbose=True)
|
||||
_last_injection_debug: Optional[InjectionDebugInfo] = None
|
||||
|
||||
|
||||
def get_last_injection_debug() -> Optional[InjectionDebugInfo]:
|
||||
"""Get debug info from the last memory injection operation.
|
||||
|
||||
When verbose=True in the config, this returns information about
|
||||
what memories were injected into the last completion() call.
|
||||
|
||||
Returns:
|
||||
InjectionDebugInfo if verbose mode captured injection info, None otherwise
|
||||
|
||||
Example:
|
||||
>>> from hindsight_litellm import configure, enable, completion, get_last_injection_debug
|
||||
>>> configure(bank_id="my-agent", verbose=True, use_reflect=True)
|
||||
>>> enable()
|
||||
>>> response = completion(model="gpt-4o-mini", messages=[...])
|
||||
>>> debug = get_last_injection_debug()
|
||||
>>> if debug:
|
||||
... print(f"Injected {debug.results_count} memories via {debug.mode}")
|
||||
... print(f"Reflect text: {debug.reflect_text}")
|
||||
"""
|
||||
return _last_injection_debug
|
||||
|
||||
|
||||
def clear_injection_debug() -> None:
|
||||
"""Clear the stored injection debug info."""
|
||||
global _last_injection_debug
|
||||
_last_injection_debug = None
|
||||
|
||||
|
||||
def _inject_memories(messages: List[dict]) -> List[dict]:
|
||||
"""Inject memories into messages list.
|
||||
|
||||
Returns the modified messages list with memories injected into the system message.
|
||||
Uses reflect API when config.use_reflect=True, otherwise uses recall API.
|
||||
|
||||
When verbose=True in config, stores debug info retrievable via get_last_injection_debug().
|
||||
"""
|
||||
global _last_injection_debug
|
||||
import logging
|
||||
|
||||
# Clear previous debug info
|
||||
_last_injection_debug = None
|
||||
|
||||
if not is_configured():
|
||||
return messages
|
||||
|
||||
config = get_config()
|
||||
if not config or not config.enabled or not config.inject_memories:
|
||||
return messages
|
||||
|
||||
if not messages:
|
||||
return messages
|
||||
|
||||
# Extract user query from last user message
|
||||
user_query = None
|
||||
for msg in reversed(messages):
|
||||
if msg.get("role") == "user":
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str):
|
||||
user_query = content
|
||||
break
|
||||
|
||||
if not user_query:
|
||||
return messages
|
||||
|
||||
try:
|
||||
from hindsight_client import Hindsight
|
||||
|
||||
# Use bank_id directly (no entity scoping)
|
||||
bank_id = config.bank_id
|
||||
|
||||
# Track debug info
|
||||
mode = "reflect" if config.use_reflect else "recall"
|
||||
reflect_text = None
|
||||
reflect_facts = None
|
||||
recall_results = None
|
||||
results_count = 0
|
||||
memory_context = ""
|
||||
|
||||
# Create client
|
||||
client = Hindsight(base_url=config.hindsight_api_url, timeout=30.0)
|
||||
|
||||
# Use reflect API if use_reflect is enabled
|
||||
if config.use_reflect:
|
||||
# If reflect_include_facts is enabled, use the API directly to include facts
|
||||
if config.reflect_include_facts:
|
||||
from hindsight_client_api.models import reflect_request, reflect_include_options
|
||||
request_obj = reflect_request.ReflectRequest(
|
||||
query=user_query,
|
||||
budget=config.recall_budget or "mid",
|
||||
include=reflect_include_options.ReflectIncludeOptions(facts={}),
|
||||
)
|
||||
import asyncio
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
result = loop.run_until_complete(client._api.reflect(bank_id, request_obj))
|
||||
# Extract facts from based_on
|
||||
if hasattr(result, 'based_on') and result.based_on:
|
||||
reflect_facts = [
|
||||
{
|
||||
"text": f.text if hasattr(f, 'text') else str(f),
|
||||
"type": getattr(f, 'type', None),
|
||||
"context": getattr(f, 'context', None),
|
||||
}
|
||||
for f in result.based_on
|
||||
]
|
||||
else:
|
||||
result = client.reflect(
|
||||
bank_id=bank_id,
|
||||
query=user_query,
|
||||
budget=config.recall_budget or "mid",
|
||||
)
|
||||
reflect_text = result.text if hasattr(result, 'text') else str(result)
|
||||
|
||||
if not reflect_text:
|
||||
# Store debug info for empty result
|
||||
if config.verbose:
|
||||
_last_injection_debug = InjectionDebugInfo(
|
||||
mode=mode,
|
||||
query=user_query,
|
||||
bank_id=bank_id,
|
||||
memory_context="",
|
||||
reflect_text="",
|
||||
reflect_facts=reflect_facts,
|
||||
results_count=0,
|
||||
injected=False,
|
||||
)
|
||||
return messages
|
||||
|
||||
results_count = 1 # reflect returns a single synthesized response
|
||||
memory_context = (
|
||||
"# Relevant Context from Memory\n"
|
||||
f"{reflect_text}"
|
||||
)
|
||||
else:
|
||||
# Use recall API (original behavior)
|
||||
result = client.recall(
|
||||
bank_id=bank_id,
|
||||
query=user_query,
|
||||
budget=config.recall_budget or "mid",
|
||||
max_tokens=config.max_memory_tokens or 4096,
|
||||
types=config.fact_types,
|
||||
)
|
||||
# client.recall() returns a list directly, not an object with .results
|
||||
if isinstance(result, list):
|
||||
results = result
|
||||
elif hasattr(result, 'results'):
|
||||
results = result.results
|
||||
else:
|
||||
results = []
|
||||
# Convert to dicts for debug info
|
||||
recall_results = [
|
||||
{
|
||||
"text": r.text if hasattr(r, 'text') else str(r),
|
||||
"type": getattr(r, 'type', 'world'),
|
||||
}
|
||||
for r in results
|
||||
]
|
||||
|
||||
if not results:
|
||||
# Store debug info for empty result
|
||||
if config.verbose:
|
||||
_last_injection_debug = InjectionDebugInfo(
|
||||
mode=mode,
|
||||
query=user_query,
|
||||
bank_id=bank_id,
|
||||
memory_context="",
|
||||
recall_results=[],
|
||||
results_count=0,
|
||||
injected=False,
|
||||
)
|
||||
return messages
|
||||
|
||||
# Format memories (apply limit if set, otherwise use all)
|
||||
results_to_use = results[:config.max_memories] if config.max_memories else results
|
||||
memory_lines = []
|
||||
for i, r in enumerate(results_to_use, 1):
|
||||
text = r.text if hasattr(r, 'text') else str(r)
|
||||
fact_type = getattr(r, 'type', 'world')
|
||||
if text:
|
||||
type_label = fact_type.upper() if fact_type else "MEMORY"
|
||||
memory_lines.append(f"{i}. [{type_label}] {text}")
|
||||
|
||||
if not memory_lines:
|
||||
if config.verbose:
|
||||
_last_injection_debug = InjectionDebugInfo(
|
||||
mode=mode,
|
||||
query=user_query,
|
||||
bank_id=bank_id,
|
||||
memory_context="",
|
||||
recall_results=recall_results,
|
||||
results_count=0,
|
||||
injected=False,
|
||||
)
|
||||
return messages
|
||||
|
||||
results_count = len(memory_lines)
|
||||
memory_context = (
|
||||
"# Relevant Memories\n"
|
||||
"The following information from memory may be relevant:\n\n"
|
||||
+ "\n".join(memory_lines)
|
||||
)
|
||||
|
||||
# Inject into messages
|
||||
updated_messages = list(messages)
|
||||
|
||||
# Find existing system message or create new one
|
||||
found_system = False
|
||||
for i, msg in enumerate(updated_messages):
|
||||
if msg.get("role") == "system":
|
||||
existing_content = msg.get("content", "")
|
||||
updated_messages[i] = {
|
||||
**msg,
|
||||
"content": f"{existing_content}\n\n{memory_context}"
|
||||
}
|
||||
found_system = True
|
||||
break
|
||||
|
||||
if not found_system:
|
||||
updated_messages.insert(0, {
|
||||
"role": "system",
|
||||
"content": memory_context
|
||||
})
|
||||
|
||||
# Store debug info when verbose
|
||||
if config.verbose:
|
||||
_last_injection_debug = InjectionDebugInfo(
|
||||
mode=mode,
|
||||
query=user_query,
|
||||
bank_id=bank_id,
|
||||
memory_context=memory_context,
|
||||
reflect_text=reflect_text,
|
||||
reflect_facts=reflect_facts,
|
||||
recall_results=recall_results,
|
||||
results_count=results_count,
|
||||
injected=True,
|
||||
)
|
||||
logger = logging.getLogger("hindsight_litellm")
|
||||
logger.info(f"Injected memories using {mode} into prompt")
|
||||
|
||||
return updated_messages
|
||||
|
||||
except ImportError as e:
|
||||
if config.verbose:
|
||||
logging.getLogger("hindsight_litellm").warning(
|
||||
f"hindsight_client not installed: {e}. Install with: pip install hindsight-client"
|
||||
)
|
||||
_last_injection_debug = InjectionDebugInfo(
|
||||
mode="reflect" if config.use_reflect else "recall",
|
||||
query=user_query or "",
|
||||
bank_id=config.bank_id or "",
|
||||
memory_context="",
|
||||
results_count=0,
|
||||
injected=False,
|
||||
error=f"hindsight_client not installed: {e}",
|
||||
)
|
||||
return messages
|
||||
except Exception as e:
|
||||
# Always set debug info on error when verbose mode is on
|
||||
if config.verbose:
|
||||
logging.getLogger("hindsight_litellm").warning(f"Failed to inject memories: {e}")
|
||||
_last_injection_debug = InjectionDebugInfo(
|
||||
mode="reflect" if config.use_reflect else "recall",
|
||||
query=user_query or "",
|
||||
bank_id=config.bank_id or "",
|
||||
memory_context="",
|
||||
results_count=0,
|
||||
injected=False,
|
||||
error=str(e),
|
||||
)
|
||||
return messages
|
||||
|
||||
|
||||
def _wrapped_completion(*args, **kwargs):
|
||||
"""Wrapper for litellm.completion that injects memories before the call."""
|
||||
# Inject memories into messages
|
||||
if "messages" in kwargs:
|
||||
kwargs["messages"] = _inject_memories(kwargs["messages"])
|
||||
elif args and len(args) > 1:
|
||||
# messages might be second positional arg after model
|
||||
args = list(args)
|
||||
if isinstance(args[1], list):
|
||||
args[1] = _inject_memories(args[1])
|
||||
args = tuple(args)
|
||||
|
||||
# Call original
|
||||
return _original_completion(*args, **kwargs)
|
||||
|
||||
|
||||
async def _wrapped_acompletion(*args, **kwargs):
|
||||
"""Wrapper for litellm.acompletion that injects memories before the call."""
|
||||
# Inject memories into messages
|
||||
if "messages" in kwargs:
|
||||
kwargs["messages"] = _inject_memories(kwargs["messages"])
|
||||
elif args and len(args) > 1:
|
||||
args = list(args)
|
||||
if isinstance(args[1], list):
|
||||
args[1] = _inject_memories(args[1])
|
||||
args = tuple(args)
|
||||
|
||||
# Call original
|
||||
return await _original_acompletion(*args, **kwargs)
|
||||
|
||||
|
||||
def enable() -> None:
|
||||
"""Enable Hindsight memory integration with LiteLLM.
|
||||
|
||||
This monkeypatches LiteLLM functions to:
|
||||
1. Inject relevant memories into prompts before LLM calls
|
||||
2. Store conversations to Hindsight after successful LLM calls
|
||||
|
||||
Must be called after configure() to take effect.
|
||||
|
||||
Example:
|
||||
>>> from hindsight_litellm import configure, enable
|
||||
>>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
>>> enable()
|
||||
>>>
|
||||
>>> # Now all LiteLLM calls will have memory integration
|
||||
>>> import litellm
|
||||
>>> response = litellm.completion(model="gpt-4", messages=[...])
|
||||
"""
|
||||
global _enabled, _original_completion, _original_acompletion
|
||||
|
||||
if _enabled:
|
||||
return # Already enabled
|
||||
|
||||
if not is_configured():
|
||||
raise RuntimeError(
|
||||
"Hindsight not configured. Call configure() before enable()."
|
||||
)
|
||||
|
||||
# Store original functions and monkeypatch for memory injection
|
||||
_original_completion = litellm.completion
|
||||
_original_acompletion = litellm.acompletion
|
||||
litellm.completion = _wrapped_completion
|
||||
litellm.acompletion = _wrapped_acompletion
|
||||
|
||||
# Get or create the callback instance for storing conversations
|
||||
callback = get_callback()
|
||||
|
||||
# Register callback using litellm.callbacks for conversation storage
|
||||
if callback not in litellm.callbacks:
|
||||
litellm.callbacks.append(callback)
|
||||
|
||||
_enabled = True
|
||||
|
||||
config = get_config()
|
||||
if config and config.verbose:
|
||||
print(f"Hindsight memory enabled for bank: {config.bank_id}")
|
||||
|
||||
|
||||
def disable() -> None:
|
||||
"""Disable Hindsight memory integration with LiteLLM.
|
||||
|
||||
This restores the original LiteLLM functions and removes callbacks,
|
||||
stopping memory injection and conversation storage.
|
||||
|
||||
Example:
|
||||
>>> from hindsight_litellm import disable
|
||||
>>> disable() # Stop memory integration
|
||||
"""
|
||||
global _enabled, _original_completion, _original_acompletion
|
||||
|
||||
if not _enabled:
|
||||
return # Already disabled
|
||||
|
||||
# Restore original functions
|
||||
if _original_completion is not None:
|
||||
litellm.completion = _original_completion
|
||||
_original_completion = None
|
||||
if _original_acompletion is not None:
|
||||
litellm.acompletion = _original_acompletion
|
||||
_original_acompletion = None
|
||||
|
||||
# Remove callback from litellm.callbacks
|
||||
callback = get_callback()
|
||||
if callback in litellm.callbacks:
|
||||
litellm.callbacks.remove(callback)
|
||||
|
||||
_enabled = False
|
||||
|
||||
config = get_config()
|
||||
if config and config.verbose:
|
||||
print("Hindsight memory disabled")
|
||||
|
||||
|
||||
def is_enabled() -> bool:
|
||||
"""Check if Hindsight memory integration is currently enabled.
|
||||
|
||||
Returns:
|
||||
True if enable() has been called and not subsequently disabled
|
||||
"""
|
||||
return _enabled
|
||||
|
||||
|
||||
def cleanup() -> None:
|
||||
"""Clean up all Hindsight resources.
|
||||
|
||||
This disables the integration and closes any open connections.
|
||||
Call this when shutting down your application.
|
||||
|
||||
Example:
|
||||
>>> from hindsight_litellm import cleanup
|
||||
>>> cleanup() # Clean up when done
|
||||
"""
|
||||
disable()
|
||||
cleanup_callback()
|
||||
reset_config()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Convenience wrappers - use hindsight_litellm.completion() directly
|
||||
# =============================================================================
|
||||
|
||||
def completion(*args, **kwargs):
|
||||
"""Call LiteLLM completion with Hindsight memory integration.
|
||||
|
||||
This is a convenience wrapper that delegates to litellm.completion().
|
||||
Memory injection and storage happen automatically if configured and enabled.
|
||||
|
||||
Args:
|
||||
*args: Positional arguments passed to litellm.completion()
|
||||
**kwargs: Keyword arguments passed to litellm.completion()
|
||||
|
||||
Returns:
|
||||
LiteLLM ModelResponse object
|
||||
|
||||
Example:
|
||||
>>> import hindsight_litellm
|
||||
>>>
|
||||
>>> hindsight_litellm.configure(
|
||||
... hindsight_api_url="http://localhost:8888",
|
||||
... bank_id="my-agent",
|
||||
... )
|
||||
>>> hindsight_litellm.enable()
|
||||
>>>
|
||||
>>> # Use directly - no need to import litellm separately
|
||||
>>> response = hindsight_litellm.completion(
|
||||
... model="gpt-4o-mini",
|
||||
... messages=[{"role": "user", "content": "Hello!"}]
|
||||
... )
|
||||
"""
|
||||
return litellm.completion(*args, **kwargs)
|
||||
|
||||
|
||||
async def acompletion(*args, **kwargs):
|
||||
"""Call LiteLLM async completion with Hindsight memory integration.
|
||||
|
||||
This is a convenience wrapper that delegates to litellm.acompletion().
|
||||
Memory injection and storage happen automatically if configured and enabled.
|
||||
|
||||
Args:
|
||||
*args: Positional arguments passed to litellm.acompletion()
|
||||
**kwargs: Keyword arguments passed to litellm.acompletion()
|
||||
|
||||
Returns:
|
||||
LiteLLM ModelResponse object
|
||||
|
||||
Example:
|
||||
>>> import hindsight_litellm
|
||||
>>> import asyncio
|
||||
>>>
|
||||
>>> hindsight_litellm.configure(
|
||||
... hindsight_api_url="http://localhost:8888",
|
||||
... bank_id="my-agent",
|
||||
... )
|
||||
>>> hindsight_litellm.enable()
|
||||
>>>
|
||||
>>> async def main():
|
||||
... response = await hindsight_litellm.acompletion(
|
||||
... model="gpt-4o-mini",
|
||||
... messages=[{"role": "user", "content": "Hello!"}]
|
||||
... )
|
||||
... return response
|
||||
>>>
|
||||
>>> asyncio.run(main())
|
||||
"""
|
||||
return await litellm.acompletion(*args, **kwargs)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def hindsight_memory(
|
||||
hindsight_api_url: str = "http://localhost:8888",
|
||||
bank_id: Optional[str] = None,
|
||||
api_key: Optional[str] = None,
|
||||
store_conversations: bool = True,
|
||||
inject_memories: bool = True,
|
||||
injection_mode: MemoryInjectionMode = MemoryInjectionMode.SYSTEM_MESSAGE,
|
||||
max_memories: Optional[int] = None,
|
||||
max_memory_tokens: int = 4096,
|
||||
recall_budget: str = "mid",
|
||||
fact_types: Optional[List[str]] = None,
|
||||
document_id: Optional[str] = None,
|
||||
excluded_models: Optional[List[str]] = None,
|
||||
verbose: bool = False,
|
||||
bank_name: Optional[str] = None,
|
||||
background: Optional[str] = None,
|
||||
):
|
||||
"""Context manager for temporary Hindsight memory integration.
|
||||
|
||||
Use this to enable memory integration for a specific block of code,
|
||||
automatically cleaning up afterwards.
|
||||
|
||||
Args:
|
||||
hindsight_api_url: URL of the Hindsight API server
|
||||
bank_id: Memory bank ID for memory operations (required). For multi-user
|
||||
support, use different bank_ids per user (e.g., f"user-{user_id}")
|
||||
api_key: Optional API key for Hindsight authentication
|
||||
store_conversations: Whether to store conversations
|
||||
inject_memories: Whether to inject relevant memories
|
||||
injection_mode: How to inject memories
|
||||
max_memories: Maximum number of memories to inject (None = unlimited)
|
||||
max_memory_tokens: Maximum tokens for memory context
|
||||
recall_budget: Budget for memory recall (low, mid, high)
|
||||
fact_types: List of fact types to filter (world, agent, opinion, observation)
|
||||
document_id: Optional document ID for grouping conversations
|
||||
excluded_models: List of model patterns to exclude
|
||||
verbose: Enable verbose logging
|
||||
bank_name: Optional display name for the memory bank
|
||||
background: Optional background/instructions for memory extraction
|
||||
|
||||
Example:
|
||||
>>> from hindsight_litellm import hindsight_memory
|
||||
>>> import litellm
|
||||
>>>
|
||||
>>> with hindsight_memory(bank_id="user-123"):
|
||||
... response = litellm.completion(model="gpt-4", messages=[...])
|
||||
>>> # Memory integration automatically disabled after context
|
||||
"""
|
||||
# Save previous state
|
||||
was_enabled = is_enabled()
|
||||
previous_config = get_config()
|
||||
|
||||
try:
|
||||
# Configure and enable
|
||||
configure(
|
||||
hindsight_api_url=hindsight_api_url,
|
||||
bank_id=bank_id,
|
||||
api_key=api_key,
|
||||
store_conversations=store_conversations,
|
||||
inject_memories=inject_memories,
|
||||
injection_mode=injection_mode,
|
||||
max_memories=max_memories,
|
||||
max_memory_tokens=max_memory_tokens,
|
||||
recall_budget=recall_budget,
|
||||
fact_types=fact_types,
|
||||
document_id=document_id,
|
||||
excluded_models=excluded_models,
|
||||
verbose=verbose,
|
||||
bank_name=bank_name,
|
||||
background=background,
|
||||
)
|
||||
enable()
|
||||
yield
|
||||
finally:
|
||||
# Restore previous state
|
||||
disable()
|
||||
if previous_config:
|
||||
configure(
|
||||
hindsight_api_url=previous_config.hindsight_api_url,
|
||||
bank_id=previous_config.bank_id,
|
||||
api_key=previous_config.api_key,
|
||||
store_conversations=previous_config.store_conversations,
|
||||
inject_memories=previous_config.inject_memories,
|
||||
injection_mode=previous_config.injection_mode,
|
||||
max_memories=previous_config.max_memories,
|
||||
max_memory_tokens=previous_config.max_memory_tokens,
|
||||
recall_budget=previous_config.recall_budget,
|
||||
fact_types=previous_config.fact_types,
|
||||
document_id=previous_config.document_id,
|
||||
excluded_models=previous_config.excluded_models,
|
||||
verbose=previous_config.verbose,
|
||||
bank_name=previous_config.bank_name,
|
||||
background=previous_config.background,
|
||||
)
|
||||
if was_enabled:
|
||||
enable()
|
||||
else:
|
||||
reset_config()
|
||||
|
||||
|
||||
__all__ = [
|
||||
# Main API
|
||||
"configure",
|
||||
"enable",
|
||||
"disable",
|
||||
"is_enabled",
|
||||
"cleanup",
|
||||
"hindsight_memory",
|
||||
# LLM completion wrappers (convenience)
|
||||
"completion",
|
||||
"acompletion",
|
||||
# Direct memory APIs
|
||||
"recall",
|
||||
"arecall",
|
||||
"RecallResult",
|
||||
"reflect",
|
||||
"areflect",
|
||||
"ReflectResult",
|
||||
"retain",
|
||||
"aretain",
|
||||
"RetainResult",
|
||||
# Native client wrappers
|
||||
"wrap_openai",
|
||||
"wrap_anthropic",
|
||||
"HindsightOpenAI",
|
||||
"HindsightAnthropic",
|
||||
# Configuration
|
||||
"get_config",
|
||||
"is_configured",
|
||||
"reset_config",
|
||||
"HindsightConfig",
|
||||
"MemoryInjectionMode",
|
||||
# Injection debug (verbose mode)
|
||||
"get_last_injection_debug",
|
||||
"clear_injection_debug",
|
||||
"InjectionDebugInfo",
|
||||
# Callback (for advanced usage)
|
||||
"HindsightCallback",
|
||||
"get_callback",
|
||||
"cleanup_callback",
|
||||
]
|
||||
@@ -0,0 +1,640 @@
|
||||
"""LiteLLM callback handlers for Hindsight memory integration.
|
||||
|
||||
This module implements LiteLLM's CustomLogger interface to intercept
|
||||
LLM calls and integrate with Hindsight for memory injection and storage.
|
||||
|
||||
Uses direct HTTP calls via requests/httpx to avoid async event loop conflicts
|
||||
when the hindsight_client's async methods are called from LiteLLM callbacks.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import fnmatch
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
import asyncio
|
||||
import threading
|
||||
import concurrent.futures
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
from .config import get_config, is_configured, HindsightConfig, MemoryInjectionMode
|
||||
|
||||
# Use requests for sync HTTP calls to avoid async event loop issues
|
||||
try:
|
||||
import requests
|
||||
HAS_REQUESTS = True
|
||||
except ImportError:
|
||||
HAS_REQUESTS = False
|
||||
|
||||
try:
|
||||
import httpx
|
||||
HAS_HTTPX = True
|
||||
except ImportError:
|
||||
HAS_HTTPX = False
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Thread pool for running async operations in background
|
||||
_executor = concurrent.futures.ThreadPoolExecutor(max_workers=4, thread_name_prefix="hindsight-")
|
||||
|
||||
|
||||
class HindsightCallback(CustomLogger):
|
||||
"""LiteLLM custom logger that integrates with Hindsight memory system.
|
||||
|
||||
This callback handler:
|
||||
1. Injects relevant memories into prompts before LLM calls
|
||||
2. Stores conversations to Hindsight after successful LLM calls
|
||||
|
||||
Features:
|
||||
- Works with 100+ LLM providers via LiteLLM
|
||||
- Deduplication to avoid storing duplicate conversations
|
||||
- Configurable memory injection modes
|
||||
- Support for entity observations in recall
|
||||
|
||||
Usage:
|
||||
>>> from hindsight_litellm import configure, enable
|
||||
>>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
>>> enable()
|
||||
>>>
|
||||
>>> # Now all LiteLLM calls will have memory integration
|
||||
>>> import litellm
|
||||
>>> response = litellm.completion(
|
||||
... model="gpt-4",
|
||||
... messages=[{"role": "user", "content": "What did we discuss?"}]
|
||||
... )
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the Hindsight callback handler."""
|
||||
super().__init__()
|
||||
self._http_session = None
|
||||
self._http_lock = threading.Lock()
|
||||
# Track recently stored conversation hashes for deduplication
|
||||
self._recent_hashes: Set[str] = set()
|
||||
self._max_hash_cache = 1000
|
||||
|
||||
def _get_http_session(self):
|
||||
"""Get or create a requests Session (thread-safe)."""
|
||||
if self._http_session is None:
|
||||
with self._http_lock:
|
||||
if self._http_session is None:
|
||||
if HAS_REQUESTS:
|
||||
self._http_session = requests.Session()
|
||||
elif HAS_HTTPX:
|
||||
self._http_session = httpx.Client(timeout=30.0)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"Neither 'requests' nor 'httpx' is installed. "
|
||||
"Please install one: pip install requests"
|
||||
)
|
||||
return self._http_session
|
||||
|
||||
def _http_post(self, url: str, json_data: dict, config: HindsightConfig) -> Optional[dict]:
|
||||
"""Make a synchronous HTTP POST request."""
|
||||
try:
|
||||
session = self._get_http_session()
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if config.api_key:
|
||||
headers["Authorization"] = f"Bearer {config.api_key}"
|
||||
|
||||
if HAS_REQUESTS:
|
||||
response = session.post(url, json=json_data, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
elif HAS_HTTPX:
|
||||
response = session.post(url, json=json_data, headers=headers)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except Exception as e:
|
||||
if config.verbose:
|
||||
logger.warning(f"HTTP POST failed: {e}")
|
||||
return None
|
||||
|
||||
def _should_skip_model(self, model: str, config: HindsightConfig) -> bool:
|
||||
"""Check if this model should be excluded from interception."""
|
||||
for pattern in config.excluded_models:
|
||||
if fnmatch.fnmatch(model.lower(), pattern.lower()):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _extract_user_query(self, messages: List[Dict[str, Any]]) -> Optional[str]:
|
||||
"""Extract the user's query from the last user message."""
|
||||
for msg in reversed(messages):
|
||||
role = msg.get("role", "")
|
||||
if role == "user":
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
elif isinstance(content, list):
|
||||
# Handle structured content (e.g., vision messages)
|
||||
text_parts = []
|
||||
for item in content:
|
||||
if isinstance(item, dict) and item.get("type") == "text":
|
||||
text_parts.append(item.get("text", ""))
|
||||
if text_parts:
|
||||
return " ".join(text_parts)
|
||||
return None
|
||||
|
||||
def _compute_conversation_hash(
|
||||
self,
|
||||
user_input: str,
|
||||
assistant_output: str,
|
||||
) -> str:
|
||||
"""Compute a hash for deduplication."""
|
||||
content = f"{user_input.strip().lower()}|{assistant_output.strip().lower()}"
|
||||
return hashlib.md5(content.encode()).hexdigest()[:16]
|
||||
|
||||
def _is_duplicate(self, conv_hash: str) -> bool:
|
||||
"""Check if this conversation was recently stored."""
|
||||
if conv_hash in self._recent_hashes:
|
||||
return True
|
||||
|
||||
# Add to cache, evict oldest if full
|
||||
self._recent_hashes.add(conv_hash)
|
||||
if len(self._recent_hashes) > self._max_hash_cache:
|
||||
# Remove oldest (arbitrary since set, but good enough)
|
||||
self._recent_hashes.pop()
|
||||
|
||||
return False
|
||||
|
||||
def _format_memories(
|
||||
self,
|
||||
results: List[Any],
|
||||
config: HindsightConfig
|
||||
) -> str:
|
||||
"""Format memory recall results into a context string.
|
||||
|
||||
Results can be RecallResult objects (with .text, .type attributes)
|
||||
or dicts (with get() method).
|
||||
"""
|
||||
if not results:
|
||||
return ""
|
||||
|
||||
# Apply limit if set, otherwise use all results
|
||||
results_to_use = results[:config.max_memories] if config.max_memories else results
|
||||
memory_lines = []
|
||||
for i, result in enumerate(results_to_use, 1):
|
||||
# Handle both RecallResult objects and dicts
|
||||
if hasattr(result, 'text'):
|
||||
text = result.text or ""
|
||||
fact_type = getattr(result, 'type', 'world') or "world"
|
||||
weight = getattr(result, 'weight', 0.0) or 0.0
|
||||
else:
|
||||
text = result.get("text", "")
|
||||
fact_type = result.get("type", result.get("fact_type", "world"))
|
||||
weight = result.get("weight", 0.0)
|
||||
|
||||
if text:
|
||||
# Include metadata for context
|
||||
type_label = fact_type.upper() if fact_type else "MEMORY"
|
||||
line = f"{i}. [{type_label}] {text}"
|
||||
if weight > 0 and config.verbose:
|
||||
line += f" (relevance: {weight:.2f})"
|
||||
memory_lines.append(line)
|
||||
|
||||
if not memory_lines:
|
||||
return ""
|
||||
|
||||
return (
|
||||
"# Relevant Memories\n"
|
||||
"The following information from memory may be relevant:\n\n"
|
||||
+ "\n".join(memory_lines)
|
||||
)
|
||||
|
||||
def _inject_memories_into_messages(
|
||||
self,
|
||||
messages: List[Dict[str, Any]],
|
||||
memory_context: str,
|
||||
config: HindsightConfig,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Inject memory context into the messages list."""
|
||||
if not memory_context:
|
||||
return messages
|
||||
|
||||
updated_messages = list(messages) # Make a copy
|
||||
|
||||
if config.injection_mode == MemoryInjectionMode.SYSTEM_MESSAGE:
|
||||
# Find existing system message or create new one
|
||||
for i, msg in enumerate(updated_messages):
|
||||
if msg.get("role") == "system":
|
||||
# Append to existing system message
|
||||
existing_content = msg.get("content", "")
|
||||
updated_messages[i] = {
|
||||
**msg,
|
||||
"content": f"{existing_content}\n\n{memory_context}"
|
||||
}
|
||||
return updated_messages
|
||||
|
||||
# No system message found, prepend one
|
||||
updated_messages.insert(0, {
|
||||
"role": "system",
|
||||
"content": memory_context
|
||||
})
|
||||
|
||||
elif config.injection_mode == MemoryInjectionMode.PREPEND_USER:
|
||||
# Find the last user message and prepend context
|
||||
for i in range(len(updated_messages) - 1, -1, -1):
|
||||
if updated_messages[i].get("role") == "user":
|
||||
original_content = updated_messages[i].get("content", "")
|
||||
if isinstance(original_content, str):
|
||||
updated_messages[i] = {
|
||||
**updated_messages[i],
|
||||
"content": f"{memory_context}\n\n---\n\n{original_content}"
|
||||
}
|
||||
break
|
||||
|
||||
return updated_messages
|
||||
|
||||
def _get_bank_id(self, config: HindsightConfig) -> str:
|
||||
"""Get the bank_id for API calls."""
|
||||
return config.bank_id
|
||||
|
||||
def _recall_memories_sync(
|
||||
self,
|
||||
query: str,
|
||||
config: HindsightConfig
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Recall relevant memories from Hindsight (sync) using direct HTTP."""
|
||||
try:
|
||||
bank_id = self._get_bank_id(config)
|
||||
url = f"{config.hindsight_api_url}/v1/default/banks/{bank_id}/memories/recall"
|
||||
|
||||
request_data = {
|
||||
"query": query,
|
||||
"budget": config.recall_budget or "mid",
|
||||
"max_tokens": config.max_memory_tokens or 4096,
|
||||
}
|
||||
if config.fact_types:
|
||||
request_data["types"] = config.fact_types
|
||||
|
||||
response = self._http_post(url, request_data, config)
|
||||
if response and "results" in response:
|
||||
return response["results"]
|
||||
return []
|
||||
|
||||
except Exception as e:
|
||||
if config.verbose:
|
||||
logger.warning(f"Failed to recall memories: {e}")
|
||||
return []
|
||||
|
||||
async def _recall_memories_async(
|
||||
self,
|
||||
query: str,
|
||||
config: HindsightConfig
|
||||
) -> List[Any]:
|
||||
"""Recall relevant memories from Hindsight (async).
|
||||
|
||||
Uses thread pool executor with sync HTTP to avoid event loop conflicts.
|
||||
"""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
results = await loop.run_in_executor(
|
||||
_executor,
|
||||
self._recall_memories_sync,
|
||||
query,
|
||||
config
|
||||
)
|
||||
|
||||
return results if isinstance(results, list) else []
|
||||
|
||||
except Exception as e:
|
||||
if config.verbose:
|
||||
logger.warning(f"Failed to recall memories: {e}")
|
||||
return []
|
||||
|
||||
def _store_conversation_sync(
|
||||
self,
|
||||
messages: List[Dict[str, Any]],
|
||||
response: ModelResponse,
|
||||
model: str,
|
||||
config: HindsightConfig,
|
||||
) -> None:
|
||||
"""Store the conversation to Hindsight (sync) using direct HTTP.
|
||||
|
||||
By default, stores the full conversation history passed to the LLM.
|
||||
Each message is stored as a separate item, all linked by document_id.
|
||||
|
||||
Hindsight will process the document as a whole for memory extraction.
|
||||
"""
|
||||
try:
|
||||
# Extract assistant response from the LLM response
|
||||
assistant_output = ""
|
||||
if response.choices and len(response.choices) > 0:
|
||||
choice = response.choices[0]
|
||||
if hasattr(choice, "message") and choice.message:
|
||||
assistant_output = choice.message.content or ""
|
||||
|
||||
if not assistant_output:
|
||||
return
|
||||
|
||||
# Build conversation items - each message becomes a separate item
|
||||
# All linked by document_id for Hindsight to process together
|
||||
items = []
|
||||
for msg in messages:
|
||||
role = msg.get("role", "").upper()
|
||||
content = msg.get("content", "")
|
||||
|
||||
# Skip system messages - they're instructions, not conversation
|
||||
if role == "SYSTEM":
|
||||
continue
|
||||
|
||||
# Skip if this looks like our injected memory context
|
||||
if isinstance(content, str) and content.startswith("# Relevant Memories"):
|
||||
continue
|
||||
|
||||
# Handle structured content (e.g., vision messages)
|
||||
if isinstance(content, list):
|
||||
text_parts = []
|
||||
for item in content:
|
||||
if isinstance(item, dict) and item.get("type") == "text":
|
||||
text_parts.append(item.get("text", ""))
|
||||
content = " ".join(text_parts)
|
||||
|
||||
if content:
|
||||
# Map roles to clearer labels
|
||||
label = "USER" if role == "USER" else "ASSISTANT"
|
||||
items.append(f"{label}: {content}")
|
||||
|
||||
# Add the new assistant response
|
||||
items.append(f"ASSISTANT: {assistant_output}")
|
||||
|
||||
if not items:
|
||||
return
|
||||
|
||||
# Use last user message for deduplication hash
|
||||
user_input = self._extract_user_query(messages) or ""
|
||||
|
||||
# Deduplication check
|
||||
conv_hash = self._compute_conversation_hash(user_input, assistant_output)
|
||||
if self._is_duplicate(conv_hash):
|
||||
if config.verbose:
|
||||
logger.debug(f"Skipping duplicate conversation: {conv_hash}")
|
||||
return
|
||||
|
||||
# Build the full conversation as a single item for now
|
||||
# (Future: could store each message as separate item in same document)
|
||||
conversation_text = "\n\n".join(items)
|
||||
|
||||
# Build metadata
|
||||
metadata = {
|
||||
"source": "litellm",
|
||||
"model": model,
|
||||
}
|
||||
|
||||
# Add token usage if available
|
||||
if hasattr(response, "usage") and response.usage:
|
||||
if hasattr(response.usage, "total_tokens"):
|
||||
metadata["tokens"] = str(response.usage.total_tokens)
|
||||
|
||||
bank_id = self._get_bank_id(config)
|
||||
url = f"{config.hindsight_api_url}/v1/default/banks/{bank_id}/memories"
|
||||
|
||||
request_data = {
|
||||
"items": [
|
||||
{
|
||||
"content": conversation_text,
|
||||
"context": f"conversation:litellm:{model}",
|
||||
"metadata": metadata,
|
||||
"document_id": config.document_id, # Group by document
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
self._http_post(url, request_data, config)
|
||||
|
||||
if config.verbose:
|
||||
logger.info(f"Stored conversation to Hindsight bank: {config.bank_id}")
|
||||
|
||||
except Exception as e:
|
||||
if config.verbose:
|
||||
logger.warning(f"Failed to store conversation: {e}")
|
||||
|
||||
async def _store_conversation_async(
|
||||
self,
|
||||
messages: List[Dict[str, Any]],
|
||||
response: ModelResponse,
|
||||
model: str,
|
||||
config: HindsightConfig,
|
||||
) -> None:
|
||||
"""Store the conversation to Hindsight (async).
|
||||
|
||||
Uses thread pool executor with sync HTTP to avoid event loop conflicts.
|
||||
"""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(
|
||||
_executor,
|
||||
self._store_conversation_sync,
|
||||
messages,
|
||||
response,
|
||||
model,
|
||||
config
|
||||
)
|
||||
except Exception as e:
|
||||
if config.verbose:
|
||||
logger.warning(f"Failed to store conversation: {e}")
|
||||
|
||||
# ========== LiteLLM CustomLogger Interface ==========
|
||||
|
||||
def log_pre_api_call(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[Dict[str, Any]],
|
||||
kwargs: Dict[str, Any],
|
||||
) -> None:
|
||||
"""Called before making the API call (sync).
|
||||
|
||||
This is where we inject memories into the messages.
|
||||
"""
|
||||
if not is_configured():
|
||||
return
|
||||
|
||||
config = get_config()
|
||||
if not config or not config.enabled or not config.inject_memories:
|
||||
return
|
||||
|
||||
if self._should_skip_model(model, config):
|
||||
return
|
||||
|
||||
# Extract user query
|
||||
user_query = self._extract_user_query(messages)
|
||||
if not user_query:
|
||||
return
|
||||
|
||||
# Recall relevant memories
|
||||
memories = self._recall_memories_sync(user_query, config)
|
||||
if not memories:
|
||||
return
|
||||
|
||||
# Format and inject memories
|
||||
memory_context = self._format_memories(memories, config)
|
||||
updated_messages = self._inject_memories_into_messages(
|
||||
messages, memory_context, config
|
||||
)
|
||||
|
||||
# Modify messages list IN-PLACE (don't just reassign kwargs)
|
||||
messages.clear()
|
||||
messages.extend(updated_messages)
|
||||
|
||||
if config.verbose:
|
||||
logger.info(f"Injected {len(memories)} memories into prompt")
|
||||
|
||||
async def async_log_pre_api_call(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[Dict[str, Any]],
|
||||
kwargs: Dict[str, Any],
|
||||
) -> None:
|
||||
"""Called before making the API call (async).
|
||||
|
||||
This is where we inject memories into the messages.
|
||||
"""
|
||||
if not is_configured():
|
||||
return
|
||||
|
||||
config = get_config()
|
||||
if not config or not config.enabled or not config.inject_memories:
|
||||
return
|
||||
|
||||
if self._should_skip_model(model, config):
|
||||
return
|
||||
|
||||
# Extract user query
|
||||
user_query = self._extract_user_query(messages)
|
||||
if not user_query:
|
||||
return
|
||||
|
||||
# Recall relevant memories
|
||||
memories = await self._recall_memories_async(user_query, config)
|
||||
if not memories:
|
||||
return
|
||||
|
||||
# Format and inject memories
|
||||
memory_context = self._format_memories(memories, config)
|
||||
updated_messages = self._inject_memories_into_messages(
|
||||
messages, memory_context, config
|
||||
)
|
||||
|
||||
# Modify messages list IN-PLACE (don't just reassign kwargs)
|
||||
messages.clear()
|
||||
messages.extend(updated_messages)
|
||||
|
||||
if config.verbose:
|
||||
logger.info(f"Injected {len(memories)} memories into prompt")
|
||||
|
||||
def log_success_event(
|
||||
self,
|
||||
kwargs: Dict[str, Any],
|
||||
response_obj: Any,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
) -> None:
|
||||
"""Called after successful API call (sync).
|
||||
|
||||
This is where we store the conversation.
|
||||
"""
|
||||
if not is_configured():
|
||||
return
|
||||
|
||||
config = get_config()
|
||||
if not config or not config.enabled or not config.store_conversations:
|
||||
return
|
||||
|
||||
model = kwargs.get("model", "unknown")
|
||||
if self._should_skip_model(model, config):
|
||||
return
|
||||
|
||||
messages = kwargs.get("messages", [])
|
||||
if not messages:
|
||||
return
|
||||
|
||||
# Store the conversation
|
||||
self._store_conversation_sync(messages, response_obj, model, config)
|
||||
|
||||
async def async_log_success_event(
|
||||
self,
|
||||
kwargs: Dict[str, Any],
|
||||
response_obj: Any,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
) -> None:
|
||||
"""Called after successful API call (async).
|
||||
|
||||
This is where we store the conversation.
|
||||
"""
|
||||
if not is_configured():
|
||||
return
|
||||
|
||||
config = get_config()
|
||||
if not config or not config.enabled or not config.store_conversations:
|
||||
return
|
||||
|
||||
model = kwargs.get("model", "unknown")
|
||||
if self._should_skip_model(model, config):
|
||||
return
|
||||
|
||||
messages = kwargs.get("messages", [])
|
||||
if not messages:
|
||||
return
|
||||
|
||||
# Store the conversation
|
||||
await self._store_conversation_async(messages, response_obj, model, config)
|
||||
|
||||
def log_failure_event(
|
||||
self,
|
||||
kwargs: Dict[str, Any],
|
||||
response_obj: Any,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
) -> None:
|
||||
"""Called after failed API call (sync)."""
|
||||
# We don't store failed conversations
|
||||
pass
|
||||
|
||||
async def async_log_failure_event(
|
||||
self,
|
||||
kwargs: Dict[str, Any],
|
||||
response_obj: Any,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
) -> None:
|
||||
"""Called after failed API call (async)."""
|
||||
# We don't store failed conversations
|
||||
pass
|
||||
|
||||
def close(self) -> None:
|
||||
"""Clean up resources."""
|
||||
with self._http_lock:
|
||||
if self._http_session is not None:
|
||||
try:
|
||||
if HAS_REQUESTS:
|
||||
self._http_session.close()
|
||||
elif HAS_HTTPX:
|
||||
self._http_session.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._http_session = None
|
||||
self._recent_hashes.clear()
|
||||
|
||||
|
||||
# Global callback instance
|
||||
_callback: Optional[HindsightCallback] = None
|
||||
|
||||
|
||||
def get_callback() -> HindsightCallback:
|
||||
"""Get the global callback instance, creating it if necessary."""
|
||||
global _callback
|
||||
if _callback is None:
|
||||
_callback = HindsightCallback()
|
||||
return _callback
|
||||
|
||||
|
||||
def cleanup_callback() -> None:
|
||||
"""Clean up the global callback instance."""
|
||||
global _callback
|
||||
if _callback is not None:
|
||||
_callback.close()
|
||||
_callback = None
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Global configuration for Hindsight-LiteLLM integration."""
|
||||
|
||||
from typing import Optional, List
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class MemoryInjectionMode(str, Enum):
|
||||
"""How memories should be injected into the prompt."""
|
||||
SYSTEM_MESSAGE = "system_message" # Add as system message
|
||||
PREPEND_USER = "prepend_user" # Prepend to user message
|
||||
DISABLED = "disabled" # Don't inject memories
|
||||
|
||||
|
||||
@dataclass
|
||||
class HindsightConfig:
|
||||
"""Configuration for Hindsight integration with LiteLLM.
|
||||
|
||||
Attributes:
|
||||
hindsight_api_url: URL of the Hindsight API server
|
||||
bank_id: Memory bank ID for memory operations (required). For multi-user
|
||||
support, use different bank_ids per user (e.g., f"user-{user_id}")
|
||||
api_key: Optional API key for Hindsight authentication
|
||||
store_conversations: Whether to store conversations to Hindsight
|
||||
inject_memories: Whether to inject relevant memories into prompts
|
||||
injection_mode: How to inject memories (system_message or prepend_user)
|
||||
max_memories: Maximum number of memories to inject
|
||||
max_memory_tokens: Maximum tokens for injected memory context
|
||||
recall_budget: Budget level for memory recall (low, mid, high)
|
||||
fact_types: List of fact types to filter recall (world, agent, opinion, observation)
|
||||
document_id: Optional document ID for grouping stored conversations
|
||||
enabled: Master switch to enable/disable Hindsight integration
|
||||
excluded_models: List of model patterns to exclude from interception
|
||||
verbose: Enable verbose logging
|
||||
bank_name: Optional display name for the memory bank
|
||||
background: Optional background/instructions for memory extraction
|
||||
use_reflect: Use reflect API instead of recall for memory injection (synthesizes answer)
|
||||
"""
|
||||
|
||||
hindsight_api_url: str = "http://localhost:8888"
|
||||
bank_id: Optional[str] = None
|
||||
api_key: Optional[str] = None
|
||||
store_conversations: bool = True
|
||||
inject_memories: bool = True
|
||||
injection_mode: MemoryInjectionMode = MemoryInjectionMode.SYSTEM_MESSAGE
|
||||
max_memories: Optional[int] = None # None = no limit (use all results from API)
|
||||
max_memory_tokens: int = 4096
|
||||
recall_budget: str = "mid" # low, mid, high
|
||||
fact_types: Optional[List[str]] = None # world, agent, opinion, observation
|
||||
document_id: Optional[str] = None
|
||||
enabled: bool = True
|
||||
excluded_models: List[str] = field(default_factory=list)
|
||||
verbose: bool = False
|
||||
bank_name: Optional[str] = None # Display name for the memory bank
|
||||
background: Optional[str] = None # Background/instructions for memory extraction
|
||||
use_reflect: bool = False # Use reflect instead of recall for memory injection
|
||||
reflect_include_facts: bool = False # Include facts used by reflect in debug info
|
||||
|
||||
|
||||
# Global configuration instance
|
||||
_global_config: Optional[HindsightConfig] = None
|
||||
|
||||
|
||||
def configure(
|
||||
hindsight_api_url: str = "http://localhost:8888",
|
||||
bank_id: Optional[str] = None,
|
||||
api_key: Optional[str] = None,
|
||||
store_conversations: bool = True,
|
||||
inject_memories: bool = True,
|
||||
injection_mode: MemoryInjectionMode = MemoryInjectionMode.SYSTEM_MESSAGE,
|
||||
max_memories: Optional[int] = None,
|
||||
max_memory_tokens: int = 4096,
|
||||
recall_budget: str = "mid",
|
||||
fact_types: Optional[List[str]] = None,
|
||||
document_id: Optional[str] = None,
|
||||
enabled: bool = True,
|
||||
excluded_models: Optional[List[str]] = None,
|
||||
verbose: bool = False,
|
||||
bank_name: Optional[str] = None,
|
||||
background: Optional[str] = None,
|
||||
use_reflect: bool = False,
|
||||
reflect_include_facts: bool = False,
|
||||
) -> HindsightConfig:
|
||||
"""Configure global Hindsight integration settings for LiteLLM.
|
||||
|
||||
This function sets up the global configuration that will be used by the
|
||||
LiteLLM callbacks to inject memories and store conversations.
|
||||
|
||||
Args:
|
||||
hindsight_api_url: URL of the Hindsight API server
|
||||
bank_id: Memory bank ID for memory operations (required). For multi-user
|
||||
support, use different bank_ids per user (e.g., f"user-{user_id}")
|
||||
api_key: Optional API key for Hindsight authentication
|
||||
store_conversations: Whether to store conversations to Hindsight
|
||||
inject_memories: Whether to inject relevant memories into prompts
|
||||
injection_mode: How to inject memories into the prompt
|
||||
max_memories: Maximum number of memories to inject
|
||||
max_memory_tokens: Maximum tokens for injected memory context
|
||||
recall_budget: Budget level for memory recall (low, mid, high)
|
||||
fact_types: List of fact types to filter (world, agent, opinion, observation)
|
||||
document_id: Optional document ID for grouping stored conversations
|
||||
enabled: Master switch to enable/disable Hindsight integration
|
||||
excluded_models: List of model patterns to exclude from interception
|
||||
verbose: Enable verbose logging
|
||||
bank_name: Optional display name for the memory bank
|
||||
background: Optional background/instructions that help Hindsight understand
|
||||
what information is important to extract and remember from conversations.
|
||||
This is passed to create_bank() to configure the memory bank.
|
||||
use_reflect: Use reflect API instead of recall for memory injection.
|
||||
When True, Hindsight will synthesize a contextual answer based on
|
||||
memories rather than returning raw memory facts.
|
||||
reflect_include_facts: When use_reflect=True, include the facts that
|
||||
were used to generate the reflect response in the debug info.
|
||||
This is useful for debugging what memories the reflect API used.
|
||||
|
||||
Returns:
|
||||
The configured HindsightConfig instance
|
||||
|
||||
Example:
|
||||
>>> from hindsight_litellm import configure, enable
|
||||
>>> configure(
|
||||
... hindsight_api_url="http://localhost:8888",
|
||||
... bank_id="user-123", # Per-user bank for multi-user support
|
||||
... store_conversations=True,
|
||||
... inject_memories=True,
|
||||
... background="This agent routes customer requests to support channels. "
|
||||
... "Remember which types of issues should go to which channels.",
|
||||
... )
|
||||
>>> enable() # Register callbacks with LiteLLM
|
||||
"""
|
||||
global _global_config
|
||||
|
||||
_global_config = HindsightConfig(
|
||||
hindsight_api_url=hindsight_api_url,
|
||||
bank_id=bank_id,
|
||||
api_key=api_key,
|
||||
store_conversations=store_conversations,
|
||||
inject_memories=inject_memories,
|
||||
injection_mode=injection_mode,
|
||||
max_memories=max_memories,
|
||||
max_memory_tokens=max_memory_tokens,
|
||||
recall_budget=recall_budget,
|
||||
fact_types=fact_types,
|
||||
document_id=document_id,
|
||||
enabled=enabled,
|
||||
excluded_models=excluded_models or [],
|
||||
verbose=verbose,
|
||||
bank_name=bank_name,
|
||||
background=background,
|
||||
use_reflect=use_reflect,
|
||||
reflect_include_facts=reflect_include_facts,
|
||||
)
|
||||
|
||||
# If background or bank_name is provided, create/update the bank
|
||||
if bank_id and (background or bank_name):
|
||||
_create_or_update_bank(
|
||||
hindsight_api_url=hindsight_api_url,
|
||||
bank_id=bank_id,
|
||||
name=bank_name,
|
||||
background=background,
|
||||
verbose=verbose,
|
||||
)
|
||||
|
||||
return _global_config
|
||||
|
||||
|
||||
def _create_or_update_bank(
|
||||
hindsight_api_url: str,
|
||||
bank_id: str,
|
||||
name: Optional[str] = None,
|
||||
background: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
) -> None:
|
||||
"""Create or update a memory bank with the given configuration.
|
||||
|
||||
This is called automatically by configure() when background or bank_name is provided.
|
||||
"""
|
||||
try:
|
||||
from hindsight_client import Hindsight
|
||||
|
||||
client = Hindsight(hindsight_api_url)
|
||||
client.create_bank(
|
||||
bank_id=bank_id,
|
||||
name=name,
|
||||
background=background,
|
||||
)
|
||||
if verbose:
|
||||
import logging
|
||||
logging.getLogger("hindsight_litellm").info(
|
||||
f"Created/updated bank '{bank_id}' with background"
|
||||
)
|
||||
except ImportError:
|
||||
if verbose:
|
||||
import logging
|
||||
logging.getLogger("hindsight_litellm").warning(
|
||||
"hindsight_client not installed. Cannot create bank with background. "
|
||||
"Install with: pip install hindsight-client"
|
||||
)
|
||||
except Exception as e:
|
||||
if verbose:
|
||||
import logging
|
||||
logging.getLogger("hindsight_litellm").warning(
|
||||
f"Failed to create/update bank: {e}"
|
||||
)
|
||||
|
||||
|
||||
def get_config() -> Optional[HindsightConfig]:
|
||||
"""Get the current global configuration.
|
||||
|
||||
Returns:
|
||||
The current HindsightConfig instance, or None if not configured
|
||||
"""
|
||||
return _global_config
|
||||
|
||||
|
||||
def is_configured() -> bool:
|
||||
"""Check if Hindsight has been configured.
|
||||
|
||||
Returns:
|
||||
True if configure() has been called with a valid bank_id
|
||||
"""
|
||||
return (
|
||||
_global_config is not None
|
||||
and _global_config.enabled
|
||||
and _global_config.bank_id is not None
|
||||
)
|
||||
|
||||
|
||||
def reset_config() -> None:
|
||||
"""Reset the global configuration to None."""
|
||||
global _global_config
|
||||
_global_config = None
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,59 @@
|
||||
[project]
|
||||
name = "hindsight-litellm"
|
||||
version = "0.1.5"
|
||||
description = "Universal LLM memory integration via LiteLLM - works with 100+ providers"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
license = { text = "MIT" }
|
||||
authors = [
|
||||
{ name = "Vectorize", email = "[email protected]" }
|
||||
]
|
||||
keywords = [
|
||||
"ai",
|
||||
"memory",
|
||||
"llm",
|
||||
"litellm",
|
||||
"openai",
|
||||
"anthropic",
|
||||
"groq",
|
||||
"langchain",
|
||||
"agents",
|
||||
"hindsight",
|
||||
]
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Intended Audience :: Developers",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
||||
]
|
||||
|
||||
dependencies = [
|
||||
"litellm>=1.40.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=7.0.0",
|
||||
"pytest-asyncio>=0.21.0",
|
||||
"pytest-mock>=3.10.0",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/vectorize-io/hindsight"
|
||||
Documentation = "https://github.com/vectorize-io/hindsight/tree/main/hindsight-integrations/litellm"
|
||||
Repository = "https://github.com/vectorize-io/hindsight"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["hindsight_litellm"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_mode = "auto"
|
||||
testpaths = ["tests"]
|
||||
@@ -0,0 +1 @@
|
||||
# Tests for hindsight-litellm
|
||||
@@ -0,0 +1,471 @@
|
||||
"""Integration tests for hindsight-litellm."""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
from typing import List, Dict, Any
|
||||
|
||||
from hindsight_litellm import (
|
||||
configure,
|
||||
enable,
|
||||
disable,
|
||||
is_enabled,
|
||||
cleanup,
|
||||
get_config,
|
||||
is_configured,
|
||||
reset_config,
|
||||
HindsightConfig,
|
||||
MemoryInjectionMode,
|
||||
)
|
||||
from hindsight_litellm.callbacks import HindsightCallback, get_callback, cleanup_callback
|
||||
|
||||
|
||||
class TestConfiguration:
|
||||
"""Tests for configuration management."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset config before each test."""
|
||||
reset_config()
|
||||
disable()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
cleanup()
|
||||
|
||||
def test_configure_creates_config(self):
|
||||
"""Test that configure creates a config object."""
|
||||
config = configure(
|
||||
bank_id="test-agent",
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
)
|
||||
|
||||
assert config is not None
|
||||
assert config.bank_id == "test-agent"
|
||||
assert config.hindsight_api_url == "http://localhost:8888"
|
||||
assert config.enabled is True
|
||||
|
||||
def test_configure_with_all_options(self):
|
||||
"""Test configure with all options."""
|
||||
config = configure(
|
||||
hindsight_api_url="http://custom:9999",
|
||||
bank_id="custom-agent",
|
||||
api_key="secret-key",
|
||||
store_conversations=False,
|
||||
inject_memories=False,
|
||||
injection_mode=MemoryInjectionMode.PREPEND_USER,
|
||||
max_memories=5,
|
||||
max_memory_tokens=1000,
|
||||
recall_budget="high",
|
||||
fact_types=["world", "opinion"],
|
||||
document_id="doc-123",
|
||||
enabled=True,
|
||||
excluded_models=["gpt-3.5*"],
|
||||
verbose=True,
|
||||
)
|
||||
|
||||
assert config.hindsight_api_url == "http://custom:9999"
|
||||
assert config.bank_id == "custom-agent"
|
||||
assert config.api_key == "secret-key"
|
||||
assert config.store_conversations is False
|
||||
assert config.inject_memories is False
|
||||
assert config.injection_mode == MemoryInjectionMode.PREPEND_USER
|
||||
assert config.max_memories == 5
|
||||
assert config.max_memory_tokens == 1000
|
||||
assert config.recall_budget == "high"
|
||||
assert config.fact_types == ["world", "opinion"]
|
||||
assert config.document_id == "doc-123"
|
||||
assert config.excluded_models == ["gpt-3.5*"]
|
||||
assert config.verbose is True
|
||||
|
||||
def test_is_configured_without_bank_id(self):
|
||||
"""Test is_configured returns False without bank_id."""
|
||||
configure(hindsight_api_url="http://localhost:8888")
|
||||
assert is_configured() is False
|
||||
|
||||
def test_is_configured_with_bank_id(self):
|
||||
"""Test is_configured returns True with bank_id."""
|
||||
configure(bank_id="test-agent")
|
||||
assert is_configured() is True
|
||||
|
||||
def test_reset_config(self):
|
||||
"""Test reset_config clears the configuration."""
|
||||
configure(bank_id="test-agent")
|
||||
assert is_configured() is True
|
||||
|
||||
reset_config()
|
||||
assert get_config() is None
|
||||
assert is_configured() is False
|
||||
|
||||
|
||||
class TestEnableDisable:
|
||||
"""Tests for enable/disable functionality."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset state before each test."""
|
||||
cleanup()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
cleanup()
|
||||
|
||||
def test_enable_without_config_raises(self):
|
||||
"""Test enable raises error without configuration."""
|
||||
with pytest.raises(RuntimeError, match="not configured"):
|
||||
enable()
|
||||
|
||||
def test_enable_registers_callback(self):
|
||||
"""Test enable registers callback with LiteLLM."""
|
||||
import litellm
|
||||
|
||||
configure(bank_id="test-agent")
|
||||
enable()
|
||||
|
||||
callback = get_callback()
|
||||
assert callback in litellm.callbacks
|
||||
assert is_enabled() is True
|
||||
|
||||
def test_disable_removes_callback(self):
|
||||
"""Test disable removes callback from LiteLLM."""
|
||||
import litellm
|
||||
|
||||
configure(bank_id="test-agent")
|
||||
enable()
|
||||
assert is_enabled() is True
|
||||
|
||||
disable()
|
||||
callback = get_callback()
|
||||
assert callback not in litellm.callbacks
|
||||
assert is_enabled() is False
|
||||
|
||||
def test_enable_idempotent(self):
|
||||
"""Test enable is idempotent (can be called multiple times)."""
|
||||
import litellm
|
||||
|
||||
configure(bank_id="test-agent")
|
||||
|
||||
# Enable multiple times
|
||||
enable()
|
||||
enable()
|
||||
enable()
|
||||
|
||||
# Should only have one callback
|
||||
callback = get_callback()
|
||||
assert litellm.callbacks.count(callback) == 1
|
||||
|
||||
|
||||
class TestCallback:
|
||||
"""Tests for the HindsightCallback class."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset state before each test."""
|
||||
cleanup()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
cleanup()
|
||||
|
||||
def test_extract_user_query_simple(self):
|
||||
"""Test extracting user query from simple messages."""
|
||||
callback = HindsightCallback()
|
||||
messages = [
|
||||
{"role": "system", "content": "You are helpful."},
|
||||
{"role": "user", "content": "What is the capital of France?"},
|
||||
]
|
||||
|
||||
query = callback._extract_user_query(messages)
|
||||
assert query == "What is the capital of France?"
|
||||
|
||||
def test_extract_user_query_from_last_user_message(self):
|
||||
"""Test extracting query from last user message."""
|
||||
callback = HindsightCallback()
|
||||
messages = [
|
||||
{"role": "user", "content": "First question"},
|
||||
{"role": "assistant", "content": "First answer"},
|
||||
{"role": "user", "content": "Second question"},
|
||||
]
|
||||
|
||||
query = callback._extract_user_query(messages)
|
||||
assert query == "Second question"
|
||||
|
||||
def test_extract_user_query_structured_content(self):
|
||||
"""Test extracting query from structured content (vision)."""
|
||||
callback = HindsightCallback()
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
{"type": "image_url", "image_url": {"url": "http://example.com/img.png"}},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
query = callback._extract_user_query(messages)
|
||||
assert query == "What's in this image?"
|
||||
|
||||
def test_extract_user_query_multiple_text_parts(self):
|
||||
"""Test extracting query with multiple text parts."""
|
||||
callback = HindsightCallback()
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "First part."},
|
||||
{"type": "text", "text": "Second part."},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
query = callback._extract_user_query(messages)
|
||||
assert query == "First part. Second part."
|
||||
|
||||
def test_format_memories(self):
|
||||
"""Test formatting memories into context string."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(bank_id="test", max_memories=10, verbose=False)
|
||||
|
||||
memories = [
|
||||
{"text": "User likes Python", "fact_type": "world", "weight": 0.95},
|
||||
{"text": "User works at Google", "fact_type": "world", "weight": 0.8},
|
||||
]
|
||||
|
||||
formatted = callback._format_memories(memories, config)
|
||||
|
||||
assert "Relevant Memories" in formatted
|
||||
assert "User likes Python" in formatted
|
||||
assert "User works at Google" in formatted
|
||||
assert "[WORLD]" in formatted
|
||||
|
||||
def test_format_memories_with_verbose(self):
|
||||
"""Test formatting memories with verbose mode shows weights."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(bank_id="test", max_memories=10, verbose=True)
|
||||
|
||||
memories = [
|
||||
{"text": "User likes Python", "fact_type": "world", "weight": 0.95},
|
||||
]
|
||||
|
||||
formatted = callback._format_memories(memories, config)
|
||||
|
||||
assert "relevance: 0.95" in formatted
|
||||
|
||||
def test_inject_memories_as_system_message(self):
|
||||
"""Test injecting memories as system message."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(
|
||||
bank_id="test",
|
||||
injection_mode=MemoryInjectionMode.SYSTEM_MESSAGE,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
]
|
||||
memory_context = "# Relevant Memories\n1. User is John"
|
||||
|
||||
result = callback._inject_memories_into_messages(messages, memory_context, config)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0]["role"] == "system"
|
||||
assert "Relevant Memories" in result[0]["content"]
|
||||
assert result[1]["role"] == "user"
|
||||
|
||||
def test_inject_memories_prepend_to_existing_system(self):
|
||||
"""Test injecting memories appends to existing system message."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(
|
||||
bank_id="test",
|
||||
injection_mode=MemoryInjectionMode.SYSTEM_MESSAGE,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": "You are helpful."},
|
||||
{"role": "user", "content": "Hello"},
|
||||
]
|
||||
memory_context = "# Relevant Memories\n1. User is John"
|
||||
|
||||
result = callback._inject_memories_into_messages(messages, memory_context, config)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0]["role"] == "system"
|
||||
assert "You are helpful." in result[0]["content"]
|
||||
assert "Relevant Memories" in result[0]["content"]
|
||||
|
||||
def test_inject_memories_prepend_user_mode(self):
|
||||
"""Test injecting memories in prepend_user mode."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(
|
||||
bank_id="test",
|
||||
injection_mode=MemoryInjectionMode.PREPEND_USER,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "What's my name?"},
|
||||
]
|
||||
memory_context = "# Relevant Memories\n1. User is John"
|
||||
|
||||
result = callback._inject_memories_into_messages(messages, memory_context, config)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["role"] == "user"
|
||||
assert "Relevant Memories" in result[0]["content"]
|
||||
assert "What's my name?" in result[0]["content"]
|
||||
|
||||
def test_should_skip_model_exact_match(self):
|
||||
"""Test model exclusion with exact match."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(
|
||||
bank_id="test",
|
||||
excluded_models=["gpt-3.5-turbo"],
|
||||
)
|
||||
|
||||
assert callback._should_skip_model("gpt-3.5-turbo", config) is True
|
||||
assert callback._should_skip_model("gpt-4", config) is False
|
||||
|
||||
def test_should_skip_model_wildcard(self):
|
||||
"""Test model exclusion with wildcard pattern."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(
|
||||
bank_id="test",
|
||||
excluded_models=["gpt-3.5*", "claude-instant-*"],
|
||||
)
|
||||
|
||||
assert callback._should_skip_model("gpt-3.5-turbo", config) is True
|
||||
assert callback._should_skip_model("gpt-3.5-turbo-16k", config) is True
|
||||
assert callback._should_skip_model("claude-instant-1.2", config) is True
|
||||
assert callback._should_skip_model("gpt-4", config) is False
|
||||
assert callback._should_skip_model("claude-3-opus", config) is False
|
||||
|
||||
|
||||
class TestDeduplication:
|
||||
"""Tests for conversation deduplication."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset state before each test."""
|
||||
cleanup()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
cleanup()
|
||||
|
||||
def test_compute_conversation_hash(self):
|
||||
"""Test computing conversation hash."""
|
||||
callback = HindsightCallback()
|
||||
|
||||
hash1 = callback._compute_conversation_hash("Hello", "Hi there!")
|
||||
hash2 = callback._compute_conversation_hash("Hello", "Hi there!")
|
||||
hash3 = callback._compute_conversation_hash("Hello", "Different response")
|
||||
|
||||
# Same content should produce same hash
|
||||
assert hash1 == hash2
|
||||
# Different content should produce different hash
|
||||
assert hash1 != hash3
|
||||
|
||||
def test_compute_conversation_hash_case_insensitive(self):
|
||||
"""Test that hash is case insensitive."""
|
||||
callback = HindsightCallback()
|
||||
|
||||
hash1 = callback._compute_conversation_hash("HELLO", "HI THERE!")
|
||||
hash2 = callback._compute_conversation_hash("hello", "hi there!")
|
||||
|
||||
assert hash1 == hash2
|
||||
|
||||
def test_is_duplicate_first_time(self):
|
||||
"""Test first occurrence is not a duplicate."""
|
||||
callback = HindsightCallback()
|
||||
|
||||
result = callback._is_duplicate("abc123")
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_is_duplicate_second_time(self):
|
||||
"""Test second occurrence is a duplicate."""
|
||||
callback = HindsightCallback()
|
||||
|
||||
callback._is_duplicate("abc123") # First time
|
||||
result = callback._is_duplicate("abc123") # Second time
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_is_duplicate_different_hashes(self):
|
||||
"""Test different hashes are not duplicates."""
|
||||
callback = HindsightCallback()
|
||||
|
||||
callback._is_duplicate("abc123")
|
||||
result = callback._is_duplicate("xyz789")
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestContextManager:
|
||||
"""Tests for the hindsight_memory context manager."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset state before each test."""
|
||||
cleanup()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
cleanup()
|
||||
|
||||
def test_context_manager_enables_and_disables(self):
|
||||
"""Test context manager enables and disables correctly."""
|
||||
from hindsight_litellm import hindsight_memory
|
||||
|
||||
assert is_enabled() is False
|
||||
|
||||
with hindsight_memory(bank_id="test-agent"):
|
||||
assert is_enabled() is True
|
||||
assert get_config().bank_id == "test-agent"
|
||||
|
||||
assert is_enabled() is False
|
||||
|
||||
def test_context_manager_restores_previous_config(self):
|
||||
"""Test context manager restores previous configuration."""
|
||||
from hindsight_litellm import hindsight_memory
|
||||
|
||||
# Set up initial config
|
||||
configure(bank_id="original-agent")
|
||||
enable()
|
||||
assert get_config().bank_id == "original-agent"
|
||||
|
||||
# Use context manager with different config
|
||||
with hindsight_memory(bank_id="temporary-agent"):
|
||||
assert get_config().bank_id == "temporary-agent"
|
||||
|
||||
# Should restore original config
|
||||
assert get_config().bank_id == "original-agent"
|
||||
assert is_enabled() is True
|
||||
|
||||
def test_context_manager_with_fact_types(self):
|
||||
"""Test context manager with fact_types parameter."""
|
||||
from hindsight_litellm import hindsight_memory
|
||||
|
||||
with hindsight_memory(bank_id="test-agent", fact_types=["world", "opinion"]):
|
||||
config = get_config()
|
||||
assert config.fact_types == ["world", "opinion"]
|
||||
|
||||
|
||||
class TestFactTypes:
|
||||
"""Tests for fact_types configuration."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset config before each test."""
|
||||
reset_config()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
cleanup()
|
||||
|
||||
def test_configure_with_fact_types(self):
|
||||
"""Test configuring with fact_types."""
|
||||
config = configure(
|
||||
bank_id="test-agent",
|
||||
fact_types=["world", "agent", "opinion"],
|
||||
)
|
||||
|
||||
assert config.fact_types == ["world", "agent", "opinion"]
|
||||
|
||||
def test_configure_without_fact_types(self):
|
||||
"""Test configuring without fact_types defaults to None."""
|
||||
config = configure(bank_id="test-agent")
|
||||
|
||||
assert config.fact_types is None
|
||||
Generated
+2188
File diff suppressed because it is too large
Load Diff
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "hindsight-all"
|
||||
version = "0.1.4"
|
||||
version = "0.1.5"
|
||||
description = "All-in-one package for Hindsight - Semantic memory system with personality-driven thinking"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
|
||||
+9281
-235
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"name": "hindsight",
|
||||
"private": true,
|
||||
"workspaces": [
|
||||
"hindsight-clients/typescript",
|
||||
"hindsight-control-plane",
|
||||
"hindsight-docs"
|
||||
]
|
||||
}
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")/../.."
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "Usage: $0 VERSION [--model MODEL]"
|
||||
echo ""
|
||||
echo "Generate changelog entry for a release."
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 1.0.5"
|
||||
echo " $0 v1.0.5"
|
||||
echo " $0 1.0.5 --model gpt-4o"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$OPENAI_API_KEY" ]; then
|
||||
ENV_FILE=".env"
|
||||
if [ -f "$ENV_FILE" ]; then
|
||||
echo "Loading environment from $ENV_FILE"
|
||||
set -a
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
else
|
||||
echo "Error: OPENAI_API_KEY not set and no .env file found"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
cd hindsight-dev
|
||||
uv run generate-changelog "$@"
|
||||
@@ -13,13 +13,10 @@ if [ ! -f "$ROOT_DIR/.env" ]; then
|
||||
fi
|
||||
|
||||
echo "🔨 Building TypeScript SDK first to ensure it's up to date..."
|
||||
cd "$ROOT_DIR/hindsight-clients/typescript" || exit 1
|
||||
npm run build
|
||||
npm run build -w @vectorize-io/hindsight-client
|
||||
echo "✅ SDK built successfully"
|
||||
echo ""
|
||||
|
||||
cd "$ROOT_DIR/hindsight-control-plane" || exit 1
|
||||
|
||||
echo "🚀 Starting Control Plane (Next.js dev server)..."
|
||||
if [ -f "$ROOT_DIR/.env" ]; then
|
||||
echo "📄 Loading environment from $ROOT_DIR/.env"
|
||||
@@ -33,4 +30,4 @@ fi
|
||||
export HOSTNAME="${HINDSIGHT_CP_HOSTNAME:-0.0.0.0}"
|
||||
|
||||
# Run dev server
|
||||
npm run dev
|
||||
npm run dev -w hindsight-control-plane
|
||||
@@ -7,22 +7,11 @@ set -e
|
||||
|
||||
# Get the project root directory
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
DOCS_DIR="$PROJECT_ROOT/hindsight-docs"
|
||||
cd "$PROJECT_ROOT" || exit 1
|
||||
|
||||
echo "Starting documentation server..."
|
||||
echo "Documentation directory: $DOCS_DIR"
|
||||
|
||||
# Check if node_modules exists
|
||||
if [ ! -d "$DOCS_DIR/node_modules" ]; then
|
||||
echo "Installing documentation dependencies..."
|
||||
cd "$DOCS_DIR"
|
||||
npm install
|
||||
fi
|
||||
|
||||
# Start the Docusaurus dev server
|
||||
cd "$DOCS_DIR"
|
||||
echo ""
|
||||
echo "Starting Docusaurus development server..."
|
||||
echo "Documentation will be available at: http://localhost:3000"
|
||||
echo ""
|
||||
npm run start
|
||||
npm run start -w hindsight-docs
|
||||
|
||||
+2
-2
@@ -65,7 +65,7 @@ fi
|
||||
print_info "Updating version in all components..."
|
||||
|
||||
# Update Python packages
|
||||
PYTHON_PACKAGES=("hindsight-api" "hindsight-dev" "hindsight-dev/benchmarks" "hindsight")
|
||||
PYTHON_PACKAGES=("hindsight-api" "hindsight-dev" "hindsight-dev/benchmarks" "hindsight" "hindsight-integrations/litellm")
|
||||
for package in "${PYTHON_PACKAGES[@]}"; do
|
||||
PYPROJECT_FILE="$package/pyproject.toml"
|
||||
if [ -f "$PYPROJECT_FILE" ]; then
|
||||
@@ -148,7 +148,7 @@ git add -A
|
||||
git commit -m "Release v$VERSION
|
||||
|
||||
- Update version to $VERSION in all components
|
||||
- Python packages: hindsight-api, hindsight-dev, hindsight-dev/benchmarks, hindsight-all
|
||||
- Python packages: hindsight-api, hindsight-dev, hindsight-dev/benchmarks, hindsight-all, hindsight-litellm
|
||||
- Python client: hindsight-clients/python
|
||||
- TypeScript client: hindsight-clients/typescript
|
||||
- Rust CLI: hindsight-cli
|
||||
|
||||
@@ -0,0 +1,798 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Documentation Example Tester
|
||||
|
||||
Tests code examples from documentation by running them directly.
|
||||
Uses deterministic transformations (no LLM) for test generation.
|
||||
LLM is only used to analyze failures and determine if they're real doc bugs.
|
||||
|
||||
Usage:
|
||||
python scripts/test-doc-examples.py
|
||||
|
||||
Environment variables:
|
||||
OPENAI_API_KEY: Required for failure analysis
|
||||
HINDSIGHT_API_URL: URL of running Hindsight server (default: http://localhost:8888)
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import site
|
||||
import json
|
||||
import glob
|
||||
import subprocess
|
||||
import tempfile
|
||||
import traceback
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import threading
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
# Thread-safe print
|
||||
print_lock = threading.Lock()
|
||||
|
||||
def safe_print(*args, **kwargs):
|
||||
with print_lock:
|
||||
print(*args, **kwargs)
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
@dataclass
|
||||
class CodeExample:
|
||||
file_path: str
|
||||
language: str
|
||||
code: str
|
||||
context: str
|
||||
line_number: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestResult:
|
||||
example: CodeExample
|
||||
success: bool
|
||||
output: str
|
||||
error: Optional[str] = None
|
||||
transformed_code: Optional[str] = None
|
||||
skip_reason: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestReport:
|
||||
total: int = 0
|
||||
passed: int = 0
|
||||
failed: int = 0
|
||||
skipped: int = 0
|
||||
results: list[TestResult] = field(default_factory=list)
|
||||
|
||||
def add_result(self, result: TestResult):
|
||||
self.total += 1
|
||||
self.results.append(result)
|
||||
if result.skip_reason:
|
||||
self.skipped += 1
|
||||
elif result.success:
|
||||
self.passed += 1
|
||||
else:
|
||||
self.failed += 1
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# STEP 1: Extract code blocks from markdown
|
||||
# =============================================================================
|
||||
|
||||
def find_markdown_files(repo_root: str) -> list[str]:
|
||||
"""Find all markdown files, excluding auto-generated docs."""
|
||||
skip_patterns = [
|
||||
"node_modules", ".git", "venv", "__pycache__",
|
||||
"hindsight_client_api/docs", "hindsight-clients/typescript/docs",
|
||||
"target/", "dist/",
|
||||
]
|
||||
md_files = []
|
||||
for pattern in ["*.md", "**/*.md"]:
|
||||
for f in glob.glob(os.path.join(repo_root, pattern), recursive=True):
|
||||
if os.path.islink(f):
|
||||
continue
|
||||
if any(skip in f for skip in skip_patterns):
|
||||
continue
|
||||
md_files.append(f)
|
||||
return sorted(set(md_files))
|
||||
|
||||
|
||||
def extract_code_blocks(file_path: str) -> list[CodeExample]:
|
||||
"""Extract code blocks from a markdown file."""
|
||||
with open(file_path, "r") as f:
|
||||
content = f.read()
|
||||
|
||||
examples = []
|
||||
pattern = r"```(\w+)\n(.*?)```"
|
||||
|
||||
for match in re.finditer(pattern, content, re.DOTALL):
|
||||
language = match.group(1).lower()
|
||||
code = match.group(2).strip()
|
||||
line_number = content[:match.start()].count('\n') + 1
|
||||
|
||||
if language in ["python", "typescript", "javascript", "bash", "sh"]:
|
||||
start = max(0, match.start() - 150)
|
||||
end = min(len(content), match.end() + 150)
|
||||
context = content[start:end]
|
||||
|
||||
examples.append(CodeExample(
|
||||
file_path=file_path,
|
||||
language=language,
|
||||
code=code,
|
||||
context=context,
|
||||
line_number=line_number
|
||||
))
|
||||
|
||||
return examples
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# STEP 2: Determine if example should be skipped (no LLM needed)
|
||||
# =============================================================================
|
||||
|
||||
def should_skip(code: str, language: str) -> Optional[str]:
|
||||
"""Determine if example should be skipped. Returns reason or None."""
|
||||
code_lower = code.lower().strip()
|
||||
|
||||
# Installation/setup commands
|
||||
if language in ["bash", "sh"]:
|
||||
if code_lower.startswith(("pip install", "npm install", "yarn add", "uv pip", "cargo install", "curl ", "wget ")):
|
||||
return "Installation command"
|
||||
if "docker" in code_lower or "docker-compose" in code_lower:
|
||||
return "Docker command"
|
||||
if code_lower.startswith("helm "):
|
||||
return "Helm command"
|
||||
if code_lower.startswith(("cargo build", "cargo test")):
|
||||
return "Cargo command"
|
||||
if "pytest" in code_lower:
|
||||
return "Test suite command"
|
||||
if code_lower.startswith("git clone"):
|
||||
return "Git clone"
|
||||
if "./scripts/" in code_lower:
|
||||
return "Development script"
|
||||
if any(x in code_lower for x in ["npm run dev", "npm run start", "npm run build", "npm run deploy"]):
|
||||
return "NPM script"
|
||||
if code_lower.startswith("cd ") and not code_lower.startswith("cd /tmp"):
|
||||
return "Directory change"
|
||||
if code_lower.startswith("export "):
|
||||
return "Environment variable"
|
||||
|
||||
# Config files
|
||||
if language in ["yaml", "toml", "json", "env"]:
|
||||
return "Configuration file"
|
||||
|
||||
# Too short
|
||||
if len(code.strip()) < 20:
|
||||
return "Too short"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# STEP 3: Transform code (LLM adds setup/cleanup around sacred doc code)
|
||||
# =============================================================================
|
||||
|
||||
def transform_code(client: OpenAI, example: CodeExample, hindsight_url: str, cli_available: bool, model: str) -> tuple[str, Optional[str]]:
|
||||
"""Use LLM to add setup/cleanup around doc code. The doc code itself is not modified."""
|
||||
|
||||
bank_id = f"doc-test-{uuid.uuid4()}"
|
||||
|
||||
# Skip CLI examples if CLI not available
|
||||
if not cli_available and example.language in ["bash", "sh"] and "hindsight " in example.code.lower():
|
||||
return "", "CLI not available"
|
||||
|
||||
if example.language == "python":
|
||||
output_format = f"""Output a Python script (.py):
|
||||
- The doc code goes inside a try block
|
||||
- Add cleanup in finally: requests.delete("{hindsight_url}/v1/default/banks/{bank_id}")
|
||||
- End with: print("TEST PASSED")
|
||||
- Do NOT use async/await - the Hindsight client is synchronous"""
|
||||
|
||||
elif example.language in ["typescript", "javascript"]:
|
||||
output_format = f"""Output a JavaScript ES module (.mjs):
|
||||
- Remove TypeScript type annotations
|
||||
- Wrap in async IIFE: (async () => {{ try {{ ... }} finally {{ ... }} }})();
|
||||
- Add cleanup in finally: await fetch("{hindsight_url}/v1/default/banks/{bank_id}", {{ method: "DELETE" }})
|
||||
- End with: console.log("TEST PASSED")"""
|
||||
|
||||
elif example.language in ["bash", "sh"]:
|
||||
output_format = f"""Output a Bash script:
|
||||
- Start with #!/bin/bash and set -e
|
||||
- Use trap for cleanup: curl -s -X DELETE "{hindsight_url}/v1/default/banks/{bank_id}"
|
||||
- End with: echo "TEST PASSED" """
|
||||
|
||||
else:
|
||||
return "", f"Unsupported language: {example.language}"
|
||||
|
||||
prompt = f"""The documentation code below is the TEST CASE. Your job is to make it runnable.
|
||||
|
||||
DOCUMENTATION CODE ({example.language}):
|
||||
```
|
||||
{example.code}
|
||||
```
|
||||
|
||||
RULES:
|
||||
1. The doc code is SACRED - do not modify its logic, method calls, or parameters
|
||||
2. You MAY add setup BEFORE it:
|
||||
- Import statements the code assumes exist
|
||||
- Object instantiation (e.g., if code uses 'client.foo()', create the client first)
|
||||
- Variable definitions
|
||||
3. You MAY add cleanup AFTER it
|
||||
4. Replace placeholder values:
|
||||
- URLs like localhost:8888 → {hindsight_url}
|
||||
- Bank IDs like "my-bank", "demo", <bank_id> → "{bank_id}"
|
||||
- Placeholder IDs like <entity_id>, <document_id> → "test-id"
|
||||
|
||||
{output_format}
|
||||
|
||||
Output ONLY the complete runnable code, no explanation."""
|
||||
|
||||
is_reasoning = model.startswith(("o1", "o3"))
|
||||
kwargs = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
}
|
||||
if is_reasoning:
|
||||
kwargs["max_completion_tokens"] = 4000
|
||||
else:
|
||||
kwargs["temperature"] = 0
|
||||
kwargs["max_tokens"] = 4000
|
||||
|
||||
try:
|
||||
response = client.chat.completions.create(**kwargs)
|
||||
script = response.choices[0].message.content
|
||||
|
||||
# Clean up markdown code blocks if present
|
||||
script = re.sub(r'^```\w*\n', '', script)
|
||||
script = re.sub(r'\n```$', '', script)
|
||||
script = script.strip()
|
||||
|
||||
return script, None
|
||||
except Exception as e:
|
||||
return "", f"Transform failed: {e}"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# STEP 4: Run tests
|
||||
# =============================================================================
|
||||
|
||||
def get_python_path() -> str:
|
||||
"""Get PYTHONPATH that includes all installed packages."""
|
||||
paths = []
|
||||
|
||||
# Add virtual environment site-packages if in a venv
|
||||
if hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix):
|
||||
# We're in a virtual environment
|
||||
venv_site = os.path.join(sys.prefix, 'lib', f'python{sys.version_info.major}.{sys.version_info.minor}', 'site-packages')
|
||||
if os.path.exists(venv_site):
|
||||
paths.append(venv_site)
|
||||
|
||||
# Add system site-packages
|
||||
paths.extend(site.getsitepackages())
|
||||
|
||||
# Add user site-packages
|
||||
user_site = site.getusersitepackages()
|
||||
if user_site and os.path.exists(user_site):
|
||||
paths.append(user_site)
|
||||
|
||||
# Add existing PYTHONPATH
|
||||
existing = os.environ.get("PYTHONPATH", "")
|
||||
if existing:
|
||||
paths.append(existing)
|
||||
|
||||
return ":".join(paths)
|
||||
|
||||
|
||||
def run_python(script: str, timeout: int = 60) -> tuple[bool, str, Optional[str]]:
|
||||
"""Run Python script."""
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
|
||||
f.write(script)
|
||||
f.flush()
|
||||
try:
|
||||
pythonpath = get_python_path()
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, f.name],
|
||||
capture_output=True, text=True, timeout=timeout,
|
||||
env={**os.environ, "PYTHONPATH": pythonpath}
|
||||
)
|
||||
output = result.stdout + result.stderr
|
||||
if "TEST PASSED" in output:
|
||||
return True, output, None
|
||||
return result.returncode == 0, output, result.stderr if result.returncode != 0 else None
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, "", "Timeout"
|
||||
except Exception as e:
|
||||
return False, "", str(e)
|
||||
finally:
|
||||
os.unlink(f.name)
|
||||
|
||||
|
||||
def run_javascript(script: str, timeout: int = 60) -> tuple[bool, str, Optional[str]]:
|
||||
"""Run JavaScript script."""
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.mjs', delete=False, dir='/tmp') as f:
|
||||
f.write(script)
|
||||
f.flush()
|
||||
try:
|
||||
env = {**os.environ}
|
||||
env["NODE_PATH"] = f"/tmp/node_modules:{env.get('NODE_PATH', '')}"
|
||||
|
||||
result = subprocess.run(
|
||||
["node", f.name],
|
||||
capture_output=True, text=True, timeout=timeout,
|
||||
env=env, cwd="/tmp"
|
||||
)
|
||||
output = result.stdout + result.stderr
|
||||
if "TEST PASSED" in output:
|
||||
return True, output, None
|
||||
return result.returncode == 0, output, result.stderr if result.returncode != 0 else None
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, "", "Timeout"
|
||||
except Exception as e:
|
||||
return False, "", str(e)
|
||||
finally:
|
||||
os.unlink(f.name)
|
||||
|
||||
|
||||
def run_bash(script: str, timeout: int = 60) -> tuple[bool, str, Optional[str]]:
|
||||
"""Run bash script."""
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.sh', delete=False) as f:
|
||||
f.write(script)
|
||||
f.flush()
|
||||
os.chmod(f.name, 0o755)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["bash", f.name],
|
||||
capture_output=True, text=True, timeout=timeout
|
||||
)
|
||||
output = result.stdout + result.stderr
|
||||
if "TEST PASSED" in output:
|
||||
return True, output, None
|
||||
return result.returncode == 0, output, result.stderr if result.returncode != 0 else None
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, "", "Timeout"
|
||||
except Exception as e:
|
||||
return False, "", str(e)
|
||||
finally:
|
||||
os.unlink(f.name)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# STEP 5: Analyze failures with LLM
|
||||
# =============================================================================
|
||||
|
||||
def get_source_context(example: CodeExample, repo_root: str) -> str:
|
||||
"""Get relevant source code for failure analysis."""
|
||||
parts = []
|
||||
code_lower = example.code.lower()
|
||||
|
||||
if example.language == "python":
|
||||
if "recall" in code_lower or "weight" in code_lower:
|
||||
try:
|
||||
with open(os.path.join(repo_root, "hindsight-clients/python/hindsight_client_api/models/recall_result.py")) as f:
|
||||
parts.append("=== RecallResult Model ===\n" + f.read()[:2000])
|
||||
except: pass
|
||||
if "reflect" in code_lower:
|
||||
try:
|
||||
with open(os.path.join(repo_root, "hindsight-clients/python/hindsight_client_api/models/reflect_response.py")) as f:
|
||||
parts.append("=== ReflectResponse Model ===\n" + f.read()[:2000])
|
||||
except: pass
|
||||
try:
|
||||
with open(os.path.join(repo_root, "hindsight-clients/python/hindsight_client/__init__.py")) as f:
|
||||
parts.append("=== Hindsight Client ===\n" + f.read()[:3000])
|
||||
except: pass
|
||||
|
||||
elif example.language in ["typescript", "javascript"]:
|
||||
try:
|
||||
with open(os.path.join(repo_root, "hindsight-clients/typescript/src/index.ts")) as f:
|
||||
parts.append("=== TypeScript Client ===\n" + f.read()[:4000])
|
||||
except: pass
|
||||
|
||||
elif example.language in ["bash", "sh"]:
|
||||
try:
|
||||
with open(os.path.join(repo_root, "hindsight-cli/src/main.rs")) as f:
|
||||
lines = f.read().split('\n')[:350]
|
||||
parts.append("=== CLI Commands ===\n" + '\n'.join(lines))
|
||||
except: pass
|
||||
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def get_doc_context(example: CodeExample) -> str:
|
||||
"""Get the full documentation context around the failing code example."""
|
||||
try:
|
||||
with open(example.file_path, "r") as f:
|
||||
content = f.read()
|
||||
|
||||
# Find the code block and get surrounding context (500 chars before/after)
|
||||
# This gives us the explanatory text around the code
|
||||
code_start = content.find(example.code[:50]) # Find by first 50 chars
|
||||
if code_start == -1:
|
||||
code_start = example.line_number * 50 # Rough estimate
|
||||
|
||||
start = max(0, code_start - 500)
|
||||
end = min(len(content), code_start + len(example.code) + 500)
|
||||
|
||||
return content[start:end]
|
||||
except:
|
||||
return example.context # Fall back to the small context we already have
|
||||
|
||||
|
||||
def analyze_failure(client: OpenAI, result: TestResult, repo_root: str, model: str) -> dict:
|
||||
"""Use LLM to determine if failure is a real doc bug."""
|
||||
source = get_source_context(result.example, repo_root)
|
||||
doc_context = get_doc_context(result.example)
|
||||
|
||||
prompt = f"""Analyze this documentation test failure.
|
||||
|
||||
## Documentation File: {result.example.file_path}
|
||||
|
||||
### Documentation Context (text around the code example)
|
||||
```markdown
|
||||
{doc_context}
|
||||
```
|
||||
|
||||
### The Code Example Being Tested (line {result.example.line_number})
|
||||
```{result.example.language}
|
||||
{result.example.code}
|
||||
```
|
||||
|
||||
## Error When Running
|
||||
{result.error[:800] if result.error else "Unknown"}
|
||||
|
||||
## Transformed Test Code (what we actually ran)
|
||||
```
|
||||
{result.transformed_code[:1500] if result.transformed_code else "N/A"}
|
||||
```
|
||||
|
||||
## Actual Source Code (ground truth - what the API really looks like)
|
||||
{source[:6000] if source else "Not available"}
|
||||
|
||||
## Your Task
|
||||
Compare the DOCUMENTATION against the ACTUAL SOURCE CODE.
|
||||
|
||||
1. Does the documentation show something that doesn't exist in the source code?
|
||||
- Wrong method names?
|
||||
- Wrong attribute names (e.g., .weight when there's no weight field)?
|
||||
- Wrong CLI commands?
|
||||
- Wrong parameters?
|
||||
|
||||
2. Or is the documentation correct, but our test transformation/execution failed?
|
||||
- Missing imports we didn't add?
|
||||
- Environment issues?
|
||||
- Timing/race conditions?
|
||||
|
||||
Respond JSON:
|
||||
{{
|
||||
"is_doc_bug": true/false,
|
||||
"confidence": "high/medium/low",
|
||||
"reason": "brief explanation of what's wrong",
|
||||
"fix": "if doc bug, what should the doc say instead"
|
||||
}}"""
|
||||
|
||||
is_reasoning = model.startswith(("o1", "o3"))
|
||||
kwargs = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
if is_reasoning:
|
||||
kwargs["max_completion_tokens"] = 2000
|
||||
else:
|
||||
kwargs["temperature"] = 0
|
||||
|
||||
try:
|
||||
response = client.chat.completions.create(**kwargs)
|
||||
return json.loads(response.choices[0].message.content)
|
||||
except Exception as e:
|
||||
return {"is_doc_bug": True, "confidence": "low", "reason": str(e)}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Main test runner
|
||||
# =============================================================================
|
||||
|
||||
def test_example(example: CodeExample, openai_client: OpenAI, hindsight_url: str, cli_available: bool, model: str) -> TestResult:
|
||||
"""Test a single code example."""
|
||||
|
||||
# Check if should skip
|
||||
skip = should_skip(example.code, example.language)
|
||||
if skip:
|
||||
return TestResult(example=example, success=True, output="", skip_reason=skip)
|
||||
|
||||
# Transform using LLM
|
||||
try:
|
||||
transformed, skip = transform_code(openai_client, example, hindsight_url, cli_available, model)
|
||||
if skip:
|
||||
return TestResult(example=example, success=True, output="", skip_reason=skip)
|
||||
|
||||
if not transformed:
|
||||
return TestResult(example=example, success=True, output="", skip_reason="Transform returned empty")
|
||||
|
||||
# Run based on language
|
||||
if example.language == "python":
|
||||
success, output, error = run_python(transformed)
|
||||
elif example.language in ["typescript", "javascript"]:
|
||||
success, output, error = run_javascript(transformed)
|
||||
elif example.language in ["bash", "sh"]:
|
||||
success, output, error = run_bash(transformed)
|
||||
else:
|
||||
return TestResult(example=example, success=True, output="", skip_reason=f"Unsupported: {example.language}")
|
||||
|
||||
return TestResult(
|
||||
example=example,
|
||||
success=success,
|
||||
output=output,
|
||||
error=error,
|
||||
transformed_code=transformed
|
||||
)
|
||||
except Exception as e:
|
||||
return TestResult(
|
||||
example=example,
|
||||
success=False,
|
||||
output="",
|
||||
error=f"Transform error: {e}\n{traceback.format_exc()}"
|
||||
)
|
||||
|
||||
|
||||
def check_cli_available() -> bool:
|
||||
"""Check if hindsight CLI is available."""
|
||||
try:
|
||||
result = subprocess.run(["hindsight", "--version"], capture_output=True, timeout=5)
|
||||
return result.returncode == 0
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
def check_dependencies() -> dict[str, bool]:
|
||||
"""Check which dependencies are available for doc tests."""
|
||||
deps = {}
|
||||
|
||||
# Check Python packages
|
||||
python_packages = [
|
||||
("hindsight_client", "Hindsight Python client"),
|
||||
("hindsight_litellm", "Hindsight LiteLLM integration"),
|
||||
("hindsight_openai", "Hindsight OpenAI integration"),
|
||||
("anthropic", "Anthropic SDK"),
|
||||
("openai", "OpenAI SDK"),
|
||||
]
|
||||
|
||||
for module, name in python_packages:
|
||||
try:
|
||||
__import__(module)
|
||||
deps[module] = True
|
||||
except ImportError:
|
||||
deps[module] = False
|
||||
|
||||
return deps
|
||||
|
||||
|
||||
def print_dependency_status(deps: dict[str, bool]):
|
||||
"""Print dependency availability status."""
|
||||
print("\n=== Dependencies ===")
|
||||
for name, available in deps.items():
|
||||
status = "✓" if available else "✗"
|
||||
print(f" {status} {name}")
|
||||
|
||||
# Print PYTHONPATH for debugging
|
||||
pythonpath = get_python_path()
|
||||
print(f"\nPYTHONPATH: {pythonpath[:100]}..." if len(pythonpath) > 100 else f"\nPYTHONPATH: {pythonpath}")
|
||||
print(f"Python: {sys.executable}")
|
||||
print(f"Prefix: {sys.prefix}")
|
||||
print()
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(line_buffering=True)
|
||||
|
||||
openai_key = os.environ.get("OPENAI_API_KEY")
|
||||
if not openai_key:
|
||||
print("ERROR: OPENAI_API_KEY required")
|
||||
sys.exit(1)
|
||||
|
||||
hindsight_url = os.environ.get("HINDSIGHT_API_URL", "http://localhost:8888")
|
||||
model = os.environ.get("DOC_TEST_MODEL", "gpt-4o")
|
||||
|
||||
# Find repo root - go up from script location
|
||||
script_path = os.path.abspath(__file__)
|
||||
repo_root = os.path.dirname(os.path.dirname(script_path))
|
||||
|
||||
# If running from a subdirectory (like hindsight-api), detect and fix
|
||||
if not os.path.exists(os.path.join(repo_root, "hindsight-docs")):
|
||||
# Try going up one more level
|
||||
repo_root = os.path.dirname(repo_root)
|
||||
if not os.path.exists(os.path.join(repo_root, "hindsight-docs")):
|
||||
# Fall back to REPO_ROOT env var or cwd
|
||||
repo_root = os.environ.get("REPO_ROOT", os.getcwd())
|
||||
|
||||
print(f"Repo: {repo_root}")
|
||||
print(f"API: {hindsight_url}")
|
||||
print(f"Model: {model}")
|
||||
|
||||
# Check CLI
|
||||
cli_available = check_cli_available()
|
||||
print(f"CLI: {'available' if cli_available else 'not available'}")
|
||||
|
||||
# Check and print dependencies
|
||||
deps = check_dependencies()
|
||||
print_dependency_status(deps)
|
||||
|
||||
# Warn if critical dependencies are missing
|
||||
if not deps.get("hindsight_client"):
|
||||
print("WARNING: hindsight_client not available - Python examples will fail")
|
||||
print(" Install with: pip install hindsight-client or uv pip install <path-to-client>")
|
||||
|
||||
# Check API health
|
||||
try:
|
||||
import urllib.request
|
||||
urllib.request.urlopen(f"{hindsight_url}/health", timeout=5)
|
||||
print("API: healthy")
|
||||
except Exception as e:
|
||||
print(f"API: WARNING - {e}")
|
||||
|
||||
# Initialize OpenAI client early (needed for transforms and analysis)
|
||||
client = OpenAI(api_key=openai_key)
|
||||
|
||||
# Find and extract examples
|
||||
md_files = find_markdown_files(repo_root)
|
||||
print(f"\nFound {len(md_files)} markdown files")
|
||||
|
||||
all_examples = []
|
||||
for md_file in md_files:
|
||||
examples = extract_code_blocks(md_file)
|
||||
if examples:
|
||||
all_examples.extend(examples)
|
||||
|
||||
print(f"Found {len(all_examples)} code examples")
|
||||
|
||||
# Run tests
|
||||
report = TestReport()
|
||||
max_workers = int(os.environ.get("MAX_WORKERS", "4")) # Lower default since LLM calls are slower
|
||||
|
||||
print(f"\nRunning tests with {max_workers} workers...")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {executor.submit(test_example, ex, client, hindsight_url, cli_available, model): ex for ex in all_examples}
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
report.add_result(result)
|
||||
|
||||
status = "SKIP" if result.skip_reason else ("PASS" if result.success else "FAIL")
|
||||
safe_print(f" [{status}] {result.example.file_path}:{result.example.line_number}")
|
||||
|
||||
# Print summary
|
||||
print("\n" + "=" * 60)
|
||||
print(f"Total: {report.total} | Pass: {report.passed} | Fail: {report.failed} | Skip: {report.skipped}")
|
||||
print("=" * 60)
|
||||
|
||||
# Analyze failures with LLM
|
||||
failures = [r for r in report.results if not r.success and not r.skip_reason]
|
||||
|
||||
if failures:
|
||||
print(f"\n=== Analyzing {len(failures)} failures (parallel) ===")
|
||||
|
||||
doc_bugs = []
|
||||
test_issues = []
|
||||
results_lock = threading.Lock()
|
||||
completed = [0] # Use list for mutable counter in closure
|
||||
|
||||
def analyze_one(result: TestResult) -> None:
|
||||
analysis = analyze_failure(client, result, repo_root, model)
|
||||
entry = {
|
||||
"file": result.example.file_path,
|
||||
"line": result.example.line_number,
|
||||
"error": result.error[:200] if result.error else "",
|
||||
"analysis": analysis
|
||||
}
|
||||
|
||||
with results_lock:
|
||||
completed[0] += 1
|
||||
idx = completed[0]
|
||||
if analysis.get("is_doc_bug", True):
|
||||
doc_bugs.append(entry)
|
||||
safe_print(f" [{idx}/{len(failures)}] {result.example.file_path}:{result.example.line_number}")
|
||||
safe_print(f" → DOC BUG: {analysis.get('reason', '')[:50]}")
|
||||
else:
|
||||
test_issues.append(entry)
|
||||
safe_print(f" [{idx}/{len(failures)}] {result.example.file_path}:{result.example.line_number}")
|
||||
safe_print(f" → Test issue: {analysis.get('reason', '')[:50]}")
|
||||
|
||||
# Run analysis in parallel (limit concurrency to avoid rate limits)
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
futures = [executor.submit(analyze_one, result) for result in failures]
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e:
|
||||
safe_print(f" Analysis error: {e}")
|
||||
|
||||
# Write summary
|
||||
print(f"\n=== RESULTS ===")
|
||||
print(f"Documentation bugs: {len(doc_bugs)}")
|
||||
print(f"Test/CI issues: {len(test_issues)}")
|
||||
|
||||
if doc_bugs:
|
||||
print(f"\n--- Documentation Bugs ---")
|
||||
for bug in doc_bugs:
|
||||
print(f" {bug['file']}:{bug['line']}")
|
||||
print(f" Reason: {bug['analysis'].get('reason', 'Unknown')}")
|
||||
if bug['analysis'].get('fix'):
|
||||
print(f" Fix: {bug['analysis']['fix']}")
|
||||
|
||||
if test_issues:
|
||||
print(f"\n--- Test/CI Issues (not doc bugs) ---")
|
||||
for issue in test_issues:
|
||||
print(f" {issue['file']}:{issue['line']}")
|
||||
print(f" Reason: {issue['analysis'].get('reason', 'Unknown')}")
|
||||
|
||||
# Write GitHub summary (include ALL failures for visibility)
|
||||
write_summary(report, doc_bugs, test_issues)
|
||||
|
||||
# Exit code based on real doc bugs only
|
||||
sys.exit(1 if doc_bugs else 0)
|
||||
else:
|
||||
print("\nAll tests passed!")
|
||||
write_summary(report, [], [])
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def write_summary(report: TestReport, doc_bugs: list, test_issues: list):
|
||||
"""Write GitHub Actions summary file."""
|
||||
with open("/tmp/doc-test-summary.md", "w") as f:
|
||||
# Header
|
||||
status = "❌" if doc_bugs else "✅"
|
||||
f.write(f"# {status} Documentation Test Results\n\n")
|
||||
|
||||
# Summary table
|
||||
f.write(f"| Metric | Count |\n")
|
||||
f.write(f"|--------|-------|\n")
|
||||
f.write(f"| Total | {report.total} |\n")
|
||||
f.write(f"| ✅ Passed | {report.passed} |\n")
|
||||
f.write(f"| ❌ Failed | {report.failed} |\n")
|
||||
f.write(f"| ⏭️ Skipped | {report.skipped} |\n\n")
|
||||
|
||||
if doc_bugs or test_issues:
|
||||
f.write(f"| Category | Count |\n")
|
||||
f.write(f"|----------|-------|\n")
|
||||
f.write(f"| 🐛 Documentation Bugs | {len(doc_bugs)} |\n")
|
||||
f.write(f"| ⚠️ Test/CI Issues | {len(test_issues)} |\n\n")
|
||||
|
||||
# Documentation bugs section
|
||||
if doc_bugs:
|
||||
f.write(f"## 🐛 Documentation Bugs ({len(doc_bugs)})\n\n")
|
||||
f.write("These are real issues in the documentation that need to be fixed:\n\n")
|
||||
for bug in doc_bugs:
|
||||
file_short = bug['file'].split('/hindsight/')[-1] if '/hindsight/' in bug['file'] else bug['file']
|
||||
f.write(f"### `{file_short}:{bug['line']}`\n")
|
||||
f.write(f"- **Issue**: {bug['analysis'].get('reason', 'Unknown')}\n")
|
||||
if bug['analysis'].get('fix'):
|
||||
f.write(f"- **Suggested Fix**: {bug['analysis']['fix']}\n")
|
||||
if bug.get('error'):
|
||||
f.write(f"- **Error**: `{bug['error'][:150]}...`\n")
|
||||
f.write("\n")
|
||||
|
||||
# Test/CI issues section
|
||||
if test_issues:
|
||||
f.write(f"## ⚠️ Test/CI Issues ({len(test_issues)})\n\n")
|
||||
f.write("These failures are NOT documentation bugs - they're issues with the test setup or CI environment:\n\n")
|
||||
for issue in test_issues:
|
||||
file_short = issue['file'].split('/hindsight/')[-1] if '/hindsight/' in issue['file'] else issue['file']
|
||||
f.write(f"### `{file_short}:{issue['line']}`\n")
|
||||
f.write(f"- **Reason**: {issue['analysis'].get('reason', 'Unknown')}\n")
|
||||
if issue.get('error'):
|
||||
f.write(f"- **Error**: `{issue['error'][:150]}...`\n")
|
||||
f.write("\n")
|
||||
|
||||
# No failures
|
||||
if not doc_bugs and not test_issues:
|
||||
if report.passed > 0:
|
||||
f.write(f"All {report.passed} tests passed! ({report.skipped} skipped)\n")
|
||||
else:
|
||||
f.write(f"All {report.skipped} examples were skipped (install commands, docker, etc.)\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1141,7 +1141,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "hindsight-all"
|
||||
version = "0.1.4"
|
||||
version = "0.1.5"
|
||||
source = { editable = "hindsight" }
|
||||
dependencies = [
|
||||
{ name = "hindsight-api" },
|
||||
@@ -1165,7 +1165,7 @@ provides-extras = ["test"]
|
||||
|
||||
[[package]]
|
||||
name = "hindsight-api"
|
||||
version = "0.1.4"
|
||||
version = "0.1.5"
|
||||
source = { editable = "hindsight-api" }
|
||||
dependencies = [
|
||||
{ name = "alembic" },
|
||||
@@ -1223,7 +1223,7 @@ requires-dist = [
|
||||
{ name = "asyncpg", specifier = ">=0.29.0" },
|
||||
{ name = "dateparser", specifier = ">=1.2.2" },
|
||||
{ name = "fastapi", extras = ["standard"], specifier = ">=0.120.3" },
|
||||
{ name = "fastmcp", specifier = ">=2.0.0" },
|
||||
{ name = "fastmcp", specifier = ">=2.3.0" },
|
||||
{ name = "filelock", marker = "extra == 'test'", specifier = ">=3.0.0" },
|
||||
{ name = "google-genai", specifier = ">=1.0.0" },
|
||||
{ name = "greenlet", specifier = ">=3.2.4" },
|
||||
@@ -1267,7 +1267,7 @@ dev = [
|
||||
|
||||
[[package]]
|
||||
name = "hindsight-client"
|
||||
version = "0.1.4"
|
||||
version = "0.1.5"
|
||||
source = { editable = "hindsight-clients/python" }
|
||||
dependencies = [
|
||||
{ name = "aiohttp" },
|
||||
@@ -1299,7 +1299,7 @@ provides-extras = ["test"]
|
||||
|
||||
[[package]]
|
||||
name = "hindsight-dev"
|
||||
version = "0.1.4"
|
||||
version = "0.1.5"
|
||||
source = { editable = "hindsight-dev" }
|
||||
dependencies = [
|
||||
{ name = "hindsight-api" },
|
||||
|
||||
Reference in New Issue
Block a user