Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3d66988c3b | ||
|
|
67dc160ba3 | ||
|
|
8291386387 | ||
|
|
3511062c51 | ||
|
|
a134d27137 | ||
|
|
31a53c0870 |
+20
-14
@@ -1,41 +1,47 @@
|
||||
# =============================================================================
|
||||
# MEMORA ENVIRONMENT CONFIGURATION
|
||||
# HINDSIGHT ENVIRONMENT CONFIGURATION
|
||||
# =============================================================================
|
||||
# Copy this file to .env and update with your values
|
||||
# Both services (API and Control Plane) read from this single file
|
||||
|
||||
# =============================================================================
|
||||
# API SERVICE (MEMORA_API_*)
|
||||
# API SERVICE (HINDSIGHT_API_*)
|
||||
# =============================================================================
|
||||
|
||||
# Database
|
||||
MEMORA_API_DATABASE_URL=postgresql://memora:memora_dev@localhost:5432/memora
|
||||
# Use "pg0" to start an embedded PostgreSQL instance via pg0
|
||||
# Or provide a full connection URL for external PostgreSQL
|
||||
#HINDSIGHT_API_DATABASE_URL=postgresql://hindsight:hindsight_dev@localhost:5432/hindsight
|
||||
HINDSIGHT_API_DATABASE_URL=pg0
|
||||
|
||||
# pg0 data directory (only used when HINDSIGHT_API_DATABASE_URL=pg0)
|
||||
# HINDSIGHT_API_PG0_DATA_DIR=/path/to/pg_data
|
||||
|
||||
# LLM Provider: "openai", "groq", or "ollama"
|
||||
MEMORA_API_LLM_PROVIDER=groq
|
||||
HINDSIGHT_API_LLM_PROVIDER=groq
|
||||
|
||||
# LLM Model (provider-specific)
|
||||
MEMORA_API_LLM_MODEL=openai/gpt-oss-20b
|
||||
HINDSIGHT_API_LLM_MODEL=openai/gpt-oss-20b
|
||||
|
||||
# API Key (not needed for ollama)
|
||||
MEMORA_API_LLM_API_KEY=your_api_key_here
|
||||
HINDSIGHT_API_LLM_API_KEY=your_api_key_here
|
||||
|
||||
# Optional: Custom base URL (for ollama or custom endpoints)
|
||||
# MEMORA_API_LLM_BASE_URL=http://localhost:11434/v1
|
||||
# HINDSIGHT_API_LLM_BASE_URL=http://localhost:11434/v1
|
||||
|
||||
# API Server Configuration (optional)
|
||||
# MEMORA_API_HOST=0.0.0.0
|
||||
# MEMORA_API_PORT=8080
|
||||
# HINDSIGHT_API_HOST=0.0.0.0
|
||||
# HINDSIGHT_API_PORT=8888
|
||||
|
||||
MEMORA_API_MCP_ENABLED=true
|
||||
HINDSIGHT_API_MCP_ENABLED=true
|
||||
|
||||
# =============================================================================
|
||||
# CONTROL PLANE SERVICE (MEMORA_CP_*)
|
||||
# CONTROL PLANE SERVICE (HINDSIGHT_CP_*)
|
||||
# =============================================================================
|
||||
|
||||
# Dataplane API URL (where the control plane connects to)
|
||||
MEMORA_CP_DATAPLANE_API_URL=http://localhost:8080
|
||||
HINDSIGHT_CP_DATAPLANE_API_URL=http://localhost:8888
|
||||
|
||||
# Control Plane Server Configuration (optional)
|
||||
# MEMORA_CP_PORT=3000
|
||||
# MEMORA_CP_HOSTNAME=0.0.0.0
|
||||
# HINDSIGHT_CP_PORT=3000
|
||||
# HINDSIGHT_CP_HOSTNAME=0.0.0.0
|
||||
|
||||
@@ -2,9 +2,10 @@ name: Deploy Docs to GitHub Pages
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: [main, renaming-pre-launch]
|
||||
paths:
|
||||
- 'memora-docs/**'
|
||||
- 'hindsight-docs/**'
|
||||
- '.github/workflows/deploy-docs.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
@@ -21,20 +22,19 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: memora-docs
|
||||
working-directory: hindsight-docs
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: npm
|
||||
cache-dependency-path: memora-docs/package-lock.json
|
||||
cache-dependency-path: hindsight-docs/package-lock.json
|
||||
- run: npm ci
|
||||
- run: npm run build
|
||||
- uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: memora-docs/build
|
||||
|
||||
path: hindsight-docs/build
|
||||
deploy:
|
||||
environment:
|
||||
name: github-pages
|
||||
|
||||
@@ -22,15 +22,15 @@ jobs:
|
||||
with:
|
||||
python-version-file: ".python-version"
|
||||
|
||||
- name: Build memora package
|
||||
working-directory: ./memora
|
||||
- name: Build hindsight package
|
||||
working-directory: ./hindsight
|
||||
run: uv build
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: python-memora-dist
|
||||
path: memora/dist/*
|
||||
name: python-hindsight-dist
|
||||
path: hindsight/dist/*
|
||||
retention-days: 30
|
||||
|
||||
build-rust-cli:
|
||||
@@ -40,16 +40,16 @@ jobs:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
target: x86_64-unknown-linux-gnu
|
||||
artifact_name: memora
|
||||
asset_name: memora-linux-amd64
|
||||
artifact_name: hindsight
|
||||
asset_name: hindsight-linux-amd64
|
||||
- os: macos-latest
|
||||
target: x86_64-apple-darwin
|
||||
artifact_name: memora
|
||||
asset_name: memora-darwin-amd64
|
||||
artifact_name: hindsight
|
||||
asset_name: hindsight-darwin-amd64
|
||||
- os: macos-latest
|
||||
target: aarch64-apple-darwin
|
||||
artifact_name: memora
|
||||
asset_name: memora-darwin-arm64
|
||||
artifact_name: hindsight
|
||||
asset_name: hindsight-darwin-arm64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -74,17 +74,17 @@ jobs:
|
||||
- name: Cache cargo build
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: memora-cli/target
|
||||
path: hindsight-cli/target
|
||||
key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
|
||||
|
||||
- name: Build
|
||||
working-directory: memora-cli
|
||||
working-directory: hindsight-cli
|
||||
run: cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Prepare artifact
|
||||
run: |
|
||||
mkdir -p artifacts
|
||||
cp memora-cli/target/${{ matrix.target }}/release/${{ matrix.artifact_name }} artifacts/${{ matrix.asset_name }}
|
||||
cp hindsight-cli/target/${{ matrix.target }}/release/${{ matrix.artifact_name }} artifacts/${{ matrix.asset_name }}
|
||||
chmod +x artifacts/${{ matrix.asset_name }}
|
||||
|
||||
- name: Upload artifacts
|
||||
@@ -135,7 +135,7 @@ jobs:
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ghcr.io/${{ github.repository_owner }}/memora-${{ matrix.component }}
|
||||
images: ghcr.io/${{ github.repository_owner }}/hindsight-${{ matrix.component }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}},value=${{ steps.get_version.outputs.VERSION }}
|
||||
type=semver,pattern={{major}}.{{minor}},value=${{ steps.get_version.outputs.VERSION }}
|
||||
@@ -179,11 +179,11 @@ jobs:
|
||||
|
||||
- name: Lint Helm chart
|
||||
run: |
|
||||
helm lint helm/memora
|
||||
helm lint helm/hindsight
|
||||
|
||||
- name: Package Helm chart
|
||||
run: |
|
||||
helm package helm/memora --destination ./helm-packages
|
||||
helm package helm/hindsight --destination ./helm-packages
|
||||
|
||||
- name: Upload Helm chart artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
@@ -208,26 +208,26 @@ jobs:
|
||||
- name: Download Python package
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: python-memora-dist
|
||||
path: ./artifacts/python-memora-dist
|
||||
name: python-hindsight-dist
|
||||
path: ./artifacts/python-hindsight-dist
|
||||
|
||||
- name: Download Rust CLI (Linux)
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: rust-cli-memora-linux-amd64
|
||||
path: ./artifacts/rust-cli-memora-linux-amd64
|
||||
name: rust-cli-hindsight-linux-amd64
|
||||
path: ./artifacts/rust-cli-hindsight-linux-amd64
|
||||
|
||||
- name: Download Rust CLI (macOS Intel)
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: rust-cli-memora-darwin-amd64
|
||||
path: ./artifacts/rust-cli-memora-darwin-amd64
|
||||
name: rust-cli-hindsight-darwin-amd64
|
||||
path: ./artifacts/rust-cli-hindsight-darwin-amd64
|
||||
|
||||
- name: Download Rust CLI (macOS ARM)
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: rust-cli-memora-darwin-arm64
|
||||
path: ./artifacts/rust-cli-memora-darwin-arm64
|
||||
name: rust-cli-hindsight-darwin-arm64
|
||||
path: ./artifacts/rust-cli-hindsight-darwin-arm64
|
||||
|
||||
- name: Download Helm chart
|
||||
uses: actions/download-artifact@v4
|
||||
@@ -239,11 +239,11 @@ jobs:
|
||||
run: |
|
||||
mkdir -p release-assets
|
||||
# Python package
|
||||
cp artifacts/python-memora-dist/* release-assets/
|
||||
cp artifacts/python-hindsight-dist/* release-assets/
|
||||
# Rust CLI binaries
|
||||
cp artifacts/rust-cli-memora-linux-amd64/memora-linux-amd64 release-assets/
|
||||
cp artifacts/rust-cli-memora-darwin-amd64/memora-darwin-amd64 release-assets/
|
||||
cp artifacts/rust-cli-memora-darwin-arm64/memora-darwin-arm64 release-assets/
|
||||
cp artifacts/rust-cli-hindsight-linux-amd64/hindsight-linux-amd64 release-assets/
|
||||
cp artifacts/rust-cli-hindsight-darwin-amd64/hindsight-darwin-amd64 release-assets/
|
||||
cp artifacts/rust-cli-hindsight-darwin-arm64/hindsight-darwin-arm64 release-assets/
|
||||
# Helm chart
|
||||
cp artifacts/helm-chart/*.tgz release-assets/
|
||||
|
||||
@@ -251,68 +251,68 @@ jobs:
|
||||
id: release_notes
|
||||
run: |
|
||||
cat << EOF > release-notes.md
|
||||
# Memora v${{ steps.get_version.outputs.VERSION }}
|
||||
# Hindsight v${{ steps.get_version.outputs.VERSION }}
|
||||
|
||||
## 📦 Release Artifacts
|
||||
|
||||
### Python Package
|
||||
- \`memora-${{ steps.get_version.outputs.VERSION }}-py3-none-any.whl\`
|
||||
- \`memora-${{ steps.get_version.outputs.VERSION }}.tar.gz\`
|
||||
- \`hindsight-${{ steps.get_version.outputs.VERSION }}-py3-none-any.whl\`
|
||||
- \`hindsight-${{ steps.get_version.outputs.VERSION }}.tar.gz\`
|
||||
|
||||
### CLI Binaries
|
||||
- \`memora-linux-amd64\` - Linux x86_64
|
||||
- \`memora-darwin-amd64\` - macOS Intel
|
||||
- \`memora-darwin-arm64\` - macOS Apple Silicon
|
||||
- \`hindsight-linux-amd64\` - Linux x86_64
|
||||
- \`hindsight-darwin-amd64\` - macOS Intel
|
||||
- \`hindsight-darwin-arm64\` - macOS Apple Silicon
|
||||
|
||||
### Helm Chart
|
||||
- \`memora-${{ steps.get_version.outputs.VERSION }}.tgz\`
|
||||
- \`hindsight-${{ steps.get_version.outputs.VERSION }}.tgz\`
|
||||
|
||||
### Docker Images
|
||||
Docker images are published to GitHub Container Registry:
|
||||
- \`ghcr.io/${{ github.repository_owner }}/memora-api:${{ steps.get_version.outputs.VERSION }}\`
|
||||
- \`ghcr.io/${{ github.repository_owner }}/memora-control-plane:${{ steps.get_version.outputs.VERSION }}\`
|
||||
- \`ghcr.io/${{ github.repository_owner }}/hindsight-api:${{ steps.get_version.outputs.VERSION }}\`
|
||||
- \`ghcr.io/${{ github.repository_owner }}/hindsight-control-plane:${{ steps.get_version.outputs.VERSION }}\`
|
||||
|
||||
## 🚀 Installation
|
||||
|
||||
### Python Package
|
||||
\`\`\`bash
|
||||
pip install memora==${{ steps.get_version.outputs.VERSION }}
|
||||
pip install hindsight==${{ steps.get_version.outputs.VERSION }}
|
||||
\`\`\`
|
||||
|
||||
### CLI
|
||||
\`\`\`bash
|
||||
# macOS (Apple Silicon)
|
||||
curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/memora-darwin-arm64 -o memora
|
||||
chmod +x memora
|
||||
sudo mv memora /usr/local/bin/
|
||||
curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/hindsight-darwin-arm64 -o hindsight
|
||||
chmod +x hindsight
|
||||
sudo mv hindsight /usr/local/bin/
|
||||
|
||||
# macOS (Intel)
|
||||
curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/memora-darwin-amd64 -o memora
|
||||
chmod +x memora
|
||||
sudo mv memora /usr/local/bin/
|
||||
curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/hindsight-darwin-amd64 -o hindsight
|
||||
chmod +x hindsight
|
||||
sudo mv hindsight /usr/local/bin/
|
||||
|
||||
# Linux
|
||||
curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/memora-linux-amd64 -o memora
|
||||
chmod +x memora
|
||||
sudo mv memora /usr/local/bin/
|
||||
curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/hindsight-linux-amd64 -o hindsight
|
||||
chmod +x hindsight
|
||||
sudo mv hindsight /usr/local/bin/
|
||||
\`\`\`
|
||||
|
||||
### Helm Chart
|
||||
\`\`\`bash
|
||||
helm install memora memora-${{ steps.get_version.outputs.VERSION }}.tgz
|
||||
helm install hindsight hindsight-${{ steps.get_version.outputs.VERSION }}.tgz
|
||||
\`\`\`
|
||||
|
||||
### Docker
|
||||
\`\`\`bash
|
||||
# Pull API image
|
||||
docker pull ghcr.io/${{ github.repository_owner }}/memora-api:${{ steps.get_version.outputs.VERSION }}
|
||||
docker pull ghcr.io/${{ github.repository_owner }}/hindsight-api:${{ steps.get_version.outputs.VERSION }}
|
||||
|
||||
# Pull Control Plane image
|
||||
docker pull ghcr.io/${{ github.repository_owner }}/memora-control-plane:${{ steps.get_version.outputs.VERSION }}
|
||||
docker pull ghcr.io/${{ github.repository_owner }}/hindsight-control-plane:${{ steps.get_version.outputs.VERSION }}
|
||||
|
||||
# Or use latest
|
||||
docker pull ghcr.io/${{ github.repository_owner }}/memora-api:latest
|
||||
docker pull ghcr.io/${{ github.repository_owner }}/memora-control-plane:latest
|
||||
docker pull ghcr.io/${{ github.repository_owner }}/hindsight-api:latest
|
||||
docker pull ghcr.io/${{ github.repository_owner }}/hindsight-control-plane:latest
|
||||
\`\`\`
|
||||
EOF
|
||||
cat release-notes.md
|
||||
@@ -333,7 +333,7 @@ jobs:
|
||||
echo "# Release v${{ steps.get_version.outputs.VERSION }} Published Successfully" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "## 📦 Components" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ Python package (memora)" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ Python package (hindsight)" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ Rust CLI (Linux amd64, macOS amd64, macOS arm64)" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ Docker images (API, Control Plane)" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ Helm chart" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: memora_test
|
||||
POSTGRES_DB: hindsight_test
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
@@ -26,10 +26,10 @@ jobs:
|
||||
- 5432:5432
|
||||
|
||||
env:
|
||||
MEMORA_API_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/memora_test
|
||||
MEMORA_API_LLM_PROVIDER: groq
|
||||
MEMORA_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
MEMORA_API_LLM_MODEL: openai/gpt-oss-20b
|
||||
HINDSIGHT_API_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/hindsight_test
|
||||
HINDSIGHT_API_LLM_PROVIDER: groq
|
||||
HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -48,9 +48,9 @@ jobs:
|
||||
run: uv sync --extra test
|
||||
|
||||
- name: Run migrations
|
||||
working-directory: ./memora
|
||||
working-directory: ./hindsight
|
||||
run: |
|
||||
uv run alembic upgrade head
|
||||
|
||||
- name: Run tests
|
||||
run: uv run pytest memora/tests -v
|
||||
run: uv run pytest hindsight/tests -v
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
# Documentation
|
||||
Do not write any markdown file, just write the code.
|
||||
|
||||
# Workflow
|
||||
- After your changes, make sure everything is working fine by running the tests.
|
||||
- keep the readme.md architecture section up to date when you change the implementation
|
||||
- when changing an implemetation, do not keep the old one as fallback
|
||||
- to run test, use uv run pytest tests
|
||||
@@ -1,64 +1,57 @@
|
||||
# Memora
|
||||
# Hindsight
|
||||
|
||||
**Long-term memory for AI agents.**
|
||||
|
||||
AI assistants forget everything between sessions. Memora fixes that with a memory system that handles temporal reasoning, entity connections, and personality-aware responses.
|
||||
AI assistants forget everything between sessions. Hindsight fixes that with a memory system that handles temporal reasoning, entity connections, and personality-aware responses.
|
||||
|
||||
## Why Memora?
|
||||
## Why Hindsight?
|
||||
|
||||
- **Temporal queries** — "What did Alice do last spring?" requires more than vector search
|
||||
- **Entity connections** — Knowing "Alice works at Google" + "Google is in Mountain View" = "Alice works in Mountain View"
|
||||
- **Agent opinions** — Agents form and recall beliefs with confidence scores
|
||||
- **Personality** — Big Five traits influence how agents process and respond to information
|
||||
|
||||
## 5-Minute Setup
|
||||
## 60-seconds step
|
||||
|
||||
### 1. Start the server
|
||||
|
||||
### 1. Install the Hindsight All package (client + API)
|
||||
|
||||
```bash
|
||||
# Clone and start with Docker
|
||||
git clone https://github.com/anthropics/memora.git
|
||||
cd memora/docker
|
||||
./start.sh
|
||||
pip install hindsight-all
|
||||
```
|
||||
|
||||
Server runs at `http://localhost:8080`
|
||||
|
||||
### 2. Install the Python client
|
||||
|
||||
### 2. Import your OpenAI API key
|
||||
```bash
|
||||
pip install memora-client
|
||||
export OPENAI_API_KEY=xx
|
||||
```
|
||||
|
||||
### 3. Use it
|
||||
### 3. Run embedded server and client
|
||||
|
||||
```python
|
||||
from memora_client import Memora
|
||||
import os
|
||||
from hindsight import HindsightServer, HindsightClient
|
||||
|
||||
client = Memora(base_url="http://localhost:8080")
|
||||
with HindsightServer(llm_provider="openai", llm_model="gpt-5.1-mini", llm_api_key=os.environ["OPENAI_API_KEY"]) as server:
|
||||
client = HindsightClient(base_url=server.url)
|
||||
|
||||
# Store memories
|
||||
client.store(agent_id="my-agent", content="Alice works at Google")
|
||||
client.store(agent_id="my-agent", content="Bob prefers Python over JavaScript")
|
||||
|
||||
# Search memories
|
||||
results = client.search(agent_id="my-agent", query="What does Alice do?")
|
||||
for r in results:
|
||||
print(f"{r['text']} ({r['weight']:.2f})")
|
||||
|
||||
# Generate personality-aware responses
|
||||
answer = client.think(agent_id="my-agent", query="Tell me about Alice")
|
||||
print(answer["text"])
|
||||
client.put(agent_id="my-agent", content="Alice works at Google")
|
||||
client.put(agent_id="my-agent", content="Bob prefers Python over JavaScript")
|
||||
|
||||
client.search(agent_id="my-agent", query="What does Alice do?")
|
||||
|
||||
client.think(agent_id="my-agent", query="Tell me about Alice")
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Documentation
|
||||
|
||||
Full documentation: [memora-docs](./memora-docs)
|
||||
Full documentation: [hindsight-docs](./hindsight-docs)
|
||||
|
||||
- [Architecture](./memora-docs/docs/developer/architecture.md) — How ingestion, storage, and retrieval work
|
||||
- [Python Client](./memora-docs/docs/sdks/python.md) — Full API reference
|
||||
- [API Reference](./memora-docs/docs/api-reference/index.md) — REST API endpoints
|
||||
- [Personality](./memora-docs/docs/developer/personality.md) — Big Five traits and opinion formation
|
||||
- [Architecture](./hindsight-docs/docs/developer/architecture.md) — How ingestion, storage, and retrieval work
|
||||
- [Python Client](./hindsight-docs/docs/sdks/python.md) — Full API reference
|
||||
- [API Reference](./hindsight-docs/docs/api-reference/index.md) — REST API endpoints
|
||||
- [Personality](./hindsight-docs/docs/developer/personality.md) — Big Five traits and opinion formation
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+9
-9
@@ -6,7 +6,7 @@
|
||||
|
||||
```bash
|
||||
uv sync
|
||||
cd memora-dev
|
||||
cd hindsight-dev
|
||||
uv run generate-openapi
|
||||
cd ..
|
||||
```
|
||||
@@ -24,7 +24,7 @@ This regenerates Python and TypeScript clients from `openapi.json`.
|
||||
### 3. Commit Everything
|
||||
|
||||
```bash
|
||||
git add openapi.json memora-clients/
|
||||
git add openapi.json hindsight-clients/
|
||||
git commit -m "Update OpenAPI spec and regenerate clients"
|
||||
```
|
||||
|
||||
@@ -47,7 +47,7 @@ This will:
|
||||
### Publish Python Client to PyPI
|
||||
|
||||
```bash
|
||||
cd memora-clients/python
|
||||
cd hindsight-clients/python
|
||||
uv build
|
||||
uv publish
|
||||
```
|
||||
@@ -55,7 +55,7 @@ uv publish
|
||||
### Publish TypeScript Client to NPM
|
||||
|
||||
```bash
|
||||
cd memora-clients/typescript
|
||||
cd hindsight-clients/typescript
|
||||
npm install
|
||||
npm run build
|
||||
npm publish --access public
|
||||
@@ -65,7 +65,7 @@ npm publish --access public
|
||||
|
||||
## Pre-Release Checklist
|
||||
|
||||
- [ ] Tests passing: `cd memora && uv run pytest tests`
|
||||
- [ ] Tests passing: `cd hindsight-api && uv run pytest tests`
|
||||
- [ ] No uncommitted changes: `git status`
|
||||
- [ ] On `main` branch
|
||||
|
||||
@@ -98,7 +98,7 @@ git status
|
||||
```
|
||||
|
||||
**GitHub Actions failed:**
|
||||
- Check: https://github.com/nicoloboschi/memora/actions
|
||||
- Check: https://github.com/vectorize-io/hindsight/actions
|
||||
- Re-run failed jobs or fix and release new patch version
|
||||
|
||||
**Rollback:**
|
||||
@@ -116,13 +116,13 @@ git push
|
||||
```bash
|
||||
# Full release workflow
|
||||
uv sync
|
||||
cd memora-dev && uv run generate-openapi && cd ..
|
||||
cd hindsight-dev && uv run generate-openapi && cd ..
|
||||
./scripts/generate-clients.sh
|
||||
git add openapi.json memora-clients/
|
||||
git add openapi.json hindsight-clients/
|
||||
git commit -m "Update OpenAPI spec and regenerate clients"
|
||||
./scripts/release.sh 0.0.6
|
||||
|
||||
# After GH Actions complete:
|
||||
cd memora-clients/python && uv build && uv publish
|
||||
cd hindsight-clients/python && uv build && uv publish
|
||||
cd ../typescript && npm run build && npm publish --access public
|
||||
```
|
||||
|
||||
@@ -1,175 +0,0 @@
|
||||
# Memora Docker Setup
|
||||
|
||||
Complete Docker Compose setup for running all Memora services locally.
|
||||
|
||||
## Services
|
||||
|
||||
This setup includes:
|
||||
- **PostgreSQL** with pgvector extension (port 5432)
|
||||
- **API Service** - FastAPI backend (port 8080)
|
||||
- **Control Plane** - Next.js web UI (port 3000)
|
||||
|
||||
## Quick Start
|
||||
|
||||
1. **Configure environment variables:**
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# Edit .env and set your API keys
|
||||
```
|
||||
|
||||
2. **Start all services:**
|
||||
```bash
|
||||
./start.sh
|
||||
```
|
||||
|
||||
3. **Access the services:**
|
||||
- Control Plane: http://localhost:3000
|
||||
- API: http://localhost:8080
|
||||
- PostgreSQL: localhost:5432
|
||||
|
||||
## Scripts
|
||||
|
||||
### `./start.sh`
|
||||
Build and start all services. Waits for all services to be healthy.
|
||||
|
||||
### `./stop.sh`
|
||||
Stop all services (keeps data).
|
||||
|
||||
### `./clean.sh`
|
||||
Stop all services and remove all data (destructive).
|
||||
|
||||
### `./logs.sh [service]`
|
||||
View logs for all services or a specific service:
|
||||
```bash
|
||||
./logs.sh # All services
|
||||
./logs.sh api # API only
|
||||
./logs.sh postgres # PostgreSQL only
|
||||
./logs.sh control-plane # Control plane only
|
||||
```
|
||||
|
||||
## Manual Docker Compose Commands
|
||||
|
||||
```bash
|
||||
# Start services
|
||||
docker-compose up -d
|
||||
|
||||
# Stop services
|
||||
docker-compose down
|
||||
|
||||
# Rebuild and start
|
||||
docker-compose up --build -d
|
||||
|
||||
# View logs
|
||||
docker-compose logs -f
|
||||
|
||||
# Remove everything including data
|
||||
docker-compose down -v
|
||||
```
|
||||
|
||||
## Database
|
||||
|
||||
### Connection Info
|
||||
- **Host:** localhost
|
||||
- **Port:** 5432
|
||||
- **Database:** memora
|
||||
- **User:** memora
|
||||
- **Password:** memora_dev
|
||||
|
||||
### Migrations
|
||||
|
||||
Database migrations run automatically when the API service starts. The API uses Alembic to:
|
||||
1. Check the current schema version
|
||||
2. Run any pending migrations
|
||||
3. Initialize the database if it's empty
|
||||
|
||||
Extensions (pgvector, uuid-ossp) are created automatically by the first migration.
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Required in `.env` file:
|
||||
|
||||
```bash
|
||||
# API Service Configuration
|
||||
MEMORA_API_DATABASE_URL=postgresql://memora:memora_dev@localhost:5432/memora
|
||||
MEMORA_API_LLM_PROVIDER=groq
|
||||
MEMORA_API_LLM_API_KEY=your-api-key-here
|
||||
MEMORA_API_LLM_MODEL=openai/gpt-oss-120b
|
||||
|
||||
# Optional: Custom LLM endpoint
|
||||
# MEMORA_API_LLM_BASE_URL=http://localhost:11434/v1
|
||||
|
||||
# Control Plane Configuration
|
||||
MEMORA_CP_DATAPLANE_API_URL=http://localhost:8080
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Services won't start
|
||||
Check logs for errors:
|
||||
```bash
|
||||
./logs.sh
|
||||
```
|
||||
|
||||
### Database connection issues
|
||||
Ensure PostgreSQL is healthy:
|
||||
```bash
|
||||
docker exec memora-postgres pg_isready -U memora
|
||||
```
|
||||
|
||||
### API won't connect to database
|
||||
Check if migrations ran successfully:
|
||||
```bash
|
||||
./logs.sh api
|
||||
```
|
||||
|
||||
### Control plane can't reach API
|
||||
Verify the API is running:
|
||||
```bash
|
||||
curl http://localhost:8080/
|
||||
```
|
||||
|
||||
### Reset everything
|
||||
```bash
|
||||
./clean.sh
|
||||
./start.sh
|
||||
```
|
||||
|
||||
## Development
|
||||
|
||||
### Rebuilding after code changes
|
||||
|
||||
**API changes:**
|
||||
```bash
|
||||
docker-compose up --build -d api
|
||||
```
|
||||
|
||||
**Control Plane changes:**
|
||||
```bash
|
||||
docker-compose up --build -d control-plane
|
||||
```
|
||||
|
||||
### Accessing the database
|
||||
```bash
|
||||
docker exec -it memora-postgres psql -U memora -d memora
|
||||
```
|
||||
|
||||
### Inspecting containers
|
||||
```bash
|
||||
docker-compose ps
|
||||
docker-compose exec api bash
|
||||
docker-compose exec control-plane sh
|
||||
```
|
||||
|
||||
## Data Persistence
|
||||
|
||||
PostgreSQL data is persisted in a Docker volume named `postgres_data`. This data survives container restarts but not `docker-compose down -v`.
|
||||
|
||||
To backup data:
|
||||
```bash
|
||||
docker exec memora-postgres pg_dump -U memora memora > backup.sql
|
||||
```
|
||||
|
||||
To restore data:
|
||||
```bash
|
||||
docker exec -i memora-postgres psql -U memora memora < backup.sql
|
||||
```
|
||||
+10
-10
@@ -9,15 +9,15 @@ RUN apt-get update && apt-get install -y \
|
||||
WORKDIR /app
|
||||
|
||||
# Copy only dependency files first for better caching
|
||||
COPY memora/pyproject.toml memora/README.md /app/memora/
|
||||
COPY memora/memora /app/memora/memora
|
||||
COPY memora/alembic /app/memora/alembic
|
||||
COPY hindsight-api/pyproject.toml hindsight-api/README.md /app/hindsight-api/
|
||||
COPY hindsight-api/hindsight_api /app/hindsight-api/hindsight_api
|
||||
COPY hindsight-api/alembic /app/hindsight-api/alembic
|
||||
|
||||
# Install uv for faster dependency installation
|
||||
RUN pip install --no-cache-dir uv
|
||||
|
||||
# Install Python dependencies to a virtual environment
|
||||
WORKDIR /app/memora
|
||||
WORKDIR /app/hindsight-api
|
||||
RUN uv venv /opt/venv && \
|
||||
. /opt/venv/bin/activate && \
|
||||
uv pip install --no-cache -e .
|
||||
@@ -33,19 +33,19 @@ RUN apt-get update && apt-get install -y \
|
||||
|
||||
# Copy virtual environment from builder
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
COPY --from=builder /app/memora /app/memora
|
||||
COPY --from=builder /app/hindsight-api /app/hindsight-api
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app/memora
|
||||
WORKDIR /app/hindsight-api
|
||||
|
||||
# Expose API port
|
||||
EXPOSE 8080
|
||||
EXPOSE 8888
|
||||
|
||||
# Set environment variables
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV DATABASE_URL=postgresql://memora:memora_dev@postgres:5432/memora
|
||||
ENV DATABASE_URL=postgresql://hindsight:hindsight_dev@postgres:5432/hindsight
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
ENV PYTHONPATH=/app/memora
|
||||
ENV PYTHONPATH=/app/hindsight-api
|
||||
|
||||
# Run the API server
|
||||
CMD ["python", "-m", "memora.web.server", "--host", "0.0.0.0", "--port", "8080"]
|
||||
CMD ["python", "-m", "hindsight_api.web.server", "--host", "0.0.0.0", "--port", "8888"]
|
||||
|
||||
+2
-2
@@ -3,7 +3,7 @@ set -e
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
echo "🧹 Cleaning Memora Services"
|
||||
echo "🧹 Cleaning Services"
|
||||
echo "============================"
|
||||
echo ""
|
||||
echo "This will:"
|
||||
@@ -20,7 +20,7 @@ fi
|
||||
|
||||
echo ""
|
||||
echo "🗑️ Removing services and data..."
|
||||
docker-compose down -v
|
||||
docker compose down -v
|
||||
|
||||
echo ""
|
||||
echo "✅ All services and data removed"
|
||||
|
||||
@@ -31,9 +31,9 @@ COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
|
||||
|
||||
USER nextjs
|
||||
|
||||
EXPOSE 3000
|
||||
EXPOSE 9999
|
||||
|
||||
ENV PORT=3000
|
||||
ENV PORT=9999
|
||||
ENV HOSTNAME="0.0.0.0"
|
||||
|
||||
CMD ["node", "server.js"]
|
||||
|
||||
+26
-26
@@ -1,78 +1,78 @@
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
container_name: memora-postgres
|
||||
container_name: hindsight-postgres
|
||||
environment:
|
||||
POSTGRES_USER: memora
|
||||
POSTGRES_PASSWORD: memora_dev
|
||||
POSTGRES_DB: memora
|
||||
POSTGRES_USER: hindsight
|
||||
POSTGRES_PASSWORD: hindsight_dev
|
||||
POSTGRES_DB: hindsight
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U memora"]
|
||||
test: ["CMD-SHELL", "pg_isready -U hindsight"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- memora-network
|
||||
- hindsight-network
|
||||
|
||||
api:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/api.Dockerfile
|
||||
container_name: memora-api
|
||||
container_name: hindsight-api
|
||||
environment:
|
||||
MEMORA_API_DATABASE_URL: postgresql://memora:memora_dev@postgres:5432/memora
|
||||
MEMORA_API_LLM_PROVIDER: ${MEMORA_API_LLM_PROVIDER:-groq}
|
||||
MEMORA_API_LLM_API_KEY: ${MEMORA_API_LLM_API_KEY}
|
||||
MEMORA_API_LLM_MODEL: ${MEMORA_API_LLM_MODEL:-openai/gpt-oss-120b}
|
||||
MEMORA_API_LLM_BASE_URL: ${MEMORA_API_LLM_BASE_URL}
|
||||
MEMORA_API_HOST: ${MEMORA_API_HOST:-0.0.0.0}
|
||||
MEMORA_API_PORT: ${MEMORA_API_PORT:-8080}
|
||||
HINDSIGHT_API_DATABASE_URL: postgresql://hindsight:hindsight_dev@postgres:5432/hindsight
|
||||
HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-groq}
|
||||
HINDSIGHT_API_LLM_API_KEY: ${HINDSIGHT_API_LLM_API_KEY}
|
||||
HINDSIGHT_API_LLM_MODEL: ${HINDSIGHT_API_LLM_MODEL:-openai/gpt-oss-20b}
|
||||
HINDSIGHT_API_LLM_BASE_URL: ${HINDSIGHT_API_LLM_BASE_URL}
|
||||
HINDSIGHT_API_HOST: 0.0.0.0
|
||||
HINDSIGHT_API_PORT: 8888
|
||||
ports:
|
||||
- "8080:8080"
|
||||
- "8888:8888"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8080/api/v1/agents"]
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8888/api/v1/agents"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
networks:
|
||||
- memora-network
|
||||
- hindsight-network
|
||||
restart: unless-stopped
|
||||
|
||||
control-plane:
|
||||
build:
|
||||
context: ../memora-control-plane
|
||||
context: ../hindsight-control-plane
|
||||
dockerfile: ../docker/control-plane.Dockerfile
|
||||
container_name: memora-control-plane
|
||||
container_name: hindsight-control-plane
|
||||
environment:
|
||||
NODE_ENV: production
|
||||
MEMORA_CP_HOSTNAME: ${MEMORA_CP_HOSTNAME:-0.0.0.0}
|
||||
MEMORA_CP_PORT: ${MEMORA_CP_PORT:-3000}
|
||||
MEMORA_CP_DATAPLANE_API_URL: ${MEMORA_CP_DATAPLANE_API_URL:-http://api:8080}
|
||||
HOSTNAME: 0.0.0.0
|
||||
PORT: 9999
|
||||
HINDSIGHT_CP_DATAPLANE_API_URL: http://api:8888
|
||||
ports:
|
||||
- "3000:3000"
|
||||
- "9999:9999"
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3000/"]
|
||||
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:9999/"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
networks:
|
||||
- memora-network
|
||||
- hindsight-network
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
memora-network:
|
||||
hindsight-network:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
|
||||
+2
-2
@@ -8,9 +8,9 @@ SERVICE=$1
|
||||
if [ -z "$SERVICE" ]; then
|
||||
echo "📋 Showing logs for all services..."
|
||||
echo ""
|
||||
docker-compose logs -f
|
||||
docker compose logs -f
|
||||
else
|
||||
echo "📋 Showing logs for $SERVICE..."
|
||||
echo ""
|
||||
docker-compose logs -f "$SERVICE"
|
||||
docker compose logs -f "$SERVICE"
|
||||
fi
|
||||
|
||||
+9
-9
@@ -3,7 +3,7 @@ set -e
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
echo "🚀 Starting Memora Services"
|
||||
echo "🚀 Starting Hindsight Services"
|
||||
echo "============================"
|
||||
echo ""
|
||||
|
||||
@@ -15,14 +15,14 @@ if [ ! -f ../.env ]; then
|
||||
cp ../.env.example ../.env
|
||||
echo ""
|
||||
echo "⚠️ Please edit .env and set your API keys:"
|
||||
echo " - MEMORA_API_LLM_API_KEY"
|
||||
echo " - HINDSIGHT_API_LLM_API_KEY"
|
||||
echo ""
|
||||
echo "Then run this script again."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "📦 Building and starting services..."
|
||||
docker-compose --env-file ../.env up --build -d
|
||||
docker compose --env-file ../.env up --build -d
|
||||
|
||||
echo ""
|
||||
echo "⏳ Waiting for services to be healthy..."
|
||||
@@ -30,21 +30,21 @@ echo ""
|
||||
|
||||
# Wait for PostgreSQL
|
||||
echo " Waiting for PostgreSQL..."
|
||||
until docker exec memora-postgres pg_isready -U memora > /dev/null 2>&1; do
|
||||
until docker exec hindsight-postgres pg_isready -U hindsight > /dev/null 2>&1; do
|
||||
sleep 1
|
||||
done
|
||||
echo " ✅ PostgreSQL is ready"
|
||||
|
||||
# Wait for API
|
||||
echo " Waiting for API..."
|
||||
until curl -f http://localhost:8080/api/v1/agents > /dev/null 2>&1; do
|
||||
until curl -f http://localhost:8888/api/v1/agents > /dev/null 2>&1; do
|
||||
sleep 2
|
||||
done
|
||||
echo " ✅ API is ready"
|
||||
|
||||
# Wait for Control Plane
|
||||
echo " Waiting for Control Plane..."
|
||||
until curl -f http://localhost:3000 > /dev/null 2>&1; do
|
||||
until curl -f http://localhost:9999 > /dev/null 2>&1; do
|
||||
sleep 2
|
||||
done
|
||||
echo " ✅ Control Plane is ready"
|
||||
@@ -53,12 +53,12 @@ echo ""
|
||||
echo "✅ All services are running!"
|
||||
echo ""
|
||||
echo "📊 Service URLs:"
|
||||
echo " Control Plane: http://localhost:3000"
|
||||
echo " API: http://localhost:8080"
|
||||
echo " Control Plane: http://localhost:9999"
|
||||
echo " API: http://localhost:8888"
|
||||
echo " PostgreSQL: localhost:5432"
|
||||
echo ""
|
||||
echo "🔍 View logs:"
|
||||
echo " docker-compose logs -f"
|
||||
echo " docker compose logs -f"
|
||||
echo ""
|
||||
echo "🛑 Stop services:"
|
||||
echo " ./stop.sh"
|
||||
|
||||
+3
-3
@@ -3,15 +3,15 @@ set -e
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
echo "🛑 Stopping Memora Services"
|
||||
echo "🛑 Stopping Services"
|
||||
echo "============================"
|
||||
echo ""
|
||||
|
||||
docker-compose down
|
||||
docker compose down
|
||||
|
||||
echo ""
|
||||
echo "✅ All services stopped"
|
||||
echo ""
|
||||
echo "💡 To remove data volumes as well, run:"
|
||||
echo " docker-compose down -v"
|
||||
echo " docker compose down -v"
|
||||
echo ""
|
||||
|
||||
+28
-28
@@ -1,4 +1,4 @@
|
||||
MEMORA HELM CHART INSTALLATION GUIDE
|
||||
HINDSIGHT HELM CHART INSTALLATION GUIDE
|
||||
=====================================
|
||||
|
||||
PREREQUISITES
|
||||
@@ -13,42 +13,42 @@ BASIC INSTALLATION
|
||||
|
||||
1. Install with default values (requires external PostgreSQL):
|
||||
|
||||
helm install memora ./memora \
|
||||
helm install hindsight ./hindsight \
|
||||
--set postgresql.external.host=your-postgres-host \
|
||||
--set postgresql.external.password=your-password \
|
||||
--set api.secrets.MEMORY_LLM_API_KEY=your-api-key
|
||||
|
||||
2. Install with custom values file:
|
||||
|
||||
helm install memora ./memora -f memora/values-production.yaml
|
||||
helm install hindsight ./hindsight -f hindsight/values-production.yaml
|
||||
|
||||
3. Install in a specific namespace:
|
||||
|
||||
kubectl create namespace memora
|
||||
helm install memora ./memora -n memora
|
||||
kubectl create namespace hindsight
|
||||
helm install hindsight ./hindsight -n hindsight
|
||||
|
||||
CONFIGURATION OPTIONS
|
||||
---------------------
|
||||
|
||||
Development setup (using values-development.yaml):
|
||||
helm install memora ./memora -f memora/values-development.yaml
|
||||
helm install hindsight ./hindsight -f hindsight/values-development.yaml
|
||||
|
||||
Production setup (using values-production.yaml):
|
||||
helm install memora ./memora -f memora/values-production.yaml
|
||||
helm install hindsight ./hindsight -f hindsight/values-production.yaml
|
||||
|
||||
Custom LLM provider:
|
||||
helm install memora ./memora \
|
||||
helm install hindsight ./hindsight \
|
||||
--set api.env.MEMORY_LLM_PROVIDER=openai \
|
||||
--set api.env.MEMORY_LLM_MODEL=gpt-4 \
|
||||
--set api.secrets.MEMORY_LLM_API_KEY=sk-your-key
|
||||
|
||||
Enable ingress:
|
||||
helm install memora ./memora \
|
||||
helm install hindsight ./hindsight \
|
||||
--set ingress.enabled=true \
|
||||
--set ingress.hosts[0].host=memora.example.com
|
||||
--set ingress.hosts[0].host=hindsight.example.com
|
||||
|
||||
Enable autoscaling:
|
||||
helm install memora ./memora \
|
||||
helm install hindsight ./hindsight \
|
||||
--set autoscaling.enabled=true \
|
||||
--set autoscaling.minReplicas=2 \
|
||||
--set autoscaling.maxReplicas=10
|
||||
@@ -57,43 +57,43 @@ UPGRADE
|
||||
-------
|
||||
|
||||
Upgrade existing installation:
|
||||
helm upgrade memora ./memora
|
||||
helm upgrade hindsight ./hindsight
|
||||
|
||||
Upgrade with new values:
|
||||
helm upgrade memora ./memora -f memora/values-production.yaml
|
||||
helm upgrade hindsight ./hindsight -f hindsight/values-production.yaml
|
||||
|
||||
UNINSTALL
|
||||
---------
|
||||
|
||||
Remove the Helm release:
|
||||
helm uninstall memora
|
||||
helm uninstall hindsight
|
||||
|
||||
Remove with namespace:
|
||||
helm uninstall memora -n memora
|
||||
helm uninstall hindsight -n hindsight
|
||||
|
||||
TESTING
|
||||
-------
|
||||
|
||||
Test the installation with dry-run:
|
||||
helm install memora ./memora --dry-run --debug
|
||||
helm install hindsight ./hindsight --dry-run --debug
|
||||
|
||||
Validate templates:
|
||||
helm template memora ./memora
|
||||
helm template hindsight ./hindsight
|
||||
|
||||
Lint the chart:
|
||||
helm lint ./memora
|
||||
helm lint ./hindsight
|
||||
|
||||
ACCESSING THE SERVICES
|
||||
----------------------
|
||||
|
||||
Port-forward control plane:
|
||||
kubectl port-forward svc/memora-control-plane 3000:3000
|
||||
kubectl port-forward svc/hindsight-control-plane 3000:3000
|
||||
|
||||
Port-forward API:
|
||||
kubectl port-forward svc/memora-api 8080:8080
|
||||
kubectl port-forward svc/hindsight-api 8888:8888
|
||||
|
||||
Get service URLs:
|
||||
helm status memora
|
||||
helm status hindsight
|
||||
|
||||
DATABASE INITIALIZATION
|
||||
-----------------------
|
||||
@@ -102,16 +102,16 @@ NOTE: Database migrations now run automatically when the API service starts.
|
||||
You typically don't need to run migrations manually.
|
||||
|
||||
If you want to pre-initialize the database before deploying (optional):
|
||||
kubectl run memora-init --rm -it --restart=Never \
|
||||
--image=memora/api:latest \
|
||||
--env="DATABASE_URL=postgresql://user:pass@host:5432/memora" \
|
||||
-- python -c "from memora.migrations import run_migrations; run_migrations()"
|
||||
kubectl run hindsight-init --rm -it --restart=Never \
|
||||
--image=hindsight/api:latest \
|
||||
--env="DATABASE_URL=postgresql://user:pass@host:5432/hindsight" \
|
||||
-- python -c "from hindsight.migrations import run_migrations; run_migrations()"
|
||||
|
||||
TROUBLESHOOTING
|
||||
---------------
|
||||
|
||||
Check pod status:
|
||||
kubectl get pods -l app.kubernetes.io/name=memora
|
||||
kubectl get pods -l app.kubernetes.io/name=hindsight
|
||||
|
||||
View logs for API:
|
||||
kubectl logs -l app.kubernetes.io/component=api
|
||||
@@ -123,8 +123,8 @@ Describe a pod:
|
||||
kubectl describe pod <pod-name>
|
||||
|
||||
Check configuration:
|
||||
kubectl get configmap memora-config -o yaml
|
||||
kubectl get secret memora-secret -o yaml
|
||||
kubectl get configmap hindsight-config -o yaml
|
||||
kubectl get secret hindsight-secret -o yaml
|
||||
|
||||
NOTES
|
||||
-----
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
apiVersion: v2
|
||||
name: hindsight
|
||||
description: A Helm chart for Hindsight - temporal-semantic-entity memory system for AI agents
|
||||
type: application
|
||||
version: 0.0.7
|
||||
appVersion: "0.0.7"
|
||||
keywords:
|
||||
- ai
|
||||
- memory
|
||||
- llm
|
||||
- agents
|
||||
maintainers:
|
||||
- name: Hindsight Team
|
||||
@@ -18,13 +18,13 @@ The application is accessible via the following URL(s):
|
||||
|
||||
1. Get the Control Plane URL by running these commands:
|
||||
{{- if contains "NodePort" .Values.controlPlane.service.type }}
|
||||
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "memora.fullname" . }}-control-plane)
|
||||
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "hindsight.fullname" . }}-control-plane)
|
||||
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
|
||||
echo "Control Plane URL: http://$NODE_IP:$NODE_PORT"
|
||||
{{- else if contains "LoadBalancer" .Values.controlPlane.service.type }}
|
||||
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
|
||||
You can watch the status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "memora.fullname" . }}-control-plane'
|
||||
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "memora.fullname" . }}-control-plane --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
|
||||
You can watch the status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "hindsight.fullname" . }}-control-plane'
|
||||
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "hindsight.fullname" . }}-control-plane --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
|
||||
echo "Control Plane URL: http://$SERVICE_IP:{{ .Values.controlPlane.service.port }}"
|
||||
{{- else if contains "ClusterIP" .Values.controlPlane.service.type }}
|
||||
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/component=control-plane,app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
|
||||
@@ -35,19 +35,19 @@ The application is accessible via the following URL(s):
|
||||
|
||||
2. Get the API URL by running these commands:
|
||||
{{- if contains "NodePort" .Values.api.service.type }}
|
||||
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "memora.fullname" . }}-api)
|
||||
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "hindsight.fullname" . }}-api)
|
||||
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
|
||||
echo "API URL: http://$NODE_IP:$NODE_PORT"
|
||||
{{- else if contains "LoadBalancer" .Values.api.service.type }}
|
||||
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
|
||||
You can watch the status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "memora.fullname" . }}-api'
|
||||
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "memora.fullname" . }}-api --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
|
||||
You can watch the status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "hindsight.fullname" . }}-api'
|
||||
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "hindsight.fullname" . }}-api --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
|
||||
echo "API URL: http://$SERVICE_IP:{{ .Values.api.service.port }}"
|
||||
{{- else if contains "ClusterIP" .Values.api.service.type }}
|
||||
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/component=api,app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
|
||||
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
|
||||
echo "API URL: http://127.0.0.1:8080"
|
||||
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
|
||||
echo "API URL: http://127.0.0.1:8888"
|
||||
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8888:$CONTAINER_PORT
|
||||
{{- end }}
|
||||
|
||||
{{- end }}
|
||||
@@ -62,10 +62,10 @@ Please ensure that:
|
||||
Database migrations run automatically when the API service starts.
|
||||
|
||||
If you want to pre-initialize the database before deploying (optional):
|
||||
kubectl run --namespace {{ .Release.Namespace }} memora-init --rm -it --restart=Never \
|
||||
kubectl run --namespace {{ .Release.Namespace }} hindsight-init --rm -it --restart=Never \
|
||||
--image={{ .Values.api.image.repository }}:{{ .Values.api.image.tag }} \
|
||||
--env="DATABASE_URL={{ include "memora.databaseUrl" . }}" \
|
||||
-- python -c "from memora.migrations import run_migrations; run_migrations()"
|
||||
--env="DATABASE_URL={{ include "hindsight.databaseUrl" . }}" \
|
||||
-- python -c "from hindsight.migrations import run_migrations; run_migrations()"
|
||||
{{- end }}
|
||||
|
||||
For more information, visit: https://github.com/yourusername/memora
|
||||
For more information, visit: https://github.com/yourusername/hindsight
|
||||
+18
-18
@@ -2,16 +2,16 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "memora.fullname" . }}-api
|
||||
name: {{ include "hindsight.fullname" . }}-api
|
||||
labels:
|
||||
{{- include "memora.api.labels" . | nindent 4 }}
|
||||
{{- include "hindsight.api.labels" . | nindent 4 }}
|
||||
spec:
|
||||
{{- if not .Values.autoscaling.enabled }}
|
||||
replicas: {{ .Values.api.replicaCount }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "memora.api.selectorLabels" . | nindent 6 }}
|
||||
{{- include "hindsight.api.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
@@ -21,10 +21,10 @@ spec:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "memora.api.selectorLabels" . | nindent 8 }}
|
||||
{{- include "hindsight.api.selectorLabels" . | nindent 8 }}
|
||||
spec:
|
||||
{{- if .Values.serviceAccount.create }}
|
||||
serviceAccountName: {{ include "memora.serviceAccountName" . }}
|
||||
serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
||||
@@ -39,37 +39,37 @@ spec:
|
||||
containerPort: {{ .Values.api.service.targetPort }}
|
||||
protocol: TCP
|
||||
env:
|
||||
- name: MEMORA_API_DATABASE_URL
|
||||
value: {{ include "memora.databaseUrl" . | quote }}
|
||||
- name: HINDSIGHT_API_DATABASE_URL
|
||||
value: {{ include "hindsight.databaseUrl" . | quote }}
|
||||
{{- if not .Values.postgresql.enabled }}
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "memora.fullname" . }}-secret
|
||||
name: {{ include "hindsight.fullname" . }}-secret
|
||||
key: postgres-password
|
||||
{{- end }}
|
||||
- name: MEMORA_API_LLM_PROVIDER
|
||||
- name: HINDSIGHT_API_LLM_PROVIDER
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "memora.fullname" . }}-config
|
||||
name: {{ include "hindsight.fullname" . }}-config
|
||||
key: llm-provider
|
||||
- name: MEMORA_API_LLM_MODEL
|
||||
- name: HINDSIGHT_API_LLM_MODEL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "memora.fullname" . }}-config
|
||||
name: {{ include "hindsight.fullname" . }}-config
|
||||
key: llm-model
|
||||
{{- if and .Values.api.secrets (hasKey .Values.api.secrets "MEMORA_API_LLM_API_KEY") }}
|
||||
- name: MEMORA_API_LLM_API_KEY
|
||||
{{- if and .Values.api.secrets (hasKey .Values.api.secrets "HINDSIGHT_API_LLM_API_KEY") }}
|
||||
- name: HINDSIGHT_API_LLM_API_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "memora.fullname" . }}-secret
|
||||
name: {{ include "hindsight.fullname" . }}-secret
|
||||
key: llm-api-key
|
||||
{{- end }}
|
||||
{{- if and .Values.api.secrets (hasKey .Values.api.secrets "MEMORA_API_LLM_BASE_URL") }}
|
||||
- name: MEMORA_API_LLM_BASE_URL
|
||||
{{- if and .Values.api.secrets (hasKey .Values.api.secrets "HINDSIGHT_API_LLM_BASE_URL") }}
|
||||
- name: HINDSIGHT_API_LLM_BASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "memora.fullname" . }}-secret
|
||||
name: {{ include "hindsight.fullname" . }}-secret
|
||||
key: llm-base-url
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
@@ -2,9 +2,9 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "memora.fullname" . }}-api
|
||||
name: {{ include "hindsight.fullname" . }}-api
|
||||
labels:
|
||||
{{- include "memora.api.labels" . | nindent 4 }}
|
||||
{{- include "hindsight.api.labels" . | nindent 4 }}
|
||||
spec:
|
||||
type: {{ .Values.api.service.type }}
|
||||
ports:
|
||||
@@ -13,5 +13,5 @@ spec:
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
{{- include "memora.api.selectorLabels" . | nindent 4 }}
|
||||
{{- include "hindsight.api.selectorLabels" . | nindent 4 }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hindsight.fullname" . }}-config
|
||||
labels:
|
||||
{{- include "hindsight.labels" . | nindent 4 }}
|
||||
data:
|
||||
# API configuration
|
||||
llm-provider: {{ .Values.api.env.HINDSIGHT_API_LLM_PROVIDER | quote }}
|
||||
llm-model: {{ .Values.api.env.HINDSIGHT_API_LLM_MODEL | quote }}
|
||||
|
||||
# Control plane configuration
|
||||
node-env: {{ .Values.controlPlane.env.NODE_ENV | quote }}
|
||||
hostname: {{ .Values.controlPlane.env.HINDSIGHT_CP_HOSTNAME | quote }}
|
||||
control-plane-port: {{ .Values.controlPlane.env.HINDSIGHT_CP_PORT | quote }}
|
||||
+12
-12
@@ -2,16 +2,16 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "memora.fullname" . }}-control-plane
|
||||
name: {{ include "hindsight.fullname" . }}-control-plane
|
||||
labels:
|
||||
{{- include "memora.controlPlane.labels" . | nindent 4 }}
|
||||
{{- include "hindsight.controlPlane.labels" . | nindent 4 }}
|
||||
spec:
|
||||
{{- if not .Values.autoscaling.enabled }}
|
||||
replicas: {{ .Values.controlPlane.replicaCount }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "memora.controlPlane.selectorLabels" . | nindent 6 }}
|
||||
{{- include "hindsight.controlPlane.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
@@ -20,10 +20,10 @@ spec:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "memora.controlPlane.selectorLabels" . | nindent 8 }}
|
||||
{{- include "hindsight.controlPlane.selectorLabels" . | nindent 8 }}
|
||||
spec:
|
||||
{{- if .Values.serviceAccount.create }}
|
||||
serviceAccountName: {{ include "memora.serviceAccountName" . }}
|
||||
serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
||||
@@ -41,20 +41,20 @@ spec:
|
||||
- name: NODE_ENV
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "memora.fullname" . }}-config
|
||||
name: {{ include "hindsight.fullname" . }}-config
|
||||
key: node-env
|
||||
- name: MEMORA_CP_HOSTNAME
|
||||
- name: HINDSIGHT_CP_HOSTNAME
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "memora.fullname" . }}-config
|
||||
name: {{ include "hindsight.fullname" . }}-config
|
||||
key: hostname
|
||||
- name: MEMORA_CP_PORT
|
||||
- name: HINDSIGHT_CP_PORT
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: {{ include "memora.fullname" . }}-config
|
||||
name: {{ include "hindsight.fullname" . }}-config
|
||||
key: control-plane-port
|
||||
- name: MEMORA_CP_DATAPLANE_API_URL
|
||||
value: {{ include "memora.apiUrl" . | quote }}
|
||||
- name: HINDSIGHT_CP_DATAPLANE_API_URL
|
||||
value: {{ include "hindsight.apiUrl" . | quote }}
|
||||
livenessProbe:
|
||||
{{- toYaml .Values.controlPlane.livenessProbe | nindent 10 }}
|
||||
readinessProbe:
|
||||
+3
-3
@@ -2,9 +2,9 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "memora.fullname" . }}-control-plane
|
||||
name: {{ include "hindsight.fullname" . }}-control-plane
|
||||
labels:
|
||||
{{- include "memora.controlPlane.labels" . | nindent 4 }}
|
||||
{{- include "hindsight.controlPlane.labels" . | nindent 4 }}
|
||||
spec:
|
||||
type: {{ .Values.controlPlane.service.type }}
|
||||
ports:
|
||||
@@ -13,5 +13,5 @@ spec:
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
{{- include "memora.controlPlane.selectorLabels" . | nindent 4 }}
|
||||
{{- include "hindsight.controlPlane.selectorLabels" . | nindent 4 }}
|
||||
{{- end }}
|
||||
@@ -3,14 +3,14 @@
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: {{ include "memora.fullname" . }}-api
|
||||
name: {{ include "hindsight.fullname" . }}-api
|
||||
labels:
|
||||
{{- include "memora.api.labels" . | nindent 4 }}
|
||||
{{- include "hindsight.api.labels" . | nindent 4 }}
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: {{ include "memora.fullname" . }}-api
|
||||
name: {{ include "hindsight.fullname" . }}-api
|
||||
minReplicas: {{ .Values.autoscaling.minReplicas }}
|
||||
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
|
||||
metrics:
|
||||
@@ -34,14 +34,14 @@ spec:
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: {{ include "memora.fullname" . }}-control-plane
|
||||
name: {{ include "hindsight.fullname" . }}-control-plane
|
||||
labels:
|
||||
{{- include "memora.controlPlane.labels" . | nindent 4 }}
|
||||
{{- include "hindsight.controlPlane.labels" . | nindent 4 }}
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: {{ include "memora.fullname" . }}-control-plane
|
||||
name: {{ include "hindsight.fullname" . }}-control-plane
|
||||
minReplicas: {{ .Values.autoscaling.minReplicas }}
|
||||
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
|
||||
metrics:
|
||||
@@ -2,9 +2,9 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "memora.fullname" . }}
|
||||
name: {{ include "hindsight.fullname" . }}
|
||||
labels:
|
||||
{{- include "memora.labels" . | nindent 4 }}
|
||||
{{- include "hindsight.labels" . | nindent 4 }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
@@ -34,11 +34,11 @@ spec:
|
||||
backend:
|
||||
service:
|
||||
{{- if eq .service "api" }}
|
||||
name: {{ include "memora.fullname" $ }}-api
|
||||
name: {{ include "hindsight.fullname" $ }}-api
|
||||
port:
|
||||
number: {{ $.Values.api.service.port }}
|
||||
{{- else if eq .service "controlPlane" }}
|
||||
name: {{ include "memora.fullname" $ }}-control-plane
|
||||
name: {{ include "hindsight.fullname" $ }}-control-plane
|
||||
port:
|
||||
number: {{ $.Values.controlPlane.service.port }}
|
||||
{{- end }}
|
||||
@@ -1,9 +1,9 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "memora.fullname" . }}-secret
|
||||
name: {{ include "hindsight.fullname" . }}-secret
|
||||
labels:
|
||||
{{- include "memora.labels" . | nindent 4 }}
|
||||
{{- include "hindsight.labels" . | nindent 4 }}
|
||||
type: Opaque
|
||||
data:
|
||||
{{- if and .Values.api.secrets (hasKey .Values.api.secrets "MEMORY_LLM_API_KEY") }}
|
||||
+2
-2
@@ -2,9 +2,9 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "memora.serviceAccountName" . }}
|
||||
name: {{ include "hindsight.serviceAccountName" . }}
|
||||
labels:
|
||||
{{- include "memora.labels" . | nindent 4 }}
|
||||
{{- include "hindsight.labels" . | nindent 4 }}
|
||||
{{- with .Values.serviceAccount.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
@@ -1,4 +1,4 @@
|
||||
# Default values for memora
|
||||
# Default values for hindsight
|
||||
|
||||
# Global settings
|
||||
replicaCount: 1
|
||||
@@ -8,14 +8,14 @@ api:
|
||||
enabled: true
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: memora/api
|
||||
repository: hindsight/api
|
||||
pullPolicy: IfNotPresent
|
||||
tag: "latest"
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
port: 8888
|
||||
targetPort: 8888
|
||||
|
||||
# Resource limits and requests
|
||||
resources:
|
||||
@@ -30,7 +30,7 @@ api:
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 8080
|
||||
port: 8888
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
@@ -39,7 +39,7 @@ api:
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 8080
|
||||
port: 8888
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
@@ -47,20 +47,20 @@ api:
|
||||
|
||||
# Environment variables
|
||||
env:
|
||||
MEMORA_API_LLM_PROVIDER: "groq"
|
||||
MEMORA_API_LLM_MODEL: "openai/gpt-oss-120b"
|
||||
HINDSIGHT_API_LLM_PROVIDER: "groq"
|
||||
HINDSIGHT_API_LLM_MODEL: "openai/gpt-oss-120b"
|
||||
|
||||
# Secret environment variables
|
||||
secrets:
|
||||
# MEMORA_API_LLM_API_KEY: "your-api-key"
|
||||
# MEMORA_API_LLM_BASE_URL: "https://api.groq.com/openai/v1"
|
||||
# HINDSIGHT_API_LLM_API_KEY: "your-api-key"
|
||||
# HINDSIGHT_API_LLM_BASE_URL: "https://api.groq.com/openai/v1"
|
||||
|
||||
# Image settings for control plane
|
||||
controlPlane:
|
||||
enabled: true
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: memora/memora-control-plane
|
||||
repository: hindsight/hindsight-control-plane
|
||||
pullPolicy: IfNotPresent
|
||||
tag: "latest"
|
||||
|
||||
@@ -100,8 +100,8 @@ controlPlane:
|
||||
# Environment variables
|
||||
env:
|
||||
NODE_ENV: "production"
|
||||
MEMORA_CP_HOSTNAME: "0.0.0.0"
|
||||
MEMORA_CP_PORT: "3000"
|
||||
HINDSIGHT_CP_HOSTNAME: "0.0.0.0"
|
||||
HINDSIGHT_CP_PORT: "3000"
|
||||
|
||||
# PostgreSQL configuration
|
||||
postgresql:
|
||||
@@ -113,8 +113,8 @@ postgresql:
|
||||
external:
|
||||
host: "postgresql"
|
||||
port: 5432
|
||||
database: "memora"
|
||||
username: "memora"
|
||||
database: "hindsight"
|
||||
username: "hindsight"
|
||||
# Password should be provided via secret
|
||||
# password: ""
|
||||
|
||||
@@ -130,7 +130,7 @@ ingress:
|
||||
# nginx.ingress.kubernetes.io/ssl-redirect: "true"
|
||||
|
||||
hosts:
|
||||
- host: memora.example.com
|
||||
- host: hindsight.example.com
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
@@ -140,9 +140,9 @@ ingress:
|
||||
service: api
|
||||
|
||||
tls: []
|
||||
# - secretName: memora-tls
|
||||
# - secretName: hindsight-tls
|
||||
# hosts:
|
||||
# - memora.example.com
|
||||
# - hindsight.example.com
|
||||
|
||||
# Service Account
|
||||
serviceAccount:
|
||||
@@ -1,13 +0,0 @@
|
||||
apiVersion: v2
|
||||
name: memora
|
||||
description: A Helm chart for Memora - temporal-semantic-entity memory system for AI agents
|
||||
type: application
|
||||
version: 0.0.7
|
||||
appVersion: "0.0.7"
|
||||
keywords:
|
||||
- ai
|
||||
- memory
|
||||
- llm
|
||||
- agents
|
||||
maintainers:
|
||||
- name: Memora Team
|
||||
@@ -1,15 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "memora.fullname" . }}-config
|
||||
labels:
|
||||
{{- include "memora.labels" . | nindent 4 }}
|
||||
data:
|
||||
# API configuration
|
||||
llm-provider: {{ .Values.api.env.MEMORA_API_LLM_PROVIDER | quote }}
|
||||
llm-model: {{ .Values.api.env.MEMORA_API_LLM_MODEL | quote }}
|
||||
|
||||
# Control plane configuration
|
||||
node-env: {{ .Values.controlPlane.env.NODE_ENV | quote }}
|
||||
hostname: {{ .Values.controlPlane.env.MEMORA_CP_HOSTNAME | quote }}
|
||||
control-plane-port: {{ .Values.controlPlane.env.MEMORA_CP_PORT | quote }}
|
||||
@@ -14,13 +14,13 @@ from alembic import context
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Import your models here
|
||||
from memora.models import Base
|
||||
from hindsight_api.models import Base
|
||||
|
||||
# Load environment variables based on MEMORA_API_DATABASE_URL env var or default to local
|
||||
# Load environment variables based on HINDSIGHT_API_DATABASE_URL env var or default to local
|
||||
def load_env():
|
||||
"""Load environment variables from .env"""
|
||||
# Check if MEMORA_API_DATABASE_URL is already set (e.g., by CI/CD)
|
||||
if os.getenv("MEMORA_API_DATABASE_URL"):
|
||||
# Check if HINDSIGHT_API_DATABASE_URL is already set (e.g., by CI/CD)
|
||||
if os.getenv("HINDSIGHT_API_DATABASE_URL"):
|
||||
return
|
||||
|
||||
# Look for .env file in the parent directory (root of the workspace)
|
||||
@@ -58,11 +58,11 @@ def get_database_url() -> str:
|
||||
# Get database URL from config (set programmatically) or environment
|
||||
database_url = config.get_main_option("sqlalchemy.url")
|
||||
if not database_url:
|
||||
database_url = os.getenv("MEMORA_API_DATABASE_URL")
|
||||
database_url = os.getenv("HINDSIGHT_API_DATABASE_URL")
|
||||
if not database_url:
|
||||
raise ValueError(
|
||||
"Database URL not found. "
|
||||
"Set MEMORA_API_DATABASE_URL environment variable or pass database_url to run_migrations()."
|
||||
"Set HINDSIGHT_API_DATABASE_URL environment variable or pass database_url to run_migrations()."
|
||||
)
|
||||
|
||||
# For migrations, use psycopg2 (sync driver) to avoid pgbouncer prepared statement issues
|
||||
@@ -3,8 +3,8 @@ Memory System for AI Agents.
|
||||
|
||||
Temporal + Semantic Memory Architecture using PostgreSQL with pgvector.
|
||||
"""
|
||||
from .temporal_semantic_memory import TemporalSemanticMemory
|
||||
from .search_trace import (
|
||||
from .engine.memory_engine import MemoryEngine
|
||||
from .engine.search_trace import (
|
||||
SearchTrace,
|
||||
QueryInfo,
|
||||
EntryPoint,
|
||||
@@ -15,11 +15,12 @@ from .search_trace import (
|
||||
SearchSummary,
|
||||
SearchPhaseMetrics,
|
||||
)
|
||||
from .search_tracer import SearchTracer
|
||||
from .embeddings import Embeddings, SentenceTransformersEmbeddings
|
||||
from .engine.search_tracer import SearchTracer
|
||||
from .engine.embeddings import Embeddings, SentenceTransformersEmbeddings
|
||||
from .engine.llm_wrapper import LLMConfig
|
||||
|
||||
__all__ = [
|
||||
"TemporalSemanticMemory",
|
||||
"MemoryEngine",
|
||||
"SearchTrace",
|
||||
"SearchTracer",
|
||||
"QueryInfo",
|
||||
@@ -32,5 +33,6 @@ __all__ = [
|
||||
"SearchPhaseMetrics",
|
||||
"Embeddings",
|
||||
"SentenceTransformersEmbeddings",
|
||||
"LLMConfig",
|
||||
]
|
||||
__version__ = "0.1.0"
|
||||
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
Unified API module for Memora.
|
||||
Unified API module for Hindsight.
|
||||
|
||||
Provides both HTTP REST API and MCP (Model Context Protocol) server.
|
||||
"""
|
||||
@@ -7,13 +7,13 @@ import logging
|
||||
from typing import Optional
|
||||
from fastapi import FastAPI
|
||||
|
||||
from memora import TemporalSemanticMemory
|
||||
from hindsight_api import MemoryEngine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_app(
|
||||
memory: TemporalSemanticMemory,
|
||||
memory: MemoryEngine,
|
||||
http_api_enabled: bool = True,
|
||||
mcp_api_enabled: bool = False,
|
||||
mcp_mount_path: str = "/mcp",
|
||||
@@ -21,10 +21,10 @@ def create_app(
|
||||
initialize_memory: bool = True
|
||||
) -> FastAPI:
|
||||
"""
|
||||
Create and configure the unified Memora API application.
|
||||
Create and configure the unified Hindsight API application.
|
||||
|
||||
Args:
|
||||
memory: TemporalSemanticMemory instance (already initialized with required parameters)
|
||||
memory: MemoryEngine instance (already initialized with required parameters)
|
||||
http_api_enabled: Whether to enable HTTP REST API endpoints (default: True)
|
||||
mcp_api_enabled: Whether to enable MCP server (default: False)
|
||||
mcp_mount_path: Path to mount MCP server (default: /mcp)
|
||||
@@ -56,7 +56,7 @@ def create_app(
|
||||
logger.info("HTTP REST API enabled")
|
||||
else:
|
||||
# Create minimal FastAPI app
|
||||
app = FastAPI(title="Memora API", version="0.0.7")
|
||||
app = FastAPI(title="Hindsight API", version="0.0.7")
|
||||
logger.info("HTTP REST API disabled")
|
||||
|
||||
# Mount MCP server if enabled
|
||||
@@ -67,12 +67,12 @@ def create_app(
|
||||
# Create MCP server with shared memory instance
|
||||
mcp_server = create_mcp_server(memory=memory)
|
||||
|
||||
# Mount at specified path
|
||||
app.mount(mcp_mount_path, mcp_server.sse_app())
|
||||
logger.info(f"MCP server enabled at {mcp_mount_path}/sse")
|
||||
# Mount at specified path using http_app (modern non-SSE alternative)
|
||||
app.mount(mcp_mount_path, mcp_server.http_app())
|
||||
logger.info(f"MCP server enabled at {mcp_mount_path}")
|
||||
except ImportError as e:
|
||||
logger.error(f"MCP server requested but dependencies not available: {e}")
|
||||
logger.error("Install with: pip install memora[mcp]")
|
||||
logger.error("Install with: pip install hindsight-api[mcp]")
|
||||
raise
|
||||
|
||||
return app
|
||||
@@ -87,6 +87,8 @@ from .http import (
|
||||
BatchPutRequest,
|
||||
ThinkRequest,
|
||||
ThinkResponse,
|
||||
CreateAgentRequest,
|
||||
PersonalityTraits,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
@@ -98,4 +100,6 @@ __all__ = [
|
||||
"BatchPutRequest",
|
||||
"ThinkRequest",
|
||||
"ThinkResponse",
|
||||
"CreateAgentRequest",
|
||||
"PersonalityTraits",
|
||||
]
|
||||
@@ -14,29 +14,41 @@ from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI, HTTPException, Query
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, ConfigDict
|
||||
|
||||
from memora import TemporalSemanticMemory
|
||||
from hindsight_api import MemoryEngine
|
||||
from hindsight_api.engine.db_utils import acquire_with_retry
|
||||
|
||||
|
||||
class MetadataFilter(BaseModel):
|
||||
"""Filter for metadata fields. Matches records where (key=value) OR (key not set) when match_unset=True."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"key": "source",
|
||||
"value": "slack",
|
||||
"match_unset": True
|
||||
}
|
||||
})
|
||||
|
||||
key: str = Field(description="Metadata key to filter on")
|
||||
value: Optional[str] = Field(default=None, description="Value to match. If None with match_unset=True, matches any record where key is not set.")
|
||||
match_unset: bool = Field(default=True, description="If True, also match records where this metadata key is not set")
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"key": "source",
|
||||
"value": "slack",
|
||||
"match_unset": True
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
"""Request model for search endpoint."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"query": "What did Alice say about machine learning?",
|
||||
"fact_type": ["world", "agent"],
|
||||
"thinking_budget": 100,
|
||||
"max_tokens": 4096,
|
||||
"trace": True,
|
||||
"question_date": "2023-05-30T23:40:00",
|
||||
"metadata_filter": [{"key": "source", "value": "slack", "match_unset": True}]
|
||||
}
|
||||
})
|
||||
|
||||
query: str
|
||||
fact_type: Optional[List[str]] = None # List of fact types to search (defaults to all if not specified)
|
||||
thinking_budget: int = 100
|
||||
@@ -45,19 +57,6 @@ class SearchRequest(BaseModel):
|
||||
question_date: Optional[str] = None # ISO format date string (e.g., "2023-05-30T23:40:00")
|
||||
metadata_filter: Optional[List[MetadataFilter]] = Field(default=None, description="Filter by metadata. Multiple filters are ANDed together.")
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"query": "What did Alice say about machine learning?",
|
||||
"fact_type": ["world", "agent"],
|
||||
"thinking_budget": 100,
|
||||
"max_tokens": 4096,
|
||||
"trace": True,
|
||||
"question_date": "2023-05-30T23:40:00",
|
||||
"metadata_filter": [{"key": "source", "value": "slack", "match_unset": True}]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class SearchResult(BaseModel):
|
||||
"""Single search result item."""
|
||||
@@ -87,93 +86,100 @@ class SearchResult(BaseModel):
|
||||
|
||||
class SearchResponse(BaseModel):
|
||||
"""Response model for search endpoints."""
|
||||
results: List[SearchResult]
|
||||
trace: Optional[Dict[str, Any]] = None
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"results": [
|
||||
{
|
||||
"id": "123e4567-e89b-12d3-a456-426614174000",
|
||||
"text": "Alice works at Google on the AI team",
|
||||
"type": "world",
|
||||
"context": "work info",
|
||||
"event_date": "2024-01-15T10:30:00Z"
|
||||
}
|
||||
],
|
||||
"trace": {
|
||||
"query": "What did Alice say about machine learning?",
|
||||
"num_results": 1,
|
||||
"time_seconds": 0.123
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"results": [
|
||||
{
|
||||
"id": "123e4567-e89b-12d3-a456-426614174000",
|
||||
"text": "Alice works at Google on the AI team",
|
||||
"type": "world",
|
||||
"context": "work info",
|
||||
"event_date": "2024-01-15T10:30:00Z"
|
||||
}
|
||||
],
|
||||
"trace": {
|
||||
"query": "What did Alice say about machine learning?",
|
||||
"num_results": 1,
|
||||
"time_seconds": 0.123
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
results: List[SearchResult]
|
||||
trace: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class MemoryItem(BaseModel):
|
||||
"""Single memory item for batch put."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"content": "Alice mentioned she's working on a new ML model",
|
||||
"event_date": "2024-01-15T10:30:00Z",
|
||||
"context": "team meeting",
|
||||
"metadata": {"source": "slack", "channel": "engineering"}
|
||||
}
|
||||
})
|
||||
|
||||
content: str
|
||||
event_date: Optional[datetime] = None
|
||||
context: Optional[str] = None
|
||||
metadata: Optional[Dict[str, str]] = None
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"content": "Alice mentioned she's working on a new ML model",
|
||||
"event_date": "2024-01-15T10:30:00Z",
|
||||
"context": "team meeting",
|
||||
"metadata": {"source": "slack", "channel": "engineering"}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class BatchPutRequest(BaseModel):
|
||||
"""Request model for batch put endpoint."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"items": [
|
||||
{
|
||||
"content": "Alice works at Google",
|
||||
"context": "work"
|
||||
},
|
||||
{
|
||||
"content": "Bob went hiking yesterday",
|
||||
"event_date": "2024-01-15T10:00:00Z"
|
||||
}
|
||||
],
|
||||
"document_id": "conversation_123"
|
||||
}
|
||||
})
|
||||
|
||||
items: List[MemoryItem]
|
||||
document_id: Optional[str] = None
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"items": [
|
||||
{
|
||||
"content": "Alice works at Google",
|
||||
"context": "work"
|
||||
},
|
||||
{
|
||||
"content": "Bob went hiking yesterday",
|
||||
"event_date": "2024-01-15T10:00:00Z"
|
||||
}
|
||||
],
|
||||
"document_id": "conversation_123"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class BatchPutResponse(BaseModel):
|
||||
"""Response model for batch put endpoint."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"success": True,
|
||||
"message": "Successfully stored 2 memory items",
|
||||
"agent_id": "user123",
|
||||
"document_id": "conversation_123",
|
||||
"items_count": 2
|
||||
}
|
||||
})
|
||||
|
||||
success: bool
|
||||
message: str
|
||||
agent_id: str
|
||||
document_id: Optional[str] = None
|
||||
items_count: int
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"success": True,
|
||||
"message": "Successfully stored 2 memory items",
|
||||
"agent_id": "user123",
|
||||
"document_id": "conversation_123",
|
||||
"items_count": 2
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class BatchPutAsyncResponse(BaseModel):
|
||||
"""Response model for async batch put endpoint."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"success": True,
|
||||
"message": "Batch put task queued for background processing",
|
||||
"agent_id": "user123",
|
||||
"document_id": "conversation_123",
|
||||
"items_count": 2,
|
||||
"queued": True
|
||||
}
|
||||
})
|
||||
|
||||
success: bool
|
||||
message: str
|
||||
agent_id: str
|
||||
@@ -181,36 +187,23 @@ class BatchPutAsyncResponse(BaseModel):
|
||||
items_count: int
|
||||
queued: bool
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"success": True,
|
||||
"message": "Batch put task queued for background processing",
|
||||
"agent_id": "user123",
|
||||
"document_id": "conversation_123",
|
||||
"items_count": 2,
|
||||
"queued": True
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class ThinkRequest(BaseModel):
|
||||
"""Request model for think endpoint."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"query": "What do you think about artificial intelligence?",
|
||||
"thinking_budget": 50,
|
||||
"context": "This is for a research paper on AI ethics",
|
||||
"metadata_filter": [{"key": "source", "value": "slack", "match_unset": True}]
|
||||
}
|
||||
})
|
||||
|
||||
query: str
|
||||
thinking_budget: int = 50
|
||||
context: Optional[str] = None
|
||||
metadata_filter: Optional[List[MetadataFilter]] = Field(default=None, description="Filter by metadata. Multiple filters are ANDed together.")
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"query": "What do you think about artificial intelligence?",
|
||||
"thinking_budget": 50,
|
||||
"context": "This is for a research paper on AI ethics",
|
||||
"metadata_filter": [{"key": "source", "value": "slack", "match_unset": True}]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class OpinionItem(BaseModel):
|
||||
"""Model for an opinion with confidence score."""
|
||||
@@ -220,67 +213,75 @@ class OpinionItem(BaseModel):
|
||||
|
||||
class ThinkFact(BaseModel):
|
||||
"""A fact used in think response."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"id": "123e4567-e89b-12d3-a456-426614174000",
|
||||
"text": "AI is used in healthcare",
|
||||
"type": "world",
|
||||
"context": "healthcare discussion",
|
||||
"event_date": "2024-01-15T10:30:00Z"
|
||||
}
|
||||
})
|
||||
|
||||
id: Optional[str] = None
|
||||
text: str
|
||||
type: Optional[str] = None # fact type: world, agent, opinion
|
||||
context: Optional[str] = None
|
||||
event_date: Optional[str] = None
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"id": "123e4567-e89b-12d3-a456-426614174000",
|
||||
"text": "AI is used in healthcare",
|
||||
"type": "world",
|
||||
"context": "healthcare discussion",
|
||||
"event_date": "2024-01-15T10:30:00Z"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class ThinkResponse(BaseModel):
|
||||
"""Response model for think endpoint."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"text": "Based on my understanding, AI is a transformative technology...",
|
||||
"based_on": [
|
||||
{
|
||||
"id": "123",
|
||||
"text": "AI is used in healthcare",
|
||||
"type": "world"
|
||||
},
|
||||
{
|
||||
"id": "456",
|
||||
"text": "I discussed AI applications last week",
|
||||
"type": "agent"
|
||||
}
|
||||
],
|
||||
"new_opinions": [
|
||||
"AI has great potential when used responsibly"
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
text: str
|
||||
based_on: List[ThinkFact] = [] # Facts used to generate the response
|
||||
new_opinions: List[str] = [] # Simplified to list of opinion strings
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"text": "Based on my understanding, AI is a transformative technology...",
|
||||
"based_on": [
|
||||
{
|
||||
"id": "123",
|
||||
"text": "AI is used in healthcare",
|
||||
"type": "world"
|
||||
},
|
||||
{
|
||||
"id": "456",
|
||||
"text": "I discussed AI applications last week",
|
||||
"type": "agent"
|
||||
}
|
||||
],
|
||||
"new_opinions": [
|
||||
"AI has great potential when used responsibly"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class AgentsResponse(BaseModel):
|
||||
"""Response model for agents list endpoint."""
|
||||
agents: List[str]
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"agents": ["user123", "agent_alice", "agent_bob"]
|
||||
}
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"agents": ["user123", "agent_alice", "agent_bob"]
|
||||
}
|
||||
})
|
||||
|
||||
agents: List[str]
|
||||
|
||||
|
||||
class PersonalityTraits(BaseModel):
|
||||
"""Personality traits based on Big Five model."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"openness": 0.8,
|
||||
"conscientiousness": 0.6,
|
||||
"extraversion": 0.5,
|
||||
"agreeableness": 0.7,
|
||||
"neuroticism": 0.3,
|
||||
"bias_strength": 0.7
|
||||
}
|
||||
})
|
||||
|
||||
openness: float = Field(ge=0.0, le=1.0, description="Openness to experience (0-1)")
|
||||
conscientiousness: float = Field(ge=0.0, le=1.0, description="Conscientiousness (0-1)")
|
||||
extraversion: float = Field(ge=0.0, le=1.0, description="Extraversion (0-1)")
|
||||
@@ -288,43 +289,30 @@ class PersonalityTraits(BaseModel):
|
||||
neuroticism: float = Field(ge=0.0, le=1.0, description="Neuroticism (0-1)")
|
||||
bias_strength: float = Field(ge=0.0, le=1.0, description="How strongly personality influences opinions (0-1)")
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
|
||||
class AgentProfileResponse(BaseModel):
|
||||
"""Response model for agent profile."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"agent_id": "user123",
|
||||
"name": "Alice",
|
||||
"personality": {
|
||||
"openness": 0.8,
|
||||
"conscientiousness": 0.6,
|
||||
"extraversion": 0.5,
|
||||
"agreeableness": 0.7,
|
||||
"neuroticism": 0.3,
|
||||
"bias_strength": 0.7
|
||||
}
|
||||
},
|
||||
"background": "I am a software engineer with 10 years of experience in startups"
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
class AgentProfileResponse(BaseModel):
|
||||
"""Response model for agent profile."""
|
||||
agent_id: str
|
||||
name: str
|
||||
personality: PersonalityTraits
|
||||
background: str
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"agent_id": "user123",
|
||||
"name": "Alice",
|
||||
"personality": {
|
||||
"openness": 0.8,
|
||||
"conscientiousness": 0.6,
|
||||
"extraversion": 0.5,
|
||||
"agreeableness": 0.7,
|
||||
"neuroticism": 0.3,
|
||||
"bias_strength": 0.7
|
||||
},
|
||||
"background": "I am a software engineer with 10 years of experience in startups"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class UpdatePersonalityRequest(BaseModel):
|
||||
"""Request model for updating personality traits."""
|
||||
@@ -333,40 +321,38 @@ class UpdatePersonalityRequest(BaseModel):
|
||||
|
||||
class AddBackgroundRequest(BaseModel):
|
||||
"""Request model for adding/merging background information."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"content": "I was born in Texas",
|
||||
"update_personality": True
|
||||
}
|
||||
})
|
||||
|
||||
content: str = Field(description="New background information to add or merge")
|
||||
update_personality: bool = Field(
|
||||
default=True,
|
||||
description="If true, infer Big Five personality traits from the merged background (default: true)"
|
||||
)
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"content": "I was born in Texas",
|
||||
"update_personality": True
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class BackgroundResponse(BaseModel):
|
||||
"""Response model for background update."""
|
||||
background: str
|
||||
personality: Optional[PersonalityTraits] = None
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"background": "I was born in Texas. I am a software engineer with 10 years of experience.",
|
||||
"personality": {
|
||||
"openness": 0.7,
|
||||
"conscientiousness": 0.6,
|
||||
"extraversion": 0.5,
|
||||
"agreeableness": 0.8,
|
||||
"neuroticism": 0.4,
|
||||
"bias_strength": 0.6
|
||||
}
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"background": "I was born in Texas. I am a software engineer with 10 years of experience.",
|
||||
"personality": {
|
||||
"openness": 0.7,
|
||||
"conscientiousness": 0.6,
|
||||
"extraversion": 0.5,
|
||||
"agreeableness": 0.8,
|
||||
"neuroticism": 0.4,
|
||||
"bias_strength": 0.6
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
background: str
|
||||
personality: Optional[PersonalityTraits] = None
|
||||
|
||||
|
||||
class AgentListItem(BaseModel):
|
||||
@@ -381,137 +367,144 @@ class AgentListItem(BaseModel):
|
||||
|
||||
class AgentListResponse(BaseModel):
|
||||
"""Response model for listing all agents."""
|
||||
agents: List[AgentListItem]
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"agents": [
|
||||
{
|
||||
"agent_id": "user123",
|
||||
"name": "Alice",
|
||||
"personality": {
|
||||
"openness": 0.5,
|
||||
"conscientiousness": 0.5,
|
||||
"extraversion": 0.5,
|
||||
"agreeableness": 0.5,
|
||||
"neuroticism": 0.5,
|
||||
"bias_strength": 0.5
|
||||
},
|
||||
"background": "I am a software engineer",
|
||||
"created_at": "2024-01-15T10:30:00Z",
|
||||
"updated_at": "2024-01-16T14:20:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"agents": [
|
||||
{
|
||||
"agent_id": "user123",
|
||||
"name": "Alice",
|
||||
"personality": {
|
||||
"openness": 0.5,
|
||||
"conscientiousness": 0.5,
|
||||
"extraversion": 0.5,
|
||||
"agreeableness": 0.5,
|
||||
"neuroticism": 0.5,
|
||||
"bias_strength": 0.5
|
||||
},
|
||||
"background": "I am a software engineer",
|
||||
"created_at": "2024-01-15T10:30:00Z",
|
||||
"updated_at": "2024-01-16T14:20:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
agents: List[AgentListItem]
|
||||
|
||||
|
||||
class CreateAgentRequest(BaseModel):
|
||||
"""Request model for creating/updating an agent."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"name": "Alice",
|
||||
"personality": {
|
||||
"openness": 0.8,
|
||||
"conscientiousness": 0.6,
|
||||
"extraversion": 0.5,
|
||||
"agreeableness": 0.7,
|
||||
"neuroticism": 0.3,
|
||||
"bias_strength": 0.7
|
||||
},
|
||||
"background": "I am a creative software engineer with 10 years of experience"
|
||||
}
|
||||
})
|
||||
|
||||
name: Optional[str] = None
|
||||
personality: Optional[PersonalityTraits] = None
|
||||
background: Optional[str] = None
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"name": "Alice",
|
||||
"personality": {
|
||||
"openness": 0.8,
|
||||
"conscientiousness": 0.6,
|
||||
"extraversion": 0.5,
|
||||
"agreeableness": 0.7,
|
||||
"neuroticism": 0.3,
|
||||
"bias_strength": 0.7
|
||||
},
|
||||
"background": "I am a creative software engineer with 10 years of experience"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class GraphDataResponse(BaseModel):
|
||||
"""Response model for graph data endpoint."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"nodes": [
|
||||
{"id": "1", "label": "Alice works at Google", "type": "world"},
|
||||
{"id": "2", "label": "Bob went hiking", "type": "world"}
|
||||
],
|
||||
"edges": [
|
||||
{"from": "1", "to": "2", "type": "semantic", "weight": 0.8}
|
||||
],
|
||||
"table_rows": [
|
||||
{"id": "abc12345...", "text": "Alice works at Google", "context": "Work info", "date": "2024-01-15 10:30", "entities": "Alice (PERSON), Google (ORGANIZATION)"}
|
||||
],
|
||||
"total_units": 2
|
||||
}
|
||||
})
|
||||
|
||||
nodes: List[Dict[str, Any]]
|
||||
edges: List[Dict[str, Any]]
|
||||
table_rows: List[Dict[str, Any]]
|
||||
total_units: int
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"nodes": [
|
||||
{"id": "1", "label": "Alice works at Google", "type": "world"},
|
||||
{"id": "2", "label": "Bob went hiking", "type": "world"}
|
||||
],
|
||||
"edges": [
|
||||
{"from": "1", "to": "2", "type": "semantic", "weight": 0.8}
|
||||
],
|
||||
"table_rows": [
|
||||
{"id": "abc12345...", "text": "Alice works at Google", "context": "Work info", "date": "2024-01-15 10:30", "entities": "Alice (PERSON), Google (ORGANIZATION)"}
|
||||
],
|
||||
"total_units": 2
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class ListMemoryUnitsResponse(BaseModel):
|
||||
"""Response model for list memory units endpoint."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"items": [
|
||||
{
|
||||
"id": "550e8400-e29b-41d4-a716-446655440000",
|
||||
"text": "Alice works at Google on the AI team",
|
||||
"context": "Work conversation",
|
||||
"date": "2024-01-15T10:30:00Z",
|
||||
"fact_type": "world",
|
||||
"entities": "Alice (PERSON), Google (ORGANIZATION)"
|
||||
}
|
||||
],
|
||||
"total": 150,
|
||||
"limit": 100,
|
||||
"offset": 0
|
||||
}
|
||||
})
|
||||
|
||||
items: List[Dict[str, Any]]
|
||||
total: int
|
||||
limit: int
|
||||
offset: int
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"items": [
|
||||
{
|
||||
"id": "550e8400-e29b-41d4-a716-446655440000",
|
||||
"text": "Alice works at Google on the AI team",
|
||||
"context": "Work conversation",
|
||||
"date": "2024-01-15T10:30:00Z",
|
||||
"fact_type": "world",
|
||||
"entities": "Alice (PERSON), Google (ORGANIZATION)"
|
||||
}
|
||||
],
|
||||
"total": 150,
|
||||
"limit": 100,
|
||||
"offset": 0
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class ListDocumentsResponse(BaseModel):
|
||||
"""Response model for list documents endpoint."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"items": [
|
||||
{
|
||||
"id": "session_1",
|
||||
"agent_id": "user123",
|
||||
"content_hash": "abc123",
|
||||
"created_at": "2024-01-15T10:30:00Z",
|
||||
"updated_at": "2024-01-15T10:30:00Z",
|
||||
"text_length": 5420,
|
||||
"memory_unit_count": 15
|
||||
}
|
||||
],
|
||||
"total": 50,
|
||||
"limit": 100,
|
||||
"offset": 0
|
||||
}
|
||||
})
|
||||
|
||||
items: List[Dict[str, Any]]
|
||||
total: int
|
||||
limit: int
|
||||
offset: int
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"items": [
|
||||
{
|
||||
"id": "session_1",
|
||||
"agent_id": "user123",
|
||||
"content_hash": "abc123",
|
||||
"created_at": "2024-01-15T10:30:00Z",
|
||||
"updated_at": "2024-01-15T10:30:00Z",
|
||||
"text_length": 5420,
|
||||
"memory_unit_count": 15
|
||||
}
|
||||
],
|
||||
"total": 50,
|
||||
"limit": 100,
|
||||
"offset": 0
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class DocumentResponse(BaseModel):
|
||||
"""Response model for get document endpoint."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"id": "session_1",
|
||||
"agent_id": "user123",
|
||||
"original_text": "Full document text here...",
|
||||
"content_hash": "abc123",
|
||||
"created_at": "2024-01-15T10:30:00Z",
|
||||
"updated_at": "2024-01-15T10:30:00Z",
|
||||
"memory_unit_count": 15
|
||||
}
|
||||
})
|
||||
|
||||
id: str
|
||||
agent_id: str
|
||||
original_text: str
|
||||
@@ -520,40 +513,26 @@ class DocumentResponse(BaseModel):
|
||||
updated_at: str
|
||||
memory_unit_count: int
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"id": "session_1",
|
||||
"agent_id": "user123",
|
||||
"original_text": "Full document text here...",
|
||||
"content_hash": "abc123",
|
||||
"created_at": "2024-01-15T10:30:00Z",
|
||||
"updated_at": "2024-01-15T10:30:00Z",
|
||||
"memory_unit_count": 15
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class DeleteResponse(BaseModel):
|
||||
"""Response model for delete operations."""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"success": True,
|
||||
"message": "Resource deleted successfully"
|
||||
}
|
||||
})
|
||||
|
||||
success: bool
|
||||
message: str
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"success": True,
|
||||
"message": "Resource deleted successfully"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def create_app(memory: TemporalSemanticMemory, run_migrations: bool = True, initialize_memory: bool = True) -> FastAPI:
|
||||
def create_app(memory: MemoryEngine, run_migrations: bool = True, initialize_memory: bool = True) -> FastAPI:
|
||||
"""
|
||||
Create and configure the FastAPI application.
|
||||
|
||||
Args:
|
||||
memory: TemporalSemanticMemory instance (already initialized with required parameters)
|
||||
memory: MemoryEngine instance (already initialized with required parameters)
|
||||
run_migrations: Whether to run database migrations on startup (default: True)
|
||||
initialize_memory: Whether to initialize memory system on startup (default: True)
|
||||
|
||||
@@ -572,15 +551,17 @@ def create_app(memory: TemporalSemanticMemory, run_migrations: bool = True, init
|
||||
Note: This only fires when running the app standalone, not when mounted.
|
||||
"""
|
||||
# Startup: Initialize database and memory system
|
||||
if run_migrations:
|
||||
from memora.migrations import run_migrations as do_migrations
|
||||
do_migrations(memory.db_url)
|
||||
logging.info("Database migrations applied")
|
||||
|
||||
if initialize_memory:
|
||||
await memory.initialize()
|
||||
logging.info("Memory system initialized")
|
||||
|
||||
if run_migrations:
|
||||
from hindsight_api.migrations import run_migrations as do_migrations
|
||||
do_migrations(memory.db_url)
|
||||
logging.info("Database migrations applied")
|
||||
|
||||
|
||||
|
||||
yield
|
||||
|
||||
# Shutdown: Cleanup memory system
|
||||
@@ -860,7 +841,7 @@ def _register_routes(app: FastAPI):
|
||||
"""Get statistics about memory nodes and links for an agent."""
|
||||
try:
|
||||
pool = await app.state.memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
# Get node counts by fact_type
|
||||
node_stats = await conn.fetch(
|
||||
"""
|
||||
@@ -1195,7 +1176,7 @@ This operation cannot be undone.
|
||||
|
||||
# Insert operation record into database BEFORE scheduling task
|
||||
pool = await app.state.memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO async_operations (id, agent_id, task_type, items_count, document_id)
|
||||
@@ -1245,7 +1226,7 @@ This operation cannot be undone.
|
||||
"""List all async operations (pending and failed) for an agent."""
|
||||
try:
|
||||
pool = await app.state.memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
operations = await conn.fetch(
|
||||
"""
|
||||
SELECT id, agent_id, task_type, items_count, document_id, created_at, status, error_message
|
||||
@@ -1296,7 +1277,7 @@ This operation cannot be undone.
|
||||
raise HTTPException(status_code=400, detail=f"Invalid operation_id format: {operation_id}")
|
||||
|
||||
pool = await app.state.memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
# Check if operation exists and belongs to this agent
|
||||
result = await conn.fetchrow(
|
||||
"SELECT agent_id FROM async_operations WHERE id = $1 AND agent_id = $2",
|
||||
@@ -1468,7 +1449,7 @@ This operation cannot be undone.
|
||||
# Update name if provided
|
||||
if request.name is not None:
|
||||
pool = await app.state.memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE agents
|
||||
@@ -1492,7 +1473,7 @@ This operation cannot be undone.
|
||||
# Update background if provided (replace, not merge)
|
||||
if request.background is not None:
|
||||
pool = await app.state.memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE agents
|
||||
@@ -1,30 +1,30 @@
|
||||
"""Memora MCP Server implementation using FastMCP."""
|
||||
"""Hindsight MCP Server implementation using FastMCP."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from fastmcp import FastMCP
|
||||
from memora import TemporalSemanticMemory
|
||||
from hindsight_api import MemoryEngine
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_mcp_server(memory: TemporalSemanticMemory) -> FastMCP:
|
||||
def create_mcp_server(memory: MemoryEngine) -> FastMCP:
|
||||
"""
|
||||
Create and configure the Memora MCP server.
|
||||
Create and configure the Hindsight MCP server.
|
||||
|
||||
Args:
|
||||
memory: TemporalSemanticMemory instance (required)
|
||||
memory: MemoryEngine instance (required)
|
||||
|
||||
Returns:
|
||||
Configured FastMCP server instance
|
||||
"""
|
||||
# Create FastMCP server
|
||||
mcp = FastMCP("memora-mcp-server")
|
||||
mcp = FastMCP("hindsight-mcp-server")
|
||||
|
||||
@mcp.tool()
|
||||
async def memora_put(agent_id: str, content: str, context: str, explanation: str = "") -> str:
|
||||
async def hindsight_put(agent_id: str, content: str, context: str, explanation: str = "") -> str:
|
||||
"""
|
||||
**CRITICAL: Store important user information to long-term memory.**
|
||||
|
||||
@@ -74,7 +74,7 @@ def create_mcp_server(memory: TemporalSemanticMemory) -> FastMCP:
|
||||
return f"Error: {str(e)}"
|
||||
|
||||
@mcp.tool()
|
||||
async def memora_search(agent_id: str, query: str, max_tokens: int = 4096, explanation: str = "") -> str:
|
||||
async def hindsight_search(agent_id: str, query: str, max_tokens: int = 4096, explanation: str = "") -> str:
|
||||
"""
|
||||
**CRITICAL: Search user's memory to provide personalized, context-aware responses.**
|
||||
|
||||
@@ -112,7 +112,7 @@ def create_mcp_server(memory: TemporalSemanticMemory) -> FastMCP:
|
||||
"""
|
||||
try:
|
||||
# Log all parameters for debugging
|
||||
logger.info(f"memora_search called with: query={query!r}, max_tokens={max_tokens}, explanation={explanation!r}")
|
||||
logger.info(f"hindsight_search called with: query={query!r}, max_tokens={max_tokens}, explanation={explanation!r}")
|
||||
|
||||
# Log explanation if provided
|
||||
if explanation:
|
||||
@@ -0,0 +1,47 @@
|
||||
"""
|
||||
Memory Engine - Core implementation of the memory system.
|
||||
|
||||
This package contains all the implementation details of the memory engine:
|
||||
- MemoryEngine: Main class for memory operations
|
||||
- Utility modules: embedding_utils, link_utils, think_utils, agent_utils
|
||||
- Supporting modules: embeddings, cross_encoder, entity_resolver, etc.
|
||||
"""
|
||||
|
||||
from .memory_engine import MemoryEngine
|
||||
from .db_utils import acquire_with_retry
|
||||
from .embeddings import Embeddings, SentenceTransformersEmbeddings
|
||||
from .search_trace import (
|
||||
SearchTrace,
|
||||
QueryInfo,
|
||||
EntryPoint,
|
||||
NodeVisit,
|
||||
WeightComponents,
|
||||
LinkInfo,
|
||||
PruningDecision,
|
||||
SearchSummary,
|
||||
SearchPhaseMetrics,
|
||||
)
|
||||
from .search_tracer import SearchTracer
|
||||
from .llm_wrapper import LLMConfig
|
||||
from .response_models import SearchResult, ThinkResult, MemoryFact
|
||||
|
||||
__all__ = [
|
||||
"MemoryEngine",
|
||||
"acquire_with_retry",
|
||||
"Embeddings",
|
||||
"SentenceTransformersEmbeddings",
|
||||
"SearchTrace",
|
||||
"SearchTracer",
|
||||
"QueryInfo",
|
||||
"EntryPoint",
|
||||
"NodeVisit",
|
||||
"WeightComponents",
|
||||
"LinkInfo",
|
||||
"PruningDecision",
|
||||
"SearchSummary",
|
||||
"SearchPhaseMetrics",
|
||||
"LLMConfig",
|
||||
"SearchResult",
|
||||
"ThinkResult",
|
||||
"MemoryFact",
|
||||
]
|
||||
@@ -0,0 +1,426 @@
|
||||
"""
|
||||
Agent profile utilities for personality and background management.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Dict, Optional
|
||||
from pydantic import BaseModel, Field
|
||||
from .db_utils import acquire_with_retry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_PERSONALITY = {
|
||||
"openness": 0.5,
|
||||
"conscientiousness": 0.5,
|
||||
"extraversion": 0.5,
|
||||
"agreeableness": 0.5,
|
||||
"neuroticism": 0.5,
|
||||
"bias_strength": 0.5,
|
||||
}
|
||||
|
||||
|
||||
class PersonalityTraits(BaseModel):
|
||||
"""Big Five personality traits with bias strength (all values 0.0-1.0)."""
|
||||
openness: float = Field(description="Creativity, curiosity, openness to new ideas (0.0-1.0)")
|
||||
conscientiousness: float = Field(description="Organization, discipline, goal-directed (0.0-1.0)")
|
||||
extraversion: float = Field(description="Sociability, assertiveness, energy from others (0.0-1.0)")
|
||||
agreeableness: float = Field(description="Cooperation, empathy, consideration (0.0-1.0)")
|
||||
neuroticism: float = Field(description="Emotional sensitivity, anxiety, stress response (0.0-1.0)")
|
||||
bias_strength: float = Field(description="How much personality influences opinions (0.0-1.0)")
|
||||
|
||||
|
||||
class BackgroundMergeResponse(BaseModel):
|
||||
"""LLM response for background merge with personality inference."""
|
||||
background: str = Field(description="Merged background in first person perspective")
|
||||
personality: PersonalityTraits = Field(description="Inferred Big Five personality traits")
|
||||
|
||||
|
||||
async def get_agent_profile(pool, agent_id: str) -> Dict:
|
||||
"""
|
||||
Get agent profile (name, personality + background).
|
||||
Auto-creates agent with default values if not exists.
|
||||
|
||||
Args:
|
||||
pool: Database connection pool
|
||||
agent_id: Agent identifier
|
||||
|
||||
Returns:
|
||||
Dict with 'name' (str), 'personality' (dict) and 'background' (str) keys
|
||||
"""
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
# Try to get existing agent
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT name, personality, background
|
||||
FROM agents
|
||||
WHERE agent_id = $1
|
||||
""",
|
||||
agent_id
|
||||
)
|
||||
|
||||
if row:
|
||||
# asyncpg returns JSONB as a string, so parse it
|
||||
personality_data = row["personality"]
|
||||
if isinstance(personality_data, str):
|
||||
personality_data = json.loads(personality_data)
|
||||
|
||||
return {
|
||||
"name": row["name"],
|
||||
"personality": personality_data,
|
||||
"background": row["background"]
|
||||
}
|
||||
|
||||
# Agent doesn't exist, create with defaults
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO agents (agent_id, name, personality, background)
|
||||
VALUES ($1, $2, $3::jsonb, $4)
|
||||
ON CONFLICT (agent_id) DO NOTHING
|
||||
""",
|
||||
agent_id,
|
||||
agent_id, # Default name is the agent_id
|
||||
json.dumps(DEFAULT_PERSONALITY),
|
||||
""
|
||||
)
|
||||
|
||||
return {
|
||||
"name": agent_id,
|
||||
"personality": DEFAULT_PERSONALITY.copy(),
|
||||
"background": ""
|
||||
}
|
||||
|
||||
|
||||
async def update_agent_personality(
|
||||
pool,
|
||||
agent_id: str,
|
||||
personality: Dict[str, float]
|
||||
) -> None:
|
||||
"""
|
||||
Update agent personality traits.
|
||||
|
||||
Args:
|
||||
pool: Database connection pool
|
||||
agent_id: Agent identifier
|
||||
personality: Dict with Big Five traits + bias_strength (all 0-1)
|
||||
"""
|
||||
# Ensure agent exists first
|
||||
await get_agent_profile(pool, agent_id)
|
||||
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE agents
|
||||
SET personality = $2::jsonb,
|
||||
updated_at = NOW()
|
||||
WHERE agent_id = $1
|
||||
""",
|
||||
agent_id,
|
||||
json.dumps(personality)
|
||||
)
|
||||
|
||||
|
||||
async def merge_agent_background(
|
||||
pool,
|
||||
llm_config,
|
||||
agent_id: str,
|
||||
new_info: str,
|
||||
update_personality: bool = True
|
||||
) -> dict:
|
||||
"""
|
||||
Merge new background information with existing background using LLM.
|
||||
Normalizes to first person ("I") and resolves conflicts.
|
||||
Optionally infers personality traits from the merged background.
|
||||
|
||||
Args:
|
||||
pool: Database connection pool
|
||||
llm_config: LLM configuration for background merging
|
||||
agent_id: Agent identifier
|
||||
new_info: New background information to add/merge
|
||||
update_personality: If True, infer Big Five traits from background (default: True)
|
||||
|
||||
Returns:
|
||||
Dict with 'background' (str) and optionally 'personality' (dict) keys
|
||||
"""
|
||||
# Get current profile
|
||||
profile = await get_agent_profile(pool, agent_id)
|
||||
current_background = profile["background"]
|
||||
|
||||
# Use LLM to merge backgrounds and optionally infer personality
|
||||
result = await _llm_merge_background(
|
||||
llm_config,
|
||||
current_background,
|
||||
new_info,
|
||||
infer_personality=update_personality
|
||||
)
|
||||
|
||||
merged_background = result["background"]
|
||||
inferred_personality = result.get("personality")
|
||||
|
||||
# Update in database
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
if inferred_personality:
|
||||
# Update both background and personality
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE agents
|
||||
SET background = $2,
|
||||
personality = $3::jsonb,
|
||||
updated_at = NOW()
|
||||
WHERE agent_id = $1
|
||||
""",
|
||||
agent_id,
|
||||
merged_background,
|
||||
json.dumps(inferred_personality)
|
||||
)
|
||||
else:
|
||||
# Update only background
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE agents
|
||||
SET background = $2,
|
||||
updated_at = NOW()
|
||||
WHERE agent_id = $1
|
||||
""",
|
||||
agent_id,
|
||||
merged_background
|
||||
)
|
||||
|
||||
response = {"background": merged_background}
|
||||
if inferred_personality:
|
||||
response["personality"] = inferred_personality
|
||||
|
||||
return response
|
||||
|
||||
|
||||
async def _llm_merge_background(
|
||||
llm_config,
|
||||
current: str,
|
||||
new_info: str,
|
||||
infer_personality: bool = False
|
||||
) -> dict:
|
||||
"""
|
||||
Use LLM to intelligently merge background information.
|
||||
Optionally infer Big Five personality traits from the merged background.
|
||||
|
||||
Args:
|
||||
llm_config: LLM configuration to use
|
||||
current: Current background text
|
||||
new_info: New information to merge
|
||||
infer_personality: If True, also infer personality traits
|
||||
|
||||
Returns:
|
||||
Dict with 'background' (str) and optionally 'personality' (dict) keys
|
||||
"""
|
||||
if infer_personality:
|
||||
prompt = f"""You are helping maintain an agent's background/profile and infer their personality. You MUST respond with ONLY valid JSON.
|
||||
|
||||
Current background: {current if current else "(empty)"}
|
||||
|
||||
New information to add: {new_info}
|
||||
|
||||
Instructions:
|
||||
1. Merge the new information with the current background
|
||||
2. If there are conflicts (e.g., different birthplaces), the NEW information overwrites the old
|
||||
3. Keep additions that don't conflict
|
||||
4. Output in FIRST PERSON ("I") perspective
|
||||
5. Be concise - keep merged background under 500 characters
|
||||
6. Infer Big Five personality traits from the merged background:
|
||||
- Openness: 0.0-1.0 (creativity, curiosity, openness to new ideas)
|
||||
- Conscientiousness: 0.0-1.0 (organization, discipline, goal-directed)
|
||||
- Extraversion: 0.0-1.0 (sociability, assertiveness, energy from others)
|
||||
- Agreeableness: 0.0-1.0 (cooperation, empathy, consideration)
|
||||
- Neuroticism: 0.0-1.0 (emotional sensitivity, anxiety, stress response)
|
||||
- Bias Strength: 0.0-1.0 (how much personality influences opinions)
|
||||
|
||||
CRITICAL: You MUST respond with ONLY a valid JSON object. No markdown, no code blocks, no explanations. Just the JSON.
|
||||
|
||||
Format:
|
||||
{{
|
||||
"background": "the merged background text in first person",
|
||||
"personality": {{
|
||||
"openness": 0.7,
|
||||
"conscientiousness": 0.6,
|
||||
"extraversion": 0.5,
|
||||
"agreeableness": 0.8,
|
||||
"neuroticism": 0.4,
|
||||
"bias_strength": 0.6
|
||||
}}
|
||||
}}
|
||||
|
||||
Trait inference examples:
|
||||
- "creative artist" → openness: 0.8+, bias_strength: 0.6
|
||||
- "organized engineer" → conscientiousness: 0.8+, openness: 0.5-0.6
|
||||
- "startup founder" → openness: 0.8+, extraversion: 0.7+, neuroticism: 0.3-0.4
|
||||
- "risk-averse analyst" → openness: 0.3-0.4, conscientiousness: 0.8+, neuroticism: 0.6+
|
||||
- "rational and diligent" → conscientiousness: 0.7+, openness: 0.6+
|
||||
- "passionate and dramatic" → extraversion: 0.7+, neuroticism: 0.6+, openness: 0.7+"""
|
||||
else:
|
||||
prompt = f"""You are helping maintain an agent's background/profile.
|
||||
|
||||
Current background: {current if current else "(empty)"}
|
||||
|
||||
New information to add: {new_info}
|
||||
|
||||
Instructions:
|
||||
1. Merge the new information with the current background
|
||||
2. If there are conflicts (e.g., different birthplaces), the NEW information overwrites the old
|
||||
3. Keep additions that don't conflict
|
||||
4. Output in FIRST PERSON ("I") perspective
|
||||
5. Be concise - keep it under 500 characters
|
||||
6. Return ONLY the merged background text, no explanations
|
||||
|
||||
Merged background:"""
|
||||
|
||||
try:
|
||||
# Prepare messages
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
|
||||
if infer_personality:
|
||||
# Use structured output with Pydantic model for personality inference
|
||||
try:
|
||||
parsed = await llm_config.call(
|
||||
messages=messages,
|
||||
response_format=BackgroundMergeResponse,
|
||||
scope="agent_background",
|
||||
temperature=0.3,
|
||||
max_tokens=8192
|
||||
)
|
||||
logger.info(f"Successfully got structured response: background={parsed.background[:100]}")
|
||||
|
||||
# Convert Pydantic model to dict format
|
||||
return {
|
||||
"background": parsed.background,
|
||||
"personality": parsed.personality.model_dump()
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"Structured output failed, falling back to manual parsing: {e}")
|
||||
# Fall through to manual parsing below
|
||||
|
||||
# Manual parsing fallback or non-personality merge
|
||||
content = await llm_config.call(
|
||||
messages=messages,
|
||||
scope="agent_background",
|
||||
temperature=0.3,
|
||||
max_tokens=8192
|
||||
)
|
||||
|
||||
logger.info(f"LLM response for background merge (first 500 chars): {content[:500]}")
|
||||
|
||||
if infer_personality:
|
||||
# Parse JSON response - try multiple extraction methods
|
||||
result = None
|
||||
|
||||
# Method 1: Direct parse
|
||||
try:
|
||||
result = json.loads(content)
|
||||
logger.info("Successfully parsed JSON directly")
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Method 2: Extract from markdown code blocks
|
||||
if result is None:
|
||||
# Remove markdown code blocks
|
||||
code_block_match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', content, re.DOTALL)
|
||||
if code_block_match:
|
||||
try:
|
||||
result = json.loads(code_block_match.group(1))
|
||||
logger.info("Successfully extracted JSON from markdown code block")
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Method 3: Find nested JSON structure
|
||||
if result is None:
|
||||
# Look for JSON object with nested structure
|
||||
json_match = re.search(r'\{[^{}]*"background"[^{}]*"personality"[^{}]*\{[^{}]*\}[^{}]*\}', content, re.DOTALL)
|
||||
if json_match:
|
||||
try:
|
||||
result = json.loads(json_match.group())
|
||||
logger.info("Successfully extracted JSON using nested pattern")
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# All parsing methods failed - use fallback
|
||||
if result is None:
|
||||
logger.warning(f"Failed to extract JSON from LLM response. Raw content: {content[:200]}")
|
||||
# Fallback: use new_info as background with default personality
|
||||
return {
|
||||
"background": new_info if new_info else current if current else "",
|
||||
"personality": DEFAULT_PERSONALITY.copy()
|
||||
}
|
||||
|
||||
# Validate personality values
|
||||
personality = result.get("personality", {})
|
||||
for key in ["openness", "conscientiousness", "extraversion",
|
||||
"agreeableness", "neuroticism", "bias_strength"]:
|
||||
if key not in personality:
|
||||
personality[key] = 0.5 # Default to neutral
|
||||
else:
|
||||
# Clamp to [0, 1]
|
||||
personality[key] = max(0.0, min(1.0, float(personality[key])))
|
||||
|
||||
result["personality"] = personality
|
||||
|
||||
# Ensure background exists
|
||||
if "background" not in result or not result["background"]:
|
||||
result["background"] = new_info if new_info else ""
|
||||
|
||||
return result
|
||||
else:
|
||||
# Just background merge
|
||||
merged = content
|
||||
if not merged or merged.lower() in ["(empty)", "none", "n/a"]:
|
||||
merged = new_info if new_info else ""
|
||||
return {"background": merged}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error merging background with LLM: {e}")
|
||||
# Fallback: just append new info
|
||||
if current:
|
||||
merged = f"{current} {new_info}".strip()
|
||||
else:
|
||||
merged = new_info
|
||||
|
||||
result = {"background": merged}
|
||||
if infer_personality:
|
||||
result["personality"] = DEFAULT_PERSONALITY.copy()
|
||||
return result
|
||||
|
||||
|
||||
async def list_agents(pool) -> list:
|
||||
"""
|
||||
List all agents in the system.
|
||||
|
||||
Args:
|
||||
pool: Database connection pool
|
||||
|
||||
Returns:
|
||||
List of dicts with agent_id, name, personality, background, created_at, updated_at
|
||||
"""
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT agent_id, name, personality, background, created_at, updated_at
|
||||
FROM agents
|
||||
ORDER BY updated_at DESC
|
||||
"""
|
||||
)
|
||||
|
||||
result = []
|
||||
for row in rows:
|
||||
# asyncpg returns JSONB as a string, so parse it
|
||||
personality_data = row["personality"]
|
||||
if isinstance(personality_data, str):
|
||||
personality_data = json.loads(personality_data)
|
||||
|
||||
result.append({
|
||||
"agent_id": row["agent_id"],
|
||||
"name": row["name"],
|
||||
"personality": personality_data,
|
||||
"background": row["background"],
|
||||
"created_at": row["created_at"].isoformat() if row["created_at"] else None,
|
||||
"updated_at": row["updated_at"].isoformat() if row["updated_at"] else None,
|
||||
})
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,93 @@
|
||||
"""
|
||||
Database utility functions for connection management with retry logic.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
import asyncpg
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default retry configuration for database operations
|
||||
DEFAULT_MAX_RETRIES = 3
|
||||
DEFAULT_BASE_DELAY = 0.5 # seconds
|
||||
DEFAULT_MAX_DELAY = 5.0 # seconds
|
||||
|
||||
# Exceptions that indicate transient connection issues worth retrying
|
||||
RETRYABLE_EXCEPTIONS = (
|
||||
asyncpg.exceptions.InterfaceError,
|
||||
asyncpg.exceptions.ConnectionDoesNotExistError,
|
||||
asyncpg.exceptions.TooManyConnectionsError,
|
||||
OSError,
|
||||
ConnectionError,
|
||||
asyncio.TimeoutError,
|
||||
)
|
||||
|
||||
|
||||
async def retry_with_backoff(
|
||||
func,
|
||||
max_retries: int = DEFAULT_MAX_RETRIES,
|
||||
base_delay: float = DEFAULT_BASE_DELAY,
|
||||
max_delay: float = DEFAULT_MAX_DELAY,
|
||||
retryable_exceptions: tuple = RETRYABLE_EXCEPTIONS,
|
||||
):
|
||||
"""
|
||||
Execute an async function with exponential backoff retry.
|
||||
|
||||
Args:
|
||||
func: Async function to execute
|
||||
max_retries: Maximum number of retry attempts
|
||||
base_delay: Initial delay between retries (seconds)
|
||||
max_delay: Maximum delay between retries (seconds)
|
||||
retryable_exceptions: Tuple of exception types to retry on
|
||||
|
||||
Returns:
|
||||
Result of the function
|
||||
|
||||
Raises:
|
||||
The last exception if all retries fail
|
||||
"""
|
||||
last_exception = None
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
return await func()
|
||||
except retryable_exceptions as e:
|
||||
last_exception = e
|
||||
if attempt < max_retries:
|
||||
delay = min(base_delay * (2 ** attempt), max_delay)
|
||||
logger.warning(
|
||||
f"Database operation failed (attempt {attempt + 1}/{max_retries + 1}): {e}. "
|
||||
f"Retrying in {delay:.1f}s..."
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
else:
|
||||
logger.error(
|
||||
f"Database operation failed after {max_retries + 1} attempts: {e}"
|
||||
)
|
||||
raise last_exception
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def acquire_with_retry(pool: asyncpg.Pool, max_retries: int = DEFAULT_MAX_RETRIES):
|
||||
"""
|
||||
Async context manager to acquire a connection with retry logic.
|
||||
|
||||
Usage:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
await conn.execute(...)
|
||||
|
||||
Args:
|
||||
pool: The asyncpg connection pool
|
||||
max_retries: Maximum number of retry attempts
|
||||
|
||||
Yields:
|
||||
An asyncpg connection
|
||||
"""
|
||||
async def acquire():
|
||||
return await pool.acquire()
|
||||
|
||||
conn = await retry_with_backoff(acquire, max_retries=max_retries)
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
await pool.release(conn)
|
||||
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
Embedding generation utilities for memory units.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def generate_embedding(embeddings_backend, text: str) -> List[float]:
|
||||
"""
|
||||
Generate embedding for text using the provided embeddings backend.
|
||||
|
||||
Args:
|
||||
embeddings_backend: Embeddings instance to use for encoding
|
||||
text: Text to embed
|
||||
|
||||
Returns:
|
||||
Embedding vector (dimension depends on embeddings backend)
|
||||
"""
|
||||
try:
|
||||
embeddings = embeddings_backend.encode([text])
|
||||
return embeddings[0]
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to generate embedding: {str(e)}")
|
||||
|
||||
|
||||
async def generate_embeddings_batch(embeddings_backend, texts: List[str]) -> List[List[float]]:
|
||||
"""
|
||||
Generate embeddings for multiple texts using the provided embeddings backend.
|
||||
|
||||
Runs the embedding generation in a thread pool to avoid blocking the event loop
|
||||
for CPU-bound operations.
|
||||
|
||||
Args:
|
||||
embeddings_backend: Embeddings instance to use for encoding
|
||||
texts: List of texts to embed
|
||||
|
||||
Returns:
|
||||
List of embeddings in same order as input texts
|
||||
"""
|
||||
try:
|
||||
# Run embeddings in thread pool to avoid blocking event loop
|
||||
loop = asyncio.get_event_loop()
|
||||
embeddings = await loop.run_in_executor(
|
||||
None, # Use default thread pool
|
||||
embeddings_backend.encode,
|
||||
texts
|
||||
)
|
||||
return embeddings
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to generate batch embeddings: {str(e)}")
|
||||
+7
-6
@@ -8,6 +8,7 @@ import asyncpg
|
||||
from typing import List, Dict, Optional, Set
|
||||
from difflib import SequenceMatcher
|
||||
from datetime import datetime, timezone
|
||||
from .db_utils import acquire_with_retry
|
||||
|
||||
|
||||
# Load spaCy model (singleton)
|
||||
@@ -56,7 +57,7 @@ class EntityResolver:
|
||||
return []
|
||||
|
||||
if conn is None:
|
||||
async with self.pool.acquire() as conn:
|
||||
async with acquire_with_retry(self.pool) as conn:
|
||||
return await self._resolve_entities_batch_impl(conn, agent_id, entities_data, context, unit_event_date)
|
||||
else:
|
||||
return await self._resolve_entities_batch_impl(conn, agent_id, entities_data, context, unit_event_date)
|
||||
@@ -220,7 +221,7 @@ class EntityResolver:
|
||||
Returns:
|
||||
Entity ID (creates new entity if needed)
|
||||
"""
|
||||
async with self.pool.acquire() as conn:
|
||||
async with acquire_with_retry(self.pool) as conn:
|
||||
# Find candidate entities with similar name
|
||||
candidates = await conn.fetch(
|
||||
"""
|
||||
@@ -366,7 +367,7 @@ class EntityResolver:
|
||||
unit_id: Memory unit ID
|
||||
entity_id: Entity ID
|
||||
"""
|
||||
async with self.pool.acquire() as conn:
|
||||
async with acquire_with_retry(self.pool) as conn:
|
||||
# Insert unit-entity link
|
||||
await conn.execute(
|
||||
"""
|
||||
@@ -434,7 +435,7 @@ class EntityResolver:
|
||||
return
|
||||
|
||||
if conn is None:
|
||||
async with self.pool.acquire() as conn:
|
||||
async with acquire_with_retry(self.pool) as conn:
|
||||
return await self._link_units_to_entities_batch_impl(conn, unit_entity_pairs)
|
||||
else:
|
||||
return await self._link_units_to_entities_batch_impl(conn, unit_entity_pairs)
|
||||
@@ -499,7 +500,7 @@ class EntityResolver:
|
||||
Returns:
|
||||
List of unit IDs
|
||||
"""
|
||||
async with self.pool.acquire() as conn:
|
||||
async with acquire_with_retry(self.pool) as conn:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT unit_id
|
||||
@@ -527,7 +528,7 @@ class EntityResolver:
|
||||
Returns:
|
||||
Entity ID if found, None otherwise
|
||||
"""
|
||||
async with self.pool.acquire() as conn:
|
||||
async with acquire_with_retry(self.pool) as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT id FROM entities
|
||||
@@ -0,0 +1,541 @@
|
||||
"""
|
||||
Link creation utilities for temporal, semantic, and entity links.
|
||||
"""
|
||||
|
||||
import time
|
||||
import logging
|
||||
from typing import List
|
||||
from datetime import timedelta
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _log(log_buffer, message, level='info'):
|
||||
"""Helper to log to buffer if available, otherwise use logger."""
|
||||
if log_buffer is not None:
|
||||
log_buffer.append(message)
|
||||
else:
|
||||
if level == 'info':
|
||||
logger.info(message)
|
||||
else:
|
||||
logger.debug(message)
|
||||
|
||||
|
||||
async def extract_entities_batch_optimized(
|
||||
entity_resolver,
|
||||
conn,
|
||||
agent_id: str,
|
||||
unit_ids: List[str],
|
||||
sentences: List[str],
|
||||
context: str,
|
||||
fact_dates: List,
|
||||
llm_entities: List[List[dict]],
|
||||
log_buffer: List[str] = None,
|
||||
) -> List[tuple]:
|
||||
"""
|
||||
Process LLM-extracted entities for ALL facts in batch.
|
||||
|
||||
Uses entities provided by the LLM (no spaCy needed), then resolves
|
||||
and links them in bulk.
|
||||
|
||||
Args:
|
||||
entity_resolver: EntityResolver instance for entity resolution
|
||||
conn: Database connection
|
||||
agent_id: Agent identifier
|
||||
unit_ids: List of unit IDs
|
||||
sentences: List of fact sentences
|
||||
context: Context string
|
||||
fact_dates: List of fact dates
|
||||
llm_entities: List of entity lists from LLM extraction
|
||||
log_buffer: Optional buffer for logging
|
||||
|
||||
Returns:
|
||||
List of tuples for batch insertion: (from_unit_id, to_unit_id, link_type, weight, entity_id)
|
||||
"""
|
||||
try:
|
||||
# Step 1: Convert LLM entities to the format expected by entity resolver
|
||||
substep_start = time.time()
|
||||
all_entities = []
|
||||
for entity_list in llm_entities:
|
||||
# Convert List[Entity] or List[dict] to List[Dict] format
|
||||
formatted_entities = []
|
||||
for ent in entity_list:
|
||||
# Handle both Entity objects and dicts
|
||||
if hasattr(ent, 'text'):
|
||||
formatted_entities.append({'text': ent.text, 'type': ent.type})
|
||||
elif isinstance(ent, dict):
|
||||
formatted_entities.append({'text': ent.get('text', ''), 'type': ent.get('type', 'CONCEPT')})
|
||||
all_entities.append(formatted_entities)
|
||||
|
||||
total_entities = sum(len(ents) for ents in all_entities)
|
||||
_log(log_buffer, f" [6.1] Process LLM entities: {total_entities} entities from {len(sentences)} facts in {time.time() - substep_start:.3f}s")
|
||||
|
||||
# Step 2: Resolve entities in BATCH (much faster!)
|
||||
substep_start = time.time()
|
||||
step_6_2_start = time.time()
|
||||
|
||||
# [6.2.1] Prepare all entities for batch resolution
|
||||
substep_6_2_1_start = time.time()
|
||||
all_entities_flat = []
|
||||
entity_to_unit = [] # Maps flat index to (unit_id, local_index)
|
||||
|
||||
for unit_id, entities, fact_date in zip(unit_ids, all_entities, fact_dates):
|
||||
if not entities:
|
||||
continue
|
||||
|
||||
for local_idx, entity in enumerate(entities):
|
||||
all_entities_flat.append({
|
||||
'text': entity['text'],
|
||||
'type': entity['type'],
|
||||
'nearby_entities': entities,
|
||||
})
|
||||
entity_to_unit.append((unit_id, local_idx, fact_date))
|
||||
_log(log_buffer, f" [6.2.1] Prepare entities: {len(all_entities_flat)} entities in {time.time() - substep_6_2_1_start:.3f}s")
|
||||
|
||||
# Resolve ALL entities in one batch call
|
||||
if all_entities_flat:
|
||||
# [6.2.2] Batch resolve entities
|
||||
substep_6_2_2_start = time.time()
|
||||
# Group by date for batch resolution (most will have same date)
|
||||
entities_by_date = {}
|
||||
for idx, (unit_id, local_idx, fact_date) in enumerate(entity_to_unit):
|
||||
date_key = fact_date
|
||||
if date_key not in entities_by_date:
|
||||
entities_by_date[date_key] = []
|
||||
entities_by_date[date_key].append((idx, all_entities_flat[idx]))
|
||||
|
||||
_log(log_buffer, f" [6.2.2] Grouped into {len(entities_by_date)} date buckets, resolving...")
|
||||
|
||||
# Resolve each date group in batch
|
||||
resolved_entity_ids = [None] * len(all_entities_flat)
|
||||
for date_idx, (fact_date, entities_group) in enumerate(entities_by_date.items(), 1):
|
||||
date_bucket_start = time.time()
|
||||
indices = [idx for idx, _ in entities_group]
|
||||
entities_data = [entity_data for _, entity_data in entities_group]
|
||||
|
||||
batch_resolved = await entity_resolver.resolve_entities_batch(
|
||||
agent_id=agent_id,
|
||||
entities_data=entities_data,
|
||||
context=context,
|
||||
unit_event_date=fact_date,
|
||||
conn=conn
|
||||
)
|
||||
|
||||
for idx, entity_id in zip(indices, batch_resolved):
|
||||
resolved_entity_ids[idx] = entity_id
|
||||
|
||||
_log(log_buffer, f" [6.2.2.{date_idx}] Resolved {len(entities_data)} entities in {time.time() - date_bucket_start:.3f}s")
|
||||
|
||||
_log(log_buffer, f" [6.2.2] Resolve entities: {len(all_entities_flat)} entities in {time.time() - substep_6_2_2_start:.3f}s")
|
||||
|
||||
# [6.2.3] Create unit-entity links in BATCH
|
||||
substep_6_2_3_start = time.time()
|
||||
# Map resolved entities back to units and collect all (unit, entity) pairs
|
||||
unit_to_entity_ids = {}
|
||||
unit_entity_pairs = []
|
||||
for idx, (unit_id, local_idx, fact_date) in enumerate(entity_to_unit):
|
||||
if unit_id not in unit_to_entity_ids:
|
||||
unit_to_entity_ids[unit_id] = []
|
||||
|
||||
entity_id = resolved_entity_ids[idx]
|
||||
unit_to_entity_ids[unit_id].append(entity_id)
|
||||
unit_entity_pairs.append((unit_id, entity_id))
|
||||
|
||||
# Batch insert all unit-entity links (MUCH faster!)
|
||||
await entity_resolver.link_units_to_entities_batch(unit_entity_pairs, conn=conn)
|
||||
_log(log_buffer, f" [6.2.3] Create unit-entity links (batched): {len(unit_entity_pairs)} links in {time.time() - substep_6_2_3_start:.3f}s")
|
||||
|
||||
_log(log_buffer, f" [6.2] Entity resolution (batched): {len(all_entities_flat)} entities resolved in {time.time() - step_6_2_start:.3f}s")
|
||||
else:
|
||||
unit_to_entity_ids = {}
|
||||
_log(log_buffer, f" [6.2] Entity resolution (batched): 0 entities in {time.time() - step_6_2_start:.3f}s")
|
||||
|
||||
# Step 3: Create entity links between units that share entities
|
||||
substep_start = time.time()
|
||||
# Collect all unique entity IDs
|
||||
all_entity_ids = set()
|
||||
for entity_ids in unit_to_entity_ids.values():
|
||||
all_entity_ids.update(entity_ids)
|
||||
|
||||
_log(log_buffer, f" [6.3] Creating entity links for {len(all_entity_ids)} unique entities...")
|
||||
|
||||
# Find all units that reference these entities (ONE batched query)
|
||||
entity_to_units = {}
|
||||
if all_entity_ids:
|
||||
query_start = time.time()
|
||||
import uuid
|
||||
entity_id_list = [uuid.UUID(eid) if isinstance(eid, str) else eid for eid in all_entity_ids]
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT entity_id, unit_id
|
||||
FROM unit_entities
|
||||
WHERE entity_id = ANY($1::uuid[])
|
||||
""",
|
||||
entity_id_list
|
||||
)
|
||||
_log(log_buffer, f" [6.3.1] Query unit_entities: {len(rows)} rows in {time.time() - query_start:.3f}s")
|
||||
|
||||
# Group by entity_id
|
||||
group_start = time.time()
|
||||
for row in rows:
|
||||
entity_id = row['entity_id']
|
||||
if entity_id not in entity_to_units:
|
||||
entity_to_units[entity_id] = []
|
||||
entity_to_units[entity_id].append(row['unit_id'])
|
||||
_log(log_buffer, f" [6.3.2] Group by entity_id: {time.time() - group_start:.3f}s")
|
||||
|
||||
# Create bidirectional links between units that share entities
|
||||
link_gen_start = time.time()
|
||||
links = []
|
||||
for entity_id, units_with_entity in entity_to_units.items():
|
||||
# For each pair of units with this entity, create bidirectional links
|
||||
for i, unit_id_1 in enumerate(units_with_entity):
|
||||
for unit_id_2 in units_with_entity[i+1:]:
|
||||
# Bidirectional links
|
||||
links.append((unit_id_1, unit_id_2, 'entity', 1.0, entity_id))
|
||||
links.append((unit_id_2, unit_id_1, 'entity', 1.0, entity_id))
|
||||
|
||||
_log(log_buffer, f" [6.3.3] Generate {len(links)} links: {time.time() - link_gen_start:.3f}s")
|
||||
_log(log_buffer, f" [6.3] Entity link creation: {len(links)} links for {len(all_entity_ids)} unique entities in {time.time() - substep_start:.3f}s")
|
||||
|
||||
return links
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to extract entities in batch: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise
|
||||
|
||||
|
||||
async def create_temporal_links_batch_per_fact(
|
||||
conn,
|
||||
agent_id: str,
|
||||
unit_ids: List[str],
|
||||
time_window_hours: int = 24,
|
||||
log_buffer: List[str] = None,
|
||||
):
|
||||
"""
|
||||
Create temporal links for multiple units, each with their own event_date.
|
||||
|
||||
Queries the event_date for each unit from the database and creates temporal
|
||||
links based on individual dates (supports per-fact dating).
|
||||
|
||||
Args:
|
||||
conn: Database connection
|
||||
agent_id: Agent identifier
|
||||
unit_ids: List of unit IDs
|
||||
time_window_hours: Time window in hours for temporal links
|
||||
log_buffer: Optional buffer for logging
|
||||
"""
|
||||
if not unit_ids:
|
||||
return
|
||||
|
||||
try:
|
||||
import time as time_mod
|
||||
|
||||
# Get the event_date for each new unit
|
||||
fetch_dates_start = time_mod.time()
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT id, event_date
|
||||
FROM memory_units
|
||||
WHERE id::text = ANY($1)
|
||||
""",
|
||||
unit_ids
|
||||
)
|
||||
new_units = {str(row['id']): row['event_date'] for row in rows}
|
||||
_log(log_buffer, f" [7.1] Fetch event_dates for {len(unit_ids)} units: {time_mod.time() - fetch_dates_start:.3f}s")
|
||||
|
||||
# Fetch ALL potential temporal neighbors in ONE query (much faster!)
|
||||
# Get time range across all units
|
||||
all_dates = list(new_units.values())
|
||||
min_date = min(all_dates) - timedelta(hours=time_window_hours)
|
||||
max_date = max(all_dates) + timedelta(hours=time_window_hours)
|
||||
|
||||
fetch_neighbors_start = time_mod.time()
|
||||
all_candidates = await conn.fetch(
|
||||
"""
|
||||
SELECT id, event_date
|
||||
FROM memory_units
|
||||
WHERE agent_id = $1
|
||||
AND event_date BETWEEN $2 AND $3
|
||||
AND id::text != ALL($4)
|
||||
ORDER BY event_date DESC
|
||||
""",
|
||||
agent_id,
|
||||
min_date,
|
||||
max_date,
|
||||
unit_ids
|
||||
)
|
||||
_log(log_buffer, f" [7.2] Fetch {len(all_candidates)} candidate neighbors (1 query): {time_mod.time() - fetch_neighbors_start:.3f}s")
|
||||
|
||||
# Filter and create links in memory (much faster than N queries)
|
||||
link_gen_start = time_mod.time()
|
||||
links = []
|
||||
for unit_id, unit_event_date in new_units.items():
|
||||
# Filter candidates within this unit's time window
|
||||
time_lower = unit_event_date - timedelta(hours=time_window_hours)
|
||||
time_upper = unit_event_date + timedelta(hours=time_window_hours)
|
||||
|
||||
matching_neighbors = [
|
||||
(row['id'], row['event_date'])
|
||||
for row in all_candidates
|
||||
if time_lower <= row['event_date'] <= time_upper
|
||||
][:10] # Limit to top 10
|
||||
|
||||
for recent_id, recent_event_date in matching_neighbors:
|
||||
# Calculate temporal proximity weight
|
||||
time_diff_hours = abs((unit_event_date - recent_event_date).total_seconds() / 3600)
|
||||
weight = max(0.3, 1.0 - (time_diff_hours / time_window_hours))
|
||||
links.append((unit_id, str(recent_id), 'temporal', weight, None))
|
||||
|
||||
_log(log_buffer, f" [7.3] Generate {len(links)} temporal links: {time_mod.time() - link_gen_start:.3f}s")
|
||||
|
||||
if links:
|
||||
insert_start = time_mod.time()
|
||||
await conn.executemany(
|
||||
"""
|
||||
INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
|
||||
""",
|
||||
links
|
||||
)
|
||||
_log(log_buffer, f" [7.4] Insert {len(links)} temporal links: {time_mod.time() - insert_start:.3f}s")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create temporal links: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise
|
||||
|
||||
|
||||
async def create_semantic_links_batch(
|
||||
conn,
|
||||
agent_id: str,
|
||||
unit_ids: List[str],
|
||||
embeddings: List[List[float]],
|
||||
top_k: int = 5,
|
||||
threshold: float = 0.7,
|
||||
log_buffer: List[str] = None,
|
||||
):
|
||||
"""
|
||||
Create semantic links for multiple units efficiently.
|
||||
|
||||
For each unit, finds similar units and creates links.
|
||||
|
||||
Args:
|
||||
conn: Database connection
|
||||
agent_id: Agent identifier
|
||||
unit_ids: List of unit IDs
|
||||
embeddings: List of embedding vectors
|
||||
top_k: Number of top similar units to link
|
||||
threshold: Minimum similarity threshold
|
||||
log_buffer: Optional buffer for logging
|
||||
"""
|
||||
if not unit_ids or not embeddings:
|
||||
return
|
||||
|
||||
try:
|
||||
import time as time_mod
|
||||
import numpy as np
|
||||
|
||||
# Fetch ALL existing units with embeddings in ONE query
|
||||
fetch_start = time_mod.time()
|
||||
all_existing = await conn.fetch(
|
||||
"""
|
||||
SELECT id, embedding
|
||||
FROM memory_units
|
||||
WHERE agent_id = $1
|
||||
AND embedding IS NOT NULL
|
||||
AND id::text != ALL($2)
|
||||
""",
|
||||
agent_id,
|
||||
unit_ids
|
||||
)
|
||||
_log(log_buffer, f" [8.1] Fetch {len(all_existing)} existing embeddings (1 query): {time_mod.time() - fetch_start:.3f}s")
|
||||
|
||||
# Convert to numpy for vectorized similarity computation
|
||||
compute_start = time_mod.time()
|
||||
all_links = []
|
||||
|
||||
if all_existing:
|
||||
# Convert existing embeddings to numpy array
|
||||
existing_ids = [str(row['id']) for row in all_existing]
|
||||
# Stack embeddings as 2D array: (num_embeddings, embedding_dim)
|
||||
embedding_arrays = []
|
||||
for row in all_existing:
|
||||
raw_emb = row['embedding']
|
||||
# Handle different pgvector formats
|
||||
if isinstance(raw_emb, str):
|
||||
# Parse string format: "[1.0, 2.0, ...]"
|
||||
import json
|
||||
emb = np.array(json.loads(raw_emb), dtype=np.float32)
|
||||
elif isinstance(raw_emb, (list, tuple)):
|
||||
emb = np.array(raw_emb, dtype=np.float32)
|
||||
else:
|
||||
# Try direct conversion (works for numpy arrays, pgvector objects, etc.)
|
||||
emb = np.array(raw_emb, dtype=np.float32)
|
||||
|
||||
# Ensure it's 1D
|
||||
if emb.ndim != 1:
|
||||
raise ValueError(f"Expected 1D embedding, got shape {emb.shape}")
|
||||
embedding_arrays.append(emb)
|
||||
|
||||
if not embedding_arrays:
|
||||
existing_embeddings = np.array([])
|
||||
elif len(embedding_arrays) == 1:
|
||||
# Single embedding: reshape to (1, dim)
|
||||
existing_embeddings = embedding_arrays[0].reshape(1, -1)
|
||||
else:
|
||||
# Multiple embeddings: vstack
|
||||
existing_embeddings = np.vstack(embedding_arrays)
|
||||
|
||||
# For each new unit, compute similarities with ALL existing units
|
||||
for unit_id, new_embedding in zip(unit_ids, embeddings):
|
||||
new_emb_array = np.array(new_embedding)
|
||||
|
||||
# Compute cosine similarities (dot product for normalized vectors)
|
||||
similarities = np.dot(existing_embeddings, new_emb_array)
|
||||
|
||||
# Find top-k above threshold
|
||||
# Get indices of similarities above threshold
|
||||
above_threshold = np.where(similarities >= threshold)[0]
|
||||
|
||||
if len(above_threshold) > 0:
|
||||
# Sort by similarity (descending) and take top-k
|
||||
sorted_indices = above_threshold[np.argsort(-similarities[above_threshold])][:top_k]
|
||||
|
||||
for idx in sorted_indices:
|
||||
similar_id = existing_ids[idx]
|
||||
similarity = float(similarities[idx])
|
||||
all_links.append((unit_id, similar_id, 'semantic', similarity, None))
|
||||
|
||||
_log(log_buffer, f" [8.2] Compute similarities & generate {len(all_links)} semantic links: {time_mod.time() - compute_start:.3f}s")
|
||||
|
||||
if all_links:
|
||||
insert_start = time_mod.time()
|
||||
await conn.executemany(
|
||||
"""
|
||||
INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
|
||||
""",
|
||||
all_links
|
||||
)
|
||||
_log(log_buffer, f" [8.3] Insert {len(all_links)} semantic links: {time_mod.time() - insert_start:.3f}s")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create semantic links: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise
|
||||
|
||||
|
||||
async def insert_entity_links_batch(conn, links: List[tuple]):
|
||||
"""
|
||||
Insert all entity links in a single batch.
|
||||
|
||||
Args:
|
||||
conn: Database connection
|
||||
links: List of tuples (from_unit_id, to_unit_id, link_type, weight, entity_id)
|
||||
"""
|
||||
if not links:
|
||||
return
|
||||
|
||||
try:
|
||||
await conn.executemany(
|
||||
"""
|
||||
INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
|
||||
""",
|
||||
links
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to insert entity links: {str(e)}")
|
||||
|
||||
|
||||
async def create_causal_links_batch(
|
||||
conn,
|
||||
unit_ids: List[str],
|
||||
causal_relations_per_fact: List[List[dict]],
|
||||
) -> int:
|
||||
"""
|
||||
Create causal links between facts based on LLM-extracted causal relationships.
|
||||
|
||||
Args:
|
||||
conn: Database connection
|
||||
unit_ids: List of unit IDs (in same order as causal_relations_per_fact)
|
||||
causal_relations_per_fact: List of causal relations for each fact.
|
||||
Each element is a list of dicts with:
|
||||
- target_fact_index: Index into unit_ids for the target fact
|
||||
- relation_type: "causes", "caused_by", "enables", or "prevents"
|
||||
- strength: Float in [0.0, 1.0] representing relationship strength
|
||||
|
||||
Returns:
|
||||
Number of causal links created
|
||||
|
||||
Causal link types:
|
||||
- "causes": This fact directly causes the target fact (forward causation)
|
||||
- "caused_by": This fact was caused by the target fact (backward causation)
|
||||
- "enables": This fact enables/allows the target fact (enablement)
|
||||
- "prevents": This fact prevents/blocks the target fact (prevention)
|
||||
"""
|
||||
if not unit_ids or not causal_relations_per_fact:
|
||||
return 0
|
||||
|
||||
try:
|
||||
import time as time_mod
|
||||
create_start = time_mod.time()
|
||||
|
||||
# Build links list
|
||||
links = []
|
||||
for fact_idx, causal_relations in enumerate(causal_relations_per_fact):
|
||||
if not causal_relations:
|
||||
continue
|
||||
|
||||
from_unit_id = unit_ids[fact_idx]
|
||||
|
||||
for relation in causal_relations:
|
||||
target_idx = relation['target_fact_index']
|
||||
relation_type = relation['relation_type']
|
||||
strength = relation.get('strength', 1.0)
|
||||
|
||||
# Validate target index
|
||||
if target_idx < 0 or target_idx >= len(unit_ids):
|
||||
logger.warning(f"Invalid target_fact_index {target_idx} in causal relation from fact {fact_idx}")
|
||||
continue
|
||||
|
||||
to_unit_id = unit_ids[target_idx]
|
||||
|
||||
# Don't create self-links
|
||||
if from_unit_id == to_unit_id:
|
||||
continue
|
||||
|
||||
# Add the causal link
|
||||
# link_type is the relation_type (e.g., "causes", "caused_by")
|
||||
# weight is the strength of the relationship
|
||||
links.append((from_unit_id, to_unit_id, relation_type, strength, None))
|
||||
|
||||
logger.debug(f"Generated {len(links)} causal links in {time_mod.time() - create_start:.3f}s")
|
||||
|
||||
if links:
|
||||
insert_start = time_mod.time()
|
||||
await conn.executemany(
|
||||
"""
|
||||
INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
|
||||
""",
|
||||
links
|
||||
)
|
||||
logger.debug(f"Inserted {len(links)} causal links in {time_mod.time() - insert_start:.3f}s")
|
||||
|
||||
return len(links)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create causal links: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise
|
||||
@@ -182,10 +182,10 @@ class LLMConfig:
|
||||
@classmethod
|
||||
def for_memory(cls) -> "LLMConfig":
|
||||
"""Create configuration for memory operations from environment variables."""
|
||||
provider = os.getenv("MEMORA_API_LLM_PROVIDER", "groq")
|
||||
api_key = os.getenv("MEMORA_API_LLM_API_KEY")
|
||||
base_url = os.getenv("MEMORA_API_LLM_BASE_URL")
|
||||
model = os.getenv("MEMORA_API_LLM_MODEL", "openai/gpt-oss-120b")
|
||||
provider = os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq")
|
||||
api_key = os.getenv("HINDSIGHT_API_LLM_API_KEY")
|
||||
base_url = os.getenv("HINDSIGHT_API_LLM_BASE_URL")
|
||||
model = os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b")
|
||||
|
||||
# Set default base URL if not provided
|
||||
if not base_url:
|
||||
@@ -211,10 +211,10 @@ class LLMConfig:
|
||||
Falls back to memory LLM config if judge-specific config not set.
|
||||
"""
|
||||
# Check if judge-specific config exists, otherwise fall back to memory config
|
||||
provider = os.getenv("MEMORA_API_JUDGE_LLM_PROVIDER", os.getenv("MEMORA_API_LLM_PROVIDER", "groq"))
|
||||
api_key = os.getenv("MEMORA_API_JUDGE_LLM_API_KEY", os.getenv("MEMORA_API_LLM_API_KEY"))
|
||||
base_url = os.getenv("MEMORA_API_JUDGE_LLM_BASE_URL", os.getenv("MEMORA_API_LLM_BASE_URL"))
|
||||
model = os.getenv("MEMORA_API_JUDGE_LLM_MODEL", os.getenv("MEMORA_API_LLM_MODEL", "openai/gpt-oss-120b"))
|
||||
provider = os.getenv("HINDSIGHT_API_JUDGE_LLM_PROVIDER", os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq"))
|
||||
api_key = os.getenv("HINDSIGHT_API_JUDGE_LLM_API_KEY", os.getenv("HINDSIGHT_API_LLM_API_KEY"))
|
||||
base_url = os.getenv("HINDSIGHT_API_JUDGE_LLM_BASE_URL", os.getenv("HINDSIGHT_API_LLM_BASE_URL"))
|
||||
model = os.getenv("HINDSIGHT_API_JUDGE_LLM_MODEL", os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b"))
|
||||
|
||||
# Set default base URL if not provided
|
||||
if not base_url:
|
||||
+312
-42
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
Temporal + Semantic + Entity Memory System for AI Agents.
|
||||
Memory Engine for AI Agents.
|
||||
|
||||
This implements a sophisticated memory architecture that combines:
|
||||
1. Temporal links: Memories connected by time proximity
|
||||
@@ -8,6 +8,7 @@ This implements a sophisticated memory architecture that combines:
|
||||
4. Spreading activation: Search through the graph with activation decay
|
||||
5. Dynamic weighting: Recency and frequency-based importance
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
@@ -19,18 +20,26 @@ import time
|
||||
import numpy as np
|
||||
import uuid
|
||||
import logging
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .query_analyzer import QueryAnalyzer
|
||||
from .utils import (
|
||||
extract_facts,
|
||||
calculate_recency_weight,
|
||||
calculate_frequency_weight,
|
||||
)
|
||||
from .entity_resolver import EntityResolver
|
||||
from .operations import EmbeddingOperationsMixin, LinkOperationsMixin, ThinkOperationsMixin, AgentOperationsMixin
|
||||
from . import (
|
||||
embedding_utils,
|
||||
link_utils,
|
||||
think_utils,
|
||||
agent_utils,
|
||||
)
|
||||
from .llm_wrapper import LLMConfig
|
||||
from .response_models import SearchResult as SearchResultModel, ThinkResult, MemoryFact
|
||||
from .task_backend import TaskBackend, AsyncIOQueueBackend
|
||||
from .search.reranking import CrossEncoderReranker
|
||||
from ..pg0 import EmbeddedPostgres
|
||||
|
||||
|
||||
def utcnow():
|
||||
@@ -41,8 +50,10 @@ def utcnow():
|
||||
# Logger for memory system
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Tiktoken for token budget filtering
|
||||
from .db_utils import acquire_with_retry, retry_with_backoff
|
||||
|
||||
import tiktoken
|
||||
from dateutil import parser as date_parser
|
||||
|
||||
# Cache tiktoken encoding for token budget filtering (module-level singleton)
|
||||
_TIKTOKEN_ENCODING = None
|
||||
@@ -55,20 +66,15 @@ def _get_tiktoken_encoding():
|
||||
return _TIKTOKEN_ENCODING
|
||||
|
||||
|
||||
class TemporalSemanticMemory(
|
||||
EmbeddingOperationsMixin,
|
||||
LinkOperationsMixin,
|
||||
ThinkOperationsMixin,
|
||||
AgentOperationsMixin,
|
||||
):
|
||||
class MemoryEngine:
|
||||
"""
|
||||
Advanced memory system using temporal and semantic linking with PostgreSQL.
|
||||
|
||||
Uses mixin architecture for code organization:
|
||||
- EmbeddingOperationsMixin: Embedding generation
|
||||
- LinkOperationsMixin: Entity, temporal, and semantic link creation
|
||||
- ThinkOperationsMixin: Think operations for formulating answers with opinions
|
||||
- AgentOperationsMixin: Agent profile and personality management
|
||||
This class provides:
|
||||
- Embedding generation for semantic search
|
||||
- Entity, temporal, and semantic link creation
|
||||
- Think operations for formulating answers with opinions
|
||||
- Agent profile and personality management
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -80,7 +86,7 @@ class TemporalSemanticMemory(
|
||||
memory_llm_base_url: Optional[str] = None,
|
||||
embeddings: Optional[Embeddings] = None,
|
||||
cross_encoder: Optional[CrossEncoderModel] = None,
|
||||
query_analyzer: Optional["QueryAnalyzer"] = None,
|
||||
query_analyzer: Optional[QueryAnalyzer] = None,
|
||||
pool_min_size: int = 5,
|
||||
pool_max_size: int = 100,
|
||||
task_backend: Optional[TaskBackend] = None,
|
||||
@@ -104,8 +110,15 @@ class TemporalSemanticMemory(
|
||||
Increase for parallel think/search operations (e.g., 200-300 for 100+ parallel thinks)
|
||||
task_backend: Custom task backend for async task execution. If not provided, uses AsyncIOQueueBackend
|
||||
"""
|
||||
# Track pg0 instance (if used)
|
||||
self._pg0: Optional[EmbeddedPostgres] = None
|
||||
|
||||
# Initialize PostgreSQL connection URL
|
||||
self.db_url = db_url
|
||||
# "pg0" or "embedded-pg" are special values that trigger embedded PostgreSQL via pg0
|
||||
# The actual URL will be set during initialize() after starting the server
|
||||
self._use_pg0 = db_url in ("pg0", "embedded-pg")
|
||||
self.db_url = db_url if not self._use_pg0 else None
|
||||
|
||||
|
||||
# Set default base URL if not provided
|
||||
if memory_llm_base_url is None:
|
||||
@@ -135,7 +148,7 @@ class TemporalSemanticMemory(
|
||||
if query_analyzer is not None:
|
||||
self.query_analyzer = query_analyzer
|
||||
else:
|
||||
from memora.query_analyzer import TransformerQueryAnalyzer
|
||||
from .query_analyzer import TransformerQueryAnalyzer
|
||||
self.query_analyzer = TransformerQueryAnalyzer()
|
||||
|
||||
# Initialize LLM configuration
|
||||
@@ -188,7 +201,7 @@ class TemporalSemanticMemory(
|
||||
try:
|
||||
# Convert string UUIDs to UUID type for faster matching
|
||||
uuid_list = [uuid.UUID(nid) for nid in node_ids]
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
await conn.execute(
|
||||
"UPDATE memory_units SET access_count = access_count + 1 WHERE id = ANY($1::uuid[])",
|
||||
uuid_list
|
||||
@@ -242,7 +255,7 @@ class TemporalSemanticMemory(
|
||||
if operation_id:
|
||||
try:
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
result = await conn.fetchrow(
|
||||
"SELECT id FROM async_operations WHERE id = $1",
|
||||
uuid.UUID(operation_id)
|
||||
@@ -297,7 +310,7 @@ class TemporalSemanticMemory(
|
||||
"""Helper to delete an operation record from the database."""
|
||||
try:
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
await conn.execute(
|
||||
"DELETE FROM async_operations WHERE id = $1",
|
||||
uuid.UUID(operation_id)
|
||||
@@ -314,7 +327,7 @@ class TemporalSemanticMemory(
|
||||
full_error = f"{error_message}\n\nTraceback:\n{error_traceback}"
|
||||
truncated_error = full_error[:5000] if len(full_error) > 5000 else full_error
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE async_operations
|
||||
@@ -333,6 +346,13 @@ class TemporalSemanticMemory(
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
# Start pg0 embedded PostgreSQL if configured
|
||||
if self._use_pg0:
|
||||
logger.info("Starting pg0 embedded PostgreSQL...")
|
||||
self._pg0 = EmbeddedPostgres()
|
||||
self.db_url = await self._pg0.ensure_running()
|
||||
logger.info(f"pg0 PostgreSQL running at: {self.db_url}")
|
||||
|
||||
# Create connection pool
|
||||
# For read-heavy workloads with many parallel think/search operations,
|
||||
# we need a larger pool. Read operations don't need strong isolation.
|
||||
@@ -341,7 +361,8 @@ class TemporalSemanticMemory(
|
||||
min_size=self._pool_min_size,
|
||||
max_size=self._pool_max_size,
|
||||
command_timeout=60,
|
||||
statement_cache_size=0 # Disable prepared statement cache
|
||||
statement_cache_size=0, # Disable prepared statement cache
|
||||
timeout=30, # Connection acquisition timeout (seconds)
|
||||
)
|
||||
|
||||
# Initialize entity resolver with pool
|
||||
@@ -360,6 +381,20 @@ class TemporalSemanticMemory(
|
||||
await self.initialize()
|
||||
return self._pool
|
||||
|
||||
async def _acquire_connection(self):
|
||||
"""
|
||||
Acquire a connection from the pool with retry logic.
|
||||
|
||||
Returns an async context manager that yields a connection.
|
||||
Retries on transient connection errors with exponential backoff.
|
||||
"""
|
||||
pool = await self._get_pool()
|
||||
|
||||
async def acquire():
|
||||
return await pool.acquire()
|
||||
|
||||
return await _retry_with_backoff(acquire)
|
||||
|
||||
async def close(self):
|
||||
"""Close the connection pool and shutdown background workers."""
|
||||
logger.info("close() started")
|
||||
@@ -379,6 +414,14 @@ class TemporalSemanticMemory(
|
||||
logger.debug("no pool to close")
|
||||
|
||||
self._initialized = False
|
||||
|
||||
# Stop pg0 if we started it
|
||||
if self._pg0 is not None:
|
||||
logger.info("Stopping pg0...")
|
||||
await self._pg0.stop()
|
||||
self._pg0 = None
|
||||
logger.info("pg0 stopped")
|
||||
|
||||
logger.debug("close() completed")
|
||||
|
||||
async def wait_for_background_tasks(self):
|
||||
@@ -729,7 +772,8 @@ class TemporalSemanticMemory(
|
||||
log_buffer.append(f"{'='*60}")
|
||||
|
||||
# Get agent name for fact extraction
|
||||
profile = await self.get_agent_profile(agent_id)
|
||||
pool = await self._get_pool()
|
||||
profile = await agent_utils.get_agent_profile(pool, agent_id)
|
||||
agent_name = profile["name"]
|
||||
|
||||
# Step 1: Extract facts from ALL contents in parallel
|
||||
@@ -774,7 +818,7 @@ class TemporalSemanticMemory(
|
||||
all_fact_texts.append(fact_dict['fact'])
|
||||
|
||||
# Extract temporal fields (new schema with ranges)
|
||||
from dateutil import parser as date_parser
|
||||
|
||||
try:
|
||||
# Try new schema first (occurred_start/end)
|
||||
occurred_start = date_parser.isoparse(fact_dict['occurred_start'])
|
||||
@@ -855,14 +899,14 @@ class TemporalSemanticMemory(
|
||||
|
||||
# Step 2b: Generate ALL embeddings in ONE batch using augmented texts (HUGE speedup!)
|
||||
step_start = time.time()
|
||||
all_embeddings = await self._generate_embeddings_batch(augmented_texts)
|
||||
all_embeddings = await embedding_utils.generate_embeddings_batch(self.embeddings, augmented_texts)
|
||||
log_buffer.append(f"[2] Generate embeddings (parallel): {len(all_embeddings)} embeddings in {time.time() - step_start:.3f}s")
|
||||
|
||||
# Step 3: Process everything in ONE database transaction
|
||||
logger.debug("Getting connection pool")
|
||||
pool = await self._get_pool()
|
||||
logger.debug("Acquiring connection from pool")
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
logger.debug("Starting transaction")
|
||||
async with conn.transaction():
|
||||
logger.debug("Inside transaction")
|
||||
@@ -1035,8 +1079,8 @@ class TemporalSemanticMemory(
|
||||
# Process entities for ALL units
|
||||
logger.debug("Processing entities")
|
||||
step_start = time.time()
|
||||
all_entity_links = await self._extract_entities_batch_optimized(
|
||||
conn, agent_id, created_unit_ids, filtered_sentences, "", filtered_dates, filtered_entities, log_buffer
|
||||
all_entity_links = await link_utils.extract_entities_batch_optimized(
|
||||
self.entity_resolver, conn, agent_id, created_unit_ids, filtered_sentences, "", filtered_dates, filtered_entities, log_buffer
|
||||
)
|
||||
logger.debug(f"Entity processing complete: {len(all_entity_links)} links")
|
||||
log_buffer.append(f"[6] Process entities (batched): {time.time() - step_start:.3f}s")
|
||||
@@ -1044,14 +1088,14 @@ class TemporalSemanticMemory(
|
||||
# Create temporal links
|
||||
logger.debug("Creating temporal links")
|
||||
step_start = time.time()
|
||||
await self._create_temporal_links_batch_per_fact(conn, agent_id, created_unit_ids, log_buffer=log_buffer)
|
||||
await link_utils.create_temporal_links_batch_per_fact(conn, agent_id, created_unit_ids, log_buffer=log_buffer)
|
||||
logger.debug("Temporal links complete")
|
||||
log_buffer.append(f"[7] Batch create temporal links: {time.time() - step_start:.3f}s")
|
||||
|
||||
# Create semantic links
|
||||
logger.debug("Creating semantic links")
|
||||
step_start = time.time()
|
||||
await self._create_semantic_links_batch(conn, agent_id, created_unit_ids, filtered_embeddings, log_buffer=log_buffer)
|
||||
await link_utils.create_semantic_links_batch(conn, agent_id, created_unit_ids, filtered_embeddings, log_buffer=log_buffer)
|
||||
logger.debug("Semantic links complete")
|
||||
log_buffer.append(f"[8] Batch create semantic links: {time.time() - step_start:.3f}s")
|
||||
|
||||
@@ -1059,14 +1103,14 @@ class TemporalSemanticMemory(
|
||||
logger.debug("Inserting entity links")
|
||||
step_start = time.time()
|
||||
if all_entity_links:
|
||||
await self._insert_entity_links_batch(conn, all_entity_links)
|
||||
await link_utils.insert_entity_links_batch(conn, all_entity_links)
|
||||
logger.debug("Entity links inserted")
|
||||
log_buffer.append(f"[9] Batch insert entity links: {time.time() - step_start:.3f}s")
|
||||
|
||||
# Create causal links
|
||||
logger.debug("Creating causal links")
|
||||
step_start = time.time()
|
||||
causal_link_count = await self._create_causal_links_batch(
|
||||
causal_link_count = await link_utils.create_causal_links_batch(
|
||||
conn, created_unit_ids, filtered_causal_relations
|
||||
)
|
||||
logger.debug(f"Causal links complete: {causal_link_count} links created")
|
||||
@@ -1261,7 +1305,7 @@ class TemporalSemanticMemory(
|
||||
try:
|
||||
# Step 1: Generate query embedding (for semantic search)
|
||||
step_start = time.time()
|
||||
query_embedding = self._generate_embedding(query)
|
||||
query_embedding = embedding_utils.generate_embedding(self.embeddings, query)
|
||||
step_duration = time.time() - step_start
|
||||
log_buffer.append(f" [1] Generate query embedding: {step_duration:.3f}s")
|
||||
|
||||
@@ -1593,7 +1637,7 @@ class TemporalSemanticMemory(
|
||||
Dictionary with document info or None if not found
|
||||
"""
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
doc = await conn.fetchrow(
|
||||
"""
|
||||
SELECT d.id, d.agent_id, d.original_text, d.content_hash,
|
||||
@@ -1631,7 +1675,7 @@ class TemporalSemanticMemory(
|
||||
Dictionary with counts of deleted items
|
||||
"""
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
async with conn.transaction():
|
||||
# Count units before deletion
|
||||
units_count = await conn.fetchval(
|
||||
@@ -1666,7 +1710,7 @@ class TemporalSemanticMemory(
|
||||
Dictionary with deletion result
|
||||
"""
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
async with conn.transaction():
|
||||
# Delete the memory unit (cascades to links and associations)
|
||||
deleted = await conn.fetchval(
|
||||
@@ -1700,7 +1744,7 @@ class TemporalSemanticMemory(
|
||||
Dictionary with counts of deleted items
|
||||
"""
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
async with conn.transaction():
|
||||
try:
|
||||
if fact_type:
|
||||
@@ -1751,7 +1795,7 @@ class TemporalSemanticMemory(
|
||||
Dict with nodes, edges, and table_rows
|
||||
"""
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
# Get memory units, optionally filtered by agent_id and fact_type
|
||||
query_conditions = []
|
||||
query_params = []
|
||||
@@ -1922,7 +1966,7 @@ class TemporalSemanticMemory(
|
||||
Dict with items (list of memory units) and total count
|
||||
"""
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
# Build query conditions
|
||||
query_conditions = []
|
||||
query_params = []
|
||||
@@ -2036,7 +2080,7 @@ class TemporalSemanticMemory(
|
||||
Dict with items (list of documents without original_text) and total count
|
||||
"""
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
# Build query conditions
|
||||
query_conditions = []
|
||||
query_params = []
|
||||
@@ -2153,7 +2197,7 @@ class TemporalSemanticMemory(
|
||||
Dict with document details including original_text, or None if not found
|
||||
"""
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
doc = await conn.fetchrow("""
|
||||
SELECT
|
||||
id,
|
||||
@@ -2336,7 +2380,7 @@ Guidelines:
|
||||
logger.debug(f"[REINFORCE] Starting opinion reinforcement for {len(entity_names)} entities")
|
||||
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
# Find all opinions related to these entities
|
||||
opinions = await conn.fetch(
|
||||
"""
|
||||
@@ -2439,3 +2483,229 @@ Guidelines:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
# ==================== Agent Profile Methods ====================
|
||||
|
||||
async def get_agent_profile(self, agent_id: str) -> Dict:
|
||||
"""
|
||||
Get agent profile (name, personality + background).
|
||||
Auto-creates agent with default values if not exists.
|
||||
|
||||
Args:
|
||||
agent_id: Agent identifier
|
||||
|
||||
Returns:
|
||||
Dict with 'name' (str), 'personality' (dict) and 'background' (str) keys
|
||||
"""
|
||||
pool = await self._get_pool()
|
||||
return await agent_utils.get_agent_profile(pool, agent_id)
|
||||
|
||||
async def update_agent_personality(
|
||||
self,
|
||||
agent_id: str,
|
||||
personality: Dict[str, float]
|
||||
) -> None:
|
||||
"""
|
||||
Update agent personality traits.
|
||||
|
||||
Args:
|
||||
agent_id: Agent identifier
|
||||
personality: Dict with Big Five traits + bias_strength (all 0-1)
|
||||
"""
|
||||
pool = await self._get_pool()
|
||||
await agent_utils.update_agent_personality(pool, agent_id, personality)
|
||||
|
||||
async def merge_agent_background(
|
||||
self,
|
||||
agent_id: str,
|
||||
new_info: str,
|
||||
update_personality: bool = True
|
||||
) -> dict:
|
||||
"""
|
||||
Merge new background information with existing background using LLM.
|
||||
Normalizes to first person ("I") and resolves conflicts.
|
||||
Optionally infers personality traits from the merged background.
|
||||
|
||||
Args:
|
||||
agent_id: Agent identifier
|
||||
new_info: New background information to add/merge
|
||||
update_personality: If True, infer Big Five traits from background (default: True)
|
||||
|
||||
Returns:
|
||||
Dict with 'background' (str) and optionally 'personality' (dict) keys
|
||||
"""
|
||||
pool = await self._get_pool()
|
||||
return await agent_utils.merge_agent_background(
|
||||
pool, self._llm_config, agent_id, new_info, update_personality
|
||||
)
|
||||
|
||||
async def list_agents(self) -> list:
|
||||
"""
|
||||
List all agents in the system.
|
||||
|
||||
Returns:
|
||||
List of dicts with agent_id, name, personality, background, created_at, updated_at
|
||||
"""
|
||||
pool = await self._get_pool()
|
||||
return await agent_utils.list_agents(pool)
|
||||
|
||||
# ==================== Think Methods ====================
|
||||
|
||||
async def think_async(
|
||||
self,
|
||||
agent_id: str,
|
||||
query: str,
|
||||
thinking_budget: int = 50,
|
||||
context: str = None,
|
||||
) -> ThinkResult:
|
||||
"""
|
||||
Think and formulate an answer using agent identity, world facts, and opinions.
|
||||
|
||||
This method:
|
||||
1. Retrieves agent facts (agent's identity and past actions)
|
||||
2. Retrieves world facts (general knowledge)
|
||||
3. Retrieves existing opinions (agent's formed perspectives)
|
||||
4. Uses LLM to formulate an answer
|
||||
5. Extracts and stores any new opinions formed during thinking
|
||||
6. Returns plain text answer and the facts used
|
||||
|
||||
Args:
|
||||
agent_id: Agent identifier
|
||||
query: Question to answer
|
||||
thinking_budget: Number of memory units to explore
|
||||
context: Additional context string to include in LLM prompt (not used in search)
|
||||
|
||||
Returns:
|
||||
ThinkResult containing:
|
||||
- text: Plain text answer (no markdown)
|
||||
- based_on: Dict with 'world', 'agent', and 'opinion' fact lists (MemoryFact objects)
|
||||
- new_opinions: List of newly formed opinions
|
||||
"""
|
||||
# Use cached LLM config
|
||||
if self._llm_config is None:
|
||||
raise ValueError("Memory LLM API key not set. Set HINDSIGHT_API_LLM_API_KEY environment variable.")
|
||||
|
||||
# Steps 1-3: Run multi-fact-type search (12-way retrieval: 4 methods × 3 fact types)
|
||||
search_result = await self.search_async(
|
||||
agent_id=agent_id,
|
||||
query=query,
|
||||
thinking_budget=thinking_budget,
|
||||
max_tokens=4096,
|
||||
enable_trace=False,
|
||||
fact_type=['agent', 'world', 'opinion']
|
||||
)
|
||||
|
||||
all_results = search_result.results
|
||||
logger.info(f"[THINK] Search returned {len(all_results)} results")
|
||||
|
||||
# Split results by fact type for structured response
|
||||
agent_results = [r for r in all_results if r.fact_type == 'agent']
|
||||
world_results = [r for r in all_results if r.fact_type == 'world']
|
||||
opinion_results = [r for r in all_results if r.fact_type == 'opinion']
|
||||
|
||||
logger.info(f"[THINK] Split results - agent: {len(agent_results)}, world: {len(world_results)}, opinion: {len(opinion_results)}")
|
||||
|
||||
# Format facts for LLM
|
||||
agent_facts_text = think_utils.format_facts_for_prompt(agent_results)
|
||||
world_facts_text = think_utils.format_facts_for_prompt(world_results)
|
||||
opinion_facts_text = think_utils.format_facts_for_prompt(opinion_results)
|
||||
|
||||
logger.info(f"[THINK] Formatted facts - agent: {len(agent_facts_text)} chars, world: {len(world_facts_text)} chars, opinion: {len(opinion_facts_text)} chars")
|
||||
|
||||
# Get agent profile (name, personality + background)
|
||||
profile = await self.get_agent_profile(agent_id)
|
||||
name = profile["name"]
|
||||
personality = profile["personality"]
|
||||
background = profile["background"]
|
||||
|
||||
# Build the prompt
|
||||
prompt = think_utils.build_think_prompt(
|
||||
agent_facts_text=agent_facts_text,
|
||||
world_facts_text=world_facts_text,
|
||||
opinion_facts_text=opinion_facts_text,
|
||||
query=query,
|
||||
name=name,
|
||||
personality=personality,
|
||||
background=background,
|
||||
context=context,
|
||||
)
|
||||
|
||||
logger.info(f"[THINK] Full prompt length: {len(prompt)} chars")
|
||||
logger.debug(f"[THINK] Prompt preview (first 500 chars): {prompt[:500]}")
|
||||
|
||||
system_message = think_utils.get_system_message(personality)
|
||||
|
||||
answer_text = await self._llm_config.call(
|
||||
messages=[
|
||||
{"role": "system", "content": system_message},
|
||||
{"role": "user", "content": prompt}
|
||||
],
|
||||
scope="memory_think",
|
||||
temperature=0.9,
|
||||
max_tokens=1000
|
||||
)
|
||||
|
||||
answer_text = answer_text.strip()
|
||||
|
||||
# Submit form_opinion task for background processing
|
||||
logger.debug(f"[THINK] Submitting form_opinion task for agent {agent_id}")
|
||||
await self._task_backend.submit_task({
|
||||
'type': 'form_opinion',
|
||||
'agent_id': agent_id,
|
||||
'answer_text': answer_text,
|
||||
'query': query
|
||||
})
|
||||
logger.debug(f"[THINK] form_opinion task submitted")
|
||||
|
||||
# Return response with facts split by type
|
||||
return ThinkResult(
|
||||
text=answer_text,
|
||||
based_on={
|
||||
"world": world_results,
|
||||
"agent": agent_results,
|
||||
"opinion": opinion_results
|
||||
},
|
||||
new_opinions=[] # Opinions are being extracted asynchronously
|
||||
)
|
||||
|
||||
async def _extract_and_store_opinions_async(
|
||||
self,
|
||||
agent_id: str,
|
||||
answer_text: str,
|
||||
query: str
|
||||
):
|
||||
"""
|
||||
Background task to extract and store opinions from think response.
|
||||
|
||||
This runs asynchronously and does not block the think response.
|
||||
|
||||
Args:
|
||||
agent_id: Agent identifier
|
||||
answer_text: The generated answer text
|
||||
query: The original query
|
||||
"""
|
||||
try:
|
||||
logger.debug(f"[THINK] Extracting opinions from answer for agent {agent_id}")
|
||||
# Extract opinions from the answer
|
||||
new_opinions = await think_utils.extract_opinions_from_text(
|
||||
self._llm_config, text=answer_text, query=query
|
||||
)
|
||||
logger.debug(f"[THINK] Extracted {len(new_opinions)} opinions")
|
||||
|
||||
# Store new opinions
|
||||
if new_opinions:
|
||||
from datetime import datetime, timezone
|
||||
current_time = datetime.now(timezone.utc)
|
||||
for opinion_dict in new_opinions:
|
||||
await self.put_async(
|
||||
agent_id=agent_id,
|
||||
content=opinion_dict["text"],
|
||||
context=f"formed during thinking about: {query}",
|
||||
event_date=current_time,
|
||||
fact_type_override='opinion',
|
||||
confidence_score=opinion_dict["confidence"]
|
||||
)
|
||||
|
||||
logger.debug(f"[THINK] Extracted and stored {len(new_opinions)} new opinions")
|
||||
except Exception as e:
|
||||
logger.warning(f"[THINK] Failed to extract/store opinions: {str(e)}")
|
||||
|
||||
+63
-67
@@ -1,13 +1,13 @@
|
||||
"""
|
||||
Core response models for Memora memory system.
|
||||
Core response models for Hindsight memory system.
|
||||
|
||||
These models define the structure of data returned by the core TemporalSemanticMemory class.
|
||||
These models define the structure of data returned by the core MemoryEngine class.
|
||||
API response models should be kept separate and convert from these core models to maintain
|
||||
API stability even if internal models change.
|
||||
"""
|
||||
|
||||
from typing import Optional, List, Dict, Any
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, ConfigDict
|
||||
|
||||
|
||||
class MemoryFact(BaseModel):
|
||||
@@ -17,6 +17,19 @@ class MemoryFact(BaseModel):
|
||||
This represents a unit of information stored in the memory system,
|
||||
including both the content and metadata.
|
||||
"""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"id": "123e4567-e89b-12d3-a456-426614174000",
|
||||
"text": "Alice works at Google on the AI team",
|
||||
"fact_type": "world",
|
||||
"context": "work info",
|
||||
"event_date": "2024-01-15T10:30:00Z",
|
||||
"document_id": "session_abc123",
|
||||
"metadata": {"source": "slack"},
|
||||
"activation": 0.95
|
||||
}
|
||||
})
|
||||
|
||||
id: str = Field(description="Unique identifier for the memory fact")
|
||||
text: str = Field(description="The actual text content of the memory")
|
||||
fact_type: str = Field(description="Type of fact: 'world', 'agent', or 'opinion'")
|
||||
@@ -31,20 +44,6 @@ class MemoryFact(BaseModel):
|
||||
# Internal metrics (used by system but may not be exposed in API)
|
||||
activation: Optional[float] = Field(None, description="Internal activation score")
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"id": "123e4567-e89b-12d3-a456-426614174000",
|
||||
"text": "Alice works at Google on the AI team",
|
||||
"fact_type": "world",
|
||||
"context": "work info",
|
||||
"event_date": "2024-01-15T10:30:00Z",
|
||||
"document_id": "session_abc123",
|
||||
"metadata": {"source": "slack"},
|
||||
"activation": 0.95
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class SearchResult(BaseModel):
|
||||
"""
|
||||
@@ -53,28 +52,27 @@ class SearchResult(BaseModel):
|
||||
Contains a list of matching memory facts and optional trace information
|
||||
for debugging and transparency.
|
||||
"""
|
||||
results: List[MemoryFact] = Field(description="List of memory facts matching the query")
|
||||
trace: Optional[Dict[str, Any]] = Field(None, description="Trace information for debugging")
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"results": [
|
||||
{
|
||||
"id": "123e4567-e89b-12d3-a456-426614174000",
|
||||
"text": "Alice works at Google on the AI team",
|
||||
"fact_type": "world",
|
||||
"context": "work info",
|
||||
"event_date": "2024-01-15T10:30:00Z",
|
||||
"activation": 0.95
|
||||
}
|
||||
],
|
||||
"trace": {
|
||||
"query": "What did Alice say about machine learning?",
|
||||
"num_results": 1
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"results": [
|
||||
{
|
||||
"id": "123e4567-e89b-12d3-a456-426614174000",
|
||||
"text": "Alice works at Google on the AI team",
|
||||
"fact_type": "world",
|
||||
"context": "work info",
|
||||
"event_date": "2024-01-15T10:30:00Z",
|
||||
"activation": 0.95
|
||||
}
|
||||
],
|
||||
"trace": {
|
||||
"query": "What did Alice say about machine learning?",
|
||||
"num_results": 1
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
results: List[MemoryFact] = Field(description="List of memory facts matching the query")
|
||||
trace: Optional[Dict[str, Any]] = Field(None, description="Trace information for debugging")
|
||||
|
||||
|
||||
class ThinkResult(BaseModel):
|
||||
@@ -84,6 +82,28 @@ class ThinkResult(BaseModel):
|
||||
Contains the formulated answer, the facts it was based on (organized by type),
|
||||
and any new opinions that were formed during the thinking process.
|
||||
"""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"text": "Based on my knowledge, machine learning is being actively used in healthcare...",
|
||||
"based_on": {
|
||||
"world": [
|
||||
{
|
||||
"id": "123e4567-e89b-12d3-a456-426614174000",
|
||||
"text": "Machine learning is used in medical diagnosis",
|
||||
"fact_type": "world",
|
||||
"context": "healthcare",
|
||||
"event_date": "2024-01-15T10:30:00Z"
|
||||
}
|
||||
],
|
||||
"agent": [],
|
||||
"opinion": []
|
||||
},
|
||||
"new_opinions": [
|
||||
"Machine learning has great potential in healthcare"
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
text: str = Field(description="The formulated answer text")
|
||||
based_on: Dict[str, List[MemoryFact]] = Field(
|
||||
description="Facts used to formulate the answer, organized by type (world, agent, opinion)"
|
||||
@@ -93,29 +113,6 @@ class ThinkResult(BaseModel):
|
||||
description="List of newly formed opinions during thinking"
|
||||
)
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"text": "Based on my knowledge, machine learning is being actively used in healthcare...",
|
||||
"based_on": {
|
||||
"world": [
|
||||
{
|
||||
"id": "123e4567-e89b-12d3-a456-426614174000",
|
||||
"text": "Machine learning is used in medical diagnosis",
|
||||
"fact_type": "world",
|
||||
"context": "healthcare",
|
||||
"event_date": "2024-01-15T10:30:00Z"
|
||||
}
|
||||
],
|
||||
"agent": [],
|
||||
"opinion": []
|
||||
},
|
||||
"new_opinions": [
|
||||
"Machine learning has great potential in healthcare"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class Opinion(BaseModel):
|
||||
"""
|
||||
@@ -124,13 +121,12 @@ class Opinion(BaseModel):
|
||||
Opinions represent the agent's formed perspectives on topics,
|
||||
with a confidence level indicating strength of belief.
|
||||
"""
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
"text": "Machine learning has great potential in healthcare",
|
||||
"confidence": 0.85
|
||||
}
|
||||
})
|
||||
|
||||
text: str = Field(description="The opinion text")
|
||||
confidence: float = Field(description="Confidence score between 0.0 and 1.0")
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"text": "Machine learning has great potential in healthcare",
|
||||
"confidence": 0.85
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -24,7 +24,7 @@ class CrossEncoderReranker:
|
||||
SentenceTransformersCrossEncoder with ms-marco-MiniLM-L-6-v2
|
||||
"""
|
||||
if cross_encoder is None:
|
||||
from ..cross_encoder import SentenceTransformersCrossEncoder
|
||||
from hindsight_api.engine.cross_encoder import SentenceTransformersCrossEncoder
|
||||
cross_encoder = SentenceTransformersCrossEncoder()
|
||||
self.cross_encoder = cross_encoder
|
||||
|
||||
+5
-18
@@ -11,6 +11,7 @@ Implements:
|
||||
from typing import List, Dict, Any, Tuple, Optional
|
||||
from datetime import datetime
|
||||
import asyncio
|
||||
from ..db_utils import acquire_with_retry
|
||||
|
||||
|
||||
async def retrieve_semantic(
|
||||
@@ -241,11 +242,6 @@ async def retrieve_temporal(
|
||||
if end_date.tzinfo is None:
|
||||
end_date = end_date.replace(tzinfo=timezone.utc)
|
||||
|
||||
# Find entry points: facts in date range with semantic relevance
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.info(f"Temporal retrieval: searching for facts between {start_date} and {end_date} (agent={agent_id}, fact_type={fact_type})")
|
||||
|
||||
entry_points = await conn.fetch(
|
||||
"""
|
||||
SELECT id, text, context, event_date, occurred_start, occurred_end, mentioned_at, access_count, embedding, fact_type, document_id,
|
||||
@@ -274,8 +270,6 @@ async def retrieve_temporal(
|
||||
query_emb_str, agent_id, fact_type, start_date, end_date, semantic_threshold
|
||||
)
|
||||
|
||||
logger.info(f"Temporal retrieval: found {len(entry_points)} entry points")
|
||||
|
||||
if not entry_points:
|
||||
# Check if there are ANY memories with temporal metadata for this agent
|
||||
total_with_dates = await conn.fetchval(
|
||||
@@ -284,7 +278,6 @@ async def retrieve_temporal(
|
||||
AND (occurred_start IS NOT NULL OR occurred_end IS NOT NULL OR mentioned_at IS NOT NULL)""",
|
||||
agent_id, fact_type
|
||||
)
|
||||
logger.info(f"Temporal retrieval: agent has {total_with_dates} total memories with temporal metadata (fact_type={fact_type})")
|
||||
return []
|
||||
|
||||
# Calculate temporal scores for entry points
|
||||
@@ -442,26 +435,20 @@ async def retrieve_parallel(
|
||||
query_text, reference_date=question_date, analyzer=query_analyzer
|
||||
)
|
||||
|
||||
if temporal_constraint:
|
||||
logger.info(f"Temporal constraint detected in retrieve_parallel: {temporal_constraint[0]} to {temporal_constraint[1]}")
|
||||
else:
|
||||
logger.info("No temporal constraint in retrieve_parallel")
|
||||
|
||||
# Each retrieval needs its own connection
|
||||
async def run_semantic():
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
return await retrieve_semantic(conn, query_embedding_str, agent_id, fact_type, limit=thinking_budget)
|
||||
|
||||
async def run_bm25():
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
return await retrieve_bm25(conn, query_text, agent_id, fact_type, limit=thinking_budget)
|
||||
|
||||
async def run_graph():
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
return await retrieve_graph(conn, query_embedding_str, agent_id, fact_type, budget=thinking_budget)
|
||||
|
||||
async def run_temporal(start_date, end_date):
|
||||
async with pool.acquire() as conn:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
return await retrieve_temporal(
|
||||
conn, query_embedding_str, agent_id, fact_type,
|
||||
start_date, end_date, budget=thinking_budget, semantic_threshold=0.4
|
||||
+1
-3
@@ -7,7 +7,7 @@ Handles natural language temporal expressions using transformer-based query anal
|
||||
from typing import Optional, Tuple
|
||||
from datetime import datetime
|
||||
import logging
|
||||
from memora.query_analyzer import QueryAnalyzer, TransformerQueryAnalyzer
|
||||
from hindsight_api.engine.query_analyzer import QueryAnalyzer, TransformerQueryAnalyzer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -59,8 +59,6 @@ def extract_temporal_constraint(
|
||||
analysis.temporal_constraint.start_date,
|
||||
analysis.temporal_constraint.end_date
|
||||
)
|
||||
logger.info(f"Temporal constraint extracted: {result[0].strftime('%Y-%m-%d')} to {result[1].strftime('%Y-%m-%d')}")
|
||||
return result
|
||||
|
||||
logger.info("No temporal constraint found in query")
|
||||
return None
|
||||
@@ -23,7 +23,7 @@ class TaskBackend(ABC):
|
||||
2. Execute tasks through a provided executor callback
|
||||
|
||||
The backend treats tasks as pure dictionaries that can be serialized
|
||||
and sent over the network. The executor (typically TemporalSemanticMemory.execute_task)
|
||||
and sent over the network. The executor (typically MemoryEngine.execute_task)
|
||||
receives the dict and routes it to the appropriate handler.
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
"""
|
||||
Think operation utilities for formulating answers based on agent and world facts.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Any
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .response_models import ThinkResult, MemoryFact
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Opinion(BaseModel):
|
||||
"""An opinion formed by the agent."""
|
||||
opinion: str = Field(description="The opinion or perspective with reasoning included")
|
||||
confidence: float = Field(description="Confidence score for this opinion (0.0 to 1.0, where 1.0 is very confident)")
|
||||
|
||||
|
||||
class OpinionExtractionResponse(BaseModel):
|
||||
"""Response containing extracted opinions."""
|
||||
opinions: List[Opinion] = Field(
|
||||
default_factory=list,
|
||||
description="List of opinions formed with their supporting reasons and confidence scores"
|
||||
)
|
||||
|
||||
|
||||
def describe_trait(name: str, value: float) -> str:
|
||||
"""Convert trait value to descriptive text."""
|
||||
if value >= 0.8:
|
||||
return f"very high {name}"
|
||||
elif value >= 0.6:
|
||||
return f"high {name}"
|
||||
elif value >= 0.4:
|
||||
return f"moderate {name}"
|
||||
elif value >= 0.2:
|
||||
return f"low {name}"
|
||||
else:
|
||||
return f"very low {name}"
|
||||
|
||||
|
||||
def build_personality_description(personality: Dict) -> str:
|
||||
"""Build a personality description string from personality traits."""
|
||||
return f"""Your personality traits:
|
||||
- {describe_trait('openness to new ideas', personality['openness'])}
|
||||
- {describe_trait('conscientiousness and organization', personality['conscientiousness'])}
|
||||
- {describe_trait('extraversion and sociability', personality['extraversion'])}
|
||||
- {describe_trait('agreeableness and cooperation', personality['agreeableness'])}
|
||||
- {describe_trait('emotional sensitivity', personality['neuroticism'])}
|
||||
|
||||
Personality influence strength: {int(personality['bias_strength'] * 100)}% (how much your personality shapes your opinions)"""
|
||||
|
||||
|
||||
def format_facts_for_prompt(facts: List[MemoryFact]) -> str:
|
||||
"""Format facts as JSON for LLM prompt."""
|
||||
import json
|
||||
|
||||
if not facts:
|
||||
return "[]"
|
||||
formatted = []
|
||||
for fact in facts:
|
||||
fact_obj = {
|
||||
"text": fact.text
|
||||
}
|
||||
|
||||
# Add context if available
|
||||
if fact.context:
|
||||
fact_obj["context"] = fact.context
|
||||
|
||||
# Add event_date if available
|
||||
if fact.event_date:
|
||||
event_date = fact.event_date
|
||||
if isinstance(event_date, str):
|
||||
fact_obj["event_date"] = event_date
|
||||
elif isinstance(event_date, datetime):
|
||||
fact_obj["event_date"] = event_date.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# Add activation if available
|
||||
if fact.activation is not None:
|
||||
fact_obj["score"] = fact.activation
|
||||
|
||||
formatted.append(fact_obj)
|
||||
|
||||
return json.dumps(formatted, indent=2)
|
||||
|
||||
|
||||
def build_think_prompt(
|
||||
agent_facts_text: str,
|
||||
world_facts_text: str,
|
||||
opinion_facts_text: str,
|
||||
query: str,
|
||||
name: str,
|
||||
personality: Dict,
|
||||
background: str,
|
||||
context: str = None,
|
||||
) -> str:
|
||||
"""Build the think prompt for the LLM."""
|
||||
personality_desc = build_personality_description(personality)
|
||||
|
||||
name_section = f"""
|
||||
|
||||
Your name: {name}
|
||||
"""
|
||||
|
||||
background_section = ""
|
||||
if background:
|
||||
background_section = f"""
|
||||
|
||||
Your background:
|
||||
{background}
|
||||
"""
|
||||
|
||||
context_section = ""
|
||||
if context:
|
||||
context_section = f"""
|
||||
ADDITIONAL CONTEXT:
|
||||
{context}
|
||||
|
||||
"""
|
||||
|
||||
return f"""Here's what I know and have experienced:
|
||||
|
||||
MY IDENTITY & EXPERIENCES:
|
||||
{agent_facts_text}
|
||||
|
||||
WHAT I KNOW ABOUT THE WORLD:
|
||||
{world_facts_text}
|
||||
|
||||
MY EXISTING OPINIONS & BELIEFS:
|
||||
{opinion_facts_text}
|
||||
|
||||
{context_section}{name_section}{personality_desc}{background_section}
|
||||
|
||||
QUESTION: {query}
|
||||
|
||||
Based on everything I know, believe, and who I am (including my name, personality and background), here's what I genuinely think about this question. I'll draw on my experiences, knowledge, opinions, and personal traits to give you my honest perspective."""
|
||||
|
||||
|
||||
def get_system_message(personality: Dict) -> str:
|
||||
"""Get the system message for the think LLM call."""
|
||||
bias_strength = personality['bias_strength']
|
||||
if bias_strength >= 0.7:
|
||||
personality_instruction = "Your personality strongly influences your thinking. Let your traits guide how you interpret facts and form opinions. Don't be afraid to be biased based on your personality."
|
||||
elif bias_strength >= 0.4:
|
||||
personality_instruction = "Your personality moderately influences your thinking. Balance your personal traits with objective analysis."
|
||||
else:
|
||||
personality_instruction = "Your personality has minimal influence on your thinking. Focus primarily on facts while keeping your traits in mind."
|
||||
|
||||
return f"You are a person with your own thoughts, experiences, opinions, and personality. When asked a question, respond naturally like a human would - share what you genuinely think based on what you know and have experienced. {personality_instruction} Be direct, express your views confidently, and use 'I think', 'I believe', 'in my view', etc. Respond in plain text without markdown formatting."
|
||||
|
||||
|
||||
async def extract_opinions_from_text(
|
||||
llm_config,
|
||||
text: str,
|
||||
query: str
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Extract opinions with reasons and confidence from text using LLM.
|
||||
|
||||
Args:
|
||||
llm_config: LLM configuration to use
|
||||
text: Text to extract opinions from
|
||||
query: The original query that prompted this response
|
||||
|
||||
Returns:
|
||||
List of dicts with keys: 'text' (opinion with reasons), 'confidence' (score 0-1)
|
||||
"""
|
||||
extraction_prompt = f"""Extract any NEW opinions or perspectives from the answer below and rewrite them in FIRST-PERSON as if YOU are stating the opinion directly.
|
||||
|
||||
ORIGINAL QUESTION:
|
||||
{query}
|
||||
|
||||
ANSWER PROVIDED:
|
||||
{text}
|
||||
|
||||
Your task: Find opinions in the answer and rewrite them AS IF YOU ARE THE ONE SAYING THEM.
|
||||
|
||||
An opinion is a judgment, viewpoint, or conclusion that goes beyond just stating facts.
|
||||
|
||||
IMPORTANT: Do NOT extract statements like:
|
||||
- "I don't have enough information"
|
||||
- "The facts don't contain information about X"
|
||||
- "I cannot answer because..."
|
||||
|
||||
ONLY extract actual opinions about substantive topics.
|
||||
|
||||
CRITICAL FORMAT REQUIREMENTS:
|
||||
1. **ALWAYS start with first-person phrases**: "I think...", "I believe...", "In my view...", "I've come to believe...", "Previously I thought... but now..."
|
||||
2. **NEVER use third-person**: Do NOT say "The speaker thinks..." or "They believe..." - always use "I"
|
||||
3. Include the reasoning naturally within the statement
|
||||
4. Provide a confidence score (0.0 to 1.0)
|
||||
|
||||
CORRECT Examples (✓ FIRST-PERSON):
|
||||
- "I think Alice is more reliable because she consistently delivers on time and writes clean code"
|
||||
- "Previously I thought all engineers were equal, but now I feel that experience and track record really matter"
|
||||
- "I believe reliability is best measured by consistent output over time"
|
||||
- "I've come to believe that track records are more important than potential"
|
||||
|
||||
WRONG Examples (✗ THIRD-PERSON - DO NOT USE):
|
||||
- "The speaker thinks Alice is more reliable"
|
||||
- "They believe reliability matters"
|
||||
- "It is believed that Alice is better"
|
||||
|
||||
If no genuine opinions are expressed (e.g., the response just says "I don't know"), return an empty list."""
|
||||
|
||||
try:
|
||||
result = await llm_config.call(
|
||||
messages=[
|
||||
{"role": "system", "content": "You are converting opinions from text into first-person statements. Always use 'I think', 'I believe', 'I feel', etc. NEVER use third-person like 'The speaker' or 'They'."},
|
||||
{"role": "user", "content": extraction_prompt}
|
||||
],
|
||||
response_format=OpinionExtractionResponse,
|
||||
scope="memory_extract_opinion"
|
||||
)
|
||||
|
||||
# Format opinions with confidence score and convert to first-person
|
||||
formatted_opinions = []
|
||||
for op in result.opinions:
|
||||
# Convert third-person to first-person if needed
|
||||
opinion_text = op.opinion
|
||||
|
||||
# Replace common third-person patterns with first-person
|
||||
def singularize_verb(verb):
|
||||
if verb.endswith('es'):
|
||||
return verb[:-1] # believes -> believe
|
||||
elif verb.endswith('s'):
|
||||
return verb[:-1] # thinks -> think
|
||||
return verb
|
||||
|
||||
# Pattern: "The speaker/user [verb]..." -> "I [verb]..."
|
||||
match = re.match(r'^(The speaker|The user|They|It is believed) (believes?|thinks?|feels?|says|asserts?|considers?)(\s+that)?(.*)$', opinion_text, re.IGNORECASE)
|
||||
if match:
|
||||
verb = singularize_verb(match.group(2))
|
||||
that_part = match.group(3) or "" # Keep " that" if present
|
||||
rest = match.group(4)
|
||||
opinion_text = f"I {verb}{that_part}{rest}"
|
||||
|
||||
# If still doesn't start with first-person, prepend "I believe that "
|
||||
first_person_starters = ["I think", "I believe", "I feel", "In my view", "I've come to believe", "Previously I"]
|
||||
if not any(opinion_text.startswith(starter) for starter in first_person_starters):
|
||||
opinion_text = "I believe that " + opinion_text[0].lower() + opinion_text[1:]
|
||||
|
||||
formatted_opinions.append({
|
||||
"text": opinion_text,
|
||||
"confidence": op.confidence
|
||||
})
|
||||
|
||||
return formatted_opinions
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to extract opinions: {str(e)}")
|
||||
return []
|
||||
@@ -37,14 +37,14 @@ def run_migrations(database_url: str, script_location: Optional[str] = None) ->
|
||||
Args:
|
||||
database_url: SQLAlchemy database URL (e.g., "postgresql://user:pass@host/db")
|
||||
script_location: Path to alembic migrations directory (e.g., "/path/to/alembic").
|
||||
If None, defaults to memora/alembic directory.
|
||||
If None, defaults to hindsight-api/alembic directory.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If migrations fail to complete
|
||||
FileNotFoundError: If script_location doesn't exist
|
||||
|
||||
Example:
|
||||
# Using default location (memora package)
|
||||
# Using default location (hindsight_api package)
|
||||
run_migrations("postgresql://user:pass@host/db")
|
||||
|
||||
# Using custom location (when importing from another project)
|
||||
@@ -56,9 +56,9 @@ def run_migrations(database_url: str, script_location: Optional[str] = None) ->
|
||||
try:
|
||||
# Determine script location
|
||||
if script_location is None:
|
||||
# Default: use the alembic directory in the memora package
|
||||
# This file is in: memora/memora/migrations.py
|
||||
# Default location is: memora/alembic
|
||||
# Default: use the alembic directory in the hindsight_api package
|
||||
# This file is in: hindsight-api/hindsight_api/migrations.py
|
||||
# Default location is: hindsight-api/alembic
|
||||
package_root = Path(__file__).parent.parent
|
||||
script_location = str(package_root / "alembic")
|
||||
|
||||
@@ -86,6 +86,9 @@ def run_migrations(database_url: str, script_location: Optional[str] = None) ->
|
||||
# Uses Python's logging system instead of alembic.ini
|
||||
alembic_cfg.set_main_option("prepend_sys_path", ".")
|
||||
|
||||
# Set path_separator to avoid deprecation warning
|
||||
alembic_cfg.set_main_option("path_separator", "os")
|
||||
|
||||
# Run migrations to head (latest version)
|
||||
# Note: Alembic may call sys.exit() on errors instead of raising exceptions
|
||||
# We rely on the outer try/except and logging to catch issues
|
||||
@@ -110,7 +113,7 @@ def check_migration_status(database_url: Optional[str] = None, script_location:
|
||||
Check current database schema version and latest available version.
|
||||
|
||||
Args:
|
||||
database_url: SQLAlchemy database URL. If None, uses MEMORA_API_DATABASE_URL env var.
|
||||
database_url: SQLAlchemy database URL. If None, uses HINDSIGHT_API_DATABASE_URL env var.
|
||||
script_location: Path to alembic migrations directory. If None, uses default location.
|
||||
|
||||
Returns:
|
||||
@@ -124,9 +127,9 @@ def check_migration_status(database_url: Optional[str] = None, script_location:
|
||||
|
||||
# Get database URL
|
||||
if database_url is None:
|
||||
database_url = os.getenv("MEMORA_API_DATABASE_URL")
|
||||
database_url = os.getenv("HINDSIGHT_API_DATABASE_URL")
|
||||
if not database_url:
|
||||
logger.warning("Database URL not provided and MEMORA_API_DATABASE_URL not set, cannot check migration status")
|
||||
logger.warning("Database URL not provided and HINDSIGHT_API_DATABASE_URL not set, cannot check migration status")
|
||||
return None, None
|
||||
|
||||
# Get current revision from database
|
||||
@@ -148,6 +151,7 @@ def check_migration_status(database_url: Optional[str] = None, script_location:
|
||||
# Create config programmatically
|
||||
alembic_cfg = Config()
|
||||
alembic_cfg.set_main_option("script_location", script_location)
|
||||
alembic_cfg.set_main_option("path_separator", "os")
|
||||
|
||||
script = ScriptDirectory.from_config(alembic_cfg)
|
||||
head_rev = script.get_current_head()
|
||||
@@ -0,0 +1,416 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_DATA_DIR = Path(os.environ.get("HINDSIGHT_API_PG0_DATA_DIR", Path.home() / ".hindsight" / "pg_data"))
|
||||
DEFAULT_INSTALL_DIR = Path.home() / ".hindsight" / "bin"
|
||||
BINARY_NAME = "pg0"
|
||||
DEFAULT_PORT = 5555
|
||||
DEFAULT_USERNAME = "hindsight"
|
||||
DEFAULT_PASSWORD = "hindsight"
|
||||
DEFAULT_DATABASE = "hindsight"
|
||||
|
||||
|
||||
def get_platform_binary_name() -> str:
|
||||
"""Get the appropriate binary name for the current platform.
|
||||
|
||||
Supported platforms:
|
||||
- macOS ARM64 (darwin-aarch64)
|
||||
- Linux x86_64
|
||||
- Windows x86_64
|
||||
"""
|
||||
system = platform.system().lower()
|
||||
machine = platform.machine().lower()
|
||||
|
||||
# Normalize architecture names
|
||||
if machine in ("x86_64", "amd64"):
|
||||
arch = "x86_64"
|
||||
elif machine in ("arm64", "aarch64"):
|
||||
arch = "aarch64"
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Embedded PostgreSQL is not supported on architecture: {machine}. "
|
||||
f"Supported architectures: x86_64/amd64 (Linux, Windows), aarch64/arm64 (macOS)"
|
||||
)
|
||||
|
||||
if system == "darwin" and arch == "aarch64":
|
||||
return "pg0-darwin-aarch64"
|
||||
elif system == "linux" and arch == "x86_64":
|
||||
return "pg0-linux-x86_64"
|
||||
elif system == "windows" and arch == "x86_64":
|
||||
return "pg0-windows-x86_64.exe"
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Embedded PostgreSQL is not supported on {system}-{arch}. "
|
||||
f"Supported platforms: darwin-aarch64 (macOS ARM), linux-x86_64, windows-x86_64"
|
||||
)
|
||||
|
||||
|
||||
def get_download_url(
|
||||
version: str = "latest",
|
||||
repo: str = "vectorize-io/pg0",
|
||||
) -> str:
|
||||
"""
|
||||
"""
|
||||
# Check for direct URL override
|
||||
binary_name = get_platform_binary_name()
|
||||
|
||||
if version == "latest":
|
||||
return f"https://github.com/{repo}/releases/latest/download/{binary_name}"
|
||||
else:
|
||||
return f"https://github.com/{repo}/releases/download/{version}/{binary_name}"
|
||||
|
||||
|
||||
class EmbeddedPostgres:
|
||||
"""
|
||||
Manages an embedded PostgreSQL server instance.
|
||||
|
||||
This class handles:
|
||||
- Downloading and installing the embedded-postgres CLI
|
||||
- Starting/stopping the PostgreSQL server
|
||||
- Getting the connection URI
|
||||
|
||||
Example:
|
||||
pg = EmbeddedPostgres(data_dir="~/.myapp/data")
|
||||
await pg.ensure_installed()
|
||||
await pg.start()
|
||||
uri = await pg.get_uri()
|
||||
# ... use uri with asyncpg ...
|
||||
await pg.stop()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
data_dir: Optional[Path] = None,
|
||||
install_dir: Optional[Path] = None,
|
||||
version: str = "latest",
|
||||
port: int = DEFAULT_PORT,
|
||||
username: str = DEFAULT_USERNAME,
|
||||
password: str = DEFAULT_PASSWORD,
|
||||
database: str = DEFAULT_DATABASE,
|
||||
):
|
||||
"""
|
||||
Initialize the embedded PostgreSQL manager.
|
||||
|
||||
Args:
|
||||
data_dir: Directory to store PostgreSQL data. Defaults to ~/.hindsight/pg_data
|
||||
install_dir: Directory to install the CLI binary. Defaults to ~/.hindsight/bin
|
||||
version: Version of embedded-postgres to use. Defaults to "latest"
|
||||
port: Port to listen on. Defaults to 5555
|
||||
username: Username for the database. Defaults to "hindsight"
|
||||
password: Password for the database. Defaults to "hindsight"
|
||||
database: Database name to create. Defaults to "hindsight"
|
||||
"""
|
||||
self.data_dir = Path(data_dir or DEFAULT_DATA_DIR).expanduser()
|
||||
self.install_dir = Path(install_dir or DEFAULT_INSTALL_DIR).expanduser()
|
||||
self.version = version
|
||||
self.port = port
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.database = database
|
||||
|
||||
# Binary path
|
||||
binary_name = "pg0.exe" if platform.system() == "Windows" else "pg0"
|
||||
self.binary_path = self.install_dir / binary_name
|
||||
|
||||
self._process: Optional[subprocess.Popen] = None
|
||||
|
||||
def is_installed(self) -> bool:
|
||||
"""Check if the embedded-postgres CLI is installed."""
|
||||
return self.binary_path.exists() and os.access(self.binary_path, os.X_OK)
|
||||
|
||||
async def ensure_installed(self) -> None:
|
||||
"""
|
||||
Ensure the embedded-postgres CLI is installed.
|
||||
|
||||
Downloads and installs the binary if not already present.
|
||||
"""
|
||||
if self.is_installed():
|
||||
logger.debug(f"pg0 already installed at {self.binary_path}")
|
||||
return
|
||||
|
||||
logger.info("Installing pg0 CLI...")
|
||||
|
||||
# Create install directory
|
||||
self.install_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Download the binary
|
||||
download_url = get_download_url(self.version)
|
||||
logger.info(f"Downloading from {download_url}")
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(follow_redirects=True, timeout=300.0) as client:
|
||||
response = await client.get(download_url)
|
||||
response.raise_for_status()
|
||||
|
||||
# Write binary to disk
|
||||
with open(self.binary_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
|
||||
# Make executable on Unix
|
||||
if platform.system() != "Windows":
|
||||
st = os.stat(self.binary_path)
|
||||
os.chmod(self.binary_path, st.st_mode | stat.S_IEXEC)
|
||||
|
||||
logger.info(f"Installed pg0 to {self.binary_path}")
|
||||
|
||||
except httpx.HTTPError as e:
|
||||
raise RuntimeError(f"Failed to download pg0: {e}") from e
|
||||
|
||||
def _run_command(self, *args: str, capture_output: bool = True) -> subprocess.CompletedProcess:
|
||||
"""Run an embedded-postgres command synchronously."""
|
||||
cmd = [str(self.binary_path), *args]
|
||||
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
capture_output=capture_output,
|
||||
text=True,
|
||||
)
|
||||
|
||||
async def _run_command_async(self, *args: str) -> tuple[int, str, str]:
|
||||
"""Run an embedded-postgres command asynchronously."""
|
||||
cmd = [str(self.binary_path), *args]
|
||||
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
|
||||
stdout, stderr = await process.communicate()
|
||||
return process.returncode, stdout.decode(), stderr.decode()
|
||||
|
||||
async def start(self) -> str:
|
||||
"""
|
||||
Start the PostgreSQL server.
|
||||
|
||||
Returns:
|
||||
The connection URI for the started server.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the server fails to start.
|
||||
"""
|
||||
if not self.is_installed():
|
||||
raise RuntimeError("pg0 is not installed. Call ensure_installed() first.")
|
||||
|
||||
# Create data directory
|
||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(f"Starting embedded PostgreSQL (data: {self.data_dir}, port: {self.port})...")
|
||||
|
||||
returncode, stdout, stderr = await self._run_command_async(
|
||||
"start",
|
||||
"--port", str(self.port),
|
||||
"--username", self.username,
|
||||
"--password", self.password,
|
||||
"--database", self.database,
|
||||
"--data-dir", self.data_dir.as_posix()
|
||||
)
|
||||
|
||||
if returncode != 0:
|
||||
raise RuntimeError(f"Failed to start PostgreSQL: {stderr}")
|
||||
|
||||
logger.info("Embedded PostgreSQL started")
|
||||
|
||||
# Get and return the URI
|
||||
return await self.get_uri()
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""
|
||||
Stop the PostgreSQL server.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the server fails to stop.
|
||||
"""
|
||||
if not self.is_installed():
|
||||
return
|
||||
|
||||
logger.info("Stopping embedded PostgreSQL...")
|
||||
|
||||
returncode, stdout, stderr = await self._run_command_async("stop")
|
||||
|
||||
if returncode != 0:
|
||||
# Don't raise if server wasn't running
|
||||
if "not running" in stderr.lower():
|
||||
logger.debug("PostgreSQL was not running")
|
||||
return
|
||||
raise RuntimeError(f"Failed to stop PostgreSQL: {stderr}")
|
||||
|
||||
logger.info("Embedded PostgreSQL stopped")
|
||||
|
||||
async def _get_info(self) -> dict:
|
||||
"""
|
||||
Get info from pg0 using the `info -o json` command.
|
||||
|
||||
Returns:
|
||||
Dictionary with 'running' (bool) and 'uri' (str) keys.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If unable to get info.
|
||||
"""
|
||||
if not self.is_installed():
|
||||
raise RuntimeError("pg0 is not installed.")
|
||||
|
||||
returncode, stdout, stderr = await self._run_command_async(
|
||||
"info", "-o", "json")
|
||||
|
||||
if returncode != 0:
|
||||
raise RuntimeError(f"Failed to get PostgreSQL info: {stderr}")
|
||||
|
||||
try:
|
||||
return json.loads(stdout.strip())
|
||||
except json.JSONDecodeError as e:
|
||||
raise RuntimeError(f"Failed to parse pg0 info output: {e}")
|
||||
|
||||
async def get_uri(self) -> str:
|
||||
"""
|
||||
Get the connection URI for the PostgreSQL server.
|
||||
|
||||
Returns:
|
||||
PostgreSQL connection URI (e.g., postgresql://user:pass@localhost:5432/db)
|
||||
|
||||
Raises:
|
||||
RuntimeError: If unable to get the URI or server is not running.
|
||||
"""
|
||||
info = await self._get_info()
|
||||
uri = info.get("uri")
|
||||
if not uri:
|
||||
raise RuntimeError("PostgreSQL server is not running or URI not available")
|
||||
return uri
|
||||
|
||||
async def status(self) -> dict:
|
||||
"""
|
||||
Get the status of the PostgreSQL server.
|
||||
|
||||
Returns:
|
||||
Dictionary with status information including 'running' boolean and 'uri'.
|
||||
"""
|
||||
if not self.is_installed():
|
||||
return {"installed": False, "running": False}
|
||||
|
||||
try:
|
||||
info = await self._get_info()
|
||||
return {
|
||||
"installed": True,
|
||||
"running": info.get("running", False),
|
||||
"uri": info.get("uri"),
|
||||
"data_dir": str(self.data_dir),
|
||||
"binary_path": str(self.binary_path),
|
||||
}
|
||||
except RuntimeError:
|
||||
return {
|
||||
"installed": True,
|
||||
"running": False,
|
||||
"data_dir": str(self.data_dir),
|
||||
"binary_path": str(self.binary_path),
|
||||
}
|
||||
|
||||
async def is_running(self) -> bool:
|
||||
"""Check if the PostgreSQL server is currently running."""
|
||||
if not self.is_installed():
|
||||
return False
|
||||
try:
|
||||
info = await self._get_info()
|
||||
return info.get("running", False)
|
||||
except RuntimeError:
|
||||
return False
|
||||
|
||||
async def ensure_running(self) -> str:
|
||||
"""
|
||||
Ensure the PostgreSQL server is running.
|
||||
|
||||
Installs if needed, starts if not running.
|
||||
|
||||
Returns:
|
||||
The connection URI.
|
||||
"""
|
||||
await self.ensure_installed()
|
||||
|
||||
if await self.is_running():
|
||||
return await self.get_uri()
|
||||
|
||||
return await self.start()
|
||||
|
||||
def uninstall(self) -> None:
|
||||
"""Remove the embedded-postgres binary."""
|
||||
if self.binary_path.exists():
|
||||
self.binary_path.unlink()
|
||||
logger.info(f"Removed {self.binary_path}")
|
||||
|
||||
def clear_data(self) -> None:
|
||||
"""Remove all PostgreSQL data (destructive!)."""
|
||||
if self.data_dir.exists():
|
||||
shutil.rmtree(self.data_dir)
|
||||
logger.info(f"Removed data directory {self.data_dir}")
|
||||
|
||||
|
||||
# Convenience functions for simple usage
|
||||
|
||||
_default_instance: Optional[EmbeddedPostgres] = None
|
||||
|
||||
|
||||
def get_embedded_postgres(
|
||||
data_dir: Optional[Path] = None,
|
||||
install_dir: Optional[Path] = None,
|
||||
) -> EmbeddedPostgres:
|
||||
"""
|
||||
Get or create the default EmbeddedPostgres instance.
|
||||
|
||||
Args:
|
||||
data_dir: Override default data directory
|
||||
install_dir: Override default install directory
|
||||
|
||||
Returns:
|
||||
EmbeddedPostgres instance
|
||||
"""
|
||||
global _default_instance
|
||||
|
||||
if _default_instance is None or data_dir or install_dir:
|
||||
_default_instance = EmbeddedPostgres(
|
||||
data_dir=data_dir,
|
||||
install_dir=install_dir,
|
||||
)
|
||||
|
||||
return _default_instance
|
||||
|
||||
|
||||
async def start_embedded_postgres(
|
||||
data_dir: Optional[Path] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Quick start function for embedded PostgreSQL.
|
||||
|
||||
Downloads, installs, and starts PostgreSQL in one call.
|
||||
|
||||
Args:
|
||||
data_dir: Directory to store PostgreSQL data
|
||||
|
||||
Returns:
|
||||
Connection URI string
|
||||
|
||||
Example:
|
||||
db_url = await start_embedded_postgres()
|
||||
conn = await asyncpg.connect(db_url)
|
||||
"""
|
||||
pg = get_embedded_postgres(data_dir=data_dir)
|
||||
return await pg.ensure_running()
|
||||
|
||||
|
||||
async def stop_embedded_postgres() -> None:
|
||||
"""Stop the default embedded PostgreSQL instance."""
|
||||
global _default_instance
|
||||
|
||||
if _default_instance:
|
||||
await _default_instance.stop()
|
||||
@@ -3,10 +3,10 @@ Web interface for memory system.
|
||||
|
||||
Provides FastAPI app and visualization interface.
|
||||
"""
|
||||
from memora.api import create_app
|
||||
from hindsight_api.api import create_app
|
||||
|
||||
# Note: Don't import app from .server here to avoid circular import warnings
|
||||
# when running with `python -m memora.web.server`
|
||||
# If you need the app, import it directly: from memora.web.server import app
|
||||
# when running with `python -m hindsight_api.web.server`
|
||||
# If you need the app, import it directly: from hindsight_api.web.server import app
|
||||
|
||||
__all__ = ["create_app"]
|
||||
@@ -4,27 +4,68 @@ FastAPI server for memory graph visualization and API.
|
||||
Provides REST API endpoints for memory operations and serves
|
||||
the interactive visualization interface.
|
||||
"""
|
||||
import warnings
|
||||
|
||||
# Filter deprecation warnings from third-party libraries
|
||||
warnings.filterwarnings("ignore", message="websockets.legacy is deprecated")
|
||||
warnings.filterwarnings("ignore", message="websockets.server.WebSocketServerProtocol is deprecated")
|
||||
|
||||
import asyncio
|
||||
import atexit
|
||||
import logging
|
||||
import os
|
||||
import argparse
|
||||
import signal
|
||||
import sys
|
||||
|
||||
from memora import TemporalSemanticMemory
|
||||
from memora.api import create_app
|
||||
from hindsight_api import MemoryEngine
|
||||
from hindsight_api.api import create_app
|
||||
|
||||
# Disable tokenizers parallelism to avoid warnings
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||
|
||||
|
||||
def _cleanup_pg0():
|
||||
"""Synchronous cleanup function to stop pg0 on exit."""
|
||||
global _memory
|
||||
if _memory is not None and _memory._pg0 is not None:
|
||||
try:
|
||||
# Run async stop in a new event loop
|
||||
loop = asyncio.new_event_loop()
|
||||
loop.run_until_complete(_memory._pg0.stop())
|
||||
loop.close()
|
||||
print("\npg0 stopped.")
|
||||
except Exception as e:
|
||||
print(f"\nError stopping pg0: {e}")
|
||||
|
||||
|
||||
# Register cleanup on normal exit
|
||||
atexit.register(_cleanup_pg0)
|
||||
|
||||
|
||||
def _signal_handler(signum, frame):
|
||||
"""Handle SIGINT/SIGTERM to ensure pg0 cleanup."""
|
||||
print(f"\nReceived signal {signum}, shutting down...")
|
||||
_cleanup_pg0()
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
# Register signal handlers for graceful shutdown
|
||||
signal.signal(signal.SIGINT, _signal_handler)
|
||||
signal.signal(signal.SIGTERM, _signal_handler)
|
||||
|
||||
|
||||
# Create app at module level (required for uvicorn import string)
|
||||
_memory = TemporalSemanticMemory(
|
||||
db_url=os.getenv("MEMORA_API_DATABASE_URL"),
|
||||
memory_llm_provider=os.getenv("MEMORA_API_LLM_PROVIDER", "groq"),
|
||||
memory_llm_api_key=os.getenv("MEMORA_API_LLM_API_KEY"),
|
||||
memory_llm_model=os.getenv("MEMORA_API_LLM_MODEL", "openai/gpt-oss-120b"),
|
||||
memory_llm_base_url=os.getenv("MEMORA_API_LLM_BASE_URL") or None,
|
||||
_memory = MemoryEngine(
|
||||
db_url=os.getenv("HINDSIGHT_API_DATABASE_URL", "pg0"),
|
||||
memory_llm_provider=os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq"),
|
||||
memory_llm_api_key=os.getenv("HINDSIGHT_API_LLM_API_KEY"),
|
||||
memory_llm_model=os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b"),
|
||||
memory_llm_base_url=os.getenv("HINDSIGHT_API_LLM_BASE_URL") or None,
|
||||
)
|
||||
|
||||
# Check if MCP should be enabled
|
||||
mcp_enabled = os.getenv("MEMORA_API_MCP_ENABLED", "true").lower() == "true"
|
||||
mcp_enabled = os.getenv("HINDSIGHT_API_MCP_ENABLED", "true").lower() == "true"
|
||||
|
||||
# Create unified app with both HTTP and optionally MCP
|
||||
app = create_app(
|
||||
@@ -43,7 +84,7 @@ if __name__ == "__main__":
|
||||
# Parse CLI arguments
|
||||
parser = argparse.ArgumentParser(description="Memory Graph API Server")
|
||||
parser.add_argument("--host", default="0.0.0.0", help="Host to bind to (default: 0.0.0.0)")
|
||||
parser.add_argument("--port", type=int, default=8080, help="Port to bind to (default: 8080)")
|
||||
parser.add_argument("--port", type=int, default=8888, help="Port to bind to (default: 8888)")
|
||||
parser.add_argument("--reload", action="store_true", help="Enable auto-reload on code changes")
|
||||
parser.add_argument("--workers", type=int, default=1, help="Number of worker processes (default: 1)")
|
||||
parser.add_argument("--log-level", default="info", choices=["critical", "error", "warning", "info", "debug", "trace"],
|
||||
@@ -58,7 +99,7 @@ if __name__ == "__main__":
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
app_ref = "memora.web.server:app"
|
||||
app_ref = "hindsight_api.web.server:app"
|
||||
|
||||
# Prepare uvicorn config
|
||||
uvicorn_config = {
|
||||
@@ -3,7 +3,7 @@ requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "memora"
|
||||
name = "hindsight-api"
|
||||
version = "0.0.7"
|
||||
description = "Temporal + Semantic + Entity Memory System for AI agents using PostgreSQL"
|
||||
readme = "README.md"
|
||||
@@ -36,15 +36,24 @@ test = [
|
||||
"pytest>=7.0.0",
|
||||
"pytest-asyncio>=0.21.0",
|
||||
"pytest-timeout>=2.4.0",
|
||||
"pytest-xdist>=3.0.0",
|
||||
"filelock>=3.0.0",
|
||||
"testcontainers[postgres]>=4.0.0",
|
||||
]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["memora"]
|
||||
packages = ["hindsight_api"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
log_cli = true
|
||||
log_cli_level = "INFO"
|
||||
log_cli_format = "%(asctime)s %(levelname)s %(message)s"
|
||||
log_cli_date_format = "%Y-%m-%d %H:%M:%S"
|
||||
addopts = "--timeout 60 -p no:warnings"
|
||||
addopts = "--timeout 60 -n auto --durations=10 -v"
|
||||
asyncio_mode = "auto"
|
||||
asyncio_default_fixture_loop_scope = "function"
|
||||
log_auto_indent = true
|
||||
filterwarnings = [
|
||||
"ignore:The @wait_container_is_ready decorator is deprecated:DeprecationWarning",
|
||||
"ignore::RuntimeWarning:asyncio",
|
||||
]
|
||||
@@ -0,0 +1,130 @@
|
||||
"""
|
||||
Pytest configuration and shared fixtures.
|
||||
"""
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import os
|
||||
import filelock
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
from hindsight_api import MemoryEngine, LLMConfig, SentenceTransformersEmbeddings
|
||||
import asyncpg
|
||||
from testcontainers.postgres import PostgresContainer
|
||||
|
||||
from hindsight_api.engine.cross_encoder import SentenceTransformersCrossEncoder
|
||||
from hindsight_api.engine.query_analyzer import TransformerQueryAnalyzer
|
||||
|
||||
|
||||
# Load environment variables from .env at the start of test session
|
||||
def pytest_configure(config):
|
||||
"""Load environment variables before running tests."""
|
||||
# Look for .env in the workspace root (two levels up from tests dir)
|
||||
env_file = Path(__file__).parent.parent.parent / ".env"
|
||||
if env_file.exists():
|
||||
load_dotenv(env_file)
|
||||
else:
|
||||
print(f"Warning: {env_file} not found, tests may fail without proper configuration")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def postgres_container(tmp_path_factory, worker_id):
|
||||
"""
|
||||
Start a postgres container shared across all test workers.
|
||||
Uses filelock to ensure only one worker starts the container.
|
||||
|
||||
- worker_id == "master": running without -n (single process)
|
||||
- worker_id == "gw0", "gw1", etc.: running with -n (parallel workers)
|
||||
"""
|
||||
# Get shared temp dir (same for all workers)
|
||||
if worker_id == "master":
|
||||
root_tmp_dir = tmp_path_factory.getbasetemp()
|
||||
else:
|
||||
root_tmp_dir = tmp_path_factory.getbasetemp().parent
|
||||
|
||||
db_url_file = root_tmp_dir / "postgres_url"
|
||||
lock_file = root_tmp_dir / "postgres.lock"
|
||||
container = None
|
||||
|
||||
with filelock.FileLock(str(lock_file)):
|
||||
if db_url_file.exists():
|
||||
# Another worker already started the container
|
||||
db_url = db_url_file.read_text()
|
||||
else:
|
||||
# First worker - start the container
|
||||
container = PostgresContainer("pgvector/pgvector:pg16")
|
||||
container.start()
|
||||
db_url = container.get_connection_url().replace("postgresql+psycopg2://", "postgresql://")
|
||||
db_url_file.write_text(db_url)
|
||||
|
||||
# Run migrations
|
||||
from hindsight_api.migrations import run_migrations
|
||||
run_migrations(db_url)
|
||||
|
||||
os.environ["HINDSIGHT_API_DATABASE_URL"] = db_url
|
||||
yield db_url
|
||||
|
||||
# Only the worker that started the container stops it
|
||||
if container is not None:
|
||||
container.stop()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def llm_config():
|
||||
"""
|
||||
Provide LLM configuration for tests.
|
||||
This can be used by tests that need to call LLM directly without memory system.
|
||||
"""
|
||||
return LLMConfig.for_memory()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def embeddings():
|
||||
|
||||
return SentenceTransformersEmbeddings("BAAI/bge-small-en-v1.5")
|
||||
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def cross_encoder():
|
||||
|
||||
return SentenceTransformersCrossEncoder()
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def query_analyzer():
|
||||
return TransformerQueryAnalyzer()
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope="function")
|
||||
async def memory(postgres_container, embeddings, cross_encoder, query_analyzer):
|
||||
"""
|
||||
Provide a MemoryEngine instance for each test.
|
||||
|
||||
Must be function-scoped because:
|
||||
1. pytest-xdist runs tests in separate processes with different event loops
|
||||
2. asyncpg pools are bound to the event loop that created them
|
||||
3. Each test needs its own pool in its own event loop
|
||||
|
||||
Uses small pool sizes since tests run in parallel and share a single
|
||||
testcontainer PostgreSQL instance with limited resources.
|
||||
"""
|
||||
mem = MemoryEngine(
|
||||
db_url=postgres_container,
|
||||
memory_llm_provider=os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq"),
|
||||
memory_llm_api_key=os.getenv("HINDSIGHT_API_LLM_API_KEY"),
|
||||
memory_llm_model=os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b"),
|
||||
memory_llm_base_url=os.getenv("HINDSIGHT_API_LLM_BASE_URL") or None,
|
||||
embeddings=embeddings,
|
||||
cross_encoder=cross_encoder,
|
||||
query_analyzer=query_analyzer,
|
||||
pool_min_size=1,
|
||||
pool_max_size=5,
|
||||
)
|
||||
await mem.initialize()
|
||||
yield mem
|
||||
try:
|
||||
if mem._pool and not mem._pool._closing:
|
||||
await mem.close()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,254 @@
|
||||
"""
|
||||
Tests for agent management API (profile, personality, background).
|
||||
"""
|
||||
import pytest
|
||||
import uuid
|
||||
from hindsight_api import MemoryEngine
|
||||
from hindsight_api.api import CreateAgentRequest, PersonalityTraits
|
||||
|
||||
|
||||
def unique_agent_id(prefix: str) -> str:
|
||||
"""Generate a unique agent ID for testing."""
|
||||
return f"{prefix}_{uuid.uuid4().hex[:8]}"
|
||||
|
||||
|
||||
class TestAgentProfile:
|
||||
"""Tests for agent profile management."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_agent_profile_creates_default(self, memory: MemoryEngine):
|
||||
"""Test that getting a profile for a new agent creates default personality."""
|
||||
agent_id = unique_agent_id("test_profile_default")
|
||||
|
||||
profile = await memory.get_agent_profile(agent_id)
|
||||
|
||||
assert profile is not None
|
||||
assert "personality" in profile
|
||||
assert "background" in profile
|
||||
|
||||
personality = profile["personality"]
|
||||
assert personality["openness"] == 0.5
|
||||
assert personality["conscientiousness"] == 0.5
|
||||
assert personality["extraversion"] == 0.5
|
||||
assert personality["agreeableness"] == 0.5
|
||||
assert personality["neuroticism"] == 0.5
|
||||
assert personality["bias_strength"] == 0.5
|
||||
|
||||
assert profile["background"] == ""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_agent_personality(self, memory: MemoryEngine):
|
||||
"""Test updating agent personality traits."""
|
||||
agent_id = unique_agent_id("test_profile_update")
|
||||
|
||||
profile = await memory.get_agent_profile(agent_id)
|
||||
assert profile["personality"]["openness"] == 0.5
|
||||
|
||||
new_personality = {
|
||||
"openness": 0.8,
|
||||
"conscientiousness": 0.6,
|
||||
"extraversion": 0.7,
|
||||
"agreeableness": 0.4,
|
||||
"neuroticism": 0.3,
|
||||
"bias_strength": 0.9,
|
||||
}
|
||||
await memory.update_agent_personality(agent_id, new_personality)
|
||||
|
||||
updated_profile = await memory.get_agent_profile(agent_id)
|
||||
for key in new_personality:
|
||||
assert abs(updated_profile["personality"][key] - new_personality[key]) < 0.001
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_agents(self, memory: MemoryEngine):
|
||||
"""Test listing all agents."""
|
||||
agent_id_1 = unique_agent_id("test_list")
|
||||
agent_id_2 = unique_agent_id("test_list")
|
||||
agent_id_3 = unique_agent_id("test_list")
|
||||
|
||||
await memory.get_agent_profile(agent_id_1)
|
||||
await memory.get_agent_profile(agent_id_2)
|
||||
await memory.get_agent_profile(agent_id_3)
|
||||
|
||||
agents = await memory.list_agents()
|
||||
|
||||
agent_ids = [a["agent_id"] for a in agents]
|
||||
assert agent_id_1 in agent_ids
|
||||
assert agent_id_2 in agent_ids
|
||||
assert agent_id_3 in agent_ids
|
||||
|
||||
for agent in agents:
|
||||
assert "agent_id" in agent
|
||||
assert "personality" in agent
|
||||
assert "background" in agent
|
||||
assert "created_at" in agent
|
||||
assert "updated_at" in agent
|
||||
|
||||
|
||||
class TestAgentBackground:
|
||||
"""Tests for agent background management."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_merge_agent_background(self, memory: MemoryEngine):
|
||||
"""Test merging agent background information."""
|
||||
agent_id = unique_agent_id("test_profile_merge")
|
||||
|
||||
profile = await memory.get_agent_profile(agent_id)
|
||||
assert profile["background"] == ""
|
||||
|
||||
result1 = await memory.merge_agent_background(
|
||||
agent_id,
|
||||
"I was born in Texas",
|
||||
update_personality=False
|
||||
)
|
||||
assert "Texas" in result1["background"]
|
||||
|
||||
result2 = await memory.merge_agent_background(
|
||||
agent_id,
|
||||
"I have 10 years of startup experience",
|
||||
update_personality=False
|
||||
)
|
||||
assert "Texas" in result2["background"] or "startup" in result2["background"]
|
||||
|
||||
final_profile = await memory.get_agent_profile(agent_id)
|
||||
assert final_profile["background"] != ""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_merge_background_handles_conflicts(self, memory: MemoryEngine):
|
||||
"""Test that merging background handles conflicts (new overwrites old)."""
|
||||
agent_id = unique_agent_id("test_profile_conflict")
|
||||
|
||||
result1 = await memory.merge_agent_background(
|
||||
agent_id,
|
||||
"I was born in Colorado",
|
||||
update_personality=False
|
||||
)
|
||||
assert "Colorado" in result1["background"]
|
||||
|
||||
result2 = await memory.merge_agent_background(
|
||||
agent_id,
|
||||
"You were born in Texas",
|
||||
update_personality=False
|
||||
)
|
||||
assert "Texas" in result2["background"]
|
||||
|
||||
|
||||
class TestAgentEndpoint:
|
||||
"""Tests for agent PUT endpoint logic."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_agent_create(self, memory: MemoryEngine):
|
||||
"""Test creating an agent via PUT endpoint."""
|
||||
agent_id = unique_agent_id("test_put_create")
|
||||
|
||||
request = CreateAgentRequest(
|
||||
personality=PersonalityTraits(
|
||||
openness=0.8,
|
||||
conscientiousness=0.6,
|
||||
extraversion=0.5,
|
||||
agreeableness=0.7,
|
||||
neuroticism=0.3,
|
||||
bias_strength=0.7
|
||||
),
|
||||
background="I am a creative software engineer"
|
||||
)
|
||||
|
||||
profile = await memory.get_agent_profile(agent_id)
|
||||
|
||||
if request.personality is not None:
|
||||
await memory.update_agent_personality(
|
||||
agent_id,
|
||||
request.personality.model_dump()
|
||||
)
|
||||
|
||||
if request.background is not None:
|
||||
pool = await memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE agents
|
||||
SET background = $2,
|
||||
updated_at = NOW()
|
||||
WHERE agent_id = $1
|
||||
""",
|
||||
agent_id,
|
||||
request.background
|
||||
)
|
||||
|
||||
final_profile = await memory.get_agent_profile(agent_id)
|
||||
|
||||
assert final_profile["personality"]["openness"] == 0.8
|
||||
assert final_profile["personality"]["bias_strength"] == 0.7
|
||||
assert final_profile["background"] == "I am a creative software engineer"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_agent_partial_update(self, memory: MemoryEngine):
|
||||
"""Test updating only background."""
|
||||
agent_id = unique_agent_id("test_put_partial")
|
||||
|
||||
request = CreateAgentRequest(
|
||||
background="I am a data scientist"
|
||||
)
|
||||
|
||||
profile = await memory.get_agent_profile(agent_id)
|
||||
|
||||
if request.background is not None:
|
||||
pool = await memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE agents
|
||||
SET background = $2,
|
||||
updated_at = NOW()
|
||||
WHERE agent_id = $1
|
||||
""",
|
||||
agent_id,
|
||||
request.background
|
||||
)
|
||||
|
||||
final_profile = await memory.get_agent_profile(agent_id)
|
||||
|
||||
assert final_profile["personality"]["openness"] == 0.5
|
||||
assert final_profile["background"] == "I am a data scientist"
|
||||
|
||||
|
||||
class TestAgentPersonalityIntegration:
|
||||
"""Tests for personality integration with other features."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_think_uses_personality(self, memory: MemoryEngine):
|
||||
"""Test that THINK operation uses agent personality."""
|
||||
agent_id = unique_agent_id("test_think")
|
||||
|
||||
personality = {
|
||||
"openness": 0.9,
|
||||
"conscientiousness": 0.2,
|
||||
"extraversion": 0.8,
|
||||
"agreeableness": 0.1,
|
||||
"neuroticism": 0.7,
|
||||
"bias_strength": 0.9,
|
||||
}
|
||||
await memory.update_agent_personality(agent_id, personality)
|
||||
|
||||
await memory.merge_agent_background(
|
||||
agent_id,
|
||||
"I am a creative artist who values innovation over tradition",
|
||||
update_personality=False
|
||||
)
|
||||
|
||||
await memory.put_batch_async(
|
||||
agent_id=agent_id,
|
||||
contents=[
|
||||
{"content": "Traditional painting techniques have been used for centuries"},
|
||||
{"content": "Modern digital art is changing the art world"}
|
||||
],
|
||||
document_id="art_facts"
|
||||
)
|
||||
|
||||
result = await memory.think_async(
|
||||
agent_id=agent_id,
|
||||
query="What do you think about traditional vs modern art?",
|
||||
thinking_budget=50
|
||||
)
|
||||
|
||||
assert result.text is not None
|
||||
assert len(result.text) > 0
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Test automatic batch chunking based on character count."""
|
||||
import asyncio
|
||||
import pytest
|
||||
from hindsight_api import MemoryEngine
|
||||
import os
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_large_batch_auto_chunks(memory):
|
||||
agent_id = "test_chunking_agent"
|
||||
# Create a large batch that should trigger chunking
|
||||
# Each item is ~2000 chars, so 30 items = 60k chars (exceeds 50k threshold)
|
||||
large_content = "Alice met with Bob at the coffee shop. " * 50 # ~2000 chars
|
||||
contents = [
|
||||
{"content": large_content, "context": f"conversation_{i}"}
|
||||
for i in range(30)
|
||||
]
|
||||
|
||||
# Calculate total chars
|
||||
total_chars = sum(len(item["content"]) for item in contents)
|
||||
print(f"\nTotal characters: {total_chars:,}")
|
||||
print(f"Should trigger chunking: {total_chars > 50_000}")
|
||||
|
||||
# Ingest the large batch (should auto-chunk)
|
||||
result = await memory.put_batch_async(
|
||||
agent_id=agent_id,
|
||||
contents=contents
|
||||
)
|
||||
|
||||
# Verify we got results back
|
||||
assert len(result) == 30, f"Expected 30 results, got {len(result)}"
|
||||
print(f"Successfully ingested {len(result)} items (auto-chunked)")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_small_batch_no_chunking(memory):
|
||||
agent_id = "test_no_chunking_agent"
|
||||
|
||||
# Create a small batch that should NOT trigger chunking
|
||||
contents = [
|
||||
{"content": "Alice works at Google", "context": "conversation_1"},
|
||||
{"content": "Bob loves Python", "context": "conversation_2"}
|
||||
]
|
||||
|
||||
# Calculate total chars
|
||||
total_chars = sum(len(item["content"]) for item in contents)
|
||||
print(f"\nTotal characters: {total_chars:,}")
|
||||
print(f"Should NOT trigger chunking: {total_chars <= 50_000}")
|
||||
|
||||
# Ingest the small batch (should NOT auto-chunk)
|
||||
result = await memory.put_batch_async(
|
||||
agent_id=agent_id,
|
||||
contents=contents
|
||||
)
|
||||
|
||||
# Verify we got results back
|
||||
assert len(result) == 2, f"Expected 2 results, got {len(result)}"
|
||||
print(f"Successfully ingested {len(result)} items (no chunking)")
|
||||
@@ -2,7 +2,7 @@
|
||||
Test chunking functionality for large documents.
|
||||
"""
|
||||
import pytest
|
||||
from memora.fact_extraction import chunk_text
|
||||
from hindsight_api.engine.fact_extraction import chunk_text
|
||||
|
||||
|
||||
def test_chunk_text_small():
|
||||
@@ -58,6 +58,3 @@ def test_chunk_text_64k():
|
||||
combined_length = sum(len(chunk) for chunk in chunks)
|
||||
assert combined_length >= len(text) * 0.95, "Lost too much content during chunking"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7,31 +7,14 @@ distinguish between things said earlier vs later.
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, timezone
|
||||
from memora import TemporalSemanticMemory
|
||||
from hindsight_api import MemoryEngine
|
||||
import os
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fact_ordering_within_conversation():
|
||||
"""
|
||||
Test that facts extracted from one conversation get incremental time offsets
|
||||
to preserve their ordering for retrieval.
|
||||
"""
|
||||
|
||||
# Create memory instance
|
||||
memory = TemporalSemanticMemory(
|
||||
db_url=os.getenv("MEMORA_API_DATABASE_URL"),
|
||||
memory_llm_provider=os.getenv("MEMORA_API_LLM_PROVIDER", "groq"),
|
||||
memory_llm_api_key=os.getenv("MEMORA_API_LLM_API_KEY"),
|
||||
memory_llm_model=os.getenv("MEMORA_API_LLM_MODEL", "openai/gpt-oss-20b"),
|
||||
)
|
||||
await memory.initialize()
|
||||
|
||||
async def test_fact_ordering_within_conversation(memory):
|
||||
agent_id = "test_ordering_agent"
|
||||
|
||||
# Clear any existing data
|
||||
await memory.delete_agent(agent_id)
|
||||
|
||||
# Get/create agent (auto-creates with defaults)
|
||||
await memory.get_agent_profile(agent_id)
|
||||
|
||||
@@ -74,20 +57,20 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
|
||||
max_tokens=8192
|
||||
)
|
||||
|
||||
print(f"\n=== Retrieved {len(results['results'])} facts ===")
|
||||
for i, result in enumerate(results['results']):
|
||||
print(f"{i+1}. [{result['event_date']}] {result['text'][:100]}")
|
||||
print(f"\n=== Retrieved {len(results.results)} facts ===")
|
||||
for i, result in enumerate(results.results):
|
||||
print(f"{i+1}. [{result.event_date}] {result.text[:100]}")
|
||||
|
||||
# Get all agent facts (Marcus's statements)
|
||||
agent_facts = [r for r in results['results'] if r.get('fact_type') == 'agent']
|
||||
agent_facts = [r for r in results.results if r.fact_type == 'agent']
|
||||
|
||||
print(f"\n=== Agent facts (Marcus's statements) ===")
|
||||
for i, fact in enumerate(agent_facts):
|
||||
print(f"{i+1}. [{fact['event_date']}] {fact['text']}")
|
||||
print(f"{i+1}. [{fact.event_date}] {fact.text}")
|
||||
|
||||
# Check that agent facts have different timestamps
|
||||
if len(agent_facts) >= 2:
|
||||
timestamps = [datetime.fromisoformat(f['event_date'].replace('Z', '+00:00')) for f in agent_facts]
|
||||
timestamps = [datetime.fromisoformat(f.event_date.replace('Z', '+00:00')) for f in agent_facts]
|
||||
|
||||
# Verify timestamps are different (have time offsets)
|
||||
unique_timestamps = set(timestamps)
|
||||
@@ -111,7 +94,7 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
|
||||
|
||||
# Verify that retrieval returns facts in chronological order
|
||||
# The first prediction should come before the changed prediction
|
||||
agent_texts = [f['text'].lower() for f in agent_facts]
|
||||
agent_texts = [f.text.lower() for f in agent_facts]
|
||||
|
||||
# Look for evidence of the sequence
|
||||
has_first_prediction = any('27' in text and '24' in text for text in agent_texts)
|
||||
@@ -122,8 +105,8 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
|
||||
first_idx = next(i for i, text in enumerate(agent_texts) if '27' in text and '24' in text)
|
||||
changed_idx = next(i for i, text in enumerate(agent_texts) if 'chang' in text or 'by 3' in text or 'realized' in text)
|
||||
|
||||
print(f"\nFirst prediction at index {first_idx}: {agent_facts[first_idx]['text'][:100]}")
|
||||
print(f"Changed prediction at index {changed_idx}: {agent_facts[changed_idx]['text'][:100]}")
|
||||
print(f"\nFirst prediction at index {first_idx}: {agent_facts[first_idx].text[:100]}")
|
||||
print(f"Changed prediction at index {changed_idx}: {agent_facts[changed_idx].text[:100]}")
|
||||
|
||||
# The original prediction should come before the changed one
|
||||
assert timestamps[first_idx] < timestamps[changed_idx], \
|
||||
@@ -138,24 +121,10 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_documents_ordering():
|
||||
"""
|
||||
Test that facts from different documents get separate time offsets,
|
||||
so facts within each document maintain their order.
|
||||
"""
|
||||
|
||||
memory = TemporalSemanticMemory(
|
||||
db_url=os.getenv("MEMORA_API_DATABASE_URL"),
|
||||
memory_llm_provider=os.getenv("MEMORA_API_LLM_PROVIDER", "groq"),
|
||||
memory_llm_api_key=os.getenv("MEMORA_API_LLM_API_KEY"),
|
||||
memory_llm_model=os.getenv("MEMORA_API_LLM_MODEL", "openai/gpt-oss-20b"),
|
||||
)
|
||||
await memory.initialize()
|
||||
async def test_multiple_documents_ordering(memory):
|
||||
|
||||
agent_id = "test_multi_doc_agent"
|
||||
|
||||
# Clear and create agent
|
||||
await memory.delete_agent(agent_id)
|
||||
await memory.get_agent_profile(agent_id) # Auto-creates with defaults
|
||||
|
||||
# Two separate conversations with same base time
|
||||
@@ -191,15 +160,15 @@ Alice: I reconsidered the team's experience level.
|
||||
max_tokens=8192
|
||||
)
|
||||
|
||||
print(f"\n=== Retrieved {len(results['results'])} agent facts ===")
|
||||
agent_facts = [r for r in results['results'] if r.get('fact_type') == 'agent']
|
||||
print(f"\n=== Retrieved {len(results.results)} agent facts ===")
|
||||
agent_facts = [r for r in results.results if r.fact_type == 'agent']
|
||||
|
||||
for i, fact in enumerate(agent_facts):
|
||||
print(f"{i+1}. [{fact['event_date']}] {fact['text'][:80]}")
|
||||
print(f"{i+1}. [{fact.event_date}] {fact.text[:80]}")
|
||||
|
||||
# Each conversation's facts should have different timestamps
|
||||
if len(agent_facts) >= 2:
|
||||
timestamps = [datetime.fromisoformat(f['event_date'].replace('Z', '+00:00')) for f in agent_facts]
|
||||
timestamps = [datetime.fromisoformat(f.event_date.replace('Z', '+00:00')) for f in agent_facts]
|
||||
unique_timestamps = set(timestamps)
|
||||
|
||||
assert len(unique_timestamps) >= 2, \
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
Integration test for the complete Memora API.
|
||||
Integration test for the complete Hindsight API.
|
||||
|
||||
Tests all endpoints by starting a FastAPI server and making HTTP requests.
|
||||
"""
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
import pytest_asyncio
|
||||
import httpx
|
||||
from datetime import datetime
|
||||
from memora.api import create_app
|
||||
from hindsight_api.api import create_app
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
+39
-14
@@ -1,17 +1,42 @@
|
||||
"""Test MCP server with real server and client."""
|
||||
"""
|
||||
Integration test for the MCP (Model Context Protocol) server.
|
||||
|
||||
Tests MCP endpoints by starting a FastAPI server with MCP enabled and using the MCP client.
|
||||
|
||||
Note: MCP server is integrated with the web server. These tests require HINDSIGHT_API_MCP_ENABLED=true.
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import httpx
|
||||
from mcp import ClientSession
|
||||
from mcp.client.sse import sse_client
|
||||
from hindsight_api.api import create_app
|
||||
|
||||
|
||||
# Note: MCP server tests now require the full web server to be running
|
||||
# with MEMORA_API_MCP_ENABLED=true since there's no standalone MCP server anymore.
|
||||
# These tests are kept for documentation but may need manual server setup.
|
||||
@pytest_asyncio.fixture
|
||||
async def mcp_server(memory):
|
||||
"""Start the FastAPI app with MCP enabled and return the SSE URL."""
|
||||
app = create_app(
|
||||
memory,
|
||||
run_migrations=False,
|
||||
initialize_memory=False,
|
||||
mcp_enabled=True,
|
||||
default_agent_id="test_mcp_agent"
|
||||
)
|
||||
|
||||
pytest.skip("MCP server is now integrated with web server. Run web server with MEMORA_API_MCP_ENABLED=true to test.", allow_module_level=True)
|
||||
# Use httpx to create a test server
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
# The MCP SSE endpoint is at /mcp/sse
|
||||
# We need to yield the base URL for sse_client to connect
|
||||
# However, sse_client expects a real URL, not a test client
|
||||
# So we'll start a real server on a random port
|
||||
pass
|
||||
|
||||
# For now, skip these tests as they require a real server
|
||||
# The sse_client doesn't work with ASGI test transport
|
||||
pytest.skip("MCP tests require a real running server. Run: HINDSIGHT_API_MCP_ENABLED=true uvicorn hindsight_api.api:app")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -27,12 +52,12 @@ async def test_mcp_server_tools_via_sse(mcp_server):
|
||||
tools_list = await session.list_tools()
|
||||
print(f"Tools: {tools_list}")
|
||||
tool_names = [t.name for t in tools_list.tools]
|
||||
assert "memora_search" in tool_names
|
||||
assert "memora_put" in tool_names
|
||||
assert "hindsight_search" in tool_names
|
||||
assert "hindsight_put" in tool_names
|
||||
|
||||
# Test 2: Call memora_put
|
||||
# Test 2: Call hindsight_put
|
||||
put_result = await session.call_tool(
|
||||
"memora_put",
|
||||
"hindsight_put",
|
||||
arguments={
|
||||
"content": "User loves Python programming",
|
||||
"context": "programming_preferences",
|
||||
@@ -45,9 +70,9 @@ async def test_mcp_server_tools_via_sse(mcp_server):
|
||||
# Wait a bit for indexing
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Test 3: Call memora_search
|
||||
# Test 3: Call hindsight_search
|
||||
search_result = await session.call_tool(
|
||||
"memora_search",
|
||||
"hindsight_search",
|
||||
arguments={
|
||||
"query": "What programming languages does the user like?",
|
||||
"max_tokens": 4096,
|
||||
@@ -71,7 +96,7 @@ async def test_multiple_concurrent_requests(mcp_server):
|
||||
async def make_search(idx):
|
||||
try:
|
||||
result = await session.call_tool(
|
||||
"memora_search",
|
||||
"hindsight_search",
|
||||
arguments={
|
||||
"query": f"test query {idx}",
|
||||
"explanation": f"Concurrent test {idx}"
|
||||
@@ -120,7 +145,7 @@ async def test_race_condition_with_rapid_requests(mcp_server):
|
||||
|
||||
# Make request immediately after initialization
|
||||
result = await session.call_tool(
|
||||
"memora_search",
|
||||
"hindsight_search",
|
||||
arguments={
|
||||
"query": f"rapid query {idx}",
|
||||
"max_tokens": 2048
|
||||
@@ -3,16 +3,14 @@ Test query analyzer for temporal extraction.
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
from memora.query_analyzer import TransformerQueryAnalyzer, QueryAnalysis
|
||||
from hindsight_api.engine.query_analyzer import TransformerQueryAnalyzer, QueryAnalysis
|
||||
|
||||
|
||||
def test_query_analyzer_june_2024():
|
||||
"""Test extracting 'june 2024' from query."""
|
||||
analyzer = TransformerQueryAnalyzer()
|
||||
def test_query_analyzer_june_2024(query_analyzer):
|
||||
reference_date = datetime(2025, 1, 15, 12, 0, 0)
|
||||
|
||||
query = "june 2024"
|
||||
analysis = analyzer.analyze(query, reference_date)
|
||||
analysis = query_analyzer.analyze(query, reference_date)
|
||||
|
||||
print(f"\nQuery: '{query}'")
|
||||
print(f"Analysis: {analysis}")
|
||||
@@ -26,13 +24,11 @@ def test_query_analyzer_june_2024():
|
||||
assert analysis.temporal_constraint.end_date.day == 30
|
||||
|
||||
|
||||
def test_query_analyzer_dogs_june_2023():
|
||||
"""Test extracting temporal info from 'dogs in June 2023'."""
|
||||
analyzer = TransformerQueryAnalyzer()
|
||||
def test_query_analyzer_dogs_june_2023(query_analyzer):
|
||||
reference_date = datetime(2025, 1, 15, 12, 0, 0)
|
||||
|
||||
query = "dogs in June 2023"
|
||||
analysis = analyzer.analyze(query, reference_date)
|
||||
analysis = query_analyzer.analyze(query, reference_date)
|
||||
|
||||
print(f"\nQuery: '{query}'")
|
||||
print(f"Analysis: {analysis}")
|
||||
@@ -46,13 +42,11 @@ def test_query_analyzer_dogs_june_2023():
|
||||
assert analysis.temporal_constraint.end_date.day == 30
|
||||
|
||||
|
||||
def test_query_analyzer_march_2023():
|
||||
"""Test extracting 'March 2023' from query."""
|
||||
analyzer = TransformerQueryAnalyzer()
|
||||
def test_query_analyzer_march_2023(query_analyzer):
|
||||
reference_date = datetime(2025, 1, 15, 12, 0, 0)
|
||||
|
||||
query = "March 2023"
|
||||
analysis = analyzer.analyze(query, reference_date)
|
||||
analysis = query_analyzer.analyze(query, reference_date)
|
||||
|
||||
print(f"\nQuery: '{query}'")
|
||||
print(f"Analysis: {analysis}")
|
||||
@@ -66,13 +60,11 @@ def test_query_analyzer_march_2023():
|
||||
assert analysis.temporal_constraint.end_date.day == 31
|
||||
|
||||
|
||||
def test_query_analyzer_last_year():
|
||||
"""Test extracting 'last year' from query."""
|
||||
analyzer = TransformerQueryAnalyzer()
|
||||
def test_query_analyzer_last_year(query_analyzer):
|
||||
reference_date = datetime(2025, 1, 15, 12, 0, 0)
|
||||
|
||||
query = "last year"
|
||||
analysis = analyzer.analyze(query, reference_date)
|
||||
analysis = query_analyzer.analyze(query, reference_date)
|
||||
|
||||
print(f"\nQuery: '{query}'")
|
||||
print(f"Analysis: {analysis}")
|
||||
@@ -86,13 +78,11 @@ def test_query_analyzer_last_year():
|
||||
assert analysis.temporal_constraint.end_date.day == 31
|
||||
|
||||
|
||||
def test_query_analyzer_no_temporal():
|
||||
"""Test that queries without temporal info return None."""
|
||||
analyzer = TransformerQueryAnalyzer()
|
||||
def test_query_analyzer_no_temporal(query_analyzer):
|
||||
reference_date = datetime(2025, 1, 15, 12, 0, 0)
|
||||
|
||||
query = "what is the weather"
|
||||
analysis = analyzer.analyze(query, reference_date)
|
||||
analysis = query_analyzer.analyze(query, reference_date)
|
||||
|
||||
print(f"\nQuery: '{query}'")
|
||||
print(f"Analysis: {analysis}")
|
||||
@@ -100,13 +90,11 @@ def test_query_analyzer_no_temporal():
|
||||
assert analysis.temporal_constraint is None, "Should not extract temporal constraint"
|
||||
|
||||
|
||||
def test_query_analyzer_activities_june_2024():
|
||||
"""Test extracting temporal info from 'melanie activities in june 2024'."""
|
||||
analyzer = TransformerQueryAnalyzer()
|
||||
def test_query_analyzer_activities_june_2024(query_analyzer):
|
||||
reference_date = datetime(2025, 1, 15, 12, 0, 0)
|
||||
|
||||
query = "melanie activities in june 2024"
|
||||
analysis = analyzer.analyze(query, reference_date)
|
||||
analysis = query_analyzer.analyze(query, reference_date)
|
||||
|
||||
print(f"\nQuery: '{query}'")
|
||||
print(f"Analysis: {analysis}")
|
||||
@@ -120,5 +108,3 @@ def test_query_analyzer_activities_june_2024():
|
||||
assert analysis.temporal_constraint.end_date.day == 30
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v", "-s"])
|
||||
@@ -2,7 +2,7 @@
|
||||
Test search tracing functionality.
|
||||
"""
|
||||
import pytest
|
||||
from memora.search_trace import SearchTrace
|
||||
from hindsight_api import SearchTrace
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import asyncio
|
||||
import os
|
||||
from datetime import datetime, timezone, timedelta
|
||||
import pytest
|
||||
from memora import TemporalSemanticMemory
|
||||
from hindsight_api import MemoryEngine
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -11,11 +11,11 @@ async def test_temporal_ranges_are_written():
|
||||
"""Test that occurred_start, occurred_end, and mentioned_at are actually written to database."""
|
||||
|
||||
# Initialize memory system
|
||||
memory = TemporalSemanticMemory(
|
||||
db_url=os.getenv("MEMORA_API_DATABASE_URL", "postgresql://memora:memora_dev@localhost:5432/memora"),
|
||||
memory_llm_provider=os.getenv("MEMORA_API_LLM_PROVIDER", "groq"),
|
||||
memory_llm_api_key=os.getenv("MEMORA_API_LLM_API_KEY"),
|
||||
memory_llm_model=os.getenv("MEMORA_API_LLM_MODEL", "openai/gpt-oss-20b"),
|
||||
memory = MemoryEngine(
|
||||
db_url=os.getenv("HINDSIGHT_API_DATABASE_URL", "postgresql://hindsight:hindsight_dev@localhost:5432/hindsight"),
|
||||
memory_llm_provider=os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq"),
|
||||
memory_llm_api_key=os.getenv("HINDSIGHT_API_LLM_API_KEY"),
|
||||
memory_llm_model=os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-20b"),
|
||||
)
|
||||
await memory.initialize()
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
[package]
|
||||
name = "memora-cli-rust"
|
||||
name = "hindsight-cli"
|
||||
version = "0.0.7"
|
||||
edition = "2021"
|
||||
authors = ["Memora Team"]
|
||||
description = "A beautiful CLI for Memora - semantic memory system"
|
||||
authors = ["Hindsight Team"]
|
||||
description = "A beautiful CLI for Hindsight - semantic memory system"
|
||||
license = "MIT"
|
||||
|
||||
[[bin]]
|
||||
name = "memora"
|
||||
name = "hindsight"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Build script for memora-cli-rust
|
||||
# Build script for hindsight-cli
|
||||
# This script builds optimized binaries for multiple platforms
|
||||
|
||||
# Source cargo environment if it exists
|
||||
@@ -19,7 +19,7 @@ if ! command -v cargo &> /dev/null; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Building Memora CLI for multiple platforms..."
|
||||
echo "Building Hindsight CLI for multiple platforms..."
|
||||
|
||||
# Ensure we're in the right directory
|
||||
cd "$(dirname "$0")"
|
||||
@@ -50,10 +50,10 @@ build_target() {
|
||||
|
||||
# Copy to dist
|
||||
if [[ "$target" == *"windows"* ]]; then
|
||||
cp "target/$target/release/memora.exe" "dist/$output_name.exe"
|
||||
cp "target/$target/release/hindsight.exe" "dist/$output_name.exe"
|
||||
echo "Created: dist/$output_name.exe"
|
||||
else
|
||||
cp "target/$target/release/memora" "dist/$output_name"
|
||||
cp "target/$target/release/hindsight" "dist/$output_name"
|
||||
chmod +x "dist/$output_name"
|
||||
echo "Created: dist/$output_name"
|
||||
fi
|
||||
@@ -70,19 +70,19 @@ case "$OS" in
|
||||
Darwin)
|
||||
if [[ "$ARCH" == "arm64" ]]; then
|
||||
echo "Building for macOS ARM64 (Apple Silicon)..."
|
||||
build_target "aarch64-apple-darwin" "memora-macos-arm64"
|
||||
build_target "aarch64-apple-darwin" "hindsight-macos-arm64"
|
||||
else
|
||||
echo "Building for macOS x86_64 (Intel)..."
|
||||
build_target "x86_64-apple-darwin" "memora-macos-x86_64"
|
||||
build_target "x86_64-apple-darwin" "hindsight-macos-x86_64"
|
||||
fi
|
||||
;;
|
||||
Linux)
|
||||
if [[ "$ARCH" == "x86_64" ]]; then
|
||||
echo "Building for Linux x86_64..."
|
||||
build_target "x86_64-unknown-linux-gnu" "memora-linux-x86_64"
|
||||
build_target "x86_64-unknown-linux-gnu" "hindsight-linux-x86_64"
|
||||
elif [[ "$ARCH" == "aarch64" ]]; then
|
||||
echo "Building for Linux ARM64..."
|
||||
build_target "aarch64-unknown-linux-gnu" "memora-linux-arm64"
|
||||
build_target "aarch64-unknown-linux-gnu" "hindsight-linux-arm64"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user