Compare commits

...
Author SHA1 Message Date
Nicolò Boschi 3d66988c3b add api cli 2025-11-25 19:28:12 +01:00
Nicolò Boschi 67dc160ba3 pg0 works 2025-11-25 17:05:18 +01:00
Nicolò Boschi 8291386387 hindsight all 2025-11-25 16:57:54 +01:00
Nicolò Boschi 3511062c51 refactor ts client 2025-11-25 12:17:13 +01:00
Nicolò Boschi a134d27137 fix tests and docs 2025-11-25 11:31:21 +01:00
Nicolò Boschi 31a53c0870 rename to hindsight 2025-11-25 10:11:04 +01:00
473 changed files with 16955 additions and 12779 deletions
+20 -14
View File
@@ -1,41 +1,47 @@
# =============================================================================
# MEMORA ENVIRONMENT CONFIGURATION
# HINDSIGHT ENVIRONMENT CONFIGURATION
# =============================================================================
# Copy this file to .env and update with your values
# Both services (API and Control Plane) read from this single file
# =============================================================================
# API SERVICE (MEMORA_API_*)
# API SERVICE (HINDSIGHT_API_*)
# =============================================================================
# Database
MEMORA_API_DATABASE_URL=postgresql://memora:memora_dev@localhost:5432/memora
# Use "pg0" to start an embedded PostgreSQL instance via pg0
# Or provide a full connection URL for external PostgreSQL
#HINDSIGHT_API_DATABASE_URL=postgresql://hindsight:hindsight_dev@localhost:5432/hindsight
HINDSIGHT_API_DATABASE_URL=pg0
# pg0 data directory (only used when HINDSIGHT_API_DATABASE_URL=pg0)
# HINDSIGHT_API_PG0_DATA_DIR=/path/to/pg_data
# LLM Provider: "openai", "groq", or "ollama"
MEMORA_API_LLM_PROVIDER=groq
HINDSIGHT_API_LLM_PROVIDER=groq
# LLM Model (provider-specific)
MEMORA_API_LLM_MODEL=openai/gpt-oss-20b
HINDSIGHT_API_LLM_MODEL=openai/gpt-oss-20b
# API Key (not needed for ollama)
MEMORA_API_LLM_API_KEY=your_api_key_here
HINDSIGHT_API_LLM_API_KEY=your_api_key_here
# Optional: Custom base URL (for ollama or custom endpoints)
# MEMORA_API_LLM_BASE_URL=http://localhost:11434/v1
# HINDSIGHT_API_LLM_BASE_URL=http://localhost:11434/v1
# API Server Configuration (optional)
# MEMORA_API_HOST=0.0.0.0
# MEMORA_API_PORT=8080
# HINDSIGHT_API_HOST=0.0.0.0
# HINDSIGHT_API_PORT=8888
MEMORA_API_MCP_ENABLED=true
HINDSIGHT_API_MCP_ENABLED=true
# =============================================================================
# CONTROL PLANE SERVICE (MEMORA_CP_*)
# CONTROL PLANE SERVICE (HINDSIGHT_CP_*)
# =============================================================================
# Dataplane API URL (where the control plane connects to)
MEMORA_CP_DATAPLANE_API_URL=http://localhost:8080
HINDSIGHT_CP_DATAPLANE_API_URL=http://localhost:8888
# Control Plane Server Configuration (optional)
# MEMORA_CP_PORT=3000
# MEMORA_CP_HOSTNAME=0.0.0.0
# HINDSIGHT_CP_PORT=3000
# HINDSIGHT_CP_HOSTNAME=0.0.0.0
+6 -6
View File
@@ -2,9 +2,10 @@ name: Deploy Docs to GitHub Pages
on:
push:
branches: [main]
branches: [main, renaming-pre-launch]
paths:
- 'memora-docs/**'
- 'hindsight-docs/**'
- '.github/workflows/deploy-docs.yml'
workflow_dispatch:
permissions:
@@ -21,20 +22,19 @@ jobs:
runs-on: ubuntu-latest
defaults:
run:
working-directory: memora-docs
working-directory: hindsight-docs
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: npm
cache-dependency-path: memora-docs/package-lock.json
cache-dependency-path: hindsight-docs/package-lock.json
- run: npm ci
- run: npm run build
- uses: actions/upload-pages-artifact@v3
with:
path: memora-docs/build
path: hindsight-docs/build
deploy:
environment:
name: github-pages
+53 -53
View File
@@ -22,15 +22,15 @@ jobs:
with:
python-version-file: ".python-version"
- name: Build memora package
working-directory: ./memora
- name: Build hindsight package
working-directory: ./hindsight
run: uv build
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: python-memora-dist
path: memora/dist/*
name: python-hindsight-dist
path: hindsight/dist/*
retention-days: 30
build-rust-cli:
@@ -40,16 +40,16 @@ jobs:
include:
- os: ubuntu-latest
target: x86_64-unknown-linux-gnu
artifact_name: memora
asset_name: memora-linux-amd64
artifact_name: hindsight
asset_name: hindsight-linux-amd64
- os: macos-latest
target: x86_64-apple-darwin
artifact_name: memora
asset_name: memora-darwin-amd64
artifact_name: hindsight
asset_name: hindsight-darwin-amd64
- os: macos-latest
target: aarch64-apple-darwin
artifact_name: memora
asset_name: memora-darwin-arm64
artifact_name: hindsight
asset_name: hindsight-darwin-arm64
steps:
- uses: actions/checkout@v4
@@ -74,17 +74,17 @@ jobs:
- name: Cache cargo build
uses: actions/cache@v4
with:
path: memora-cli/target
path: hindsight-cli/target
key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
- name: Build
working-directory: memora-cli
working-directory: hindsight-cli
run: cargo build --release --target ${{ matrix.target }}
- name: Prepare artifact
run: |
mkdir -p artifacts
cp memora-cli/target/${{ matrix.target }}/release/${{ matrix.artifact_name }} artifacts/${{ matrix.asset_name }}
cp hindsight-cli/target/${{ matrix.target }}/release/${{ matrix.artifact_name }} artifacts/${{ matrix.asset_name }}
chmod +x artifacts/${{ matrix.asset_name }}
- name: Upload artifacts
@@ -135,7 +135,7 @@ jobs:
id: meta
uses: docker/metadata-action@v5
with:
images: ghcr.io/${{ github.repository_owner }}/memora-${{ matrix.component }}
images: ghcr.io/${{ github.repository_owner }}/hindsight-${{ matrix.component }}
tags: |
type=semver,pattern={{version}},value=${{ steps.get_version.outputs.VERSION }}
type=semver,pattern={{major}}.{{minor}},value=${{ steps.get_version.outputs.VERSION }}
@@ -179,11 +179,11 @@ jobs:
- name: Lint Helm chart
run: |
helm lint helm/memora
helm lint helm/hindsight
- name: Package Helm chart
run: |
helm package helm/memora --destination ./helm-packages
helm package helm/hindsight --destination ./helm-packages
- name: Upload Helm chart artifact
uses: actions/upload-artifact@v4
@@ -208,26 +208,26 @@ jobs:
- name: Download Python package
uses: actions/download-artifact@v4
with:
name: python-memora-dist
path: ./artifacts/python-memora-dist
name: python-hindsight-dist
path: ./artifacts/python-hindsight-dist
- name: Download Rust CLI (Linux)
uses: actions/download-artifact@v4
with:
name: rust-cli-memora-linux-amd64
path: ./artifacts/rust-cli-memora-linux-amd64
name: rust-cli-hindsight-linux-amd64
path: ./artifacts/rust-cli-hindsight-linux-amd64
- name: Download Rust CLI (macOS Intel)
uses: actions/download-artifact@v4
with:
name: rust-cli-memora-darwin-amd64
path: ./artifacts/rust-cli-memora-darwin-amd64
name: rust-cli-hindsight-darwin-amd64
path: ./artifacts/rust-cli-hindsight-darwin-amd64
- name: Download Rust CLI (macOS ARM)
uses: actions/download-artifact@v4
with:
name: rust-cli-memora-darwin-arm64
path: ./artifacts/rust-cli-memora-darwin-arm64
name: rust-cli-hindsight-darwin-arm64
path: ./artifacts/rust-cli-hindsight-darwin-arm64
- name: Download Helm chart
uses: actions/download-artifact@v4
@@ -239,11 +239,11 @@ jobs:
run: |
mkdir -p release-assets
# Python package
cp artifacts/python-memora-dist/* release-assets/
cp artifacts/python-hindsight-dist/* release-assets/
# Rust CLI binaries
cp artifacts/rust-cli-memora-linux-amd64/memora-linux-amd64 release-assets/
cp artifacts/rust-cli-memora-darwin-amd64/memora-darwin-amd64 release-assets/
cp artifacts/rust-cli-memora-darwin-arm64/memora-darwin-arm64 release-assets/
cp artifacts/rust-cli-hindsight-linux-amd64/hindsight-linux-amd64 release-assets/
cp artifacts/rust-cli-hindsight-darwin-amd64/hindsight-darwin-amd64 release-assets/
cp artifacts/rust-cli-hindsight-darwin-arm64/hindsight-darwin-arm64 release-assets/
# Helm chart
cp artifacts/helm-chart/*.tgz release-assets/
@@ -251,68 +251,68 @@ jobs:
id: release_notes
run: |
cat << EOF > release-notes.md
# Memora v${{ steps.get_version.outputs.VERSION }}
# Hindsight v${{ steps.get_version.outputs.VERSION }}
## 📦 Release Artifacts
### Python Package
- \`memora-${{ steps.get_version.outputs.VERSION }}-py3-none-any.whl\`
- \`memora-${{ steps.get_version.outputs.VERSION }}.tar.gz\`
- \`hindsight-${{ steps.get_version.outputs.VERSION }}-py3-none-any.whl\`
- \`hindsight-${{ steps.get_version.outputs.VERSION }}.tar.gz\`
### CLI Binaries
- \`memora-linux-amd64\` - Linux x86_64
- \`memora-darwin-amd64\` - macOS Intel
- \`memora-darwin-arm64\` - macOS Apple Silicon
- \`hindsight-linux-amd64\` - Linux x86_64
- \`hindsight-darwin-amd64\` - macOS Intel
- \`hindsight-darwin-arm64\` - macOS Apple Silicon
### Helm Chart
- \`memora-${{ steps.get_version.outputs.VERSION }}.tgz\`
- \`hindsight-${{ steps.get_version.outputs.VERSION }}.tgz\`
### Docker Images
Docker images are published to GitHub Container Registry:
- \`ghcr.io/${{ github.repository_owner }}/memora-api:${{ steps.get_version.outputs.VERSION }}\`
- \`ghcr.io/${{ github.repository_owner }}/memora-control-plane:${{ steps.get_version.outputs.VERSION }}\`
- \`ghcr.io/${{ github.repository_owner }}/hindsight-api:${{ steps.get_version.outputs.VERSION }}\`
- \`ghcr.io/${{ github.repository_owner }}/hindsight-control-plane:${{ steps.get_version.outputs.VERSION }}\`
## 🚀 Installation
### Python Package
\`\`\`bash
pip install memora==${{ steps.get_version.outputs.VERSION }}
pip install hindsight==${{ steps.get_version.outputs.VERSION }}
\`\`\`
### CLI
\`\`\`bash
# macOS (Apple Silicon)
curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/memora-darwin-arm64 -o memora
chmod +x memora
sudo mv memora /usr/local/bin/
curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/hindsight-darwin-arm64 -o hindsight
chmod +x hindsight
sudo mv hindsight /usr/local/bin/
# macOS (Intel)
curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/memora-darwin-amd64 -o memora
chmod +x memora
sudo mv memora /usr/local/bin/
curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/hindsight-darwin-amd64 -o hindsight
chmod +x hindsight
sudo mv hindsight /usr/local/bin/
# Linux
curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/memora-linux-amd64 -o memora
chmod +x memora
sudo mv memora /usr/local/bin/
curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/hindsight-linux-amd64 -o hindsight
chmod +x hindsight
sudo mv hindsight /usr/local/bin/
\`\`\`
### Helm Chart
\`\`\`bash
helm install memora memora-${{ steps.get_version.outputs.VERSION }}.tgz
helm install hindsight hindsight-${{ steps.get_version.outputs.VERSION }}.tgz
\`\`\`
### Docker
\`\`\`bash
# Pull API image
docker pull ghcr.io/${{ github.repository_owner }}/memora-api:${{ steps.get_version.outputs.VERSION }}
docker pull ghcr.io/${{ github.repository_owner }}/hindsight-api:${{ steps.get_version.outputs.VERSION }}
# Pull Control Plane image
docker pull ghcr.io/${{ github.repository_owner }}/memora-control-plane:${{ steps.get_version.outputs.VERSION }}
docker pull ghcr.io/${{ github.repository_owner }}/hindsight-control-plane:${{ steps.get_version.outputs.VERSION }}
# Or use latest
docker pull ghcr.io/${{ github.repository_owner }}/memora-api:latest
docker pull ghcr.io/${{ github.repository_owner }}/memora-control-plane:latest
docker pull ghcr.io/${{ github.repository_owner }}/hindsight-api:latest
docker pull ghcr.io/${{ github.repository_owner }}/hindsight-control-plane:latest
\`\`\`
EOF
cat release-notes.md
@@ -333,7 +333,7 @@ jobs:
echo "# Release v${{ steps.get_version.outputs.VERSION }} Published Successfully" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "## 📦 Components" >> $GITHUB_STEP_SUMMARY
echo "- ✅ Python package (memora)" >> $GITHUB_STEP_SUMMARY
echo "- ✅ Python package (hindsight)" >> $GITHUB_STEP_SUMMARY
echo "- ✅ Rust CLI (Linux amd64, macOS amd64, macOS arm64)" >> $GITHUB_STEP_SUMMARY
echo "- ✅ Docker images (API, Control Plane)" >> $GITHUB_STEP_SUMMARY
echo "- ✅ Helm chart" >> $GITHUB_STEP_SUMMARY
+7 -7
View File
@@ -16,7 +16,7 @@ jobs:
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: memora_test
POSTGRES_DB: hindsight_test
options: >-
--health-cmd pg_isready
--health-interval 10s
@@ -26,10 +26,10 @@ jobs:
- 5432:5432
env:
MEMORA_API_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/memora_test
MEMORA_API_LLM_PROVIDER: groq
MEMORA_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
MEMORA_API_LLM_MODEL: openai/gpt-oss-20b
HINDSIGHT_API_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/hindsight_test
HINDSIGHT_API_LLM_PROVIDER: groq
HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
steps:
- uses: actions/checkout@v4
@@ -48,9 +48,9 @@ jobs:
run: uv sync --extra test
- name: Run migrations
working-directory: ./memora
working-directory: ./hindsight
run: |
uv run alembic upgrade head
- name: Run tests
run: uv run pytest memora/tests -v
run: uv run pytest hindsight/tests -v
-8
View File
@@ -1,8 +0,0 @@
# Documentation
Do not write any markdown file, just write the code.
# Workflow
- After your changes, make sure everything is working fine by running the tests.
- keep the readme.md architecture section up to date when you change the implementation
- when changing an implemetation, do not keep the old one as fallback
- to run test, use uv run pytest tests
+27 -34
View File
@@ -1,64 +1,57 @@
# Memora
# Hindsight
**Long-term memory for AI agents.**
AI assistants forget everything between sessions. Memora fixes that with a memory system that handles temporal reasoning, entity connections, and personality-aware responses.
AI assistants forget everything between sessions. Hindsight fixes that with a memory system that handles temporal reasoning, entity connections, and personality-aware responses.
## Why Memora?
## Why Hindsight?
- **Temporal queries** — "What did Alice do last spring?" requires more than vector search
- **Entity connections** — Knowing "Alice works at Google" + "Google is in Mountain View" = "Alice works in Mountain View"
- **Agent opinions** — Agents form and recall beliefs with confidence scores
- **Personality** — Big Five traits influence how agents process and respond to information
## 5-Minute Setup
## 60-seconds step
### 1. Start the server
### 1. Install the Hindsight All package (client + API)
```bash
# Clone and start with Docker
git clone https://github.com/anthropics/memora.git
cd memora/docker
./start.sh
pip install hindsight-all
```
Server runs at `http://localhost:8080`
### 2. Install the Python client
### 2. Import your OpenAI API key
```bash
pip install memora-client
export OPENAI_API_KEY=xx
```
### 3. Use it
### 3. Run embedded server and client
```python
from memora_client import Memora
import os
from hindsight import HindsightServer, HindsightClient
client = Memora(base_url="http://localhost:8080")
with HindsightServer(llm_provider="openai", llm_model="gpt-5.1-mini", llm_api_key=os.environ["OPENAI_API_KEY"]) as server:
client = HindsightClient(base_url=server.url)
# Store memories
client.store(agent_id="my-agent", content="Alice works at Google")
client.store(agent_id="my-agent", content="Bob prefers Python over JavaScript")
# Search memories
results = client.search(agent_id="my-agent", query="What does Alice do?")
for r in results:
print(f"{r['text']} ({r['weight']:.2f})")
# Generate personality-aware responses
answer = client.think(agent_id="my-agent", query="Tell me about Alice")
print(answer["text"])
client.put(agent_id="my-agent", content="Alice works at Google")
client.put(agent_id="my-agent", content="Bob prefers Python over JavaScript")
client.search(agent_id="my-agent", query="What does Alice do?")
client.think(agent_id="my-agent", query="Tell me about Alice")
```
## Documentation
Full documentation: [memora-docs](./memora-docs)
Full documentation: [hindsight-docs](./hindsight-docs)
- [Architecture](./memora-docs/docs/developer/architecture.md) — How ingestion, storage, and retrieval work
- [Python Client](./memora-docs/docs/sdks/python.md) — Full API reference
- [API Reference](./memora-docs/docs/api-reference/index.md) — REST API endpoints
- [Personality](./memora-docs/docs/developer/personality.md) — Big Five traits and opinion formation
- [Architecture](./hindsight-docs/docs/developer/architecture.md) — How ingestion, storage, and retrieval work
- [Python Client](./hindsight-docs/docs/sdks/python.md) — Full API reference
- [API Reference](./hindsight-docs/docs/api-reference/index.md) — REST API endpoints
- [Personality](./hindsight-docs/docs/developer/personality.md) — Big Five traits and opinion formation
## License
+9 -9
View File
@@ -6,7 +6,7 @@
```bash
uv sync
cd memora-dev
cd hindsight-dev
uv run generate-openapi
cd ..
```
@@ -24,7 +24,7 @@ This regenerates Python and TypeScript clients from `openapi.json`.
### 3. Commit Everything
```bash
git add openapi.json memora-clients/
git add openapi.json hindsight-clients/
git commit -m "Update OpenAPI spec and regenerate clients"
```
@@ -47,7 +47,7 @@ This will:
### Publish Python Client to PyPI
```bash
cd memora-clients/python
cd hindsight-clients/python
uv build
uv publish
```
@@ -55,7 +55,7 @@ uv publish
### Publish TypeScript Client to NPM
```bash
cd memora-clients/typescript
cd hindsight-clients/typescript
npm install
npm run build
npm publish --access public
@@ -65,7 +65,7 @@ npm publish --access public
## Pre-Release Checklist
- [ ] Tests passing: `cd memora && uv run pytest tests`
- [ ] Tests passing: `cd hindsight-api && uv run pytest tests`
- [ ] No uncommitted changes: `git status`
- [ ] On `main` branch
@@ -98,7 +98,7 @@ git status
```
**GitHub Actions failed:**
- Check: https://github.com/nicoloboschi/memora/actions
- Check: https://github.com/vectorize-io/hindsight/actions
- Re-run failed jobs or fix and release new patch version
**Rollback:**
@@ -116,13 +116,13 @@ git push
```bash
# Full release workflow
uv sync
cd memora-dev && uv run generate-openapi && cd ..
cd hindsight-dev && uv run generate-openapi && cd ..
./scripts/generate-clients.sh
git add openapi.json memora-clients/
git add openapi.json hindsight-clients/
git commit -m "Update OpenAPI spec and regenerate clients"
./scripts/release.sh 0.0.6
# After GH Actions complete:
cd memora-clients/python && uv build && uv publish
cd hindsight-clients/python && uv build && uv publish
cd ../typescript && npm run build && npm publish --access public
```
-175
View File
@@ -1,175 +0,0 @@
# Memora Docker Setup
Complete Docker Compose setup for running all Memora services locally.
## Services
This setup includes:
- **PostgreSQL** with pgvector extension (port 5432)
- **API Service** - FastAPI backend (port 8080)
- **Control Plane** - Next.js web UI (port 3000)
## Quick Start
1. **Configure environment variables:**
```bash
cp .env.example .env
# Edit .env and set your API keys
```
2. **Start all services:**
```bash
./start.sh
```
3. **Access the services:**
- Control Plane: http://localhost:3000
- API: http://localhost:8080
- PostgreSQL: localhost:5432
## Scripts
### `./start.sh`
Build and start all services. Waits for all services to be healthy.
### `./stop.sh`
Stop all services (keeps data).
### `./clean.sh`
Stop all services and remove all data (destructive).
### `./logs.sh [service]`
View logs for all services or a specific service:
```bash
./logs.sh # All services
./logs.sh api # API only
./logs.sh postgres # PostgreSQL only
./logs.sh control-plane # Control plane only
```
## Manual Docker Compose Commands
```bash
# Start services
docker-compose up -d
# Stop services
docker-compose down
# Rebuild and start
docker-compose up --build -d
# View logs
docker-compose logs -f
# Remove everything including data
docker-compose down -v
```
## Database
### Connection Info
- **Host:** localhost
- **Port:** 5432
- **Database:** memora
- **User:** memora
- **Password:** memora_dev
### Migrations
Database migrations run automatically when the API service starts. The API uses Alembic to:
1. Check the current schema version
2. Run any pending migrations
3. Initialize the database if it's empty
Extensions (pgvector, uuid-ossp) are created automatically by the first migration.
## Environment Variables
Required in `.env` file:
```bash
# API Service Configuration
MEMORA_API_DATABASE_URL=postgresql://memora:memora_dev@localhost:5432/memora
MEMORA_API_LLM_PROVIDER=groq
MEMORA_API_LLM_API_KEY=your-api-key-here
MEMORA_API_LLM_MODEL=openai/gpt-oss-120b
# Optional: Custom LLM endpoint
# MEMORA_API_LLM_BASE_URL=http://localhost:11434/v1
# Control Plane Configuration
MEMORA_CP_DATAPLANE_API_URL=http://localhost:8080
```
## Troubleshooting
### Services won't start
Check logs for errors:
```bash
./logs.sh
```
### Database connection issues
Ensure PostgreSQL is healthy:
```bash
docker exec memora-postgres pg_isready -U memora
```
### API won't connect to database
Check if migrations ran successfully:
```bash
./logs.sh api
```
### Control plane can't reach API
Verify the API is running:
```bash
curl http://localhost:8080/
```
### Reset everything
```bash
./clean.sh
./start.sh
```
## Development
### Rebuilding after code changes
**API changes:**
```bash
docker-compose up --build -d api
```
**Control Plane changes:**
```bash
docker-compose up --build -d control-plane
```
### Accessing the database
```bash
docker exec -it memora-postgres psql -U memora -d memora
```
### Inspecting containers
```bash
docker-compose ps
docker-compose exec api bash
docker-compose exec control-plane sh
```
## Data Persistence
PostgreSQL data is persisted in a Docker volume named `postgres_data`. This data survives container restarts but not `docker-compose down -v`.
To backup data:
```bash
docker exec memora-postgres pg_dump -U memora memora > backup.sql
```
To restore data:
```bash
docker exec -i memora-postgres psql -U memora memora < backup.sql
```
+10 -10
View File
@@ -9,15 +9,15 @@ RUN apt-get update && apt-get install -y \
WORKDIR /app
# Copy only dependency files first for better caching
COPY memora/pyproject.toml memora/README.md /app/memora/
COPY memora/memora /app/memora/memora
COPY memora/alembic /app/memora/alembic
COPY hindsight-api/pyproject.toml hindsight-api/README.md /app/hindsight-api/
COPY hindsight-api/hindsight_api /app/hindsight-api/hindsight_api
COPY hindsight-api/alembic /app/hindsight-api/alembic
# Install uv for faster dependency installation
RUN pip install --no-cache-dir uv
# Install Python dependencies to a virtual environment
WORKDIR /app/memora
WORKDIR /app/hindsight-api
RUN uv venv /opt/venv && \
. /opt/venv/bin/activate && \
uv pip install --no-cache -e .
@@ -33,19 +33,19 @@ RUN apt-get update && apt-get install -y \
# Copy virtual environment from builder
COPY --from=builder /opt/venv /opt/venv
COPY --from=builder /app/memora /app/memora
COPY --from=builder /app/hindsight-api /app/hindsight-api
# Set working directory
WORKDIR /app/memora
WORKDIR /app/hindsight-api
# Expose API port
EXPOSE 8080
EXPOSE 8888
# Set environment variables
ENV PYTHONUNBUFFERED=1
ENV DATABASE_URL=postgresql://memora:memora_dev@postgres:5432/memora
ENV DATABASE_URL=postgresql://hindsight:hindsight_dev@postgres:5432/hindsight
ENV PATH="/opt/venv/bin:$PATH"
ENV PYTHONPATH=/app/memora
ENV PYTHONPATH=/app/hindsight-api
# Run the API server
CMD ["python", "-m", "memora.web.server", "--host", "0.0.0.0", "--port", "8080"]
CMD ["python", "-m", "hindsight_api.web.server", "--host", "0.0.0.0", "--port", "8888"]
+2 -2
View File
@@ -3,7 +3,7 @@ set -e
cd "$(dirname "$0")"
echo "🧹 Cleaning Memora Services"
echo "🧹 Cleaning Services"
echo "============================"
echo ""
echo "This will:"
@@ -20,7 +20,7 @@ fi
echo ""
echo "🗑️ Removing services and data..."
docker-compose down -v
docker compose down -v
echo ""
echo "✅ All services and data removed"
+2 -2
View File
@@ -31,9 +31,9 @@ COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
USER nextjs
EXPOSE 3000
EXPOSE 9999
ENV PORT=3000
ENV PORT=9999
ENV HOSTNAME="0.0.0.0"
CMD ["node", "server.js"]
+26 -26
View File
@@ -1,78 +1,78 @@
services:
postgres:
image: pgvector/pgvector:pg16
container_name: memora-postgres
container_name: hindsight-postgres
environment:
POSTGRES_USER: memora
POSTGRES_PASSWORD: memora_dev
POSTGRES_DB: memora
POSTGRES_USER: hindsight
POSTGRES_PASSWORD: hindsight_dev
POSTGRES_DB: hindsight
ports:
- "5432:5432"
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U memora"]
test: ["CMD-SHELL", "pg_isready -U hindsight"]
interval: 5s
timeout: 5s
retries: 5
networks:
- memora-network
- hindsight-network
api:
build:
context: ..
dockerfile: docker/api.Dockerfile
container_name: memora-api
container_name: hindsight-api
environment:
MEMORA_API_DATABASE_URL: postgresql://memora:memora_dev@postgres:5432/memora
MEMORA_API_LLM_PROVIDER: ${MEMORA_API_LLM_PROVIDER:-groq}
MEMORA_API_LLM_API_KEY: ${MEMORA_API_LLM_API_KEY}
MEMORA_API_LLM_MODEL: ${MEMORA_API_LLM_MODEL:-openai/gpt-oss-120b}
MEMORA_API_LLM_BASE_URL: ${MEMORA_API_LLM_BASE_URL}
MEMORA_API_HOST: ${MEMORA_API_HOST:-0.0.0.0}
MEMORA_API_PORT: ${MEMORA_API_PORT:-8080}
HINDSIGHT_API_DATABASE_URL: postgresql://hindsight:hindsight_dev@postgres:5432/hindsight
HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-groq}
HINDSIGHT_API_LLM_API_KEY: ${HINDSIGHT_API_LLM_API_KEY}
HINDSIGHT_API_LLM_MODEL: ${HINDSIGHT_API_LLM_MODEL:-openai/gpt-oss-20b}
HINDSIGHT_API_LLM_BASE_URL: ${HINDSIGHT_API_LLM_BASE_URL}
HINDSIGHT_API_HOST: 0.0.0.0
HINDSIGHT_API_PORT: 8888
ports:
- "8080:8080"
- "8888:8888"
depends_on:
postgres:
condition: service_healthy
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8080/api/v1/agents"]
test: ["CMD", "curl", "-f", "http://localhost:8888/api/v1/agents"]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
networks:
- memora-network
- hindsight-network
restart: unless-stopped
control-plane:
build:
context: ../memora-control-plane
context: ../hindsight-control-plane
dockerfile: ../docker/control-plane.Dockerfile
container_name: memora-control-plane
container_name: hindsight-control-plane
environment:
NODE_ENV: production
MEMORA_CP_HOSTNAME: ${MEMORA_CP_HOSTNAME:-0.0.0.0}
MEMORA_CP_PORT: ${MEMORA_CP_PORT:-3000}
MEMORA_CP_DATAPLANE_API_URL: ${MEMORA_CP_DATAPLANE_API_URL:-http://api:8080}
HOSTNAME: 0.0.0.0
PORT: 9999
HINDSIGHT_CP_DATAPLANE_API_URL: http://api:8888
ports:
- "3000:3000"
- "9999:9999"
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3000/"]
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:9999/"]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
networks:
- memora-network
- hindsight-network
restart: unless-stopped
networks:
memora-network:
hindsight-network:
driver: bridge
volumes:
+2 -2
View File
@@ -8,9 +8,9 @@ SERVICE=$1
if [ -z "$SERVICE" ]; then
echo "📋 Showing logs for all services..."
echo ""
docker-compose logs -f
docker compose logs -f
else
echo "📋 Showing logs for $SERVICE..."
echo ""
docker-compose logs -f "$SERVICE"
docker compose logs -f "$SERVICE"
fi
+9 -9
View File
@@ -3,7 +3,7 @@ set -e
cd "$(dirname "$0")"
echo "🚀 Starting Memora Services"
echo "🚀 Starting Hindsight Services"
echo "============================"
echo ""
@@ -15,14 +15,14 @@ if [ ! -f ../.env ]; then
cp ../.env.example ../.env
echo ""
echo "⚠️ Please edit .env and set your API keys:"
echo " - MEMORA_API_LLM_API_KEY"
echo " - HINDSIGHT_API_LLM_API_KEY"
echo ""
echo "Then run this script again."
exit 1
fi
echo "📦 Building and starting services..."
docker-compose --env-file ../.env up --build -d
docker compose --env-file ../.env up --build -d
echo ""
echo "⏳ Waiting for services to be healthy..."
@@ -30,21 +30,21 @@ echo ""
# Wait for PostgreSQL
echo " Waiting for PostgreSQL..."
until docker exec memora-postgres pg_isready -U memora > /dev/null 2>&1; do
until docker exec hindsight-postgres pg_isready -U hindsight > /dev/null 2>&1; do
sleep 1
done
echo " ✅ PostgreSQL is ready"
# Wait for API
echo " Waiting for API..."
until curl -f http://localhost:8080/api/v1/agents > /dev/null 2>&1; do
until curl -f http://localhost:8888/api/v1/agents > /dev/null 2>&1; do
sleep 2
done
echo " ✅ API is ready"
# Wait for Control Plane
echo " Waiting for Control Plane..."
until curl -f http://localhost:3000 > /dev/null 2>&1; do
until curl -f http://localhost:9999 > /dev/null 2>&1; do
sleep 2
done
echo " ✅ Control Plane is ready"
@@ -53,12 +53,12 @@ echo ""
echo "✅ All services are running!"
echo ""
echo "📊 Service URLs:"
echo " Control Plane: http://localhost:3000"
echo " API: http://localhost:8080"
echo " Control Plane: http://localhost:9999"
echo " API: http://localhost:8888"
echo " PostgreSQL: localhost:5432"
echo ""
echo "🔍 View logs:"
echo " docker-compose logs -f"
echo " docker compose logs -f"
echo ""
echo "🛑 Stop services:"
echo " ./stop.sh"
+3 -3
View File
@@ -3,15 +3,15 @@ set -e
cd "$(dirname "$0")"
echo "🛑 Stopping Memora Services"
echo "🛑 Stopping Services"
echo "============================"
echo ""
docker-compose down
docker compose down
echo ""
echo "✅ All services stopped"
echo ""
echo "💡 To remove data volumes as well, run:"
echo " docker-compose down -v"
echo " docker compose down -v"
echo ""
+28 -28
View File
@@ -1,4 +1,4 @@
MEMORA HELM CHART INSTALLATION GUIDE
HINDSIGHT HELM CHART INSTALLATION GUIDE
=====================================
PREREQUISITES
@@ -13,42 +13,42 @@ BASIC INSTALLATION
1. Install with default values (requires external PostgreSQL):
helm install memora ./memora \
helm install hindsight ./hindsight \
--set postgresql.external.host=your-postgres-host \
--set postgresql.external.password=your-password \
--set api.secrets.MEMORY_LLM_API_KEY=your-api-key
2. Install with custom values file:
helm install memora ./memora -f memora/values-production.yaml
helm install hindsight ./hindsight -f hindsight/values-production.yaml
3. Install in a specific namespace:
kubectl create namespace memora
helm install memora ./memora -n memora
kubectl create namespace hindsight
helm install hindsight ./hindsight -n hindsight
CONFIGURATION OPTIONS
---------------------
Development setup (using values-development.yaml):
helm install memora ./memora -f memora/values-development.yaml
helm install hindsight ./hindsight -f hindsight/values-development.yaml
Production setup (using values-production.yaml):
helm install memora ./memora -f memora/values-production.yaml
helm install hindsight ./hindsight -f hindsight/values-production.yaml
Custom LLM provider:
helm install memora ./memora \
helm install hindsight ./hindsight \
--set api.env.MEMORY_LLM_PROVIDER=openai \
--set api.env.MEMORY_LLM_MODEL=gpt-4 \
--set api.secrets.MEMORY_LLM_API_KEY=sk-your-key
Enable ingress:
helm install memora ./memora \
helm install hindsight ./hindsight \
--set ingress.enabled=true \
--set ingress.hosts[0].host=memora.example.com
--set ingress.hosts[0].host=hindsight.example.com
Enable autoscaling:
helm install memora ./memora \
helm install hindsight ./hindsight \
--set autoscaling.enabled=true \
--set autoscaling.minReplicas=2 \
--set autoscaling.maxReplicas=10
@@ -57,43 +57,43 @@ UPGRADE
-------
Upgrade existing installation:
helm upgrade memora ./memora
helm upgrade hindsight ./hindsight
Upgrade with new values:
helm upgrade memora ./memora -f memora/values-production.yaml
helm upgrade hindsight ./hindsight -f hindsight/values-production.yaml
UNINSTALL
---------
Remove the Helm release:
helm uninstall memora
helm uninstall hindsight
Remove with namespace:
helm uninstall memora -n memora
helm uninstall hindsight -n hindsight
TESTING
-------
Test the installation with dry-run:
helm install memora ./memora --dry-run --debug
helm install hindsight ./hindsight --dry-run --debug
Validate templates:
helm template memora ./memora
helm template hindsight ./hindsight
Lint the chart:
helm lint ./memora
helm lint ./hindsight
ACCESSING THE SERVICES
----------------------
Port-forward control plane:
kubectl port-forward svc/memora-control-plane 3000:3000
kubectl port-forward svc/hindsight-control-plane 3000:3000
Port-forward API:
kubectl port-forward svc/memora-api 8080:8080
kubectl port-forward svc/hindsight-api 8888:8888
Get service URLs:
helm status memora
helm status hindsight
DATABASE INITIALIZATION
-----------------------
@@ -102,16 +102,16 @@ NOTE: Database migrations now run automatically when the API service starts.
You typically don't need to run migrations manually.
If you want to pre-initialize the database before deploying (optional):
kubectl run memora-init --rm -it --restart=Never \
--image=memora/api:latest \
--env="DATABASE_URL=postgresql://user:pass@host:5432/memora" \
-- python -c "from memora.migrations import run_migrations; run_migrations()"
kubectl run hindsight-init --rm -it --restart=Never \
--image=hindsight/api:latest \
--env="DATABASE_URL=postgresql://user:pass@host:5432/hindsight" \
-- python -c "from hindsight.migrations import run_migrations; run_migrations()"
TROUBLESHOOTING
---------------
Check pod status:
kubectl get pods -l app.kubernetes.io/name=memora
kubectl get pods -l app.kubernetes.io/name=hindsight
View logs for API:
kubectl logs -l app.kubernetes.io/component=api
@@ -123,8 +123,8 @@ Describe a pod:
kubectl describe pod <pod-name>
Check configuration:
kubectl get configmap memora-config -o yaml
kubectl get secret memora-secret -o yaml
kubectl get configmap hindsight-config -o yaml
kubectl get secret hindsight-secret -o yaml
NOTES
-----
+13
View File
@@ -0,0 +1,13 @@
apiVersion: v2
name: hindsight
description: A Helm chart for Hindsight - temporal-semantic-entity memory system for AI agents
type: application
version: 0.0.7
appVersion: "0.0.7"
keywords:
- ai
- memory
- llm
- agents
maintainers:
- name: Hindsight Team
@@ -18,13 +18,13 @@ The application is accessible via the following URL(s):
1. Get the Control Plane URL by running these commands:
{{- if contains "NodePort" .Values.controlPlane.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "memora.fullname" . }}-control-plane)
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "hindsight.fullname" . }}-control-plane)
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo "Control Plane URL: http://$NODE_IP:$NODE_PORT"
{{- else if contains "LoadBalancer" .Values.controlPlane.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "memora.fullname" . }}-control-plane'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "memora.fullname" . }}-control-plane --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
You can watch the status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "hindsight.fullname" . }}-control-plane'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "hindsight.fullname" . }}-control-plane --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
echo "Control Plane URL: http://$SERVICE_IP:{{ .Values.controlPlane.service.port }}"
{{- else if contains "ClusterIP" .Values.controlPlane.service.type }}
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/component=control-plane,app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
@@ -35,19 +35,19 @@ The application is accessible via the following URL(s):
2. Get the API URL by running these commands:
{{- if contains "NodePort" .Values.api.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "memora.fullname" . }}-api)
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "hindsight.fullname" . }}-api)
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo "API URL: http://$NODE_IP:$NODE_PORT"
{{- else if contains "LoadBalancer" .Values.api.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "memora.fullname" . }}-api'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "memora.fullname" . }}-api --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
You can watch the status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "hindsight.fullname" . }}-api'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "hindsight.fullname" . }}-api --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
echo "API URL: http://$SERVICE_IP:{{ .Values.api.service.port }}"
{{- else if contains "ClusterIP" .Values.api.service.type }}
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/component=api,app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
echo "API URL: http://127.0.0.1:8080"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
echo "API URL: http://127.0.0.1:8888"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8888:$CONTAINER_PORT
{{- end }}
{{- end }}
@@ -62,10 +62,10 @@ Please ensure that:
Database migrations run automatically when the API service starts.
If you want to pre-initialize the database before deploying (optional):
kubectl run --namespace {{ .Release.Namespace }} memora-init --rm -it --restart=Never \
kubectl run --namespace {{ .Release.Namespace }} hindsight-init --rm -it --restart=Never \
--image={{ .Values.api.image.repository }}:{{ .Values.api.image.tag }} \
--env="DATABASE_URL={{ include "memora.databaseUrl" . }}" \
-- python -c "from memora.migrations import run_migrations; run_migrations()"
--env="DATABASE_URL={{ include "hindsight.databaseUrl" . }}" \
-- python -c "from hindsight.migrations import run_migrations; run_migrations()"
{{- end }}
For more information, visit: https://github.com/yourusername/memora
For more information, visit: https://github.com/yourusername/hindsight
@@ -2,16 +2,16 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "memora.fullname" . }}-api
name: {{ include "hindsight.fullname" . }}-api
labels:
{{- include "memora.api.labels" . | nindent 4 }}
{{- include "hindsight.api.labels" . | nindent 4 }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.api.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "memora.api.selectorLabels" . | nindent 6 }}
{{- include "hindsight.api.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
@@ -21,10 +21,10 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "memora.api.selectorLabels" . | nindent 8 }}
{{- include "hindsight.api.selectorLabels" . | nindent 8 }}
spec:
{{- if .Values.serviceAccount.create }}
serviceAccountName: {{ include "memora.serviceAccountName" . }}
serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
{{- end }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
@@ -39,37 +39,37 @@ spec:
containerPort: {{ .Values.api.service.targetPort }}
protocol: TCP
env:
- name: MEMORA_API_DATABASE_URL
value: {{ include "memora.databaseUrl" . | quote }}
- name: HINDSIGHT_API_DATABASE_URL
value: {{ include "hindsight.databaseUrl" . | quote }}
{{- if not .Values.postgresql.enabled }}
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "memora.fullname" . }}-secret
name: {{ include "hindsight.fullname" . }}-secret
key: postgres-password
{{- end }}
- name: MEMORA_API_LLM_PROVIDER
- name: HINDSIGHT_API_LLM_PROVIDER
valueFrom:
configMapKeyRef:
name: {{ include "memora.fullname" . }}-config
name: {{ include "hindsight.fullname" . }}-config
key: llm-provider
- name: MEMORA_API_LLM_MODEL
- name: HINDSIGHT_API_LLM_MODEL
valueFrom:
configMapKeyRef:
name: {{ include "memora.fullname" . }}-config
name: {{ include "hindsight.fullname" . }}-config
key: llm-model
{{- if and .Values.api.secrets (hasKey .Values.api.secrets "MEMORA_API_LLM_API_KEY") }}
- name: MEMORA_API_LLM_API_KEY
{{- if and .Values.api.secrets (hasKey .Values.api.secrets "HINDSIGHT_API_LLM_API_KEY") }}
- name: HINDSIGHT_API_LLM_API_KEY
valueFrom:
secretKeyRef:
name: {{ include "memora.fullname" . }}-secret
name: {{ include "hindsight.fullname" . }}-secret
key: llm-api-key
{{- end }}
{{- if and .Values.api.secrets (hasKey .Values.api.secrets "MEMORA_API_LLM_BASE_URL") }}
- name: MEMORA_API_LLM_BASE_URL
{{- if and .Values.api.secrets (hasKey .Values.api.secrets "HINDSIGHT_API_LLM_BASE_URL") }}
- name: HINDSIGHT_API_LLM_BASE_URL
valueFrom:
secretKeyRef:
name: {{ include "memora.fullname" . }}-secret
name: {{ include "hindsight.fullname" . }}-secret
key: llm-base-url
{{- end }}
livenessProbe:
@@ -2,9 +2,9 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "memora.fullname" . }}-api
name: {{ include "hindsight.fullname" . }}-api
labels:
{{- include "memora.api.labels" . | nindent 4 }}
{{- include "hindsight.api.labels" . | nindent 4 }}
spec:
type: {{ .Values.api.service.type }}
ports:
@@ -13,5 +13,5 @@ spec:
protocol: TCP
name: http
selector:
{{- include "memora.api.selectorLabels" . | nindent 4 }}
{{- include "hindsight.api.selectorLabels" . | nindent 4 }}
{{- end }}
+15
View File
@@ -0,0 +1,15 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "hindsight.fullname" . }}-config
labels:
{{- include "hindsight.labels" . | nindent 4 }}
data:
# API configuration
llm-provider: {{ .Values.api.env.HINDSIGHT_API_LLM_PROVIDER | quote }}
llm-model: {{ .Values.api.env.HINDSIGHT_API_LLM_MODEL | quote }}
# Control plane configuration
node-env: {{ .Values.controlPlane.env.NODE_ENV | quote }}
hostname: {{ .Values.controlPlane.env.HINDSIGHT_CP_HOSTNAME | quote }}
control-plane-port: {{ .Values.controlPlane.env.HINDSIGHT_CP_PORT | quote }}
@@ -2,16 +2,16 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "memora.fullname" . }}-control-plane
name: {{ include "hindsight.fullname" . }}-control-plane
labels:
{{- include "memora.controlPlane.labels" . | nindent 4 }}
{{- include "hindsight.controlPlane.labels" . | nindent 4 }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.controlPlane.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "memora.controlPlane.selectorLabels" . | nindent 6 }}
{{- include "hindsight.controlPlane.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
@@ -20,10 +20,10 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "memora.controlPlane.selectorLabels" . | nindent 8 }}
{{- include "hindsight.controlPlane.selectorLabels" . | nindent 8 }}
spec:
{{- if .Values.serviceAccount.create }}
serviceAccountName: {{ include "memora.serviceAccountName" . }}
serviceAccountName: {{ include "hindsight.serviceAccountName" . }}
{{- end }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
@@ -41,20 +41,20 @@ spec:
- name: NODE_ENV
valueFrom:
configMapKeyRef:
name: {{ include "memora.fullname" . }}-config
name: {{ include "hindsight.fullname" . }}-config
key: node-env
- name: MEMORA_CP_HOSTNAME
- name: HINDSIGHT_CP_HOSTNAME
valueFrom:
configMapKeyRef:
name: {{ include "memora.fullname" . }}-config
name: {{ include "hindsight.fullname" . }}-config
key: hostname
- name: MEMORA_CP_PORT
- name: HINDSIGHT_CP_PORT
valueFrom:
configMapKeyRef:
name: {{ include "memora.fullname" . }}-config
name: {{ include "hindsight.fullname" . }}-config
key: control-plane-port
- name: MEMORA_CP_DATAPLANE_API_URL
value: {{ include "memora.apiUrl" . | quote }}
- name: HINDSIGHT_CP_DATAPLANE_API_URL
value: {{ include "hindsight.apiUrl" . | quote }}
livenessProbe:
{{- toYaml .Values.controlPlane.livenessProbe | nindent 10 }}
readinessProbe:
@@ -2,9 +2,9 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "memora.fullname" . }}-control-plane
name: {{ include "hindsight.fullname" . }}-control-plane
labels:
{{- include "memora.controlPlane.labels" . | nindent 4 }}
{{- include "hindsight.controlPlane.labels" . | nindent 4 }}
spec:
type: {{ .Values.controlPlane.service.type }}
ports:
@@ -13,5 +13,5 @@ spec:
protocol: TCP
name: http
selector:
{{- include "memora.controlPlane.selectorLabels" . | nindent 4 }}
{{- include "hindsight.controlPlane.selectorLabels" . | nindent 4 }}
{{- end }}
@@ -3,14 +3,14 @@
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "memora.fullname" . }}-api
name: {{ include "hindsight.fullname" . }}-api
labels:
{{- include "memora.api.labels" . | nindent 4 }}
{{- include "hindsight.api.labels" . | nindent 4 }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "memora.fullname" . }}-api
name: {{ include "hindsight.fullname" . }}-api
minReplicas: {{ .Values.autoscaling.minReplicas }}
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
metrics:
@@ -34,14 +34,14 @@ spec:
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "memora.fullname" . }}-control-plane
name: {{ include "hindsight.fullname" . }}-control-plane
labels:
{{- include "memora.controlPlane.labels" . | nindent 4 }}
{{- include "hindsight.controlPlane.labels" . | nindent 4 }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "memora.fullname" . }}-control-plane
name: {{ include "hindsight.fullname" . }}-control-plane
minReplicas: {{ .Values.autoscaling.minReplicas }}
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
metrics:
@@ -2,9 +2,9 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "memora.fullname" . }}
name: {{ include "hindsight.fullname" . }}
labels:
{{- include "memora.labels" . | nindent 4 }}
{{- include "hindsight.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
@@ -34,11 +34,11 @@ spec:
backend:
service:
{{- if eq .service "api" }}
name: {{ include "memora.fullname" $ }}-api
name: {{ include "hindsight.fullname" $ }}-api
port:
number: {{ $.Values.api.service.port }}
{{- else if eq .service "controlPlane" }}
name: {{ include "memora.fullname" $ }}-control-plane
name: {{ include "hindsight.fullname" $ }}-control-plane
port:
number: {{ $.Values.controlPlane.service.port }}
{{- end }}
@@ -1,9 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: {{ include "memora.fullname" . }}-secret
name: {{ include "hindsight.fullname" . }}-secret
labels:
{{- include "memora.labels" . | nindent 4 }}
{{- include "hindsight.labels" . | nindent 4 }}
type: Opaque
data:
{{- if and .Values.api.secrets (hasKey .Values.api.secrets "MEMORY_LLM_API_KEY") }}
@@ -2,9 +2,9 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "memora.serviceAccountName" . }}
name: {{ include "hindsight.serviceAccountName" . }}
labels:
{{- include "memora.labels" . | nindent 4 }}
{{- include "hindsight.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
@@ -1,4 +1,4 @@
# Default values for memora
# Default values for hindsight
# Global settings
replicaCount: 1
@@ -8,14 +8,14 @@ api:
enabled: true
replicaCount: 1
image:
repository: memora/api
repository: hindsight/api
pullPolicy: IfNotPresent
tag: "latest"
service:
type: ClusterIP
port: 8080
targetPort: 8080
port: 8888
targetPort: 8888
# Resource limits and requests
resources:
@@ -30,7 +30,7 @@ api:
livenessProbe:
httpGet:
path: /
port: 8080
port: 8888
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
@@ -39,7 +39,7 @@ api:
readinessProbe:
httpGet:
path: /
port: 8080
port: 8888
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 3
@@ -47,20 +47,20 @@ api:
# Environment variables
env:
MEMORA_API_LLM_PROVIDER: "groq"
MEMORA_API_LLM_MODEL: "openai/gpt-oss-120b"
HINDSIGHT_API_LLM_PROVIDER: "groq"
HINDSIGHT_API_LLM_MODEL: "openai/gpt-oss-120b"
# Secret environment variables
secrets:
# MEMORA_API_LLM_API_KEY: "your-api-key"
# MEMORA_API_LLM_BASE_URL: "https://api.groq.com/openai/v1"
# HINDSIGHT_API_LLM_API_KEY: "your-api-key"
# HINDSIGHT_API_LLM_BASE_URL: "https://api.groq.com/openai/v1"
# Image settings for control plane
controlPlane:
enabled: true
replicaCount: 1
image:
repository: memora/memora-control-plane
repository: hindsight/hindsight-control-plane
pullPolicy: IfNotPresent
tag: "latest"
@@ -100,8 +100,8 @@ controlPlane:
# Environment variables
env:
NODE_ENV: "production"
MEMORA_CP_HOSTNAME: "0.0.0.0"
MEMORA_CP_PORT: "3000"
HINDSIGHT_CP_HOSTNAME: "0.0.0.0"
HINDSIGHT_CP_PORT: "3000"
# PostgreSQL configuration
postgresql:
@@ -113,8 +113,8 @@ postgresql:
external:
host: "postgresql"
port: 5432
database: "memora"
username: "memora"
database: "hindsight"
username: "hindsight"
# Password should be provided via secret
# password: ""
@@ -130,7 +130,7 @@ ingress:
# nginx.ingress.kubernetes.io/ssl-redirect: "true"
hosts:
- host: memora.example.com
- host: hindsight.example.com
paths:
- path: /
pathType: Prefix
@@ -140,9 +140,9 @@ ingress:
service: api
tls: []
# - secretName: memora-tls
# - secretName: hindsight-tls
# hosts:
# - memora.example.com
# - hindsight.example.com
# Service Account
serviceAccount:
-13
View File
@@ -1,13 +0,0 @@
apiVersion: v2
name: memora
description: A Helm chart for Memora - temporal-semantic-entity memory system for AI agents
type: application
version: 0.0.7
appVersion: "0.0.7"
keywords:
- ai
- memory
- llm
- agents
maintainers:
- name: Memora Team
-15
View File
@@ -1,15 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "memora.fullname" . }}-config
labels:
{{- include "memora.labels" . | nindent 4 }}
data:
# API configuration
llm-provider: {{ .Values.api.env.MEMORA_API_LLM_PROVIDER | quote }}
llm-model: {{ .Values.api.env.MEMORA_API_LLM_MODEL | quote }}
# Control plane configuration
node-env: {{ .Values.controlPlane.env.NODE_ENV | quote }}
hostname: {{ .Values.controlPlane.env.MEMORA_CP_HOSTNAME | quote }}
control-plane-port: {{ .Values.controlPlane.env.MEMORA_CP_PORT | quote }}
@@ -14,13 +14,13 @@ from alembic import context
from dotenv import load_dotenv
# Import your models here
from memora.models import Base
from hindsight_api.models import Base
# Load environment variables based on MEMORA_API_DATABASE_URL env var or default to local
# Load environment variables based on HINDSIGHT_API_DATABASE_URL env var or default to local
def load_env():
"""Load environment variables from .env"""
# Check if MEMORA_API_DATABASE_URL is already set (e.g., by CI/CD)
if os.getenv("MEMORA_API_DATABASE_URL"):
# Check if HINDSIGHT_API_DATABASE_URL is already set (e.g., by CI/CD)
if os.getenv("HINDSIGHT_API_DATABASE_URL"):
return
# Look for .env file in the parent directory (root of the workspace)
@@ -58,11 +58,11 @@ def get_database_url() -> str:
# Get database URL from config (set programmatically) or environment
database_url = config.get_main_option("sqlalchemy.url")
if not database_url:
database_url = os.getenv("MEMORA_API_DATABASE_URL")
database_url = os.getenv("HINDSIGHT_API_DATABASE_URL")
if not database_url:
raise ValueError(
"Database URL not found. "
"Set MEMORA_API_DATABASE_URL environment variable or pass database_url to run_migrations()."
"Set HINDSIGHT_API_DATABASE_URL environment variable or pass database_url to run_migrations()."
)
# For migrations, use psycopg2 (sync driver) to avoid pgbouncer prepared statement issues
@@ -3,8 +3,8 @@ Memory System for AI Agents.
Temporal + Semantic Memory Architecture using PostgreSQL with pgvector.
"""
from .temporal_semantic_memory import TemporalSemanticMemory
from .search_trace import (
from .engine.memory_engine import MemoryEngine
from .engine.search_trace import (
SearchTrace,
QueryInfo,
EntryPoint,
@@ -15,11 +15,12 @@ from .search_trace import (
SearchSummary,
SearchPhaseMetrics,
)
from .search_tracer import SearchTracer
from .embeddings import Embeddings, SentenceTransformersEmbeddings
from .engine.search_tracer import SearchTracer
from .engine.embeddings import Embeddings, SentenceTransformersEmbeddings
from .engine.llm_wrapper import LLMConfig
__all__ = [
"TemporalSemanticMemory",
"MemoryEngine",
"SearchTrace",
"SearchTracer",
"QueryInfo",
@@ -32,5 +33,6 @@ __all__ = [
"SearchPhaseMetrics",
"Embeddings",
"SentenceTransformersEmbeddings",
"LLMConfig",
]
__version__ = "0.1.0"
@@ -1,5 +1,5 @@
"""
Unified API module for Memora.
Unified API module for Hindsight.
Provides both HTTP REST API and MCP (Model Context Protocol) server.
"""
@@ -7,13 +7,13 @@ import logging
from typing import Optional
from fastapi import FastAPI
from memora import TemporalSemanticMemory
from hindsight_api import MemoryEngine
logger = logging.getLogger(__name__)
def create_app(
memory: TemporalSemanticMemory,
memory: MemoryEngine,
http_api_enabled: bool = True,
mcp_api_enabled: bool = False,
mcp_mount_path: str = "/mcp",
@@ -21,10 +21,10 @@ def create_app(
initialize_memory: bool = True
) -> FastAPI:
"""
Create and configure the unified Memora API application.
Create and configure the unified Hindsight API application.
Args:
memory: TemporalSemanticMemory instance (already initialized with required parameters)
memory: MemoryEngine instance (already initialized with required parameters)
http_api_enabled: Whether to enable HTTP REST API endpoints (default: True)
mcp_api_enabled: Whether to enable MCP server (default: False)
mcp_mount_path: Path to mount MCP server (default: /mcp)
@@ -56,7 +56,7 @@ def create_app(
logger.info("HTTP REST API enabled")
else:
# Create minimal FastAPI app
app = FastAPI(title="Memora API", version="0.0.7")
app = FastAPI(title="Hindsight API", version="0.0.7")
logger.info("HTTP REST API disabled")
# Mount MCP server if enabled
@@ -67,12 +67,12 @@ def create_app(
# Create MCP server with shared memory instance
mcp_server = create_mcp_server(memory=memory)
# Mount at specified path
app.mount(mcp_mount_path, mcp_server.sse_app())
logger.info(f"MCP server enabled at {mcp_mount_path}/sse")
# Mount at specified path using http_app (modern non-SSE alternative)
app.mount(mcp_mount_path, mcp_server.http_app())
logger.info(f"MCP server enabled at {mcp_mount_path}")
except ImportError as e:
logger.error(f"MCP server requested but dependencies not available: {e}")
logger.error("Install with: pip install memora[mcp]")
logger.error("Install with: pip install hindsight-api[mcp]")
raise
return app
@@ -87,6 +87,8 @@ from .http import (
BatchPutRequest,
ThinkRequest,
ThinkResponse,
CreateAgentRequest,
PersonalityTraits,
)
__all__ = [
@@ -98,4 +100,6 @@ __all__ = [
"BatchPutRequest",
"ThinkRequest",
"ThinkResponse",
"CreateAgentRequest",
"PersonalityTraits",
]
@@ -14,29 +14,41 @@ from contextlib import asynccontextmanager
from fastapi import FastAPI, HTTPException, Query
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field, ConfigDict
from memora import TemporalSemanticMemory
from hindsight_api import MemoryEngine
from hindsight_api.engine.db_utils import acquire_with_retry
class MetadataFilter(BaseModel):
"""Filter for metadata fields. Matches records where (key=value) OR (key not set) when match_unset=True."""
model_config = ConfigDict(json_schema_extra={
"example": {
"key": "source",
"value": "slack",
"match_unset": True
}
})
key: str = Field(description="Metadata key to filter on")
value: Optional[str] = Field(default=None, description="Value to match. If None with match_unset=True, matches any record where key is not set.")
match_unset: bool = Field(default=True, description="If True, also match records where this metadata key is not set")
class Config:
json_schema_extra = {
"example": {
"key": "source",
"value": "slack",
"match_unset": True
}
}
class SearchRequest(BaseModel):
"""Request model for search endpoint."""
model_config = ConfigDict(json_schema_extra={
"example": {
"query": "What did Alice say about machine learning?",
"fact_type": ["world", "agent"],
"thinking_budget": 100,
"max_tokens": 4096,
"trace": True,
"question_date": "2023-05-30T23:40:00",
"metadata_filter": [{"key": "source", "value": "slack", "match_unset": True}]
}
})
query: str
fact_type: Optional[List[str]] = None # List of fact types to search (defaults to all if not specified)
thinking_budget: int = 100
@@ -45,19 +57,6 @@ class SearchRequest(BaseModel):
question_date: Optional[str] = None # ISO format date string (e.g., "2023-05-30T23:40:00")
metadata_filter: Optional[List[MetadataFilter]] = Field(default=None, description="Filter by metadata. Multiple filters are ANDed together.")
class Config:
json_schema_extra = {
"example": {
"query": "What did Alice say about machine learning?",
"fact_type": ["world", "agent"],
"thinking_budget": 100,
"max_tokens": 4096,
"trace": True,
"question_date": "2023-05-30T23:40:00",
"metadata_filter": [{"key": "source", "value": "slack", "match_unset": True}]
}
}
class SearchResult(BaseModel):
"""Single search result item."""
@@ -87,93 +86,100 @@ class SearchResult(BaseModel):
class SearchResponse(BaseModel):
"""Response model for search endpoints."""
results: List[SearchResult]
trace: Optional[Dict[str, Any]] = None
class Config:
json_schema_extra = {
"example": {
"results": [
{
"id": "123e4567-e89b-12d3-a456-426614174000",
"text": "Alice works at Google on the AI team",
"type": "world",
"context": "work info",
"event_date": "2024-01-15T10:30:00Z"
}
],
"trace": {
"query": "What did Alice say about machine learning?",
"num_results": 1,
"time_seconds": 0.123
model_config = ConfigDict(json_schema_extra={
"example": {
"results": [
{
"id": "123e4567-e89b-12d3-a456-426614174000",
"text": "Alice works at Google on the AI team",
"type": "world",
"context": "work info",
"event_date": "2024-01-15T10:30:00Z"
}
],
"trace": {
"query": "What did Alice say about machine learning?",
"num_results": 1,
"time_seconds": 0.123
}
}
})
results: List[SearchResult]
trace: Optional[Dict[str, Any]] = None
class MemoryItem(BaseModel):
"""Single memory item for batch put."""
model_config = ConfigDict(json_schema_extra={
"example": {
"content": "Alice mentioned she's working on a new ML model",
"event_date": "2024-01-15T10:30:00Z",
"context": "team meeting",
"metadata": {"source": "slack", "channel": "engineering"}
}
})
content: str
event_date: Optional[datetime] = None
context: Optional[str] = None
metadata: Optional[Dict[str, str]] = None
class Config:
json_schema_extra = {
"example": {
"content": "Alice mentioned she's working on a new ML model",
"event_date": "2024-01-15T10:30:00Z",
"context": "team meeting",
"metadata": {"source": "slack", "channel": "engineering"}
}
}
class BatchPutRequest(BaseModel):
"""Request model for batch put endpoint."""
model_config = ConfigDict(json_schema_extra={
"example": {
"items": [
{
"content": "Alice works at Google",
"context": "work"
},
{
"content": "Bob went hiking yesterday",
"event_date": "2024-01-15T10:00:00Z"
}
],
"document_id": "conversation_123"
}
})
items: List[MemoryItem]
document_id: Optional[str] = None
class Config:
json_schema_extra = {
"example": {
"items": [
{
"content": "Alice works at Google",
"context": "work"
},
{
"content": "Bob went hiking yesterday",
"event_date": "2024-01-15T10:00:00Z"
}
],
"document_id": "conversation_123"
}
}
class BatchPutResponse(BaseModel):
"""Response model for batch put endpoint."""
model_config = ConfigDict(json_schema_extra={
"example": {
"success": True,
"message": "Successfully stored 2 memory items",
"agent_id": "user123",
"document_id": "conversation_123",
"items_count": 2
}
})
success: bool
message: str
agent_id: str
document_id: Optional[str] = None
items_count: int
class Config:
json_schema_extra = {
"example": {
"success": True,
"message": "Successfully stored 2 memory items",
"agent_id": "user123",
"document_id": "conversation_123",
"items_count": 2
}
}
class BatchPutAsyncResponse(BaseModel):
"""Response model for async batch put endpoint."""
model_config = ConfigDict(json_schema_extra={
"example": {
"success": True,
"message": "Batch put task queued for background processing",
"agent_id": "user123",
"document_id": "conversation_123",
"items_count": 2,
"queued": True
}
})
success: bool
message: str
agent_id: str
@@ -181,36 +187,23 @@ class BatchPutAsyncResponse(BaseModel):
items_count: int
queued: bool
class Config:
json_schema_extra = {
"example": {
"success": True,
"message": "Batch put task queued for background processing",
"agent_id": "user123",
"document_id": "conversation_123",
"items_count": 2,
"queued": True
}
}
class ThinkRequest(BaseModel):
"""Request model for think endpoint."""
model_config = ConfigDict(json_schema_extra={
"example": {
"query": "What do you think about artificial intelligence?",
"thinking_budget": 50,
"context": "This is for a research paper on AI ethics",
"metadata_filter": [{"key": "source", "value": "slack", "match_unset": True}]
}
})
query: str
thinking_budget: int = 50
context: Optional[str] = None
metadata_filter: Optional[List[MetadataFilter]] = Field(default=None, description="Filter by metadata. Multiple filters are ANDed together.")
class Config:
json_schema_extra = {
"example": {
"query": "What do you think about artificial intelligence?",
"thinking_budget": 50,
"context": "This is for a research paper on AI ethics",
"metadata_filter": [{"key": "source", "value": "slack", "match_unset": True}]
}
}
class OpinionItem(BaseModel):
"""Model for an opinion with confidence score."""
@@ -220,67 +213,75 @@ class OpinionItem(BaseModel):
class ThinkFact(BaseModel):
"""A fact used in think response."""
model_config = ConfigDict(json_schema_extra={
"example": {
"id": "123e4567-e89b-12d3-a456-426614174000",
"text": "AI is used in healthcare",
"type": "world",
"context": "healthcare discussion",
"event_date": "2024-01-15T10:30:00Z"
}
})
id: Optional[str] = None
text: str
type: Optional[str] = None # fact type: world, agent, opinion
context: Optional[str] = None
event_date: Optional[str] = None
class Config:
json_schema_extra = {
"example": {
"id": "123e4567-e89b-12d3-a456-426614174000",
"text": "AI is used in healthcare",
"type": "world",
"context": "healthcare discussion",
"event_date": "2024-01-15T10:30:00Z"
}
}
class ThinkResponse(BaseModel):
"""Response model for think endpoint."""
model_config = ConfigDict(json_schema_extra={
"example": {
"text": "Based on my understanding, AI is a transformative technology...",
"based_on": [
{
"id": "123",
"text": "AI is used in healthcare",
"type": "world"
},
{
"id": "456",
"text": "I discussed AI applications last week",
"type": "agent"
}
],
"new_opinions": [
"AI has great potential when used responsibly"
]
}
})
text: str
based_on: List[ThinkFact] = [] # Facts used to generate the response
new_opinions: List[str] = [] # Simplified to list of opinion strings
class Config:
json_schema_extra = {
"example": {
"text": "Based on my understanding, AI is a transformative technology...",
"based_on": [
{
"id": "123",
"text": "AI is used in healthcare",
"type": "world"
},
{
"id": "456",
"text": "I discussed AI applications last week",
"type": "agent"
}
],
"new_opinions": [
"AI has great potential when used responsibly"
]
}
}
class AgentsResponse(BaseModel):
"""Response model for agents list endpoint."""
agents: List[str]
class Config:
json_schema_extra = {
"example": {
"agents": ["user123", "agent_alice", "agent_bob"]
}
model_config = ConfigDict(json_schema_extra={
"example": {
"agents": ["user123", "agent_alice", "agent_bob"]
}
})
agents: List[str]
class PersonalityTraits(BaseModel):
"""Personality traits based on Big Five model."""
model_config = ConfigDict(json_schema_extra={
"example": {
"openness": 0.8,
"conscientiousness": 0.6,
"extraversion": 0.5,
"agreeableness": 0.7,
"neuroticism": 0.3,
"bias_strength": 0.7
}
})
openness: float = Field(ge=0.0, le=1.0, description="Openness to experience (0-1)")
conscientiousness: float = Field(ge=0.0, le=1.0, description="Conscientiousness (0-1)")
extraversion: float = Field(ge=0.0, le=1.0, description="Extraversion (0-1)")
@@ -288,43 +289,30 @@ class PersonalityTraits(BaseModel):
neuroticism: float = Field(ge=0.0, le=1.0, description="Neuroticism (0-1)")
bias_strength: float = Field(ge=0.0, le=1.0, description="How strongly personality influences opinions (0-1)")
class Config:
json_schema_extra = {
"example": {
class AgentProfileResponse(BaseModel):
"""Response model for agent profile."""
model_config = ConfigDict(json_schema_extra={
"example": {
"agent_id": "user123",
"name": "Alice",
"personality": {
"openness": 0.8,
"conscientiousness": 0.6,
"extraversion": 0.5,
"agreeableness": 0.7,
"neuroticism": 0.3,
"bias_strength": 0.7
}
},
"background": "I am a software engineer with 10 years of experience in startups"
}
})
class AgentProfileResponse(BaseModel):
"""Response model for agent profile."""
agent_id: str
name: str
personality: PersonalityTraits
background: str
class Config:
json_schema_extra = {
"example": {
"agent_id": "user123",
"name": "Alice",
"personality": {
"openness": 0.8,
"conscientiousness": 0.6,
"extraversion": 0.5,
"agreeableness": 0.7,
"neuroticism": 0.3,
"bias_strength": 0.7
},
"background": "I am a software engineer with 10 years of experience in startups"
}
}
class UpdatePersonalityRequest(BaseModel):
"""Request model for updating personality traits."""
@@ -333,40 +321,38 @@ class UpdatePersonalityRequest(BaseModel):
class AddBackgroundRequest(BaseModel):
"""Request model for adding/merging background information."""
model_config = ConfigDict(json_schema_extra={
"example": {
"content": "I was born in Texas",
"update_personality": True
}
})
content: str = Field(description="New background information to add or merge")
update_personality: bool = Field(
default=True,
description="If true, infer Big Five personality traits from the merged background (default: true)"
)
class Config:
json_schema_extra = {
"example": {
"content": "I was born in Texas",
"update_personality": True
}
}
class BackgroundResponse(BaseModel):
"""Response model for background update."""
background: str
personality: Optional[PersonalityTraits] = None
class Config:
json_schema_extra = {
"example": {
"background": "I was born in Texas. I am a software engineer with 10 years of experience.",
"personality": {
"openness": 0.7,
"conscientiousness": 0.6,
"extraversion": 0.5,
"agreeableness": 0.8,
"neuroticism": 0.4,
"bias_strength": 0.6
}
model_config = ConfigDict(json_schema_extra={
"example": {
"background": "I was born in Texas. I am a software engineer with 10 years of experience.",
"personality": {
"openness": 0.7,
"conscientiousness": 0.6,
"extraversion": 0.5,
"agreeableness": 0.8,
"neuroticism": 0.4,
"bias_strength": 0.6
}
}
})
background: str
personality: Optional[PersonalityTraits] = None
class AgentListItem(BaseModel):
@@ -381,137 +367,144 @@ class AgentListItem(BaseModel):
class AgentListResponse(BaseModel):
"""Response model for listing all agents."""
agents: List[AgentListItem]
class Config:
json_schema_extra = {
"example": {
"agents": [
{
"agent_id": "user123",
"name": "Alice",
"personality": {
"openness": 0.5,
"conscientiousness": 0.5,
"extraversion": 0.5,
"agreeableness": 0.5,
"neuroticism": 0.5,
"bias_strength": 0.5
},
"background": "I am a software engineer",
"created_at": "2024-01-15T10:30:00Z",
"updated_at": "2024-01-16T14:20:00Z"
}
]
}
model_config = ConfigDict(json_schema_extra={
"example": {
"agents": [
{
"agent_id": "user123",
"name": "Alice",
"personality": {
"openness": 0.5,
"conscientiousness": 0.5,
"extraversion": 0.5,
"agreeableness": 0.5,
"neuroticism": 0.5,
"bias_strength": 0.5
},
"background": "I am a software engineer",
"created_at": "2024-01-15T10:30:00Z",
"updated_at": "2024-01-16T14:20:00Z"
}
]
}
})
agents: List[AgentListItem]
class CreateAgentRequest(BaseModel):
"""Request model for creating/updating an agent."""
model_config = ConfigDict(json_schema_extra={
"example": {
"name": "Alice",
"personality": {
"openness": 0.8,
"conscientiousness": 0.6,
"extraversion": 0.5,
"agreeableness": 0.7,
"neuroticism": 0.3,
"bias_strength": 0.7
},
"background": "I am a creative software engineer with 10 years of experience"
}
})
name: Optional[str] = None
personality: Optional[PersonalityTraits] = None
background: Optional[str] = None
class Config:
json_schema_extra = {
"example": {
"name": "Alice",
"personality": {
"openness": 0.8,
"conscientiousness": 0.6,
"extraversion": 0.5,
"agreeableness": 0.7,
"neuroticism": 0.3,
"bias_strength": 0.7
},
"background": "I am a creative software engineer with 10 years of experience"
}
}
class GraphDataResponse(BaseModel):
"""Response model for graph data endpoint."""
model_config = ConfigDict(json_schema_extra={
"example": {
"nodes": [
{"id": "1", "label": "Alice works at Google", "type": "world"},
{"id": "2", "label": "Bob went hiking", "type": "world"}
],
"edges": [
{"from": "1", "to": "2", "type": "semantic", "weight": 0.8}
],
"table_rows": [
{"id": "abc12345...", "text": "Alice works at Google", "context": "Work info", "date": "2024-01-15 10:30", "entities": "Alice (PERSON), Google (ORGANIZATION)"}
],
"total_units": 2
}
})
nodes: List[Dict[str, Any]]
edges: List[Dict[str, Any]]
table_rows: List[Dict[str, Any]]
total_units: int
class Config:
json_schema_extra = {
"example": {
"nodes": [
{"id": "1", "label": "Alice works at Google", "type": "world"},
{"id": "2", "label": "Bob went hiking", "type": "world"}
],
"edges": [
{"from": "1", "to": "2", "type": "semantic", "weight": 0.8}
],
"table_rows": [
{"id": "abc12345...", "text": "Alice works at Google", "context": "Work info", "date": "2024-01-15 10:30", "entities": "Alice (PERSON), Google (ORGANIZATION)"}
],
"total_units": 2
}
}
class ListMemoryUnitsResponse(BaseModel):
"""Response model for list memory units endpoint."""
model_config = ConfigDict(json_schema_extra={
"example": {
"items": [
{
"id": "550e8400-e29b-41d4-a716-446655440000",
"text": "Alice works at Google on the AI team",
"context": "Work conversation",
"date": "2024-01-15T10:30:00Z",
"fact_type": "world",
"entities": "Alice (PERSON), Google (ORGANIZATION)"
}
],
"total": 150,
"limit": 100,
"offset": 0
}
})
items: List[Dict[str, Any]]
total: int
limit: int
offset: int
class Config:
json_schema_extra = {
"example": {
"items": [
{
"id": "550e8400-e29b-41d4-a716-446655440000",
"text": "Alice works at Google on the AI team",
"context": "Work conversation",
"date": "2024-01-15T10:30:00Z",
"fact_type": "world",
"entities": "Alice (PERSON), Google (ORGANIZATION)"
}
],
"total": 150,
"limit": 100,
"offset": 0
}
}
class ListDocumentsResponse(BaseModel):
"""Response model for list documents endpoint."""
model_config = ConfigDict(json_schema_extra={
"example": {
"items": [
{
"id": "session_1",
"agent_id": "user123",
"content_hash": "abc123",
"created_at": "2024-01-15T10:30:00Z",
"updated_at": "2024-01-15T10:30:00Z",
"text_length": 5420,
"memory_unit_count": 15
}
],
"total": 50,
"limit": 100,
"offset": 0
}
})
items: List[Dict[str, Any]]
total: int
limit: int
offset: int
class Config:
json_schema_extra = {
"example": {
"items": [
{
"id": "session_1",
"agent_id": "user123",
"content_hash": "abc123",
"created_at": "2024-01-15T10:30:00Z",
"updated_at": "2024-01-15T10:30:00Z",
"text_length": 5420,
"memory_unit_count": 15
}
],
"total": 50,
"limit": 100,
"offset": 0
}
}
class DocumentResponse(BaseModel):
"""Response model for get document endpoint."""
model_config = ConfigDict(json_schema_extra={
"example": {
"id": "session_1",
"agent_id": "user123",
"original_text": "Full document text here...",
"content_hash": "abc123",
"created_at": "2024-01-15T10:30:00Z",
"updated_at": "2024-01-15T10:30:00Z",
"memory_unit_count": 15
}
})
id: str
agent_id: str
original_text: str
@@ -520,40 +513,26 @@ class DocumentResponse(BaseModel):
updated_at: str
memory_unit_count: int
class Config:
json_schema_extra = {
"example": {
"id": "session_1",
"agent_id": "user123",
"original_text": "Full document text here...",
"content_hash": "abc123",
"created_at": "2024-01-15T10:30:00Z",
"updated_at": "2024-01-15T10:30:00Z",
"memory_unit_count": 15
}
}
class DeleteResponse(BaseModel):
"""Response model for delete operations."""
model_config = ConfigDict(json_schema_extra={
"example": {
"success": True,
"message": "Resource deleted successfully"
}
})
success: bool
message: str
class Config:
json_schema_extra = {
"example": {
"success": True,
"message": "Resource deleted successfully"
}
}
def create_app(memory: TemporalSemanticMemory, run_migrations: bool = True, initialize_memory: bool = True) -> FastAPI:
def create_app(memory: MemoryEngine, run_migrations: bool = True, initialize_memory: bool = True) -> FastAPI:
"""
Create and configure the FastAPI application.
Args:
memory: TemporalSemanticMemory instance (already initialized with required parameters)
memory: MemoryEngine instance (already initialized with required parameters)
run_migrations: Whether to run database migrations on startup (default: True)
initialize_memory: Whether to initialize memory system on startup (default: True)
@@ -572,15 +551,17 @@ def create_app(memory: TemporalSemanticMemory, run_migrations: bool = True, init
Note: This only fires when running the app standalone, not when mounted.
"""
# Startup: Initialize database and memory system
if run_migrations:
from memora.migrations import run_migrations as do_migrations
do_migrations(memory.db_url)
logging.info("Database migrations applied")
if initialize_memory:
await memory.initialize()
logging.info("Memory system initialized")
if run_migrations:
from hindsight_api.migrations import run_migrations as do_migrations
do_migrations(memory.db_url)
logging.info("Database migrations applied")
yield
# Shutdown: Cleanup memory system
@@ -860,7 +841,7 @@ def _register_routes(app: FastAPI):
"""Get statistics about memory nodes and links for an agent."""
try:
pool = await app.state.memory._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
# Get node counts by fact_type
node_stats = await conn.fetch(
"""
@@ -1195,7 +1176,7 @@ This operation cannot be undone.
# Insert operation record into database BEFORE scheduling task
pool = await app.state.memory._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
await conn.execute(
"""
INSERT INTO async_operations (id, agent_id, task_type, items_count, document_id)
@@ -1245,7 +1226,7 @@ This operation cannot be undone.
"""List all async operations (pending and failed) for an agent."""
try:
pool = await app.state.memory._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
operations = await conn.fetch(
"""
SELECT id, agent_id, task_type, items_count, document_id, created_at, status, error_message
@@ -1296,7 +1277,7 @@ This operation cannot be undone.
raise HTTPException(status_code=400, detail=f"Invalid operation_id format: {operation_id}")
pool = await app.state.memory._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
# Check if operation exists and belongs to this agent
result = await conn.fetchrow(
"SELECT agent_id FROM async_operations WHERE id = $1 AND agent_id = $2",
@@ -1468,7 +1449,7 @@ This operation cannot be undone.
# Update name if provided
if request.name is not None:
pool = await app.state.memory._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
await conn.execute(
"""
UPDATE agents
@@ -1492,7 +1473,7 @@ This operation cannot be undone.
# Update background if provided (replace, not merge)
if request.background is not None:
pool = await app.state.memory._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
await conn.execute(
"""
UPDATE agents
@@ -1,30 +1,30 @@
"""Memora MCP Server implementation using FastMCP."""
"""Hindsight MCP Server implementation using FastMCP."""
import json
import logging
from fastmcp import FastMCP
from memora import TemporalSemanticMemory
from hindsight_api import MemoryEngine
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def create_mcp_server(memory: TemporalSemanticMemory) -> FastMCP:
def create_mcp_server(memory: MemoryEngine) -> FastMCP:
"""
Create and configure the Memora MCP server.
Create and configure the Hindsight MCP server.
Args:
memory: TemporalSemanticMemory instance (required)
memory: MemoryEngine instance (required)
Returns:
Configured FastMCP server instance
"""
# Create FastMCP server
mcp = FastMCP("memora-mcp-server")
mcp = FastMCP("hindsight-mcp-server")
@mcp.tool()
async def memora_put(agent_id: str, content: str, context: str, explanation: str = "") -> str:
async def hindsight_put(agent_id: str, content: str, context: str, explanation: str = "") -> str:
"""
**CRITICAL: Store important user information to long-term memory.**
@@ -74,7 +74,7 @@ def create_mcp_server(memory: TemporalSemanticMemory) -> FastMCP:
return f"Error: {str(e)}"
@mcp.tool()
async def memora_search(agent_id: str, query: str, max_tokens: int = 4096, explanation: str = "") -> str:
async def hindsight_search(agent_id: str, query: str, max_tokens: int = 4096, explanation: str = "") -> str:
"""
**CRITICAL: Search user's memory to provide personalized, context-aware responses.**
@@ -112,7 +112,7 @@ def create_mcp_server(memory: TemporalSemanticMemory) -> FastMCP:
"""
try:
# Log all parameters for debugging
logger.info(f"memora_search called with: query={query!r}, max_tokens={max_tokens}, explanation={explanation!r}")
logger.info(f"hindsight_search called with: query={query!r}, max_tokens={max_tokens}, explanation={explanation!r}")
# Log explanation if provided
if explanation:
@@ -0,0 +1,47 @@
"""
Memory Engine - Core implementation of the memory system.
This package contains all the implementation details of the memory engine:
- MemoryEngine: Main class for memory operations
- Utility modules: embedding_utils, link_utils, think_utils, agent_utils
- Supporting modules: embeddings, cross_encoder, entity_resolver, etc.
"""
from .memory_engine import MemoryEngine
from .db_utils import acquire_with_retry
from .embeddings import Embeddings, SentenceTransformersEmbeddings
from .search_trace import (
SearchTrace,
QueryInfo,
EntryPoint,
NodeVisit,
WeightComponents,
LinkInfo,
PruningDecision,
SearchSummary,
SearchPhaseMetrics,
)
from .search_tracer import SearchTracer
from .llm_wrapper import LLMConfig
from .response_models import SearchResult, ThinkResult, MemoryFact
__all__ = [
"MemoryEngine",
"acquire_with_retry",
"Embeddings",
"SentenceTransformersEmbeddings",
"SearchTrace",
"SearchTracer",
"QueryInfo",
"EntryPoint",
"NodeVisit",
"WeightComponents",
"LinkInfo",
"PruningDecision",
"SearchSummary",
"SearchPhaseMetrics",
"LLMConfig",
"SearchResult",
"ThinkResult",
"MemoryFact",
]
@@ -0,0 +1,426 @@
"""
Agent profile utilities for personality and background management.
"""
import json
import logging
import re
from typing import Dict, Optional
from pydantic import BaseModel, Field
from .db_utils import acquire_with_retry
logger = logging.getLogger(__name__)
DEFAULT_PERSONALITY = {
"openness": 0.5,
"conscientiousness": 0.5,
"extraversion": 0.5,
"agreeableness": 0.5,
"neuroticism": 0.5,
"bias_strength": 0.5,
}
class PersonalityTraits(BaseModel):
"""Big Five personality traits with bias strength (all values 0.0-1.0)."""
openness: float = Field(description="Creativity, curiosity, openness to new ideas (0.0-1.0)")
conscientiousness: float = Field(description="Organization, discipline, goal-directed (0.0-1.0)")
extraversion: float = Field(description="Sociability, assertiveness, energy from others (0.0-1.0)")
agreeableness: float = Field(description="Cooperation, empathy, consideration (0.0-1.0)")
neuroticism: float = Field(description="Emotional sensitivity, anxiety, stress response (0.0-1.0)")
bias_strength: float = Field(description="How much personality influences opinions (0.0-1.0)")
class BackgroundMergeResponse(BaseModel):
"""LLM response for background merge with personality inference."""
background: str = Field(description="Merged background in first person perspective")
personality: PersonalityTraits = Field(description="Inferred Big Five personality traits")
async def get_agent_profile(pool, agent_id: str) -> Dict:
"""
Get agent profile (name, personality + background).
Auto-creates agent with default values if not exists.
Args:
pool: Database connection pool
agent_id: Agent identifier
Returns:
Dict with 'name' (str), 'personality' (dict) and 'background' (str) keys
"""
async with acquire_with_retry(pool) as conn:
# Try to get existing agent
row = await conn.fetchrow(
"""
SELECT name, personality, background
FROM agents
WHERE agent_id = $1
""",
agent_id
)
if row:
# asyncpg returns JSONB as a string, so parse it
personality_data = row["personality"]
if isinstance(personality_data, str):
personality_data = json.loads(personality_data)
return {
"name": row["name"],
"personality": personality_data,
"background": row["background"]
}
# Agent doesn't exist, create with defaults
await conn.execute(
"""
INSERT INTO agents (agent_id, name, personality, background)
VALUES ($1, $2, $3::jsonb, $4)
ON CONFLICT (agent_id) DO NOTHING
""",
agent_id,
agent_id, # Default name is the agent_id
json.dumps(DEFAULT_PERSONALITY),
""
)
return {
"name": agent_id,
"personality": DEFAULT_PERSONALITY.copy(),
"background": ""
}
async def update_agent_personality(
pool,
agent_id: str,
personality: Dict[str, float]
) -> None:
"""
Update agent personality traits.
Args:
pool: Database connection pool
agent_id: Agent identifier
personality: Dict with Big Five traits + bias_strength (all 0-1)
"""
# Ensure agent exists first
await get_agent_profile(pool, agent_id)
async with acquire_with_retry(pool) as conn:
await conn.execute(
"""
UPDATE agents
SET personality = $2::jsonb,
updated_at = NOW()
WHERE agent_id = $1
""",
agent_id,
json.dumps(personality)
)
async def merge_agent_background(
pool,
llm_config,
agent_id: str,
new_info: str,
update_personality: bool = True
) -> dict:
"""
Merge new background information with existing background using LLM.
Normalizes to first person ("I") and resolves conflicts.
Optionally infers personality traits from the merged background.
Args:
pool: Database connection pool
llm_config: LLM configuration for background merging
agent_id: Agent identifier
new_info: New background information to add/merge
update_personality: If True, infer Big Five traits from background (default: True)
Returns:
Dict with 'background' (str) and optionally 'personality' (dict) keys
"""
# Get current profile
profile = await get_agent_profile(pool, agent_id)
current_background = profile["background"]
# Use LLM to merge backgrounds and optionally infer personality
result = await _llm_merge_background(
llm_config,
current_background,
new_info,
infer_personality=update_personality
)
merged_background = result["background"]
inferred_personality = result.get("personality")
# Update in database
async with acquire_with_retry(pool) as conn:
if inferred_personality:
# Update both background and personality
await conn.execute(
"""
UPDATE agents
SET background = $2,
personality = $3::jsonb,
updated_at = NOW()
WHERE agent_id = $1
""",
agent_id,
merged_background,
json.dumps(inferred_personality)
)
else:
# Update only background
await conn.execute(
"""
UPDATE agents
SET background = $2,
updated_at = NOW()
WHERE agent_id = $1
""",
agent_id,
merged_background
)
response = {"background": merged_background}
if inferred_personality:
response["personality"] = inferred_personality
return response
async def _llm_merge_background(
llm_config,
current: str,
new_info: str,
infer_personality: bool = False
) -> dict:
"""
Use LLM to intelligently merge background information.
Optionally infer Big Five personality traits from the merged background.
Args:
llm_config: LLM configuration to use
current: Current background text
new_info: New information to merge
infer_personality: If True, also infer personality traits
Returns:
Dict with 'background' (str) and optionally 'personality' (dict) keys
"""
if infer_personality:
prompt = f"""You are helping maintain an agent's background/profile and infer their personality. You MUST respond with ONLY valid JSON.
Current background: {current if current else "(empty)"}
New information to add: {new_info}
Instructions:
1. Merge the new information with the current background
2. If there are conflicts (e.g., different birthplaces), the NEW information overwrites the old
3. Keep additions that don't conflict
4. Output in FIRST PERSON ("I") perspective
5. Be concise - keep merged background under 500 characters
6. Infer Big Five personality traits from the merged background:
- Openness: 0.0-1.0 (creativity, curiosity, openness to new ideas)
- Conscientiousness: 0.0-1.0 (organization, discipline, goal-directed)
- Extraversion: 0.0-1.0 (sociability, assertiveness, energy from others)
- Agreeableness: 0.0-1.0 (cooperation, empathy, consideration)
- Neuroticism: 0.0-1.0 (emotional sensitivity, anxiety, stress response)
- Bias Strength: 0.0-1.0 (how much personality influences opinions)
CRITICAL: You MUST respond with ONLY a valid JSON object. No markdown, no code blocks, no explanations. Just the JSON.
Format:
{{
"background": "the merged background text in first person",
"personality": {{
"openness": 0.7,
"conscientiousness": 0.6,
"extraversion": 0.5,
"agreeableness": 0.8,
"neuroticism": 0.4,
"bias_strength": 0.6
}}
}}
Trait inference examples:
- "creative artist" → openness: 0.8+, bias_strength: 0.6
- "organized engineer" → conscientiousness: 0.8+, openness: 0.5-0.6
- "startup founder" → openness: 0.8+, extraversion: 0.7+, neuroticism: 0.3-0.4
- "risk-averse analyst" → openness: 0.3-0.4, conscientiousness: 0.8+, neuroticism: 0.6+
- "rational and diligent" → conscientiousness: 0.7+, openness: 0.6+
- "passionate and dramatic" → extraversion: 0.7+, neuroticism: 0.6+, openness: 0.7+"""
else:
prompt = f"""You are helping maintain an agent's background/profile.
Current background: {current if current else "(empty)"}
New information to add: {new_info}
Instructions:
1. Merge the new information with the current background
2. If there are conflicts (e.g., different birthplaces), the NEW information overwrites the old
3. Keep additions that don't conflict
4. Output in FIRST PERSON ("I") perspective
5. Be concise - keep it under 500 characters
6. Return ONLY the merged background text, no explanations
Merged background:"""
try:
# Prepare messages
messages = [{"role": "user", "content": prompt}]
if infer_personality:
# Use structured output with Pydantic model for personality inference
try:
parsed = await llm_config.call(
messages=messages,
response_format=BackgroundMergeResponse,
scope="agent_background",
temperature=0.3,
max_tokens=8192
)
logger.info(f"Successfully got structured response: background={parsed.background[:100]}")
# Convert Pydantic model to dict format
return {
"background": parsed.background,
"personality": parsed.personality.model_dump()
}
except Exception as e:
logger.warning(f"Structured output failed, falling back to manual parsing: {e}")
# Fall through to manual parsing below
# Manual parsing fallback or non-personality merge
content = await llm_config.call(
messages=messages,
scope="agent_background",
temperature=0.3,
max_tokens=8192
)
logger.info(f"LLM response for background merge (first 500 chars): {content[:500]}")
if infer_personality:
# Parse JSON response - try multiple extraction methods
result = None
# Method 1: Direct parse
try:
result = json.loads(content)
logger.info("Successfully parsed JSON directly")
except json.JSONDecodeError:
pass
# Method 2: Extract from markdown code blocks
if result is None:
# Remove markdown code blocks
code_block_match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', content, re.DOTALL)
if code_block_match:
try:
result = json.loads(code_block_match.group(1))
logger.info("Successfully extracted JSON from markdown code block")
except json.JSONDecodeError:
pass
# Method 3: Find nested JSON structure
if result is None:
# Look for JSON object with nested structure
json_match = re.search(r'\{[^{}]*"background"[^{}]*"personality"[^{}]*\{[^{}]*\}[^{}]*\}', content, re.DOTALL)
if json_match:
try:
result = json.loads(json_match.group())
logger.info("Successfully extracted JSON using nested pattern")
except json.JSONDecodeError:
pass
# All parsing methods failed - use fallback
if result is None:
logger.warning(f"Failed to extract JSON from LLM response. Raw content: {content[:200]}")
# Fallback: use new_info as background with default personality
return {
"background": new_info if new_info else current if current else "",
"personality": DEFAULT_PERSONALITY.copy()
}
# Validate personality values
personality = result.get("personality", {})
for key in ["openness", "conscientiousness", "extraversion",
"agreeableness", "neuroticism", "bias_strength"]:
if key not in personality:
personality[key] = 0.5 # Default to neutral
else:
# Clamp to [0, 1]
personality[key] = max(0.0, min(1.0, float(personality[key])))
result["personality"] = personality
# Ensure background exists
if "background" not in result or not result["background"]:
result["background"] = new_info if new_info else ""
return result
else:
# Just background merge
merged = content
if not merged or merged.lower() in ["(empty)", "none", "n/a"]:
merged = new_info if new_info else ""
return {"background": merged}
except Exception as e:
logger.error(f"Error merging background with LLM: {e}")
# Fallback: just append new info
if current:
merged = f"{current} {new_info}".strip()
else:
merged = new_info
result = {"background": merged}
if infer_personality:
result["personality"] = DEFAULT_PERSONALITY.copy()
return result
async def list_agents(pool) -> list:
"""
List all agents in the system.
Args:
pool: Database connection pool
Returns:
List of dicts with agent_id, name, personality, background, created_at, updated_at
"""
async with acquire_with_retry(pool) as conn:
rows = await conn.fetch(
"""
SELECT agent_id, name, personality, background, created_at, updated_at
FROM agents
ORDER BY updated_at DESC
"""
)
result = []
for row in rows:
# asyncpg returns JSONB as a string, so parse it
personality_data = row["personality"]
if isinstance(personality_data, str):
personality_data = json.loads(personality_data)
result.append({
"agent_id": row["agent_id"],
"name": row["name"],
"personality": personality_data,
"background": row["background"],
"created_at": row["created_at"].isoformat() if row["created_at"] else None,
"updated_at": row["updated_at"].isoformat() if row["updated_at"] else None,
})
return result
@@ -0,0 +1,93 @@
"""
Database utility functions for connection management with retry logic.
"""
import asyncio
import logging
from contextlib import asynccontextmanager
import asyncpg
logger = logging.getLogger(__name__)
# Default retry configuration for database operations
DEFAULT_MAX_RETRIES = 3
DEFAULT_BASE_DELAY = 0.5 # seconds
DEFAULT_MAX_DELAY = 5.0 # seconds
# Exceptions that indicate transient connection issues worth retrying
RETRYABLE_EXCEPTIONS = (
asyncpg.exceptions.InterfaceError,
asyncpg.exceptions.ConnectionDoesNotExistError,
asyncpg.exceptions.TooManyConnectionsError,
OSError,
ConnectionError,
asyncio.TimeoutError,
)
async def retry_with_backoff(
func,
max_retries: int = DEFAULT_MAX_RETRIES,
base_delay: float = DEFAULT_BASE_DELAY,
max_delay: float = DEFAULT_MAX_DELAY,
retryable_exceptions: tuple = RETRYABLE_EXCEPTIONS,
):
"""
Execute an async function with exponential backoff retry.
Args:
func: Async function to execute
max_retries: Maximum number of retry attempts
base_delay: Initial delay between retries (seconds)
max_delay: Maximum delay between retries (seconds)
retryable_exceptions: Tuple of exception types to retry on
Returns:
Result of the function
Raises:
The last exception if all retries fail
"""
last_exception = None
for attempt in range(max_retries + 1):
try:
return await func()
except retryable_exceptions as e:
last_exception = e
if attempt < max_retries:
delay = min(base_delay * (2 ** attempt), max_delay)
logger.warning(
f"Database operation failed (attempt {attempt + 1}/{max_retries + 1}): {e}. "
f"Retrying in {delay:.1f}s..."
)
await asyncio.sleep(delay)
else:
logger.error(
f"Database operation failed after {max_retries + 1} attempts: {e}"
)
raise last_exception
@asynccontextmanager
async def acquire_with_retry(pool: asyncpg.Pool, max_retries: int = DEFAULT_MAX_RETRIES):
"""
Async context manager to acquire a connection with retry logic.
Usage:
async with acquire_with_retry(pool) as conn:
await conn.execute(...)
Args:
pool: The asyncpg connection pool
max_retries: Maximum number of retry attempts
Yields:
An asyncpg connection
"""
async def acquire():
return await pool.acquire()
conn = await retry_with_backoff(acquire, max_retries=max_retries)
try:
yield conn
finally:
await pool.release(conn)
@@ -0,0 +1,54 @@
"""
Embedding generation utilities for memory units.
"""
import asyncio
import logging
from typing import List
logger = logging.getLogger(__name__)
def generate_embedding(embeddings_backend, text: str) -> List[float]:
"""
Generate embedding for text using the provided embeddings backend.
Args:
embeddings_backend: Embeddings instance to use for encoding
text: Text to embed
Returns:
Embedding vector (dimension depends on embeddings backend)
"""
try:
embeddings = embeddings_backend.encode([text])
return embeddings[0]
except Exception as e:
raise Exception(f"Failed to generate embedding: {str(e)}")
async def generate_embeddings_batch(embeddings_backend, texts: List[str]) -> List[List[float]]:
"""
Generate embeddings for multiple texts using the provided embeddings backend.
Runs the embedding generation in a thread pool to avoid blocking the event loop
for CPU-bound operations.
Args:
embeddings_backend: Embeddings instance to use for encoding
texts: List of texts to embed
Returns:
List of embeddings in same order as input texts
"""
try:
# Run embeddings in thread pool to avoid blocking event loop
loop = asyncio.get_event_loop()
embeddings = await loop.run_in_executor(
None, # Use default thread pool
embeddings_backend.encode,
texts
)
return embeddings
except Exception as e:
raise Exception(f"Failed to generate batch embeddings: {str(e)}")
@@ -8,6 +8,7 @@ import asyncpg
from typing import List, Dict, Optional, Set
from difflib import SequenceMatcher
from datetime import datetime, timezone
from .db_utils import acquire_with_retry
# Load spaCy model (singleton)
@@ -56,7 +57,7 @@ class EntityResolver:
return []
if conn is None:
async with self.pool.acquire() as conn:
async with acquire_with_retry(self.pool) as conn:
return await self._resolve_entities_batch_impl(conn, agent_id, entities_data, context, unit_event_date)
else:
return await self._resolve_entities_batch_impl(conn, agent_id, entities_data, context, unit_event_date)
@@ -220,7 +221,7 @@ class EntityResolver:
Returns:
Entity ID (creates new entity if needed)
"""
async with self.pool.acquire() as conn:
async with acquire_with_retry(self.pool) as conn:
# Find candidate entities with similar name
candidates = await conn.fetch(
"""
@@ -366,7 +367,7 @@ class EntityResolver:
unit_id: Memory unit ID
entity_id: Entity ID
"""
async with self.pool.acquire() as conn:
async with acquire_with_retry(self.pool) as conn:
# Insert unit-entity link
await conn.execute(
"""
@@ -434,7 +435,7 @@ class EntityResolver:
return
if conn is None:
async with self.pool.acquire() as conn:
async with acquire_with_retry(self.pool) as conn:
return await self._link_units_to_entities_batch_impl(conn, unit_entity_pairs)
else:
return await self._link_units_to_entities_batch_impl(conn, unit_entity_pairs)
@@ -499,7 +500,7 @@ class EntityResolver:
Returns:
List of unit IDs
"""
async with self.pool.acquire() as conn:
async with acquire_with_retry(self.pool) as conn:
rows = await conn.fetch(
"""
SELECT unit_id
@@ -527,7 +528,7 @@ class EntityResolver:
Returns:
Entity ID if found, None otherwise
"""
async with self.pool.acquire() as conn:
async with acquire_with_retry(self.pool) as conn:
row = await conn.fetchrow(
"""
SELECT id FROM entities
@@ -0,0 +1,541 @@
"""
Link creation utilities for temporal, semantic, and entity links.
"""
import time
import logging
from typing import List
from datetime import timedelta
logger = logging.getLogger(__name__)
def _log(log_buffer, message, level='info'):
"""Helper to log to buffer if available, otherwise use logger."""
if log_buffer is not None:
log_buffer.append(message)
else:
if level == 'info':
logger.info(message)
else:
logger.debug(message)
async def extract_entities_batch_optimized(
entity_resolver,
conn,
agent_id: str,
unit_ids: List[str],
sentences: List[str],
context: str,
fact_dates: List,
llm_entities: List[List[dict]],
log_buffer: List[str] = None,
) -> List[tuple]:
"""
Process LLM-extracted entities for ALL facts in batch.
Uses entities provided by the LLM (no spaCy needed), then resolves
and links them in bulk.
Args:
entity_resolver: EntityResolver instance for entity resolution
conn: Database connection
agent_id: Agent identifier
unit_ids: List of unit IDs
sentences: List of fact sentences
context: Context string
fact_dates: List of fact dates
llm_entities: List of entity lists from LLM extraction
log_buffer: Optional buffer for logging
Returns:
List of tuples for batch insertion: (from_unit_id, to_unit_id, link_type, weight, entity_id)
"""
try:
# Step 1: Convert LLM entities to the format expected by entity resolver
substep_start = time.time()
all_entities = []
for entity_list in llm_entities:
# Convert List[Entity] or List[dict] to List[Dict] format
formatted_entities = []
for ent in entity_list:
# Handle both Entity objects and dicts
if hasattr(ent, 'text'):
formatted_entities.append({'text': ent.text, 'type': ent.type})
elif isinstance(ent, dict):
formatted_entities.append({'text': ent.get('text', ''), 'type': ent.get('type', 'CONCEPT')})
all_entities.append(formatted_entities)
total_entities = sum(len(ents) for ents in all_entities)
_log(log_buffer, f" [6.1] Process LLM entities: {total_entities} entities from {len(sentences)} facts in {time.time() - substep_start:.3f}s")
# Step 2: Resolve entities in BATCH (much faster!)
substep_start = time.time()
step_6_2_start = time.time()
# [6.2.1] Prepare all entities for batch resolution
substep_6_2_1_start = time.time()
all_entities_flat = []
entity_to_unit = [] # Maps flat index to (unit_id, local_index)
for unit_id, entities, fact_date in zip(unit_ids, all_entities, fact_dates):
if not entities:
continue
for local_idx, entity in enumerate(entities):
all_entities_flat.append({
'text': entity['text'],
'type': entity['type'],
'nearby_entities': entities,
})
entity_to_unit.append((unit_id, local_idx, fact_date))
_log(log_buffer, f" [6.2.1] Prepare entities: {len(all_entities_flat)} entities in {time.time() - substep_6_2_1_start:.3f}s")
# Resolve ALL entities in one batch call
if all_entities_flat:
# [6.2.2] Batch resolve entities
substep_6_2_2_start = time.time()
# Group by date for batch resolution (most will have same date)
entities_by_date = {}
for idx, (unit_id, local_idx, fact_date) in enumerate(entity_to_unit):
date_key = fact_date
if date_key not in entities_by_date:
entities_by_date[date_key] = []
entities_by_date[date_key].append((idx, all_entities_flat[idx]))
_log(log_buffer, f" [6.2.2] Grouped into {len(entities_by_date)} date buckets, resolving...")
# Resolve each date group in batch
resolved_entity_ids = [None] * len(all_entities_flat)
for date_idx, (fact_date, entities_group) in enumerate(entities_by_date.items(), 1):
date_bucket_start = time.time()
indices = [idx for idx, _ in entities_group]
entities_data = [entity_data for _, entity_data in entities_group]
batch_resolved = await entity_resolver.resolve_entities_batch(
agent_id=agent_id,
entities_data=entities_data,
context=context,
unit_event_date=fact_date,
conn=conn
)
for idx, entity_id in zip(indices, batch_resolved):
resolved_entity_ids[idx] = entity_id
_log(log_buffer, f" [6.2.2.{date_idx}] Resolved {len(entities_data)} entities in {time.time() - date_bucket_start:.3f}s")
_log(log_buffer, f" [6.2.2] Resolve entities: {len(all_entities_flat)} entities in {time.time() - substep_6_2_2_start:.3f}s")
# [6.2.3] Create unit-entity links in BATCH
substep_6_2_3_start = time.time()
# Map resolved entities back to units and collect all (unit, entity) pairs
unit_to_entity_ids = {}
unit_entity_pairs = []
for idx, (unit_id, local_idx, fact_date) in enumerate(entity_to_unit):
if unit_id not in unit_to_entity_ids:
unit_to_entity_ids[unit_id] = []
entity_id = resolved_entity_ids[idx]
unit_to_entity_ids[unit_id].append(entity_id)
unit_entity_pairs.append((unit_id, entity_id))
# Batch insert all unit-entity links (MUCH faster!)
await entity_resolver.link_units_to_entities_batch(unit_entity_pairs, conn=conn)
_log(log_buffer, f" [6.2.3] Create unit-entity links (batched): {len(unit_entity_pairs)} links in {time.time() - substep_6_2_3_start:.3f}s")
_log(log_buffer, f" [6.2] Entity resolution (batched): {len(all_entities_flat)} entities resolved in {time.time() - step_6_2_start:.3f}s")
else:
unit_to_entity_ids = {}
_log(log_buffer, f" [6.2] Entity resolution (batched): 0 entities in {time.time() - step_6_2_start:.3f}s")
# Step 3: Create entity links between units that share entities
substep_start = time.time()
# Collect all unique entity IDs
all_entity_ids = set()
for entity_ids in unit_to_entity_ids.values():
all_entity_ids.update(entity_ids)
_log(log_buffer, f" [6.3] Creating entity links for {len(all_entity_ids)} unique entities...")
# Find all units that reference these entities (ONE batched query)
entity_to_units = {}
if all_entity_ids:
query_start = time.time()
import uuid
entity_id_list = [uuid.UUID(eid) if isinstance(eid, str) else eid for eid in all_entity_ids]
rows = await conn.fetch(
"""
SELECT entity_id, unit_id
FROM unit_entities
WHERE entity_id = ANY($1::uuid[])
""",
entity_id_list
)
_log(log_buffer, f" [6.3.1] Query unit_entities: {len(rows)} rows in {time.time() - query_start:.3f}s")
# Group by entity_id
group_start = time.time()
for row in rows:
entity_id = row['entity_id']
if entity_id not in entity_to_units:
entity_to_units[entity_id] = []
entity_to_units[entity_id].append(row['unit_id'])
_log(log_buffer, f" [6.3.2] Group by entity_id: {time.time() - group_start:.3f}s")
# Create bidirectional links between units that share entities
link_gen_start = time.time()
links = []
for entity_id, units_with_entity in entity_to_units.items():
# For each pair of units with this entity, create bidirectional links
for i, unit_id_1 in enumerate(units_with_entity):
for unit_id_2 in units_with_entity[i+1:]:
# Bidirectional links
links.append((unit_id_1, unit_id_2, 'entity', 1.0, entity_id))
links.append((unit_id_2, unit_id_1, 'entity', 1.0, entity_id))
_log(log_buffer, f" [6.3.3] Generate {len(links)} links: {time.time() - link_gen_start:.3f}s")
_log(log_buffer, f" [6.3] Entity link creation: {len(links)} links for {len(all_entity_ids)} unique entities in {time.time() - substep_start:.3f}s")
return links
except Exception as e:
logger.error(f"Failed to extract entities in batch: {str(e)}")
import traceback
traceback.print_exc()
raise
async def create_temporal_links_batch_per_fact(
conn,
agent_id: str,
unit_ids: List[str],
time_window_hours: int = 24,
log_buffer: List[str] = None,
):
"""
Create temporal links for multiple units, each with their own event_date.
Queries the event_date for each unit from the database and creates temporal
links based on individual dates (supports per-fact dating).
Args:
conn: Database connection
agent_id: Agent identifier
unit_ids: List of unit IDs
time_window_hours: Time window in hours for temporal links
log_buffer: Optional buffer for logging
"""
if not unit_ids:
return
try:
import time as time_mod
# Get the event_date for each new unit
fetch_dates_start = time_mod.time()
rows = await conn.fetch(
"""
SELECT id, event_date
FROM memory_units
WHERE id::text = ANY($1)
""",
unit_ids
)
new_units = {str(row['id']): row['event_date'] for row in rows}
_log(log_buffer, f" [7.1] Fetch event_dates for {len(unit_ids)} units: {time_mod.time() - fetch_dates_start:.3f}s")
# Fetch ALL potential temporal neighbors in ONE query (much faster!)
# Get time range across all units
all_dates = list(new_units.values())
min_date = min(all_dates) - timedelta(hours=time_window_hours)
max_date = max(all_dates) + timedelta(hours=time_window_hours)
fetch_neighbors_start = time_mod.time()
all_candidates = await conn.fetch(
"""
SELECT id, event_date
FROM memory_units
WHERE agent_id = $1
AND event_date BETWEEN $2 AND $3
AND id::text != ALL($4)
ORDER BY event_date DESC
""",
agent_id,
min_date,
max_date,
unit_ids
)
_log(log_buffer, f" [7.2] Fetch {len(all_candidates)} candidate neighbors (1 query): {time_mod.time() - fetch_neighbors_start:.3f}s")
# Filter and create links in memory (much faster than N queries)
link_gen_start = time_mod.time()
links = []
for unit_id, unit_event_date in new_units.items():
# Filter candidates within this unit's time window
time_lower = unit_event_date - timedelta(hours=time_window_hours)
time_upper = unit_event_date + timedelta(hours=time_window_hours)
matching_neighbors = [
(row['id'], row['event_date'])
for row in all_candidates
if time_lower <= row['event_date'] <= time_upper
][:10] # Limit to top 10
for recent_id, recent_event_date in matching_neighbors:
# Calculate temporal proximity weight
time_diff_hours = abs((unit_event_date - recent_event_date).total_seconds() / 3600)
weight = max(0.3, 1.0 - (time_diff_hours / time_window_hours))
links.append((unit_id, str(recent_id), 'temporal', weight, None))
_log(log_buffer, f" [7.3] Generate {len(links)} temporal links: {time_mod.time() - link_gen_start:.3f}s")
if links:
insert_start = time_mod.time()
await conn.executemany(
"""
INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
""",
links
)
_log(log_buffer, f" [7.4] Insert {len(links)} temporal links: {time_mod.time() - insert_start:.3f}s")
except Exception as e:
logger.error(f"Failed to create temporal links: {str(e)}")
import traceback
traceback.print_exc()
raise
async def create_semantic_links_batch(
conn,
agent_id: str,
unit_ids: List[str],
embeddings: List[List[float]],
top_k: int = 5,
threshold: float = 0.7,
log_buffer: List[str] = None,
):
"""
Create semantic links for multiple units efficiently.
For each unit, finds similar units and creates links.
Args:
conn: Database connection
agent_id: Agent identifier
unit_ids: List of unit IDs
embeddings: List of embedding vectors
top_k: Number of top similar units to link
threshold: Minimum similarity threshold
log_buffer: Optional buffer for logging
"""
if not unit_ids or not embeddings:
return
try:
import time as time_mod
import numpy as np
# Fetch ALL existing units with embeddings in ONE query
fetch_start = time_mod.time()
all_existing = await conn.fetch(
"""
SELECT id, embedding
FROM memory_units
WHERE agent_id = $1
AND embedding IS NOT NULL
AND id::text != ALL($2)
""",
agent_id,
unit_ids
)
_log(log_buffer, f" [8.1] Fetch {len(all_existing)} existing embeddings (1 query): {time_mod.time() - fetch_start:.3f}s")
# Convert to numpy for vectorized similarity computation
compute_start = time_mod.time()
all_links = []
if all_existing:
# Convert existing embeddings to numpy array
existing_ids = [str(row['id']) for row in all_existing]
# Stack embeddings as 2D array: (num_embeddings, embedding_dim)
embedding_arrays = []
for row in all_existing:
raw_emb = row['embedding']
# Handle different pgvector formats
if isinstance(raw_emb, str):
# Parse string format: "[1.0, 2.0, ...]"
import json
emb = np.array(json.loads(raw_emb), dtype=np.float32)
elif isinstance(raw_emb, (list, tuple)):
emb = np.array(raw_emb, dtype=np.float32)
else:
# Try direct conversion (works for numpy arrays, pgvector objects, etc.)
emb = np.array(raw_emb, dtype=np.float32)
# Ensure it's 1D
if emb.ndim != 1:
raise ValueError(f"Expected 1D embedding, got shape {emb.shape}")
embedding_arrays.append(emb)
if not embedding_arrays:
existing_embeddings = np.array([])
elif len(embedding_arrays) == 1:
# Single embedding: reshape to (1, dim)
existing_embeddings = embedding_arrays[0].reshape(1, -1)
else:
# Multiple embeddings: vstack
existing_embeddings = np.vstack(embedding_arrays)
# For each new unit, compute similarities with ALL existing units
for unit_id, new_embedding in zip(unit_ids, embeddings):
new_emb_array = np.array(new_embedding)
# Compute cosine similarities (dot product for normalized vectors)
similarities = np.dot(existing_embeddings, new_emb_array)
# Find top-k above threshold
# Get indices of similarities above threshold
above_threshold = np.where(similarities >= threshold)[0]
if len(above_threshold) > 0:
# Sort by similarity (descending) and take top-k
sorted_indices = above_threshold[np.argsort(-similarities[above_threshold])][:top_k]
for idx in sorted_indices:
similar_id = existing_ids[idx]
similarity = float(similarities[idx])
all_links.append((unit_id, similar_id, 'semantic', similarity, None))
_log(log_buffer, f" [8.2] Compute similarities & generate {len(all_links)} semantic links: {time_mod.time() - compute_start:.3f}s")
if all_links:
insert_start = time_mod.time()
await conn.executemany(
"""
INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
""",
all_links
)
_log(log_buffer, f" [8.3] Insert {len(all_links)} semantic links: {time_mod.time() - insert_start:.3f}s")
except Exception as e:
logger.error(f"Failed to create semantic links: {str(e)}")
import traceback
traceback.print_exc()
raise
async def insert_entity_links_batch(conn, links: List[tuple]):
"""
Insert all entity links in a single batch.
Args:
conn: Database connection
links: List of tuples (from_unit_id, to_unit_id, link_type, weight, entity_id)
"""
if not links:
return
try:
await conn.executemany(
"""
INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
""",
links
)
except Exception as e:
logger.warning(f"Failed to insert entity links: {str(e)}")
async def create_causal_links_batch(
conn,
unit_ids: List[str],
causal_relations_per_fact: List[List[dict]],
) -> int:
"""
Create causal links between facts based on LLM-extracted causal relationships.
Args:
conn: Database connection
unit_ids: List of unit IDs (in same order as causal_relations_per_fact)
causal_relations_per_fact: List of causal relations for each fact.
Each element is a list of dicts with:
- target_fact_index: Index into unit_ids for the target fact
- relation_type: "causes", "caused_by", "enables", or "prevents"
- strength: Float in [0.0, 1.0] representing relationship strength
Returns:
Number of causal links created
Causal link types:
- "causes": This fact directly causes the target fact (forward causation)
- "caused_by": This fact was caused by the target fact (backward causation)
- "enables": This fact enables/allows the target fact (enablement)
- "prevents": This fact prevents/blocks the target fact (prevention)
"""
if not unit_ids or not causal_relations_per_fact:
return 0
try:
import time as time_mod
create_start = time_mod.time()
# Build links list
links = []
for fact_idx, causal_relations in enumerate(causal_relations_per_fact):
if not causal_relations:
continue
from_unit_id = unit_ids[fact_idx]
for relation in causal_relations:
target_idx = relation['target_fact_index']
relation_type = relation['relation_type']
strength = relation.get('strength', 1.0)
# Validate target index
if target_idx < 0 or target_idx >= len(unit_ids):
logger.warning(f"Invalid target_fact_index {target_idx} in causal relation from fact {fact_idx}")
continue
to_unit_id = unit_ids[target_idx]
# Don't create self-links
if from_unit_id == to_unit_id:
continue
# Add the causal link
# link_type is the relation_type (e.g., "causes", "caused_by")
# weight is the strength of the relationship
links.append((from_unit_id, to_unit_id, relation_type, strength, None))
logger.debug(f"Generated {len(links)} causal links in {time_mod.time() - create_start:.3f}s")
if links:
insert_start = time_mod.time()
await conn.executemany(
"""
INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
""",
links
)
logger.debug(f"Inserted {len(links)} causal links in {time_mod.time() - insert_start:.3f}s")
return len(links)
except Exception as e:
logger.error(f"Failed to create causal links: {str(e)}")
import traceback
traceback.print_exc()
raise
@@ -182,10 +182,10 @@ class LLMConfig:
@classmethod
def for_memory(cls) -> "LLMConfig":
"""Create configuration for memory operations from environment variables."""
provider = os.getenv("MEMORA_API_LLM_PROVIDER", "groq")
api_key = os.getenv("MEMORA_API_LLM_API_KEY")
base_url = os.getenv("MEMORA_API_LLM_BASE_URL")
model = os.getenv("MEMORA_API_LLM_MODEL", "openai/gpt-oss-120b")
provider = os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq")
api_key = os.getenv("HINDSIGHT_API_LLM_API_KEY")
base_url = os.getenv("HINDSIGHT_API_LLM_BASE_URL")
model = os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b")
# Set default base URL if not provided
if not base_url:
@@ -211,10 +211,10 @@ class LLMConfig:
Falls back to memory LLM config if judge-specific config not set.
"""
# Check if judge-specific config exists, otherwise fall back to memory config
provider = os.getenv("MEMORA_API_JUDGE_LLM_PROVIDER", os.getenv("MEMORA_API_LLM_PROVIDER", "groq"))
api_key = os.getenv("MEMORA_API_JUDGE_LLM_API_KEY", os.getenv("MEMORA_API_LLM_API_KEY"))
base_url = os.getenv("MEMORA_API_JUDGE_LLM_BASE_URL", os.getenv("MEMORA_API_LLM_BASE_URL"))
model = os.getenv("MEMORA_API_JUDGE_LLM_MODEL", os.getenv("MEMORA_API_LLM_MODEL", "openai/gpt-oss-120b"))
provider = os.getenv("HINDSIGHT_API_JUDGE_LLM_PROVIDER", os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq"))
api_key = os.getenv("HINDSIGHT_API_JUDGE_LLM_API_KEY", os.getenv("HINDSIGHT_API_LLM_API_KEY"))
base_url = os.getenv("HINDSIGHT_API_JUDGE_LLM_BASE_URL", os.getenv("HINDSIGHT_API_LLM_BASE_URL"))
model = os.getenv("HINDSIGHT_API_JUDGE_LLM_MODEL", os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b"))
# Set default base URL if not provided
if not base_url:
@@ -1,5 +1,5 @@
"""
Temporal + Semantic + Entity Memory System for AI Agents.
Memory Engine for AI Agents.
This implements a sophisticated memory architecture that combines:
1. Temporal links: Memories connected by time proximity
@@ -8,6 +8,7 @@ This implements a sophisticated memory architecture that combines:
4. Spreading activation: Search through the graph with activation decay
5. Dynamic weighting: Recency and frequency-based importance
"""
import json
import os
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional, Tuple, Union
@@ -19,18 +20,26 @@ import time
import numpy as np
import uuid
import logging
from pydantic import BaseModel, Field
from .query_analyzer import QueryAnalyzer
from .utils import (
extract_facts,
calculate_recency_weight,
calculate_frequency_weight,
)
from .entity_resolver import EntityResolver
from .operations import EmbeddingOperationsMixin, LinkOperationsMixin, ThinkOperationsMixin, AgentOperationsMixin
from . import (
embedding_utils,
link_utils,
think_utils,
agent_utils,
)
from .llm_wrapper import LLMConfig
from .response_models import SearchResult as SearchResultModel, ThinkResult, MemoryFact
from .task_backend import TaskBackend, AsyncIOQueueBackend
from .search.reranking import CrossEncoderReranker
from ..pg0 import EmbeddedPostgres
def utcnow():
@@ -41,8 +50,10 @@ def utcnow():
# Logger for memory system
logger = logging.getLogger(__name__)
# Tiktoken for token budget filtering
from .db_utils import acquire_with_retry, retry_with_backoff
import tiktoken
from dateutil import parser as date_parser
# Cache tiktoken encoding for token budget filtering (module-level singleton)
_TIKTOKEN_ENCODING = None
@@ -55,20 +66,15 @@ def _get_tiktoken_encoding():
return _TIKTOKEN_ENCODING
class TemporalSemanticMemory(
EmbeddingOperationsMixin,
LinkOperationsMixin,
ThinkOperationsMixin,
AgentOperationsMixin,
):
class MemoryEngine:
"""
Advanced memory system using temporal and semantic linking with PostgreSQL.
Uses mixin architecture for code organization:
- EmbeddingOperationsMixin: Embedding generation
- LinkOperationsMixin: Entity, temporal, and semantic link creation
- ThinkOperationsMixin: Think operations for formulating answers with opinions
- AgentOperationsMixin: Agent profile and personality management
This class provides:
- Embedding generation for semantic search
- Entity, temporal, and semantic link creation
- Think operations for formulating answers with opinions
- Agent profile and personality management
"""
def __init__(
@@ -80,7 +86,7 @@ class TemporalSemanticMemory(
memory_llm_base_url: Optional[str] = None,
embeddings: Optional[Embeddings] = None,
cross_encoder: Optional[CrossEncoderModel] = None,
query_analyzer: Optional["QueryAnalyzer"] = None,
query_analyzer: Optional[QueryAnalyzer] = None,
pool_min_size: int = 5,
pool_max_size: int = 100,
task_backend: Optional[TaskBackend] = None,
@@ -104,8 +110,15 @@ class TemporalSemanticMemory(
Increase for parallel think/search operations (e.g., 200-300 for 100+ parallel thinks)
task_backend: Custom task backend for async task execution. If not provided, uses AsyncIOQueueBackend
"""
# Track pg0 instance (if used)
self._pg0: Optional[EmbeddedPostgres] = None
# Initialize PostgreSQL connection URL
self.db_url = db_url
# "pg0" or "embedded-pg" are special values that trigger embedded PostgreSQL via pg0
# The actual URL will be set during initialize() after starting the server
self._use_pg0 = db_url in ("pg0", "embedded-pg")
self.db_url = db_url if not self._use_pg0 else None
# Set default base URL if not provided
if memory_llm_base_url is None:
@@ -135,7 +148,7 @@ class TemporalSemanticMemory(
if query_analyzer is not None:
self.query_analyzer = query_analyzer
else:
from memora.query_analyzer import TransformerQueryAnalyzer
from .query_analyzer import TransformerQueryAnalyzer
self.query_analyzer = TransformerQueryAnalyzer()
# Initialize LLM configuration
@@ -188,7 +201,7 @@ class TemporalSemanticMemory(
try:
# Convert string UUIDs to UUID type for faster matching
uuid_list = [uuid.UUID(nid) for nid in node_ids]
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
await conn.execute(
"UPDATE memory_units SET access_count = access_count + 1 WHERE id = ANY($1::uuid[])",
uuid_list
@@ -242,7 +255,7 @@ class TemporalSemanticMemory(
if operation_id:
try:
pool = await self._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
result = await conn.fetchrow(
"SELECT id FROM async_operations WHERE id = $1",
uuid.UUID(operation_id)
@@ -297,7 +310,7 @@ class TemporalSemanticMemory(
"""Helper to delete an operation record from the database."""
try:
pool = await self._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
await conn.execute(
"DELETE FROM async_operations WHERE id = $1",
uuid.UUID(operation_id)
@@ -314,7 +327,7 @@ class TemporalSemanticMemory(
full_error = f"{error_message}\n\nTraceback:\n{error_traceback}"
truncated_error = full_error[:5000] if len(full_error) > 5000 else full_error
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
await conn.execute(
"""
UPDATE async_operations
@@ -333,6 +346,13 @@ class TemporalSemanticMemory(
if self._initialized:
return
# Start pg0 embedded PostgreSQL if configured
if self._use_pg0:
logger.info("Starting pg0 embedded PostgreSQL...")
self._pg0 = EmbeddedPostgres()
self.db_url = await self._pg0.ensure_running()
logger.info(f"pg0 PostgreSQL running at: {self.db_url}")
# Create connection pool
# For read-heavy workloads with many parallel think/search operations,
# we need a larger pool. Read operations don't need strong isolation.
@@ -341,7 +361,8 @@ class TemporalSemanticMemory(
min_size=self._pool_min_size,
max_size=self._pool_max_size,
command_timeout=60,
statement_cache_size=0 # Disable prepared statement cache
statement_cache_size=0, # Disable prepared statement cache
timeout=30, # Connection acquisition timeout (seconds)
)
# Initialize entity resolver with pool
@@ -360,6 +381,20 @@ class TemporalSemanticMemory(
await self.initialize()
return self._pool
async def _acquire_connection(self):
"""
Acquire a connection from the pool with retry logic.
Returns an async context manager that yields a connection.
Retries on transient connection errors with exponential backoff.
"""
pool = await self._get_pool()
async def acquire():
return await pool.acquire()
return await _retry_with_backoff(acquire)
async def close(self):
"""Close the connection pool and shutdown background workers."""
logger.info("close() started")
@@ -379,6 +414,14 @@ class TemporalSemanticMemory(
logger.debug("no pool to close")
self._initialized = False
# Stop pg0 if we started it
if self._pg0 is not None:
logger.info("Stopping pg0...")
await self._pg0.stop()
self._pg0 = None
logger.info("pg0 stopped")
logger.debug("close() completed")
async def wait_for_background_tasks(self):
@@ -729,7 +772,8 @@ class TemporalSemanticMemory(
log_buffer.append(f"{'='*60}")
# Get agent name for fact extraction
profile = await self.get_agent_profile(agent_id)
pool = await self._get_pool()
profile = await agent_utils.get_agent_profile(pool, agent_id)
agent_name = profile["name"]
# Step 1: Extract facts from ALL contents in parallel
@@ -774,7 +818,7 @@ class TemporalSemanticMemory(
all_fact_texts.append(fact_dict['fact'])
# Extract temporal fields (new schema with ranges)
from dateutil import parser as date_parser
try:
# Try new schema first (occurred_start/end)
occurred_start = date_parser.isoparse(fact_dict['occurred_start'])
@@ -855,14 +899,14 @@ class TemporalSemanticMemory(
# Step 2b: Generate ALL embeddings in ONE batch using augmented texts (HUGE speedup!)
step_start = time.time()
all_embeddings = await self._generate_embeddings_batch(augmented_texts)
all_embeddings = await embedding_utils.generate_embeddings_batch(self.embeddings, augmented_texts)
log_buffer.append(f"[2] Generate embeddings (parallel): {len(all_embeddings)} embeddings in {time.time() - step_start:.3f}s")
# Step 3: Process everything in ONE database transaction
logger.debug("Getting connection pool")
pool = await self._get_pool()
logger.debug("Acquiring connection from pool")
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
logger.debug("Starting transaction")
async with conn.transaction():
logger.debug("Inside transaction")
@@ -1035,8 +1079,8 @@ class TemporalSemanticMemory(
# Process entities for ALL units
logger.debug("Processing entities")
step_start = time.time()
all_entity_links = await self._extract_entities_batch_optimized(
conn, agent_id, created_unit_ids, filtered_sentences, "", filtered_dates, filtered_entities, log_buffer
all_entity_links = await link_utils.extract_entities_batch_optimized(
self.entity_resolver, conn, agent_id, created_unit_ids, filtered_sentences, "", filtered_dates, filtered_entities, log_buffer
)
logger.debug(f"Entity processing complete: {len(all_entity_links)} links")
log_buffer.append(f"[6] Process entities (batched): {time.time() - step_start:.3f}s")
@@ -1044,14 +1088,14 @@ class TemporalSemanticMemory(
# Create temporal links
logger.debug("Creating temporal links")
step_start = time.time()
await self._create_temporal_links_batch_per_fact(conn, agent_id, created_unit_ids, log_buffer=log_buffer)
await link_utils.create_temporal_links_batch_per_fact(conn, agent_id, created_unit_ids, log_buffer=log_buffer)
logger.debug("Temporal links complete")
log_buffer.append(f"[7] Batch create temporal links: {time.time() - step_start:.3f}s")
# Create semantic links
logger.debug("Creating semantic links")
step_start = time.time()
await self._create_semantic_links_batch(conn, agent_id, created_unit_ids, filtered_embeddings, log_buffer=log_buffer)
await link_utils.create_semantic_links_batch(conn, agent_id, created_unit_ids, filtered_embeddings, log_buffer=log_buffer)
logger.debug("Semantic links complete")
log_buffer.append(f"[8] Batch create semantic links: {time.time() - step_start:.3f}s")
@@ -1059,14 +1103,14 @@ class TemporalSemanticMemory(
logger.debug("Inserting entity links")
step_start = time.time()
if all_entity_links:
await self._insert_entity_links_batch(conn, all_entity_links)
await link_utils.insert_entity_links_batch(conn, all_entity_links)
logger.debug("Entity links inserted")
log_buffer.append(f"[9] Batch insert entity links: {time.time() - step_start:.3f}s")
# Create causal links
logger.debug("Creating causal links")
step_start = time.time()
causal_link_count = await self._create_causal_links_batch(
causal_link_count = await link_utils.create_causal_links_batch(
conn, created_unit_ids, filtered_causal_relations
)
logger.debug(f"Causal links complete: {causal_link_count} links created")
@@ -1261,7 +1305,7 @@ class TemporalSemanticMemory(
try:
# Step 1: Generate query embedding (for semantic search)
step_start = time.time()
query_embedding = self._generate_embedding(query)
query_embedding = embedding_utils.generate_embedding(self.embeddings, query)
step_duration = time.time() - step_start
log_buffer.append(f" [1] Generate query embedding: {step_duration:.3f}s")
@@ -1593,7 +1637,7 @@ class TemporalSemanticMemory(
Dictionary with document info or None if not found
"""
pool = await self._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
doc = await conn.fetchrow(
"""
SELECT d.id, d.agent_id, d.original_text, d.content_hash,
@@ -1631,7 +1675,7 @@ class TemporalSemanticMemory(
Dictionary with counts of deleted items
"""
pool = await self._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
async with conn.transaction():
# Count units before deletion
units_count = await conn.fetchval(
@@ -1666,7 +1710,7 @@ class TemporalSemanticMemory(
Dictionary with deletion result
"""
pool = await self._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
async with conn.transaction():
# Delete the memory unit (cascades to links and associations)
deleted = await conn.fetchval(
@@ -1700,7 +1744,7 @@ class TemporalSemanticMemory(
Dictionary with counts of deleted items
"""
pool = await self._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
async with conn.transaction():
try:
if fact_type:
@@ -1751,7 +1795,7 @@ class TemporalSemanticMemory(
Dict with nodes, edges, and table_rows
"""
pool = await self._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
# Get memory units, optionally filtered by agent_id and fact_type
query_conditions = []
query_params = []
@@ -1922,7 +1966,7 @@ class TemporalSemanticMemory(
Dict with items (list of memory units) and total count
"""
pool = await self._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
# Build query conditions
query_conditions = []
query_params = []
@@ -2036,7 +2080,7 @@ class TemporalSemanticMemory(
Dict with items (list of documents without original_text) and total count
"""
pool = await self._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
# Build query conditions
query_conditions = []
query_params = []
@@ -2153,7 +2197,7 @@ class TemporalSemanticMemory(
Dict with document details including original_text, or None if not found
"""
pool = await self._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
doc = await conn.fetchrow("""
SELECT
id,
@@ -2336,7 +2380,7 @@ Guidelines:
logger.debug(f"[REINFORCE] Starting opinion reinforcement for {len(entity_names)} entities")
pool = await self._get_pool()
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
# Find all opinions related to these entities
opinions = await conn.fetch(
"""
@@ -2439,3 +2483,229 @@ Guidelines:
import traceback
traceback.print_exc()
# ==================== Agent Profile Methods ====================
async def get_agent_profile(self, agent_id: str) -> Dict:
"""
Get agent profile (name, personality + background).
Auto-creates agent with default values if not exists.
Args:
agent_id: Agent identifier
Returns:
Dict with 'name' (str), 'personality' (dict) and 'background' (str) keys
"""
pool = await self._get_pool()
return await agent_utils.get_agent_profile(pool, agent_id)
async def update_agent_personality(
self,
agent_id: str,
personality: Dict[str, float]
) -> None:
"""
Update agent personality traits.
Args:
agent_id: Agent identifier
personality: Dict with Big Five traits + bias_strength (all 0-1)
"""
pool = await self._get_pool()
await agent_utils.update_agent_personality(pool, agent_id, personality)
async def merge_agent_background(
self,
agent_id: str,
new_info: str,
update_personality: bool = True
) -> dict:
"""
Merge new background information with existing background using LLM.
Normalizes to first person ("I") and resolves conflicts.
Optionally infers personality traits from the merged background.
Args:
agent_id: Agent identifier
new_info: New background information to add/merge
update_personality: If True, infer Big Five traits from background (default: True)
Returns:
Dict with 'background' (str) and optionally 'personality' (dict) keys
"""
pool = await self._get_pool()
return await agent_utils.merge_agent_background(
pool, self._llm_config, agent_id, new_info, update_personality
)
async def list_agents(self) -> list:
"""
List all agents in the system.
Returns:
List of dicts with agent_id, name, personality, background, created_at, updated_at
"""
pool = await self._get_pool()
return await agent_utils.list_agents(pool)
# ==================== Think Methods ====================
async def think_async(
self,
agent_id: str,
query: str,
thinking_budget: int = 50,
context: str = None,
) -> ThinkResult:
"""
Think and formulate an answer using agent identity, world facts, and opinions.
This method:
1. Retrieves agent facts (agent's identity and past actions)
2. Retrieves world facts (general knowledge)
3. Retrieves existing opinions (agent's formed perspectives)
4. Uses LLM to formulate an answer
5. Extracts and stores any new opinions formed during thinking
6. Returns plain text answer and the facts used
Args:
agent_id: Agent identifier
query: Question to answer
thinking_budget: Number of memory units to explore
context: Additional context string to include in LLM prompt (not used in search)
Returns:
ThinkResult containing:
- text: Plain text answer (no markdown)
- based_on: Dict with 'world', 'agent', and 'opinion' fact lists (MemoryFact objects)
- new_opinions: List of newly formed opinions
"""
# Use cached LLM config
if self._llm_config is None:
raise ValueError("Memory LLM API key not set. Set HINDSIGHT_API_LLM_API_KEY environment variable.")
# Steps 1-3: Run multi-fact-type search (12-way retrieval: 4 methods × 3 fact types)
search_result = await self.search_async(
agent_id=agent_id,
query=query,
thinking_budget=thinking_budget,
max_tokens=4096,
enable_trace=False,
fact_type=['agent', 'world', 'opinion']
)
all_results = search_result.results
logger.info(f"[THINK] Search returned {len(all_results)} results")
# Split results by fact type for structured response
agent_results = [r for r in all_results if r.fact_type == 'agent']
world_results = [r for r in all_results if r.fact_type == 'world']
opinion_results = [r for r in all_results if r.fact_type == 'opinion']
logger.info(f"[THINK] Split results - agent: {len(agent_results)}, world: {len(world_results)}, opinion: {len(opinion_results)}")
# Format facts for LLM
agent_facts_text = think_utils.format_facts_for_prompt(agent_results)
world_facts_text = think_utils.format_facts_for_prompt(world_results)
opinion_facts_text = think_utils.format_facts_for_prompt(opinion_results)
logger.info(f"[THINK] Formatted facts - agent: {len(agent_facts_text)} chars, world: {len(world_facts_text)} chars, opinion: {len(opinion_facts_text)} chars")
# Get agent profile (name, personality + background)
profile = await self.get_agent_profile(agent_id)
name = profile["name"]
personality = profile["personality"]
background = profile["background"]
# Build the prompt
prompt = think_utils.build_think_prompt(
agent_facts_text=agent_facts_text,
world_facts_text=world_facts_text,
opinion_facts_text=opinion_facts_text,
query=query,
name=name,
personality=personality,
background=background,
context=context,
)
logger.info(f"[THINK] Full prompt length: {len(prompt)} chars")
logger.debug(f"[THINK] Prompt preview (first 500 chars): {prompt[:500]}")
system_message = think_utils.get_system_message(personality)
answer_text = await self._llm_config.call(
messages=[
{"role": "system", "content": system_message},
{"role": "user", "content": prompt}
],
scope="memory_think",
temperature=0.9,
max_tokens=1000
)
answer_text = answer_text.strip()
# Submit form_opinion task for background processing
logger.debug(f"[THINK] Submitting form_opinion task for agent {agent_id}")
await self._task_backend.submit_task({
'type': 'form_opinion',
'agent_id': agent_id,
'answer_text': answer_text,
'query': query
})
logger.debug(f"[THINK] form_opinion task submitted")
# Return response with facts split by type
return ThinkResult(
text=answer_text,
based_on={
"world": world_results,
"agent": agent_results,
"opinion": opinion_results
},
new_opinions=[] # Opinions are being extracted asynchronously
)
async def _extract_and_store_opinions_async(
self,
agent_id: str,
answer_text: str,
query: str
):
"""
Background task to extract and store opinions from think response.
This runs asynchronously and does not block the think response.
Args:
agent_id: Agent identifier
answer_text: The generated answer text
query: The original query
"""
try:
logger.debug(f"[THINK] Extracting opinions from answer for agent {agent_id}")
# Extract opinions from the answer
new_opinions = await think_utils.extract_opinions_from_text(
self._llm_config, text=answer_text, query=query
)
logger.debug(f"[THINK] Extracted {len(new_opinions)} opinions")
# Store new opinions
if new_opinions:
from datetime import datetime, timezone
current_time = datetime.now(timezone.utc)
for opinion_dict in new_opinions:
await self.put_async(
agent_id=agent_id,
content=opinion_dict["text"],
context=f"formed during thinking about: {query}",
event_date=current_time,
fact_type_override='opinion',
confidence_score=opinion_dict["confidence"]
)
logger.debug(f"[THINK] Extracted and stored {len(new_opinions)} new opinions")
except Exception as e:
logger.warning(f"[THINK] Failed to extract/store opinions: {str(e)}")
@@ -1,13 +1,13 @@
"""
Core response models for Memora memory system.
Core response models for Hindsight memory system.
These models define the structure of data returned by the core TemporalSemanticMemory class.
These models define the structure of data returned by the core MemoryEngine class.
API response models should be kept separate and convert from these core models to maintain
API stability even if internal models change.
"""
from typing import Optional, List, Dict, Any
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field, ConfigDict
class MemoryFact(BaseModel):
@@ -17,6 +17,19 @@ class MemoryFact(BaseModel):
This represents a unit of information stored in the memory system,
including both the content and metadata.
"""
model_config = ConfigDict(json_schema_extra={
"example": {
"id": "123e4567-e89b-12d3-a456-426614174000",
"text": "Alice works at Google on the AI team",
"fact_type": "world",
"context": "work info",
"event_date": "2024-01-15T10:30:00Z",
"document_id": "session_abc123",
"metadata": {"source": "slack"},
"activation": 0.95
}
})
id: str = Field(description="Unique identifier for the memory fact")
text: str = Field(description="The actual text content of the memory")
fact_type: str = Field(description="Type of fact: 'world', 'agent', or 'opinion'")
@@ -31,20 +44,6 @@ class MemoryFact(BaseModel):
# Internal metrics (used by system but may not be exposed in API)
activation: Optional[float] = Field(None, description="Internal activation score")
class Config:
json_schema_extra = {
"example": {
"id": "123e4567-e89b-12d3-a456-426614174000",
"text": "Alice works at Google on the AI team",
"fact_type": "world",
"context": "work info",
"event_date": "2024-01-15T10:30:00Z",
"document_id": "session_abc123",
"metadata": {"source": "slack"},
"activation": 0.95
}
}
class SearchResult(BaseModel):
"""
@@ -53,28 +52,27 @@ class SearchResult(BaseModel):
Contains a list of matching memory facts and optional trace information
for debugging and transparency.
"""
results: List[MemoryFact] = Field(description="List of memory facts matching the query")
trace: Optional[Dict[str, Any]] = Field(None, description="Trace information for debugging")
class Config:
json_schema_extra = {
"example": {
"results": [
{
"id": "123e4567-e89b-12d3-a456-426614174000",
"text": "Alice works at Google on the AI team",
"fact_type": "world",
"context": "work info",
"event_date": "2024-01-15T10:30:00Z",
"activation": 0.95
}
],
"trace": {
"query": "What did Alice say about machine learning?",
"num_results": 1
model_config = ConfigDict(json_schema_extra={
"example": {
"results": [
{
"id": "123e4567-e89b-12d3-a456-426614174000",
"text": "Alice works at Google on the AI team",
"fact_type": "world",
"context": "work info",
"event_date": "2024-01-15T10:30:00Z",
"activation": 0.95
}
],
"trace": {
"query": "What did Alice say about machine learning?",
"num_results": 1
}
}
})
results: List[MemoryFact] = Field(description="List of memory facts matching the query")
trace: Optional[Dict[str, Any]] = Field(None, description="Trace information for debugging")
class ThinkResult(BaseModel):
@@ -84,6 +82,28 @@ class ThinkResult(BaseModel):
Contains the formulated answer, the facts it was based on (organized by type),
and any new opinions that were formed during the thinking process.
"""
model_config = ConfigDict(json_schema_extra={
"example": {
"text": "Based on my knowledge, machine learning is being actively used in healthcare...",
"based_on": {
"world": [
{
"id": "123e4567-e89b-12d3-a456-426614174000",
"text": "Machine learning is used in medical diagnosis",
"fact_type": "world",
"context": "healthcare",
"event_date": "2024-01-15T10:30:00Z"
}
],
"agent": [],
"opinion": []
},
"new_opinions": [
"Machine learning has great potential in healthcare"
]
}
})
text: str = Field(description="The formulated answer text")
based_on: Dict[str, List[MemoryFact]] = Field(
description="Facts used to formulate the answer, organized by type (world, agent, opinion)"
@@ -93,29 +113,6 @@ class ThinkResult(BaseModel):
description="List of newly formed opinions during thinking"
)
class Config:
json_schema_extra = {
"example": {
"text": "Based on my knowledge, machine learning is being actively used in healthcare...",
"based_on": {
"world": [
{
"id": "123e4567-e89b-12d3-a456-426614174000",
"text": "Machine learning is used in medical diagnosis",
"fact_type": "world",
"context": "healthcare",
"event_date": "2024-01-15T10:30:00Z"
}
],
"agent": [],
"opinion": []
},
"new_opinions": [
"Machine learning has great potential in healthcare"
]
}
}
class Opinion(BaseModel):
"""
@@ -124,13 +121,12 @@ class Opinion(BaseModel):
Opinions represent the agent's formed perspectives on topics,
with a confidence level indicating strength of belief.
"""
model_config = ConfigDict(json_schema_extra={
"example": {
"text": "Machine learning has great potential in healthcare",
"confidence": 0.85
}
})
text: str = Field(description="The opinion text")
confidence: float = Field(description="Confidence score between 0.0 and 1.0")
class Config:
json_schema_extra = {
"example": {
"text": "Machine learning has great potential in healthcare",
"confidence": 0.85
}
}
@@ -24,7 +24,7 @@ class CrossEncoderReranker:
SentenceTransformersCrossEncoder with ms-marco-MiniLM-L-6-v2
"""
if cross_encoder is None:
from ..cross_encoder import SentenceTransformersCrossEncoder
from hindsight_api.engine.cross_encoder import SentenceTransformersCrossEncoder
cross_encoder = SentenceTransformersCrossEncoder()
self.cross_encoder = cross_encoder
@@ -11,6 +11,7 @@ Implements:
from typing import List, Dict, Any, Tuple, Optional
from datetime import datetime
import asyncio
from ..db_utils import acquire_with_retry
async def retrieve_semantic(
@@ -241,11 +242,6 @@ async def retrieve_temporal(
if end_date.tzinfo is None:
end_date = end_date.replace(tzinfo=timezone.utc)
# Find entry points: facts in date range with semantic relevance
import logging
logger = logging.getLogger(__name__)
logger.info(f"Temporal retrieval: searching for facts between {start_date} and {end_date} (agent={agent_id}, fact_type={fact_type})")
entry_points = await conn.fetch(
"""
SELECT id, text, context, event_date, occurred_start, occurred_end, mentioned_at, access_count, embedding, fact_type, document_id,
@@ -274,8 +270,6 @@ async def retrieve_temporal(
query_emb_str, agent_id, fact_type, start_date, end_date, semantic_threshold
)
logger.info(f"Temporal retrieval: found {len(entry_points)} entry points")
if not entry_points:
# Check if there are ANY memories with temporal metadata for this agent
total_with_dates = await conn.fetchval(
@@ -284,7 +278,6 @@ async def retrieve_temporal(
AND (occurred_start IS NOT NULL OR occurred_end IS NOT NULL OR mentioned_at IS NOT NULL)""",
agent_id, fact_type
)
logger.info(f"Temporal retrieval: agent has {total_with_dates} total memories with temporal metadata (fact_type={fact_type})")
return []
# Calculate temporal scores for entry points
@@ -442,26 +435,20 @@ async def retrieve_parallel(
query_text, reference_date=question_date, analyzer=query_analyzer
)
if temporal_constraint:
logger.info(f"Temporal constraint detected in retrieve_parallel: {temporal_constraint[0]} to {temporal_constraint[1]}")
else:
logger.info("No temporal constraint in retrieve_parallel")
# Each retrieval needs its own connection
async def run_semantic():
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
return await retrieve_semantic(conn, query_embedding_str, agent_id, fact_type, limit=thinking_budget)
async def run_bm25():
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
return await retrieve_bm25(conn, query_text, agent_id, fact_type, limit=thinking_budget)
async def run_graph():
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
return await retrieve_graph(conn, query_embedding_str, agent_id, fact_type, budget=thinking_budget)
async def run_temporal(start_date, end_date):
async with pool.acquire() as conn:
async with acquire_with_retry(pool) as conn:
return await retrieve_temporal(
conn, query_embedding_str, agent_id, fact_type,
start_date, end_date, budget=thinking_budget, semantic_threshold=0.4
@@ -7,7 +7,7 @@ Handles natural language temporal expressions using transformer-based query anal
from typing import Optional, Tuple
from datetime import datetime
import logging
from memora.query_analyzer import QueryAnalyzer, TransformerQueryAnalyzer
from hindsight_api.engine.query_analyzer import QueryAnalyzer, TransformerQueryAnalyzer
logger = logging.getLogger(__name__)
@@ -59,8 +59,6 @@ def extract_temporal_constraint(
analysis.temporal_constraint.start_date,
analysis.temporal_constraint.end_date
)
logger.info(f"Temporal constraint extracted: {result[0].strftime('%Y-%m-%d')} to {result[1].strftime('%Y-%m-%d')}")
return result
logger.info("No temporal constraint found in query")
return None
@@ -23,7 +23,7 @@ class TaskBackend(ABC):
2. Execute tasks through a provided executor callback
The backend treats tasks as pure dictionaries that can be serialized
and sent over the network. The executor (typically TemporalSemanticMemory.execute_task)
and sent over the network. The executor (typically MemoryEngine.execute_task)
receives the dict and routes it to the appropriate handler.
"""
@@ -0,0 +1,255 @@
"""
Think operation utilities for formulating answers based on agent and world facts.
"""
import asyncio
import logging
import re
from datetime import datetime, timezone
from typing import Dict, List, Any
from pydantic import BaseModel, Field
from .response_models import ThinkResult, MemoryFact
logger = logging.getLogger(__name__)
class Opinion(BaseModel):
"""An opinion formed by the agent."""
opinion: str = Field(description="The opinion or perspective with reasoning included")
confidence: float = Field(description="Confidence score for this opinion (0.0 to 1.0, where 1.0 is very confident)")
class OpinionExtractionResponse(BaseModel):
"""Response containing extracted opinions."""
opinions: List[Opinion] = Field(
default_factory=list,
description="List of opinions formed with their supporting reasons and confidence scores"
)
def describe_trait(name: str, value: float) -> str:
"""Convert trait value to descriptive text."""
if value >= 0.8:
return f"very high {name}"
elif value >= 0.6:
return f"high {name}"
elif value >= 0.4:
return f"moderate {name}"
elif value >= 0.2:
return f"low {name}"
else:
return f"very low {name}"
def build_personality_description(personality: Dict) -> str:
"""Build a personality description string from personality traits."""
return f"""Your personality traits:
- {describe_trait('openness to new ideas', personality['openness'])}
- {describe_trait('conscientiousness and organization', personality['conscientiousness'])}
- {describe_trait('extraversion and sociability', personality['extraversion'])}
- {describe_trait('agreeableness and cooperation', personality['agreeableness'])}
- {describe_trait('emotional sensitivity', personality['neuroticism'])}
Personality influence strength: {int(personality['bias_strength'] * 100)}% (how much your personality shapes your opinions)"""
def format_facts_for_prompt(facts: List[MemoryFact]) -> str:
"""Format facts as JSON for LLM prompt."""
import json
if not facts:
return "[]"
formatted = []
for fact in facts:
fact_obj = {
"text": fact.text
}
# Add context if available
if fact.context:
fact_obj["context"] = fact.context
# Add event_date if available
if fact.event_date:
event_date = fact.event_date
if isinstance(event_date, str):
fact_obj["event_date"] = event_date
elif isinstance(event_date, datetime):
fact_obj["event_date"] = event_date.strftime('%Y-%m-%d %H:%M:%S')
# Add activation if available
if fact.activation is not None:
fact_obj["score"] = fact.activation
formatted.append(fact_obj)
return json.dumps(formatted, indent=2)
def build_think_prompt(
agent_facts_text: str,
world_facts_text: str,
opinion_facts_text: str,
query: str,
name: str,
personality: Dict,
background: str,
context: str = None,
) -> str:
"""Build the think prompt for the LLM."""
personality_desc = build_personality_description(personality)
name_section = f"""
Your name: {name}
"""
background_section = ""
if background:
background_section = f"""
Your background:
{background}
"""
context_section = ""
if context:
context_section = f"""
ADDITIONAL CONTEXT:
{context}
"""
return f"""Here's what I know and have experienced:
MY IDENTITY & EXPERIENCES:
{agent_facts_text}
WHAT I KNOW ABOUT THE WORLD:
{world_facts_text}
MY EXISTING OPINIONS & BELIEFS:
{opinion_facts_text}
{context_section}{name_section}{personality_desc}{background_section}
QUESTION: {query}
Based on everything I know, believe, and who I am (including my name, personality and background), here's what I genuinely think about this question. I'll draw on my experiences, knowledge, opinions, and personal traits to give you my honest perspective."""
def get_system_message(personality: Dict) -> str:
"""Get the system message for the think LLM call."""
bias_strength = personality['bias_strength']
if bias_strength >= 0.7:
personality_instruction = "Your personality strongly influences your thinking. Let your traits guide how you interpret facts and form opinions. Don't be afraid to be biased based on your personality."
elif bias_strength >= 0.4:
personality_instruction = "Your personality moderately influences your thinking. Balance your personal traits with objective analysis."
else:
personality_instruction = "Your personality has minimal influence on your thinking. Focus primarily on facts while keeping your traits in mind."
return f"You are a person with your own thoughts, experiences, opinions, and personality. When asked a question, respond naturally like a human would - share what you genuinely think based on what you know and have experienced. {personality_instruction} Be direct, express your views confidently, and use 'I think', 'I believe', 'in my view', etc. Respond in plain text without markdown formatting."
async def extract_opinions_from_text(
llm_config,
text: str,
query: str
) -> List[Dict[str, Any]]:
"""
Extract opinions with reasons and confidence from text using LLM.
Args:
llm_config: LLM configuration to use
text: Text to extract opinions from
query: The original query that prompted this response
Returns:
List of dicts with keys: 'text' (opinion with reasons), 'confidence' (score 0-1)
"""
extraction_prompt = f"""Extract any NEW opinions or perspectives from the answer below and rewrite them in FIRST-PERSON as if YOU are stating the opinion directly.
ORIGINAL QUESTION:
{query}
ANSWER PROVIDED:
{text}
Your task: Find opinions in the answer and rewrite them AS IF YOU ARE THE ONE SAYING THEM.
An opinion is a judgment, viewpoint, or conclusion that goes beyond just stating facts.
IMPORTANT: Do NOT extract statements like:
- "I don't have enough information"
- "The facts don't contain information about X"
- "I cannot answer because..."
ONLY extract actual opinions about substantive topics.
CRITICAL FORMAT REQUIREMENTS:
1. **ALWAYS start with first-person phrases**: "I think...", "I believe...", "In my view...", "I've come to believe...", "Previously I thought... but now..."
2. **NEVER use third-person**: Do NOT say "The speaker thinks..." or "They believe..." - always use "I"
3. Include the reasoning naturally within the statement
4. Provide a confidence score (0.0 to 1.0)
CORRECT Examples (✓ FIRST-PERSON):
- "I think Alice is more reliable because she consistently delivers on time and writes clean code"
- "Previously I thought all engineers were equal, but now I feel that experience and track record really matter"
- "I believe reliability is best measured by consistent output over time"
- "I've come to believe that track records are more important than potential"
WRONG Examples (✗ THIRD-PERSON - DO NOT USE):
- "The speaker thinks Alice is more reliable"
- "They believe reliability matters"
- "It is believed that Alice is better"
If no genuine opinions are expressed (e.g., the response just says "I don't know"), return an empty list."""
try:
result = await llm_config.call(
messages=[
{"role": "system", "content": "You are converting opinions from text into first-person statements. Always use 'I think', 'I believe', 'I feel', etc. NEVER use third-person like 'The speaker' or 'They'."},
{"role": "user", "content": extraction_prompt}
],
response_format=OpinionExtractionResponse,
scope="memory_extract_opinion"
)
# Format opinions with confidence score and convert to first-person
formatted_opinions = []
for op in result.opinions:
# Convert third-person to first-person if needed
opinion_text = op.opinion
# Replace common third-person patterns with first-person
def singularize_verb(verb):
if verb.endswith('es'):
return verb[:-1] # believes -> believe
elif verb.endswith('s'):
return verb[:-1] # thinks -> think
return verb
# Pattern: "The speaker/user [verb]..." -> "I [verb]..."
match = re.match(r'^(The speaker|The user|They|It is believed) (believes?|thinks?|feels?|says|asserts?|considers?)(\s+that)?(.*)$', opinion_text, re.IGNORECASE)
if match:
verb = singularize_verb(match.group(2))
that_part = match.group(3) or "" # Keep " that" if present
rest = match.group(4)
opinion_text = f"I {verb}{that_part}{rest}"
# If still doesn't start with first-person, prepend "I believe that "
first_person_starters = ["I think", "I believe", "I feel", "In my view", "I've come to believe", "Previously I"]
if not any(opinion_text.startswith(starter) for starter in first_person_starters):
opinion_text = "I believe that " + opinion_text[0].lower() + opinion_text[1:]
formatted_opinions.append({
"text": opinion_text,
"confidence": op.confidence
})
return formatted_opinions
except Exception as e:
logger.warning(f"Failed to extract opinions: {str(e)}")
return []
@@ -37,14 +37,14 @@ def run_migrations(database_url: str, script_location: Optional[str] = None) ->
Args:
database_url: SQLAlchemy database URL (e.g., "postgresql://user:pass@host/db")
script_location: Path to alembic migrations directory (e.g., "/path/to/alembic").
If None, defaults to memora/alembic directory.
If None, defaults to hindsight-api/alembic directory.
Raises:
RuntimeError: If migrations fail to complete
FileNotFoundError: If script_location doesn't exist
Example:
# Using default location (memora package)
# Using default location (hindsight_api package)
run_migrations("postgresql://user:pass@host/db")
# Using custom location (when importing from another project)
@@ -56,9 +56,9 @@ def run_migrations(database_url: str, script_location: Optional[str] = None) ->
try:
# Determine script location
if script_location is None:
# Default: use the alembic directory in the memora package
# This file is in: memora/memora/migrations.py
# Default location is: memora/alembic
# Default: use the alembic directory in the hindsight_api package
# This file is in: hindsight-api/hindsight_api/migrations.py
# Default location is: hindsight-api/alembic
package_root = Path(__file__).parent.parent
script_location = str(package_root / "alembic")
@@ -86,6 +86,9 @@ def run_migrations(database_url: str, script_location: Optional[str] = None) ->
# Uses Python's logging system instead of alembic.ini
alembic_cfg.set_main_option("prepend_sys_path", ".")
# Set path_separator to avoid deprecation warning
alembic_cfg.set_main_option("path_separator", "os")
# Run migrations to head (latest version)
# Note: Alembic may call sys.exit() on errors instead of raising exceptions
# We rely on the outer try/except and logging to catch issues
@@ -110,7 +113,7 @@ def check_migration_status(database_url: Optional[str] = None, script_location:
Check current database schema version and latest available version.
Args:
database_url: SQLAlchemy database URL. If None, uses MEMORA_API_DATABASE_URL env var.
database_url: SQLAlchemy database URL. If None, uses HINDSIGHT_API_DATABASE_URL env var.
script_location: Path to alembic migrations directory. If None, uses default location.
Returns:
@@ -124,9 +127,9 @@ def check_migration_status(database_url: Optional[str] = None, script_location:
# Get database URL
if database_url is None:
database_url = os.getenv("MEMORA_API_DATABASE_URL")
database_url = os.getenv("HINDSIGHT_API_DATABASE_URL")
if not database_url:
logger.warning("Database URL not provided and MEMORA_API_DATABASE_URL not set, cannot check migration status")
logger.warning("Database URL not provided and HINDSIGHT_API_DATABASE_URL not set, cannot check migration status")
return None, None
# Get current revision from database
@@ -148,6 +151,7 @@ def check_migration_status(database_url: Optional[str] = None, script_location:
# Create config programmatically
alembic_cfg = Config()
alembic_cfg.set_main_option("script_location", script_location)
alembic_cfg.set_main_option("path_separator", "os")
script = ScriptDirectory.from_config(alembic_cfg)
head_rev = script.get_current_head()
+416
View File
@@ -0,0 +1,416 @@
import asyncio
import json
import logging
import os
import platform
import shutil
import stat
import subprocess
import sys
from pathlib import Path
from typing import Optional
import httpx
logger = logging.getLogger(__name__)
DEFAULT_DATA_DIR = Path(os.environ.get("HINDSIGHT_API_PG0_DATA_DIR", Path.home() / ".hindsight" / "pg_data"))
DEFAULT_INSTALL_DIR = Path.home() / ".hindsight" / "bin"
BINARY_NAME = "pg0"
DEFAULT_PORT = 5555
DEFAULT_USERNAME = "hindsight"
DEFAULT_PASSWORD = "hindsight"
DEFAULT_DATABASE = "hindsight"
def get_platform_binary_name() -> str:
"""Get the appropriate binary name for the current platform.
Supported platforms:
- macOS ARM64 (darwin-aarch64)
- Linux x86_64
- Windows x86_64
"""
system = platform.system().lower()
machine = platform.machine().lower()
# Normalize architecture names
if machine in ("x86_64", "amd64"):
arch = "x86_64"
elif machine in ("arm64", "aarch64"):
arch = "aarch64"
else:
raise RuntimeError(
f"Embedded PostgreSQL is not supported on architecture: {machine}. "
f"Supported architectures: x86_64/amd64 (Linux, Windows), aarch64/arm64 (macOS)"
)
if system == "darwin" and arch == "aarch64":
return "pg0-darwin-aarch64"
elif system == "linux" and arch == "x86_64":
return "pg0-linux-x86_64"
elif system == "windows" and arch == "x86_64":
return "pg0-windows-x86_64.exe"
else:
raise RuntimeError(
f"Embedded PostgreSQL is not supported on {system}-{arch}. "
f"Supported platforms: darwin-aarch64 (macOS ARM), linux-x86_64, windows-x86_64"
)
def get_download_url(
version: str = "latest",
repo: str = "vectorize-io/pg0",
) -> str:
"""
"""
# Check for direct URL override
binary_name = get_platform_binary_name()
if version == "latest":
return f"https://github.com/{repo}/releases/latest/download/{binary_name}"
else:
return f"https://github.com/{repo}/releases/download/{version}/{binary_name}"
class EmbeddedPostgres:
"""
Manages an embedded PostgreSQL server instance.
This class handles:
- Downloading and installing the embedded-postgres CLI
- Starting/stopping the PostgreSQL server
- Getting the connection URI
Example:
pg = EmbeddedPostgres(data_dir="~/.myapp/data")
await pg.ensure_installed()
await pg.start()
uri = await pg.get_uri()
# ... use uri with asyncpg ...
await pg.stop()
"""
def __init__(
self,
data_dir: Optional[Path] = None,
install_dir: Optional[Path] = None,
version: str = "latest",
port: int = DEFAULT_PORT,
username: str = DEFAULT_USERNAME,
password: str = DEFAULT_PASSWORD,
database: str = DEFAULT_DATABASE,
):
"""
Initialize the embedded PostgreSQL manager.
Args:
data_dir: Directory to store PostgreSQL data. Defaults to ~/.hindsight/pg_data
install_dir: Directory to install the CLI binary. Defaults to ~/.hindsight/bin
version: Version of embedded-postgres to use. Defaults to "latest"
port: Port to listen on. Defaults to 5555
username: Username for the database. Defaults to "hindsight"
password: Password for the database. Defaults to "hindsight"
database: Database name to create. Defaults to "hindsight"
"""
self.data_dir = Path(data_dir or DEFAULT_DATA_DIR).expanduser()
self.install_dir = Path(install_dir or DEFAULT_INSTALL_DIR).expanduser()
self.version = version
self.port = port
self.username = username
self.password = password
self.database = database
# Binary path
binary_name = "pg0.exe" if platform.system() == "Windows" else "pg0"
self.binary_path = self.install_dir / binary_name
self._process: Optional[subprocess.Popen] = None
def is_installed(self) -> bool:
"""Check if the embedded-postgres CLI is installed."""
return self.binary_path.exists() and os.access(self.binary_path, os.X_OK)
async def ensure_installed(self) -> None:
"""
Ensure the embedded-postgres CLI is installed.
Downloads and installs the binary if not already present.
"""
if self.is_installed():
logger.debug(f"pg0 already installed at {self.binary_path}")
return
logger.info("Installing pg0 CLI...")
# Create install directory
self.install_dir.mkdir(parents=True, exist_ok=True)
# Download the binary
download_url = get_download_url(self.version)
logger.info(f"Downloading from {download_url}")
try:
async with httpx.AsyncClient(follow_redirects=True, timeout=300.0) as client:
response = await client.get(download_url)
response.raise_for_status()
# Write binary to disk
with open(self.binary_path, "wb") as f:
f.write(response.content)
# Make executable on Unix
if platform.system() != "Windows":
st = os.stat(self.binary_path)
os.chmod(self.binary_path, st.st_mode | stat.S_IEXEC)
logger.info(f"Installed pg0 to {self.binary_path}")
except httpx.HTTPError as e:
raise RuntimeError(f"Failed to download pg0: {e}") from e
def _run_command(self, *args: str, capture_output: bool = True) -> subprocess.CompletedProcess:
"""Run an embedded-postgres command synchronously."""
cmd = [str(self.binary_path), *args]
return subprocess.run(
cmd,
capture_output=capture_output,
text=True,
)
async def _run_command_async(self, *args: str) -> tuple[int, str, str]:
"""Run an embedded-postgres command asynchronously."""
cmd = [str(self.binary_path), *args]
process = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await process.communicate()
return process.returncode, stdout.decode(), stderr.decode()
async def start(self) -> str:
"""
Start the PostgreSQL server.
Returns:
The connection URI for the started server.
Raises:
RuntimeError: If the server fails to start.
"""
if not self.is_installed():
raise RuntimeError("pg0 is not installed. Call ensure_installed() first.")
# Create data directory
self.data_dir.mkdir(parents=True, exist_ok=True)
logger.info(f"Starting embedded PostgreSQL (data: {self.data_dir}, port: {self.port})...")
returncode, stdout, stderr = await self._run_command_async(
"start",
"--port", str(self.port),
"--username", self.username,
"--password", self.password,
"--database", self.database,
"--data-dir", self.data_dir.as_posix()
)
if returncode != 0:
raise RuntimeError(f"Failed to start PostgreSQL: {stderr}")
logger.info("Embedded PostgreSQL started")
# Get and return the URI
return await self.get_uri()
async def stop(self) -> None:
"""
Stop the PostgreSQL server.
Raises:
RuntimeError: If the server fails to stop.
"""
if not self.is_installed():
return
logger.info("Stopping embedded PostgreSQL...")
returncode, stdout, stderr = await self._run_command_async("stop")
if returncode != 0:
# Don't raise if server wasn't running
if "not running" in stderr.lower():
logger.debug("PostgreSQL was not running")
return
raise RuntimeError(f"Failed to stop PostgreSQL: {stderr}")
logger.info("Embedded PostgreSQL stopped")
async def _get_info(self) -> dict:
"""
Get info from pg0 using the `info -o json` command.
Returns:
Dictionary with 'running' (bool) and 'uri' (str) keys.
Raises:
RuntimeError: If unable to get info.
"""
if not self.is_installed():
raise RuntimeError("pg0 is not installed.")
returncode, stdout, stderr = await self._run_command_async(
"info", "-o", "json")
if returncode != 0:
raise RuntimeError(f"Failed to get PostgreSQL info: {stderr}")
try:
return json.loads(stdout.strip())
except json.JSONDecodeError as e:
raise RuntimeError(f"Failed to parse pg0 info output: {e}")
async def get_uri(self) -> str:
"""
Get the connection URI for the PostgreSQL server.
Returns:
PostgreSQL connection URI (e.g., postgresql://user:pass@localhost:5432/db)
Raises:
RuntimeError: If unable to get the URI or server is not running.
"""
info = await self._get_info()
uri = info.get("uri")
if not uri:
raise RuntimeError("PostgreSQL server is not running or URI not available")
return uri
async def status(self) -> dict:
"""
Get the status of the PostgreSQL server.
Returns:
Dictionary with status information including 'running' boolean and 'uri'.
"""
if not self.is_installed():
return {"installed": False, "running": False}
try:
info = await self._get_info()
return {
"installed": True,
"running": info.get("running", False),
"uri": info.get("uri"),
"data_dir": str(self.data_dir),
"binary_path": str(self.binary_path),
}
except RuntimeError:
return {
"installed": True,
"running": False,
"data_dir": str(self.data_dir),
"binary_path": str(self.binary_path),
}
async def is_running(self) -> bool:
"""Check if the PostgreSQL server is currently running."""
if not self.is_installed():
return False
try:
info = await self._get_info()
return info.get("running", False)
except RuntimeError:
return False
async def ensure_running(self) -> str:
"""
Ensure the PostgreSQL server is running.
Installs if needed, starts if not running.
Returns:
The connection URI.
"""
await self.ensure_installed()
if await self.is_running():
return await self.get_uri()
return await self.start()
def uninstall(self) -> None:
"""Remove the embedded-postgres binary."""
if self.binary_path.exists():
self.binary_path.unlink()
logger.info(f"Removed {self.binary_path}")
def clear_data(self) -> None:
"""Remove all PostgreSQL data (destructive!)."""
if self.data_dir.exists():
shutil.rmtree(self.data_dir)
logger.info(f"Removed data directory {self.data_dir}")
# Convenience functions for simple usage
_default_instance: Optional[EmbeddedPostgres] = None
def get_embedded_postgres(
data_dir: Optional[Path] = None,
install_dir: Optional[Path] = None,
) -> EmbeddedPostgres:
"""
Get or create the default EmbeddedPostgres instance.
Args:
data_dir: Override default data directory
install_dir: Override default install directory
Returns:
EmbeddedPostgres instance
"""
global _default_instance
if _default_instance is None or data_dir or install_dir:
_default_instance = EmbeddedPostgres(
data_dir=data_dir,
install_dir=install_dir,
)
return _default_instance
async def start_embedded_postgres(
data_dir: Optional[Path] = None,
) -> str:
"""
Quick start function for embedded PostgreSQL.
Downloads, installs, and starts PostgreSQL in one call.
Args:
data_dir: Directory to store PostgreSQL data
Returns:
Connection URI string
Example:
db_url = await start_embedded_postgres()
conn = await asyncpg.connect(db_url)
"""
pg = get_embedded_postgres(data_dir=data_dir)
return await pg.ensure_running()
async def stop_embedded_postgres() -> None:
"""Stop the default embedded PostgreSQL instance."""
global _default_instance
if _default_instance:
await _default_instance.stop()
@@ -3,10 +3,10 @@ Web interface for memory system.
Provides FastAPI app and visualization interface.
"""
from memora.api import create_app
from hindsight_api.api import create_app
# Note: Don't import app from .server here to avoid circular import warnings
# when running with `python -m memora.web.server`
# If you need the app, import it directly: from memora.web.server import app
# when running with `python -m hindsight_api.web.server`
# If you need the app, import it directly: from hindsight_api.web.server import app
__all__ = ["create_app"]
@@ -4,27 +4,68 @@ FastAPI server for memory graph visualization and API.
Provides REST API endpoints for memory operations and serves
the interactive visualization interface.
"""
import warnings
# Filter deprecation warnings from third-party libraries
warnings.filterwarnings("ignore", message="websockets.legacy is deprecated")
warnings.filterwarnings("ignore", message="websockets.server.WebSocketServerProtocol is deprecated")
import asyncio
import atexit
import logging
import os
import argparse
import signal
import sys
from memora import TemporalSemanticMemory
from memora.api import create_app
from hindsight_api import MemoryEngine
from hindsight_api.api import create_app
# Disable tokenizers parallelism to avoid warnings
os.environ["TOKENIZERS_PARALLELISM"] = "false"
def _cleanup_pg0():
"""Synchronous cleanup function to stop pg0 on exit."""
global _memory
if _memory is not None and _memory._pg0 is not None:
try:
# Run async stop in a new event loop
loop = asyncio.new_event_loop()
loop.run_until_complete(_memory._pg0.stop())
loop.close()
print("\npg0 stopped.")
except Exception as e:
print(f"\nError stopping pg0: {e}")
# Register cleanup on normal exit
atexit.register(_cleanup_pg0)
def _signal_handler(signum, frame):
"""Handle SIGINT/SIGTERM to ensure pg0 cleanup."""
print(f"\nReceived signal {signum}, shutting down...")
_cleanup_pg0()
sys.exit(0)
# Register signal handlers for graceful shutdown
signal.signal(signal.SIGINT, _signal_handler)
signal.signal(signal.SIGTERM, _signal_handler)
# Create app at module level (required for uvicorn import string)
_memory = TemporalSemanticMemory(
db_url=os.getenv("MEMORA_API_DATABASE_URL"),
memory_llm_provider=os.getenv("MEMORA_API_LLM_PROVIDER", "groq"),
memory_llm_api_key=os.getenv("MEMORA_API_LLM_API_KEY"),
memory_llm_model=os.getenv("MEMORA_API_LLM_MODEL", "openai/gpt-oss-120b"),
memory_llm_base_url=os.getenv("MEMORA_API_LLM_BASE_URL") or None,
_memory = MemoryEngine(
db_url=os.getenv("HINDSIGHT_API_DATABASE_URL", "pg0"),
memory_llm_provider=os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq"),
memory_llm_api_key=os.getenv("HINDSIGHT_API_LLM_API_KEY"),
memory_llm_model=os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b"),
memory_llm_base_url=os.getenv("HINDSIGHT_API_LLM_BASE_URL") or None,
)
# Check if MCP should be enabled
mcp_enabled = os.getenv("MEMORA_API_MCP_ENABLED", "true").lower() == "true"
mcp_enabled = os.getenv("HINDSIGHT_API_MCP_ENABLED", "true").lower() == "true"
# Create unified app with both HTTP and optionally MCP
app = create_app(
@@ -43,7 +84,7 @@ if __name__ == "__main__":
# Parse CLI arguments
parser = argparse.ArgumentParser(description="Memory Graph API Server")
parser.add_argument("--host", default="0.0.0.0", help="Host to bind to (default: 0.0.0.0)")
parser.add_argument("--port", type=int, default=8080, help="Port to bind to (default: 8080)")
parser.add_argument("--port", type=int, default=8888, help="Port to bind to (default: 8888)")
parser.add_argument("--reload", action="store_true", help="Enable auto-reload on code changes")
parser.add_argument("--workers", type=int, default=1, help="Number of worker processes (default: 1)")
parser.add_argument("--log-level", default="info", choices=["critical", "error", "warning", "info", "debug", "trace"],
@@ -58,7 +99,7 @@ if __name__ == "__main__":
args = parser.parse_args()
app_ref = "memora.web.server:app"
app_ref = "hindsight_api.web.server:app"
# Prepare uvicorn config
uvicorn_config = {
@@ -3,7 +3,7 @@ requires = ["hatchling"]
build-backend = "hatchling.build"
[project]
name = "memora"
name = "hindsight-api"
version = "0.0.7"
description = "Temporal + Semantic + Entity Memory System for AI agents using PostgreSQL"
readme = "README.md"
@@ -36,15 +36,24 @@ test = [
"pytest>=7.0.0",
"pytest-asyncio>=0.21.0",
"pytest-timeout>=2.4.0",
"pytest-xdist>=3.0.0",
"filelock>=3.0.0",
"testcontainers[postgres]>=4.0.0",
]
[tool.hatch.build.targets.wheel]
packages = ["memora"]
packages = ["hindsight_api"]
[tool.pytest.ini_options]
log_cli = true
log_cli_level = "INFO"
log_cli_format = "%(asctime)s %(levelname)s %(message)s"
log_cli_date_format = "%Y-%m-%d %H:%M:%S"
addopts = "--timeout 60 -p no:warnings"
addopts = "--timeout 60 -n auto --durations=10 -v"
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
log_auto_indent = true
filterwarnings = [
"ignore:The @wait_container_is_ready decorator is deprecated:DeprecationWarning",
"ignore::RuntimeWarning:asyncio",
]
+130
View File
@@ -0,0 +1,130 @@
"""
Pytest configuration and shared fixtures.
"""
import pytest
import pytest_asyncio
import os
import filelock
from pathlib import Path
from dotenv import load_dotenv
from hindsight_api import MemoryEngine, LLMConfig, SentenceTransformersEmbeddings
import asyncpg
from testcontainers.postgres import PostgresContainer
from hindsight_api.engine.cross_encoder import SentenceTransformersCrossEncoder
from hindsight_api.engine.query_analyzer import TransformerQueryAnalyzer
# Load environment variables from .env at the start of test session
def pytest_configure(config):
"""Load environment variables before running tests."""
# Look for .env in the workspace root (two levels up from tests dir)
env_file = Path(__file__).parent.parent.parent / ".env"
if env_file.exists():
load_dotenv(env_file)
else:
print(f"Warning: {env_file} not found, tests may fail without proper configuration")
@pytest.fixture(scope="session")
def postgres_container(tmp_path_factory, worker_id):
"""
Start a postgres container shared across all test workers.
Uses filelock to ensure only one worker starts the container.
- worker_id == "master": running without -n (single process)
- worker_id == "gw0", "gw1", etc.: running with -n (parallel workers)
"""
# Get shared temp dir (same for all workers)
if worker_id == "master":
root_tmp_dir = tmp_path_factory.getbasetemp()
else:
root_tmp_dir = tmp_path_factory.getbasetemp().parent
db_url_file = root_tmp_dir / "postgres_url"
lock_file = root_tmp_dir / "postgres.lock"
container = None
with filelock.FileLock(str(lock_file)):
if db_url_file.exists():
# Another worker already started the container
db_url = db_url_file.read_text()
else:
# First worker - start the container
container = PostgresContainer("pgvector/pgvector:pg16")
container.start()
db_url = container.get_connection_url().replace("postgresql+psycopg2://", "postgresql://")
db_url_file.write_text(db_url)
# Run migrations
from hindsight_api.migrations import run_migrations
run_migrations(db_url)
os.environ["HINDSIGHT_API_DATABASE_URL"] = db_url
yield db_url
# Only the worker that started the container stops it
if container is not None:
container.stop()
@pytest.fixture(scope="session")
def llm_config():
"""
Provide LLM configuration for tests.
This can be used by tests that need to call LLM directly without memory system.
"""
return LLMConfig.for_memory()
@pytest.fixture(scope="session")
def embeddings():
return SentenceTransformersEmbeddings("BAAI/bge-small-en-v1.5")
@pytest.fixture(scope="session")
def cross_encoder():
return SentenceTransformersCrossEncoder()
@pytest.fixture(scope="session")
def query_analyzer():
return TransformerQueryAnalyzer()
@pytest_asyncio.fixture(scope="function")
async def memory(postgres_container, embeddings, cross_encoder, query_analyzer):
"""
Provide a MemoryEngine instance for each test.
Must be function-scoped because:
1. pytest-xdist runs tests in separate processes with different event loops
2. asyncpg pools are bound to the event loop that created them
3. Each test needs its own pool in its own event loop
Uses small pool sizes since tests run in parallel and share a single
testcontainer PostgreSQL instance with limited resources.
"""
mem = MemoryEngine(
db_url=postgres_container,
memory_llm_provider=os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq"),
memory_llm_api_key=os.getenv("HINDSIGHT_API_LLM_API_KEY"),
memory_llm_model=os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b"),
memory_llm_base_url=os.getenv("HINDSIGHT_API_LLM_BASE_URL") or None,
embeddings=embeddings,
cross_encoder=cross_encoder,
query_analyzer=query_analyzer,
pool_min_size=1,
pool_max_size=5,
)
await mem.initialize()
yield mem
try:
if mem._pool and not mem._pool._closing:
await mem.close()
except Exception:
pass
+254
View File
@@ -0,0 +1,254 @@
"""
Tests for agent management API (profile, personality, background).
"""
import pytest
import uuid
from hindsight_api import MemoryEngine
from hindsight_api.api import CreateAgentRequest, PersonalityTraits
def unique_agent_id(prefix: str) -> str:
"""Generate a unique agent ID for testing."""
return f"{prefix}_{uuid.uuid4().hex[:8]}"
class TestAgentProfile:
"""Tests for agent profile management."""
@pytest.mark.asyncio
async def test_get_agent_profile_creates_default(self, memory: MemoryEngine):
"""Test that getting a profile for a new agent creates default personality."""
agent_id = unique_agent_id("test_profile_default")
profile = await memory.get_agent_profile(agent_id)
assert profile is not None
assert "personality" in profile
assert "background" in profile
personality = profile["personality"]
assert personality["openness"] == 0.5
assert personality["conscientiousness"] == 0.5
assert personality["extraversion"] == 0.5
assert personality["agreeableness"] == 0.5
assert personality["neuroticism"] == 0.5
assert personality["bias_strength"] == 0.5
assert profile["background"] == ""
@pytest.mark.asyncio
async def test_update_agent_personality(self, memory: MemoryEngine):
"""Test updating agent personality traits."""
agent_id = unique_agent_id("test_profile_update")
profile = await memory.get_agent_profile(agent_id)
assert profile["personality"]["openness"] == 0.5
new_personality = {
"openness": 0.8,
"conscientiousness": 0.6,
"extraversion": 0.7,
"agreeableness": 0.4,
"neuroticism": 0.3,
"bias_strength": 0.9,
}
await memory.update_agent_personality(agent_id, new_personality)
updated_profile = await memory.get_agent_profile(agent_id)
for key in new_personality:
assert abs(updated_profile["personality"][key] - new_personality[key]) < 0.001
@pytest.mark.asyncio
async def test_list_agents(self, memory: MemoryEngine):
"""Test listing all agents."""
agent_id_1 = unique_agent_id("test_list")
agent_id_2 = unique_agent_id("test_list")
agent_id_3 = unique_agent_id("test_list")
await memory.get_agent_profile(agent_id_1)
await memory.get_agent_profile(agent_id_2)
await memory.get_agent_profile(agent_id_3)
agents = await memory.list_agents()
agent_ids = [a["agent_id"] for a in agents]
assert agent_id_1 in agent_ids
assert agent_id_2 in agent_ids
assert agent_id_3 in agent_ids
for agent in agents:
assert "agent_id" in agent
assert "personality" in agent
assert "background" in agent
assert "created_at" in agent
assert "updated_at" in agent
class TestAgentBackground:
"""Tests for agent background management."""
@pytest.mark.asyncio
async def test_merge_agent_background(self, memory: MemoryEngine):
"""Test merging agent background information."""
agent_id = unique_agent_id("test_profile_merge")
profile = await memory.get_agent_profile(agent_id)
assert profile["background"] == ""
result1 = await memory.merge_agent_background(
agent_id,
"I was born in Texas",
update_personality=False
)
assert "Texas" in result1["background"]
result2 = await memory.merge_agent_background(
agent_id,
"I have 10 years of startup experience",
update_personality=False
)
assert "Texas" in result2["background"] or "startup" in result2["background"]
final_profile = await memory.get_agent_profile(agent_id)
assert final_profile["background"] != ""
@pytest.mark.asyncio
async def test_merge_background_handles_conflicts(self, memory: MemoryEngine):
"""Test that merging background handles conflicts (new overwrites old)."""
agent_id = unique_agent_id("test_profile_conflict")
result1 = await memory.merge_agent_background(
agent_id,
"I was born in Colorado",
update_personality=False
)
assert "Colorado" in result1["background"]
result2 = await memory.merge_agent_background(
agent_id,
"You were born in Texas",
update_personality=False
)
assert "Texas" in result2["background"]
class TestAgentEndpoint:
"""Tests for agent PUT endpoint logic."""
@pytest.mark.asyncio
async def test_put_agent_create(self, memory: MemoryEngine):
"""Test creating an agent via PUT endpoint."""
agent_id = unique_agent_id("test_put_create")
request = CreateAgentRequest(
personality=PersonalityTraits(
openness=0.8,
conscientiousness=0.6,
extraversion=0.5,
agreeableness=0.7,
neuroticism=0.3,
bias_strength=0.7
),
background="I am a creative software engineer"
)
profile = await memory.get_agent_profile(agent_id)
if request.personality is not None:
await memory.update_agent_personality(
agent_id,
request.personality.model_dump()
)
if request.background is not None:
pool = await memory._get_pool()
async with pool.acquire() as conn:
await conn.execute(
"""
UPDATE agents
SET background = $2,
updated_at = NOW()
WHERE agent_id = $1
""",
agent_id,
request.background
)
final_profile = await memory.get_agent_profile(agent_id)
assert final_profile["personality"]["openness"] == 0.8
assert final_profile["personality"]["bias_strength"] == 0.7
assert final_profile["background"] == "I am a creative software engineer"
@pytest.mark.asyncio
async def test_put_agent_partial_update(self, memory: MemoryEngine):
"""Test updating only background."""
agent_id = unique_agent_id("test_put_partial")
request = CreateAgentRequest(
background="I am a data scientist"
)
profile = await memory.get_agent_profile(agent_id)
if request.background is not None:
pool = await memory._get_pool()
async with pool.acquire() as conn:
await conn.execute(
"""
UPDATE agents
SET background = $2,
updated_at = NOW()
WHERE agent_id = $1
""",
agent_id,
request.background
)
final_profile = await memory.get_agent_profile(agent_id)
assert final_profile["personality"]["openness"] == 0.5
assert final_profile["background"] == "I am a data scientist"
class TestAgentPersonalityIntegration:
"""Tests for personality integration with other features."""
@pytest.mark.asyncio
async def test_think_uses_personality(self, memory: MemoryEngine):
"""Test that THINK operation uses agent personality."""
agent_id = unique_agent_id("test_think")
personality = {
"openness": 0.9,
"conscientiousness": 0.2,
"extraversion": 0.8,
"agreeableness": 0.1,
"neuroticism": 0.7,
"bias_strength": 0.9,
}
await memory.update_agent_personality(agent_id, personality)
await memory.merge_agent_background(
agent_id,
"I am a creative artist who values innovation over tradition",
update_personality=False
)
await memory.put_batch_async(
agent_id=agent_id,
contents=[
{"content": "Traditional painting techniques have been used for centuries"},
{"content": "Modern digital art is changing the art world"}
],
document_id="art_facts"
)
result = await memory.think_async(
agent_id=agent_id,
query="What do you think about traditional vs modern art?",
thinking_budget=50
)
assert result.text is not None
assert len(result.text) > 0
@@ -0,0 +1,58 @@
"""Test automatic batch chunking based on character count."""
import asyncio
import pytest
from hindsight_api import MemoryEngine
import os
@pytest.mark.asyncio
async def test_large_batch_auto_chunks(memory):
agent_id = "test_chunking_agent"
# Create a large batch that should trigger chunking
# Each item is ~2000 chars, so 30 items = 60k chars (exceeds 50k threshold)
large_content = "Alice met with Bob at the coffee shop. " * 50 # ~2000 chars
contents = [
{"content": large_content, "context": f"conversation_{i}"}
for i in range(30)
]
# Calculate total chars
total_chars = sum(len(item["content"]) for item in contents)
print(f"\nTotal characters: {total_chars:,}")
print(f"Should trigger chunking: {total_chars > 50_000}")
# Ingest the large batch (should auto-chunk)
result = await memory.put_batch_async(
agent_id=agent_id,
contents=contents
)
# Verify we got results back
assert len(result) == 30, f"Expected 30 results, got {len(result)}"
print(f"Successfully ingested {len(result)} items (auto-chunked)")
@pytest.mark.asyncio
async def test_small_batch_no_chunking(memory):
agent_id = "test_no_chunking_agent"
# Create a small batch that should NOT trigger chunking
contents = [
{"content": "Alice works at Google", "context": "conversation_1"},
{"content": "Bob loves Python", "context": "conversation_2"}
]
# Calculate total chars
total_chars = sum(len(item["content"]) for item in contents)
print(f"\nTotal characters: {total_chars:,}")
print(f"Should NOT trigger chunking: {total_chars <= 50_000}")
# Ingest the small batch (should NOT auto-chunk)
result = await memory.put_batch_async(
agent_id=agent_id,
contents=contents
)
# Verify we got results back
assert len(result) == 2, f"Expected 2 results, got {len(result)}"
print(f"Successfully ingested {len(result)} items (no chunking)")
@@ -2,7 +2,7 @@
Test chunking functionality for large documents.
"""
import pytest
from memora.fact_extraction import chunk_text
from hindsight_api.engine.fact_extraction import chunk_text
def test_chunk_text_small():
@@ -58,6 +58,3 @@ def test_chunk_text_64k():
combined_length = sum(len(chunk) for chunk in chunks)
assert combined_length >= len(text) * 0.95, "Lost too much content during chunking"
if __name__ == "__main__":
pytest.main([__file__, "-v"])
File diff suppressed because it is too large Load Diff
@@ -7,31 +7,14 @@ distinguish between things said earlier vs later.
"""
import pytest
from datetime import datetime, timezone
from memora import TemporalSemanticMemory
from hindsight_api import MemoryEngine
import os
@pytest.mark.asyncio
async def test_fact_ordering_within_conversation():
"""
Test that facts extracted from one conversation get incremental time offsets
to preserve their ordering for retrieval.
"""
# Create memory instance
memory = TemporalSemanticMemory(
db_url=os.getenv("MEMORA_API_DATABASE_URL"),
memory_llm_provider=os.getenv("MEMORA_API_LLM_PROVIDER", "groq"),
memory_llm_api_key=os.getenv("MEMORA_API_LLM_API_KEY"),
memory_llm_model=os.getenv("MEMORA_API_LLM_MODEL", "openai/gpt-oss-20b"),
)
await memory.initialize()
async def test_fact_ordering_within_conversation(memory):
agent_id = "test_ordering_agent"
# Clear any existing data
await memory.delete_agent(agent_id)
# Get/create agent (auto-creates with defaults)
await memory.get_agent_profile(agent_id)
@@ -74,20 +57,20 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
max_tokens=8192
)
print(f"\n=== Retrieved {len(results['results'])} facts ===")
for i, result in enumerate(results['results']):
print(f"{i+1}. [{result['event_date']}] {result['text'][:100]}")
print(f"\n=== Retrieved {len(results.results)} facts ===")
for i, result in enumerate(results.results):
print(f"{i+1}. [{result.event_date}] {result.text[:100]}")
# Get all agent facts (Marcus's statements)
agent_facts = [r for r in results['results'] if r.get('fact_type') == 'agent']
agent_facts = [r for r in results.results if r.fact_type == 'agent']
print(f"\n=== Agent facts (Marcus's statements) ===")
for i, fact in enumerate(agent_facts):
print(f"{i+1}. [{fact['event_date']}] {fact['text']}")
print(f"{i+1}. [{fact.event_date}] {fact.text}")
# Check that agent facts have different timestamps
if len(agent_facts) >= 2:
timestamps = [datetime.fromisoformat(f['event_date'].replace('Z', '+00:00')) for f in agent_facts]
timestamps = [datetime.fromisoformat(f.event_date.replace('Z', '+00:00')) for f in agent_facts]
# Verify timestamps are different (have time offsets)
unique_timestamps = set(timestamps)
@@ -111,7 +94,7 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
# Verify that retrieval returns facts in chronological order
# The first prediction should come before the changed prediction
agent_texts = [f['text'].lower() for f in agent_facts]
agent_texts = [f.text.lower() for f in agent_facts]
# Look for evidence of the sequence
has_first_prediction = any('27' in text and '24' in text for text in agent_texts)
@@ -122,8 +105,8 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
first_idx = next(i for i, text in enumerate(agent_texts) if '27' in text and '24' in text)
changed_idx = next(i for i, text in enumerate(agent_texts) if 'chang' in text or 'by 3' in text or 'realized' in text)
print(f"\nFirst prediction at index {first_idx}: {agent_facts[first_idx]['text'][:100]}")
print(f"Changed prediction at index {changed_idx}: {agent_facts[changed_idx]['text'][:100]}")
print(f"\nFirst prediction at index {first_idx}: {agent_facts[first_idx].text[:100]}")
print(f"Changed prediction at index {changed_idx}: {agent_facts[changed_idx].text[:100]}")
# The original prediction should come before the changed one
assert timestamps[first_idx] < timestamps[changed_idx], \
@@ -138,24 +121,10 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
@pytest.mark.asyncio
async def test_multiple_documents_ordering():
"""
Test that facts from different documents get separate time offsets,
so facts within each document maintain their order.
"""
memory = TemporalSemanticMemory(
db_url=os.getenv("MEMORA_API_DATABASE_URL"),
memory_llm_provider=os.getenv("MEMORA_API_LLM_PROVIDER", "groq"),
memory_llm_api_key=os.getenv("MEMORA_API_LLM_API_KEY"),
memory_llm_model=os.getenv("MEMORA_API_LLM_MODEL", "openai/gpt-oss-20b"),
)
await memory.initialize()
async def test_multiple_documents_ordering(memory):
agent_id = "test_multi_doc_agent"
# Clear and create agent
await memory.delete_agent(agent_id)
await memory.get_agent_profile(agent_id) # Auto-creates with defaults
# Two separate conversations with same base time
@@ -191,15 +160,15 @@ Alice: I reconsidered the team's experience level.
max_tokens=8192
)
print(f"\n=== Retrieved {len(results['results'])} agent facts ===")
agent_facts = [r for r in results['results'] if r.get('fact_type') == 'agent']
print(f"\n=== Retrieved {len(results.results)} agent facts ===")
agent_facts = [r for r in results.results if r.fact_type == 'agent']
for i, fact in enumerate(agent_facts):
print(f"{i+1}. [{fact['event_date']}] {fact['text'][:80]}")
print(f"{i+1}. [{fact.event_date}] {fact.text[:80]}")
# Each conversation's facts should have different timestamps
if len(agent_facts) >= 2:
timestamps = [datetime.fromisoformat(f['event_date'].replace('Z', '+00:00')) for f in agent_facts]
timestamps = [datetime.fromisoformat(f.event_date.replace('Z', '+00:00')) for f in agent_facts]
unique_timestamps = set(timestamps)
assert len(unique_timestamps) >= 2, \
@@ -1,5 +1,5 @@
"""
Integration test for the complete Memora API.
Integration test for the complete Hindsight API.
Tests all endpoints by starting a FastAPI server and making HTTP requests.
"""
@@ -7,7 +7,7 @@ import pytest
import pytest_asyncio
import httpx
from datetime import datetime
from memora.api import create_app
from hindsight_api.api import create_app
@pytest_asyncio.fixture
@@ -1,17 +1,42 @@
"""Test MCP server with real server and client."""
"""
Integration test for the MCP (Model Context Protocol) server.
Tests MCP endpoints by starting a FastAPI server with MCP enabled and using the MCP client.
Note: MCP server is integrated with the web server. These tests require HINDSIGHT_API_MCP_ENABLED=true.
"""
import asyncio
import os
import pytest
import pytest_asyncio
import httpx
from mcp import ClientSession
from mcp.client.sse import sse_client
from hindsight_api.api import create_app
# Note: MCP server tests now require the full web server to be running
# with MEMORA_API_MCP_ENABLED=true since there's no standalone MCP server anymore.
# These tests are kept for documentation but may need manual server setup.
@pytest_asyncio.fixture
async def mcp_server(memory):
"""Start the FastAPI app with MCP enabled and return the SSE URL."""
app = create_app(
memory,
run_migrations=False,
initialize_memory=False,
mcp_enabled=True,
default_agent_id="test_mcp_agent"
)
pytest.skip("MCP server is now integrated with web server. Run web server with MEMORA_API_MCP_ENABLED=true to test.", allow_module_level=True)
# Use httpx to create a test server
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
# The MCP SSE endpoint is at /mcp/sse
# We need to yield the base URL for sse_client to connect
# However, sse_client expects a real URL, not a test client
# So we'll start a real server on a random port
pass
# For now, skip these tests as they require a real server
# The sse_client doesn't work with ASGI test transport
pytest.skip("MCP tests require a real running server. Run: HINDSIGHT_API_MCP_ENABLED=true uvicorn hindsight_api.api:app")
@pytest.mark.asyncio
@@ -27,12 +52,12 @@ async def test_mcp_server_tools_via_sse(mcp_server):
tools_list = await session.list_tools()
print(f"Tools: {tools_list}")
tool_names = [t.name for t in tools_list.tools]
assert "memora_search" in tool_names
assert "memora_put" in tool_names
assert "hindsight_search" in tool_names
assert "hindsight_put" in tool_names
# Test 2: Call memora_put
# Test 2: Call hindsight_put
put_result = await session.call_tool(
"memora_put",
"hindsight_put",
arguments={
"content": "User loves Python programming",
"context": "programming_preferences",
@@ -45,9 +70,9 @@ async def test_mcp_server_tools_via_sse(mcp_server):
# Wait a bit for indexing
await asyncio.sleep(1)
# Test 3: Call memora_search
# Test 3: Call hindsight_search
search_result = await session.call_tool(
"memora_search",
"hindsight_search",
arguments={
"query": "What programming languages does the user like?",
"max_tokens": 4096,
@@ -71,7 +96,7 @@ async def test_multiple_concurrent_requests(mcp_server):
async def make_search(idx):
try:
result = await session.call_tool(
"memora_search",
"hindsight_search",
arguments={
"query": f"test query {idx}",
"explanation": f"Concurrent test {idx}"
@@ -120,7 +145,7 @@ async def test_race_condition_with_rapid_requests(mcp_server):
# Make request immediately after initialization
result = await session.call_tool(
"memora_search",
"hindsight_search",
arguments={
"query": f"rapid query {idx}",
"max_tokens": 2048
@@ -3,16 +3,14 @@ Test query analyzer for temporal extraction.
"""
import pytest
from datetime import datetime
from memora.query_analyzer import TransformerQueryAnalyzer, QueryAnalysis
from hindsight_api.engine.query_analyzer import TransformerQueryAnalyzer, QueryAnalysis
def test_query_analyzer_june_2024():
"""Test extracting 'june 2024' from query."""
analyzer = TransformerQueryAnalyzer()
def test_query_analyzer_june_2024(query_analyzer):
reference_date = datetime(2025, 1, 15, 12, 0, 0)
query = "june 2024"
analysis = analyzer.analyze(query, reference_date)
analysis = query_analyzer.analyze(query, reference_date)
print(f"\nQuery: '{query}'")
print(f"Analysis: {analysis}")
@@ -26,13 +24,11 @@ def test_query_analyzer_june_2024():
assert analysis.temporal_constraint.end_date.day == 30
def test_query_analyzer_dogs_june_2023():
"""Test extracting temporal info from 'dogs in June 2023'."""
analyzer = TransformerQueryAnalyzer()
def test_query_analyzer_dogs_june_2023(query_analyzer):
reference_date = datetime(2025, 1, 15, 12, 0, 0)
query = "dogs in June 2023"
analysis = analyzer.analyze(query, reference_date)
analysis = query_analyzer.analyze(query, reference_date)
print(f"\nQuery: '{query}'")
print(f"Analysis: {analysis}")
@@ -46,13 +42,11 @@ def test_query_analyzer_dogs_june_2023():
assert analysis.temporal_constraint.end_date.day == 30
def test_query_analyzer_march_2023():
"""Test extracting 'March 2023' from query."""
analyzer = TransformerQueryAnalyzer()
def test_query_analyzer_march_2023(query_analyzer):
reference_date = datetime(2025, 1, 15, 12, 0, 0)
query = "March 2023"
analysis = analyzer.analyze(query, reference_date)
analysis = query_analyzer.analyze(query, reference_date)
print(f"\nQuery: '{query}'")
print(f"Analysis: {analysis}")
@@ -66,13 +60,11 @@ def test_query_analyzer_march_2023():
assert analysis.temporal_constraint.end_date.day == 31
def test_query_analyzer_last_year():
"""Test extracting 'last year' from query."""
analyzer = TransformerQueryAnalyzer()
def test_query_analyzer_last_year(query_analyzer):
reference_date = datetime(2025, 1, 15, 12, 0, 0)
query = "last year"
analysis = analyzer.analyze(query, reference_date)
analysis = query_analyzer.analyze(query, reference_date)
print(f"\nQuery: '{query}'")
print(f"Analysis: {analysis}")
@@ -86,13 +78,11 @@ def test_query_analyzer_last_year():
assert analysis.temporal_constraint.end_date.day == 31
def test_query_analyzer_no_temporal():
"""Test that queries without temporal info return None."""
analyzer = TransformerQueryAnalyzer()
def test_query_analyzer_no_temporal(query_analyzer):
reference_date = datetime(2025, 1, 15, 12, 0, 0)
query = "what is the weather"
analysis = analyzer.analyze(query, reference_date)
analysis = query_analyzer.analyze(query, reference_date)
print(f"\nQuery: '{query}'")
print(f"Analysis: {analysis}")
@@ -100,13 +90,11 @@ def test_query_analyzer_no_temporal():
assert analysis.temporal_constraint is None, "Should not extract temporal constraint"
def test_query_analyzer_activities_june_2024():
"""Test extracting temporal info from 'melanie activities in june 2024'."""
analyzer = TransformerQueryAnalyzer()
def test_query_analyzer_activities_june_2024(query_analyzer):
reference_date = datetime(2025, 1, 15, 12, 0, 0)
query = "melanie activities in june 2024"
analysis = analyzer.analyze(query, reference_date)
analysis = query_analyzer.analyze(query, reference_date)
print(f"\nQuery: '{query}'")
print(f"Analysis: {analysis}")
@@ -120,5 +108,3 @@ def test_query_analyzer_activities_june_2024():
assert analysis.temporal_constraint.end_date.day == 30
if __name__ == "__main__":
pytest.main([__file__, "-v", "-s"])
@@ -2,7 +2,7 @@
Test search tracing functionality.
"""
import pytest
from memora.search_trace import SearchTrace
from hindsight_api import SearchTrace
from datetime import datetime, timezone
@@ -3,7 +3,7 @@ import asyncio
import os
from datetime import datetime, timezone, timedelta
import pytest
from memora import TemporalSemanticMemory
from hindsight_api import MemoryEngine
@pytest.mark.asyncio
@@ -11,11 +11,11 @@ async def test_temporal_ranges_are_written():
"""Test that occurred_start, occurred_end, and mentioned_at are actually written to database."""
# Initialize memory system
memory = TemporalSemanticMemory(
db_url=os.getenv("MEMORA_API_DATABASE_URL", "postgresql://memora:memora_dev@localhost:5432/memora"),
memory_llm_provider=os.getenv("MEMORA_API_LLM_PROVIDER", "groq"),
memory_llm_api_key=os.getenv("MEMORA_API_LLM_API_KEY"),
memory_llm_model=os.getenv("MEMORA_API_LLM_MODEL", "openai/gpt-oss-20b"),
memory = MemoryEngine(
db_url=os.getenv("HINDSIGHT_API_DATABASE_URL", "postgresql://hindsight:hindsight_dev@localhost:5432/hindsight"),
memory_llm_provider=os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq"),
memory_llm_api_key=os.getenv("HINDSIGHT_API_LLM_API_KEY"),
memory_llm_model=os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-20b"),
)
await memory.initialize()
@@ -1,13 +1,13 @@
[package]
name = "memora-cli-rust"
name = "hindsight-cli"
version = "0.0.7"
edition = "2021"
authors = ["Memora Team"]
description = "A beautiful CLI for Memora - semantic memory system"
authors = ["Hindsight Team"]
description = "A beautiful CLI for Hindsight - semantic memory system"
license = "MIT"
[[bin]]
name = "memora"
name = "hindsight"
path = "src/main.rs"
[dependencies]
@@ -1,7 +1,7 @@
#!/bin/bash
set -e
# Build script for memora-cli-rust
# Build script for hindsight-cli
# This script builds optimized binaries for multiple platforms
# Source cargo environment if it exists
@@ -19,7 +19,7 @@ if ! command -v cargo &> /dev/null; then
exit 1
fi
echo "Building Memora CLI for multiple platforms..."
echo "Building Hindsight CLI for multiple platforms..."
# Ensure we're in the right directory
cd "$(dirname "$0")"
@@ -50,10 +50,10 @@ build_target() {
# Copy to dist
if [[ "$target" == *"windows"* ]]; then
cp "target/$target/release/memora.exe" "dist/$output_name.exe"
cp "target/$target/release/hindsight.exe" "dist/$output_name.exe"
echo "Created: dist/$output_name.exe"
else
cp "target/$target/release/memora" "dist/$output_name"
cp "target/$target/release/hindsight" "dist/$output_name"
chmod +x "dist/$output_name"
echo "Created: dist/$output_name"
fi
@@ -70,19 +70,19 @@ case "$OS" in
Darwin)
if [[ "$ARCH" == "arm64" ]]; then
echo "Building for macOS ARM64 (Apple Silicon)..."
build_target "aarch64-apple-darwin" "memora-macos-arm64"
build_target "aarch64-apple-darwin" "hindsight-macos-arm64"
else
echo "Building for macOS x86_64 (Intel)..."
build_target "x86_64-apple-darwin" "memora-macos-x86_64"
build_target "x86_64-apple-darwin" "hindsight-macos-x86_64"
fi
;;
Linux)
if [[ "$ARCH" == "x86_64" ]]; then
echo "Building for Linux x86_64..."
build_target "x86_64-unknown-linux-gnu" "memora-linux-x86_64"
build_target "x86_64-unknown-linux-gnu" "hindsight-linux-x86_64"
elif [[ "$ARCH" == "aarch64" ]]; then
echo "Building for Linux ARM64..."
build_target "aarch64-unknown-linux-gnu" "memora-linux-arm64"
build_target "aarch64-unknown-linux-gnu" "hindsight-linux-arm64"
fi
;;
*)

Some files were not shown because too many files have changed in this diff Show More