make transformation less specific

.github/workflows/test.yml

@@ -154,6 +154,9 @@ jobs:
     env:
       HINDSIGHT_API_LLM_PROVIDER: groq
       HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
+      GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
+      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       # Prefer CPU-only PyTorch in CI (but keep PyPI for everything else)
@@ -469,4 +472,165 @@ jobs:
 
     - name: Run tests
       working-directory: ./hindsight-integrations/litellm
-      run: uv run pytest tests -v
+      run: uv run pytest tests -v
+
+  test-doc-examples:
+    runs-on: ubuntu-latest
+    env:
+      HINDSIGHT_API_LLM_PROVIDER: groq
+      HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
+      HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
+      HINDSIGHT_API_URL: http://localhost:8888
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      # Model for test generation and analysis (options: gpt-4o, o3-mini, o1, etc.)
+      DOC_TEST_MODEL: o3-mini
+      # Prefer CPU-only PyTorch in CI (but keep PyPI for everything else)
+      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version-file: ".python-version"
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+        cache: 'npm'
+        cache-dependency-path: package-lock.json
+
+    - name: Build API
+      working-directory: ./hindsight-api
+      run: uv build
+
+    - name: Build Python client
+      working-directory: ./hindsight-clients/python
+      run: uv build
+
+    - name: Install Python client
+      working-directory: ./hindsight-clients/python
+      run: uv sync --index-strategy unsafe-best-match
+
+    - name: Install API dependencies
+      working-directory: ./hindsight-api
+      run: uv sync --no-install-project --index-strategy unsafe-best-match
+
+    - name: Install test dependencies in API venv
+      working-directory: ./hindsight-api
+      run: |
+        uv pip install ../hindsight-clients/python requests anthropic
+        uv pip install ../hindsight-integrations/litellm
+        uv pip install ../hindsight-integrations/openai
+
+    - name: Verify Python dependencies
+      working-directory: ./hindsight-api
+      run: |
+        echo "=== Verifying Python dependencies ==="
+        uv run python -c "
+        import sys
+        print(f'Python: {sys.executable}')
+        print(f'Prefix: {sys.prefix}')
+
+        # Check required packages
+        packages = [
+            'hindsight_client',
+            'hindsight_litellm',
+            'hindsight_openai',
+            'anthropic',
+            'openai',
+        ]
+
+        missing = []
+        for pkg in packages:
+            try:
+                __import__(pkg)
+                print(f'  ✓ {pkg}')
+            except ImportError as e:
+                print(f'  ✗ {pkg}: {e}')
+                missing.append(pkg)
+
+        if missing:
+            print(f'\nERROR: Missing packages: {missing}')
+            sys.exit(1)
+        print('\nAll Python dependencies verified!')
+        "
+
+    - name: Install TypeScript client dependencies
+      run: npm ci
+
+    - name: Build TypeScript client
+      run: npm run build --workspace=hindsight-clients/typescript
+
+    - name: Install TypeScript client globally
+      working-directory: ./hindsight-clients/typescript
+      run: npm install -g .
+
+    - name: Make TypeScript client available for temp files
+      run: |
+        # ESM modules don't use NODE_PATH, so create node_modules in /tmp
+        # where test scripts are written
+        mkdir -p /tmp/node_modules/@vectorize-io
+        ln -s ${{ github.workspace }}/hindsight-clients/typescript /tmp/node_modules/@vectorize-io/hindsight-client
+
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@stable
+
+    - name: Build and install hindsight CLI
+      working-directory: ./hindsight-cli
+      run: |
+        cargo build --release
+        sudo cp target/release/hindsight /usr/local/bin/
+
+    - name: Create .env file
+      run: |
+        cat > .env << EOF
+        HINDSIGHT_API_LLM_PROVIDER=${{ env.HINDSIGHT_API_LLM_PROVIDER }}
+        HINDSIGHT_API_LLM_API_KEY=${{ env.HINDSIGHT_API_LLM_API_KEY }}
+        HINDSIGHT_API_LLM_MODEL=${{ env.HINDSIGHT_API_LLM_MODEL }}
+        EOF
+
+    - name: Start API server
+      run: |
+        ./scripts/dev/start-api.sh > /tmp/api-server.log 2>&1 &
+        echo "Waiting for API server to be ready..."
+        for i in {1..60}; do
+          if curl -sf http://localhost:8888/health > /dev/null 2>&1; then
+            echo "API server is ready after ${i}s"
+            break
+          fi
+          if [ $i -eq 60 ]; then
+            echo "API server failed to start after 60s"
+            cat /tmp/api-server.log
+            exit 1
+          fi
+          sleep 1
+        done
+
+    - name: Test documentation examples
+      working-directory: ./hindsight-api
+      env:
+        REPO_ROOT: ${{ github.workspace }}
+      run: uv run python ../scripts/test-doc-examples.py
+
+    - name: Write test summary
+      if: always()
+      run: |
+        echo "=== Documentation Test Summary ==="
+        cat /tmp/doc-test-summary.md
+        cat /tmp/doc-test-summary.md >> $GITHUB_STEP_SUMMARY
+
+    - name: Show API server logs
+      if: always()
+      run: |
+        echo "=== API Server Logs ==="
+        cat /tmp/api-server.log || echo "No API server log found"

hindsight-dev/hindsight_dev/generate_changelog.py

@@ -0,0 +1,359 @@
+#!/usr/bin/env python3
+"""
+Generate changelog entry for a new release.
+
+This script fetches the commit diff between releases, uses an LLM to summarize,
+and prepends the entry to the changelog page.
+"""
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+from openai import OpenAI
+from pydantic import BaseModel
+from rich.console import Console
+
+console = Console()
+
+GITHUB_REPO = "vectorize-io/hindsight"
+GITHUB_RELEASES_URL = f"https://github.com/{GITHUB_REPO}/releases"
+GITHUB_COMMIT_URL = f"https://github.com/{GITHUB_REPO}/commit"
+REPO_PATH = Path(__file__).parent.parent.parent
+CHANGELOG_PATH = REPO_PATH / "hindsight-docs" / "docs" / "changelog" / "index.md"
+
+
+class ChangelogEntry(BaseModel):
+    """A single changelog entry."""
+    category: str  # "feature", "improvement", "bugfix", "breaking", "other"
+    summary: str  # Brief description of the change
+    commit_id: str  # Short commit hash
+
+
+class ChangelogResponse(BaseModel):
+    """Structured response from LLM."""
+    entries: list[ChangelogEntry]
+
+
+@dataclass
+class Commit:
+    """Parsed commit from git log."""
+    hash: str
+    message: str
+
+
+def parse_semver(version: str) -> tuple[int, int, int]:
+    """Parse a semver string into (major, minor, patch)."""
+    version = version.lstrip("v")
+    match = re.match(r"^(\d+)\.(\d+)\.(\d+)", version)
+    if not match:
+        raise ValueError(f"Invalid semver: {version}")
+    return int(match.group(1)), int(match.group(2)), int(match.group(3))
+
+
+def get_git_tags() -> list[str]:
+    """Get all git tags sorted by semver (newest first)."""
+    result = subprocess.run(
+        ["git", "tag"],
+        cwd=REPO_PATH,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    tags = [t.strip() for t in result.stdout.strip().split("\n") if t.strip()]
+
+    valid_tags = []
+    for tag in tags:
+        try:
+            parse_semver(tag)
+            valid_tags.append(tag)
+        except ValueError:
+            continue
+
+    valid_tags.sort(key=lambda t: parse_semver(t), reverse=True)
+    return valid_tags
+
+
+def find_previous_version(new_version: str, existing_tags: list[str]) -> str | None:
+    """Find the previous version based on semver rules."""
+    new_major, new_minor, new_patch = parse_semver(new_version)
+
+    candidates = []
+    for tag in existing_tags:
+        try:
+            major, minor, patch = parse_semver(tag)
+        except ValueError:
+            continue
+
+        if (major, minor, patch) >= (new_major, new_minor, new_patch):
+            continue
+
+        candidates.append((tag, (major, minor, patch)))
+
+    if not candidates:
+        return None
+
+    candidates.sort(key=lambda x: x[1], reverse=True)
+    return candidates[0][0]
+
+
+def get_commits(from_ref: str | None, to_ref: str) -> list[Commit]:
+    """Get commits between two refs as structured data."""
+    if from_ref:
+        cmd = ["git", "log", "--format=%h|%s", "--no-merges", f"{from_ref}..{to_ref}"]
+    else:
+        cmd = ["git", "log", "--format=%h|%s", "--no-merges", to_ref]
+
+    result = subprocess.run(
+        cmd,
+        cwd=REPO_PATH,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+    commits = []
+    for line in result.stdout.strip().split("\n"):
+        if not line:
+            continue
+        parts = line.split("|", 1)
+        if len(parts) == 2:
+            commits.append(Commit(hash=parts[0], message=parts[1]))
+
+    return commits
+
+
+def get_detailed_diff(from_ref: str | None, to_ref: str) -> str:
+    """Get file change stats between two refs."""
+    if from_ref:
+        cmd = ["git", "diff", "--stat", f"{from_ref}..{to_ref}"]
+    else:
+        cmd = ["git", "diff", "--stat", f"{to_ref}^..{to_ref}"]
+
+    result = subprocess.run(
+        cmd,
+        cwd=REPO_PATH,
+        capture_output=True,
+        text=True,
+    )
+    return result.stdout.strip()
+
+
+def analyze_commits_with_llm(
+    client: OpenAI,
+    model: str,
+    version: str,
+    commits: list[Commit],
+    file_diff: str,
+) -> list[ChangelogEntry]:
+    """Use LLM to analyze commits and return structured changelog entries."""
+    commits_json = json.dumps(
+        [{"commit_id": c.hash, "message": c.message} for c in commits],
+        indent=2
+    )
+
+    prompt = f"""Analyze the following git commits for release {version} of Hindsight (an AI memory system).
+
+For each meaningful change, create a changelog entry with:
+- category: one of "feature", "improvement", "bugfix", "breaking", "other"
+- summary: brief one-line description of the change (user-facing, not technical)
+- commit_id: the commit hash from the input
+
+Rules:
+- Group related commits into a single entry if they're part of the same change
+- Skip trivial changes (typo fixes, formatting, internal refactoring)
+- Skip repository-only changes: README updates, CI/GitHub Actions, release scripts, changelog updates, version bumps
+- Focus on user-facing changes that affect the product functionality
+- Use the exact commit_id from the input (pick the most relevant one if grouping)
+- If no meaningful changes remain after filtering, return an empty list
+
+Commits:
+{commits_json}
+
+Files changed summary:
+{file_diff[:4000]}"""
+
+    response = client.beta.chat.completions.parse(
+        model=model,
+        messages=[{"role": "user", "content": prompt}],
+        response_format=ChangelogResponse,
+        max_completion_tokens=16000,
+    )
+
+    return response.choices[0].message.parsed.entries
+
+
+def build_changelog_markdown(
+    version: str,
+    tag: str,
+    entries: list[ChangelogEntry],
+) -> str:
+    """Build markdown changelog from structured entries."""
+    release_url = f"{GITHUB_RELEASES_URL}/tag/{tag}"
+
+    # Group entries by category
+    categories = {
+        "breaking": ("Breaking Changes", []),
+        "feature": ("Features", []),
+        "improvement": ("Improvements", []),
+        "bugfix": ("Bug Fixes", []),
+        "other": ("Other", []),
+    }
+
+    for entry in entries:
+        cat = entry.category.lower()
+        if cat in categories:
+            categories[cat][1].append(entry)
+        else:
+            categories["other"][1].append(entry)
+
+    # Build markdown
+    lines = [f"## [{version}]({release_url})", ""]
+
+    for cat_key in ["breaking", "feature", "improvement", "bugfix", "other"]:
+        cat_name, cat_entries = categories[cat_key]
+        if cat_entries:
+            lines.append(f"**{cat_name}**")
+            lines.append("")
+            for entry in cat_entries:
+                commit_url = f"{GITHUB_COMMIT_URL}/{entry.commit_id}"
+                lines.append(f"- {entry.summary} ([`{entry.commit_id}`]({commit_url}))")
+            lines.append("")
+
+    return "\n".join(lines)
+
+
+def read_existing_changelog() -> tuple[str, str]:
+    """Read existing changelog and split into header and content."""
+    if not CHANGELOG_PATH.exists():
+        header = """---
+sidebar_position: 1
+---
+
+# Changelog
+
+For full release details, see [GitHub Releases](https://github.com/vectorize-io/hindsight/releases).
+"""
+        return header, ""
+
+    content = CHANGELOG_PATH.read_text()
+
+    match = re.search(r"^## ", content, re.MULTILINE)
+    if match:
+        header = content[:match.start()].rstrip() + "\n\n"
+        releases = content[match.start():]
+    else:
+        header = content.rstrip() + "\n\n"
+        releases = ""
+
+    return header, releases
+
+
+def write_changelog(header: str, new_entry: str, existing_releases: str) -> None:
+    """Write changelog with new entry prepended."""
+    content = header + new_entry + "\n" + existing_releases
+    CHANGELOG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    CHANGELOG_PATH.write_text(content.rstrip() + "\n")
+
+
+def generate_changelog_entry(
+    version: str,
+    llm_model: str = "gpt-5.2",
+) -> None:
+    """Generate changelog entry for a specific version."""
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        console.print("[red]Error: OPENAI_API_KEY environment variable not set[/red]")
+        sys.exit(1)
+
+    client = OpenAI(api_key=api_key)
+
+    tag = version if version.startswith("v") else f"v{version}"
+    display_version = version.lstrip("v")
+
+    console.print(f"[blue]Fetching tags from repository...[/blue]")
+    existing_tags = get_git_tags()
+
+    if tag not in existing_tags and display_version not in existing_tags:
+        console.print(f"[red]Error: Tag {tag} not found in repository[/red]")
+        console.print("[red]Create the tag first before generating changelog[/red]")
+        sys.exit(1)
+
+    actual_tag = tag if tag in existing_tags else display_version
+
+    previous_tag = find_previous_version(display_version, existing_tags)
+
+    if previous_tag:
+        console.print(f"[green]Found previous version: {previous_tag}[/green]")
+    else:
+        console.print("[yellow]No previous version found, will include all commits[/yellow]")
+
+    console.print(f"[blue]Getting commits...[/blue]")
+    commits = get_commits(previous_tag, actual_tag)
+    file_diff = get_detailed_diff(previous_tag, actual_tag)
+
+    if not commits:
+        console.print("[red]Error: No commits found for this release[/red]")
+        sys.exit(1)
+
+    console.print(f"[blue]Found {len(commits)} commits[/blue]")
+
+    # Log commits
+    console.print("\n[bold]Commits:[/bold]")
+    for c in commits:
+        console.print(f"  {c.hash} {c.message}")
+
+    console.print("\n[bold]Files changed:[/bold]")
+    console.print(file_diff[:4000] if len(file_diff) > 4000 else file_diff)
+    console.print("")
+
+    console.print(f"[blue]Analyzing commits with LLM ({llm_model})...[/blue]")
+    entries = analyze_commits_with_llm(client, llm_model, display_version, commits, file_diff)
+
+    console.print(f"\n[bold]LLM identified {len(entries)} changelog entries:[/bold]")
+    for entry in entries:
+        console.print(f"  [{entry.category}] {entry.summary} ({entry.commit_id})")
+
+    new_entry = build_changelog_markdown(display_version, tag, entries)
+
+    header, existing_releases = read_existing_changelog()
+
+    if f"## [{display_version}]" in existing_releases:
+        console.print(f"[red]Error: Version {display_version} already exists in changelog[/red]")
+        sys.exit(1)
+
+    write_changelog(header, new_entry, existing_releases)
+
+    console.print(f"\n[green]Changelog updated: {CHANGELOG_PATH}[/green]")
+    console.print(f"\n[bold]New entry:[/bold]\n{new_entry}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate changelog entry for a release",
+        usage="generate-changelog VERSION [--model MODEL]",
+    )
+    parser.add_argument(
+        "version",
+        help="Version to generate changelog for (e.g., 1.0.5, v1.0.5)",
+    )
+    parser.add_argument(
+        "--model",
+        default="gpt-5.2",
+        help="OpenAI model to use (default: gpt-5.2)",
+    )
+
+    args = parser.parse_args()
+
+    generate_changelog_entry(
+        version=args.version,
+        llm_model=args.model,
+    )
+
+
+if __name__ == "__main__":
+    main()

hindsight-dev/pyproject.toml

@@ -24,3 +24,4 @@ hindsight-api = { workspace = true }
 
 [project.scripts]
 generate-openapi = "hindsight_dev.generate_openapi:generate_openapi_spec"
+generate-changelog = "hindsight_dev.generate_changelog:main"

hindsight-docs/docs/changelog/index.md

@@ -4,4 +4,35 @@ sidebar_position: 1
 
 # Changelog
 
-Coming soon.
+For full release details, see [GitHub Releases](https://github.com/vectorize-io/hindsight/releases).
+
+## [0.1.5](https://github.com/vectorize-io/hindsight/releases/tag/v0.1.5)
+
+**Features**
+
+- Added LiteLLM integration so Hindsight can capture and manage memories from LiteLLM-based LLM calls. ([`dfccbf2`](https://github.com/vectorize-io/hindsight/commit/dfccbf2))
+- Added an optional graph-based retriever (MPFP) to improve recall by leveraging relationships between memories. ([`7445cef`](https://github.com/vectorize-io/hindsight/commit/7445cef))
+
+**Improvements**
+
+- Switched the embedded Postgres layer to pg0-embedded for a smoother local/standalone experience. ([`94c2b85`](https://github.com/vectorize-io/hindsight/commit/94c2b85))
+
+**Bug Fixes**
+
+- Fixed repeated retries on 400 errors from the LLM, preventing unnecessary request loops and failures. ([`70983f5`](https://github.com/vectorize-io/hindsight/commit/70983f5))
+- Fixed recall trace visualization in the control plane so search/recall debugging displays correctly. ([`922164e`](https://github.com/vectorize-io/hindsight/commit/922164e))
+- Fixed the CLI installer to make installation more reliable. ([`158a6aa`](https://github.com/vectorize-io/hindsight/commit/158a6aa))
+- Updated Next.js to patch security vulnerabilities (CVE-2025-55184, CVE-2025-55183). ([`f018cc5`](https://github.com/vectorize-io/hindsight/commit/f018cc5))
+
+## [0.1.3](https://github.com/vectorize-io/hindsight/releases/tag/v0.1.3)
+
+**Improvements**
+
+- Improved CLI and UI branding/polish, including new banner/logo assets and updated interface styling. ([`fa554b8`](https://github.com/vectorize-io/hindsight/commit/fa554b8))
+
+
+## [0.1.2](https://github.com/vectorize-io/hindsight/releases/tag/v0.1.2)
+
+**Bug Fixes**
+
+- Fixed the standalone Docker image so it builds/runs correctly. ([`1056a20`](https://github.com/vectorize-io/hindsight/commit/1056a20))

hindsight-docs/docs/developer/models.md

@@ -6,14 +6,70 @@ Hindsight uses several machine learning models for different tasks.
 
 | Model Type | Purpose | Default | Configurable |
 |------------|---------|---------|--------------|
+| **LLM** | Fact extraction, reasoning, generation | Provider-specific | Yes |
 | **Embedding** | Vector representations for semantic search | `BAAI/bge-small-en-v1.5` | Yes |
 | **Cross-Encoder** | Reranking search results | `cross-encoder/ms-marco-MiniLM-L-6-v2` | Yes |
-| **LLM** | Fact extraction, reasoning, generation | Provider-specific | Yes |
 
 All local models (embedding, cross-encoder) are automatically downloaded from HuggingFace on first run.
 
 ---
 
+## LLM
+
+Used for fact extraction, entity resolution, opinion generation, and answer synthesis.
+
+**Supported providers:** OpenAI, Gemini, Groq, Ollama
+
+### Tested Models
+
+The following models have been tested and verified to work correctly with Hindsight:
+
+| Provider | Model |
+|----------|-------|
+| **OpenAI** | `gpt-5` |
+| **OpenAI** | `gpt-5-mini` |
+| **OpenAI** | `gpt-5-nano` |
+| **OpenAI** | `gpt-4.1-mini` |
+| **OpenAI** | `gpt-4.1-nano` |
+| **OpenAI** | `gpt-4o-mini` |
+| **Gemini** | `gemini-2.5-flash` |
+| **Gemini** | `gemini-2.5-flash-lite` |
+| **Groq** | `openai/gpt-oss-120b` |
+| **Groq** | `openai/gpt-oss-20b` |
+| **Groq** | `llama-3.3-70b-versatile` |
+
+### Using Other Models
+
+Other LLM models not listed above may work with Hindsight, but they must support **at least 65,000 output tokens** to ensure reliable fact extraction. If you need support for a specific model that doesn't meet this requirement, please [open an issue](https://github.com/hindsight-ai/hindsight/issues) to request an exception.
+
+### Configuration
+
+```bash
+# Groq (recommended)
+export HINDSIGHT_API_LLM_PROVIDER=groq
+export HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx
+export HINDSIGHT_API_LLM_MODEL=openai/gpt-oss-20b
+
+# OpenAI
+export HINDSIGHT_API_LLM_PROVIDER=openai
+export HINDSIGHT_API_LLM_API_KEY=sk-xxxxxxxxxxxx
+export HINDSIGHT_API_LLM_MODEL=gpt-4o
+
+# Gemini
+export HINDSIGHT_API_LLM_PROVIDER=gemini
+export HINDSIGHT_API_LLM_API_KEY=xxxxxxxxxxxx
+export HINDSIGHT_API_LLM_MODEL=gemini-2.0-flash
+
+# Ollama (local)
+export HINDSIGHT_API_LLM_PROVIDER=ollama
+export HINDSIGHT_API_LLM_BASE_URL=http://localhost:11434/v1
+export HINDSIGHT_API_LLM_MODEL=llama3.1
+```
+
+**Note:** The LLM is the primary bottleneck for retain operations. See [Performance](./performance) for optimization strategies.
+
+---
+
 ## Embedding Model
 
 Converts text into dense vector representations for semantic similarity search.
@@ -22,14 +78,13 @@ Converts text into dense vector representations for semantic similarity search.
 
 **Alternatives:**
 
-| Model | Dimensions | Use Case |
-|-------|------------|----------|
-| `BAAI/bge-small-en-v1.5` | 384 | Default, fast, good quality |
-| `BAAI/bge-base-en-v1.5` | 768 | Higher accuracy, slower |
-| `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2` | 384 | Multilingual (50+ languages) |
+| Model | Use Case |
+|-------|----------|
+| `BAAI/bge-small-en-v1.5` | Default, fast, good quality |
+| `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2` | Multilingual (50+ languages) |
 
 :::warning
-All embedding models must produce 384-dimensional vectors to match the database schema.
+All embedding models must produce **384-dimensional vectors** to match the database schema.
 :::
 
 **Configuration:**
@@ -71,44 +126,3 @@ export HINDSIGHT_API_RERANKER_LOCAL_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2
 export HINDSIGHT_API_RERANKER_PROVIDER=tei
 export HINDSIGHT_API_RERANKER_TEI_URL=http://localhost:8081
 ```
-
----
-
-## LLM
-
-Used for fact extraction, entity resolution, opinion generation, and answer synthesis.
-
-**Supported providers:** Groq, OpenAI, Gemini, Ollama
-
-| Provider | Recommended Model | Best For |
-|----------|------------------|----------|
-| **Groq** | `openai/gpt-oss-20b` | Fast inference, high throughput (recommended) |
-| **OpenAI** | `gpt-4o` | Good quality |
-| **Gemini** | `gemini-2.0-flash` | Good quality, cost effective |
-| **Ollama** | `llama3.1` | Local deployment, privacy |
-
-**Configuration:**
-
-```bash
-# Groq (recommended)
-export HINDSIGHT_API_LLM_PROVIDER=groq
-export HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx
-export HINDSIGHT_API_LLM_MODEL=openai/gpt-oss-20b
-
-# OpenAI
-export HINDSIGHT_API_LLM_PROVIDER=openai
-export HINDSIGHT_API_LLM_API_KEY=sk-xxxxxxxxxxxx
-export HINDSIGHT_API_LLM_MODEL=gpt-4o
-
-# Gemini
-export HINDSIGHT_API_LLM_PROVIDER=gemini
-export HINDSIGHT_API_LLM_API_KEY=xxxxxxxxxxxx
-export HINDSIGHT_API_LLM_MODEL=gemini-2.0-flash
-
-# Ollama (local)
-export HINDSIGHT_API_LLM_PROVIDER=ollama
-export HINDSIGHT_API_LLM_BASE_URL=http://localhost:11434/v1
-export HINDSIGHT_API_LLM_MODEL=llama3.1
-```
-
-**Note:** The LLM is the primary bottleneck for retain operations. See [Performance](./performance) for optimization strategies.

hindsight-docs/src/css/custom.css

@@ -514,6 +514,27 @@ article a:not(.button):not([class*="hash-link"]):hover {
   text-decoration-color: var(--hindsight-gradient-start);
 }
 
+/* Links inside code blocks - use solid color instead of gradient */
+code a,
+pre a,
+article code a,
+article pre a {
+  background: none !important;
+  -webkit-background-clip: unset !important;
+  -webkit-text-fill-color: var(--ifm-color-primary) !important;
+  background-clip: unset !important;
+  color: var(--ifm-color-primary) !important;
+  text-decoration: underline;
+}
+
+code a:hover,
+pre a:hover,
+article code a:hover,
+article pre a:hover {
+  color: var(--ifm-color-primary-dark) !important;
+  -webkit-text-fill-color: var(--ifm-color-primary-dark) !important;
+}
+
 /* Admonitions - gradient themed */
 .theme-admonition,
 [class*="admonition_"] {

hindsight-docs/static/llms-full.txt

@@ -3,7 +3,7 @@
 > Agent Memory that Works Like Human Memory
 
 This file contains the complete Hindsight documentation for LLM consumption.
-Generated: 2025-12-15T09:47:27.854Z
+Generated: 2025-12-15T13:52:15.559Z
 
 ---
 
@@ -2750,14 +2750,70 @@ Hindsight uses several machine learning models for different tasks.
 
 | Model Type | Purpose | Default | Configurable |
 |------------|---------|---------|--------------|
+| **LLM** | Fact extraction, reasoning, generation | Provider-specific | Yes |
 | **Embedding** | Vector representations for semantic search | `BAAI/bge-small-en-v1.5` | Yes |
 | **Cross-Encoder** | Reranking search results | `cross-encoder/ms-marco-MiniLM-L-6-v2` | Yes |
-| **LLM** | Fact extraction, reasoning, generation | Provider-specific | Yes |
 
 All local models (embedding, cross-encoder) are automatically downloaded from HuggingFace on first run.
 
 ---
 
+## LLM
+
+Used for fact extraction, entity resolution, opinion generation, and answer synthesis.
+
+**Supported providers:** OpenAI, Gemini, Groq, Ollama
+
+### Tested Models
+
+The following models have been tested and verified to work correctly with Hindsight:
+
+| Provider | Model |
+|----------|-------|
+| **OpenAI** | `gpt-5` |
+| **OpenAI** | `gpt-5-mini` |
+| **OpenAI** | `gpt-5-nano` |
+| **OpenAI** | `gpt-4.1-mini` |
+| **OpenAI** | `gpt-4.1-nano` |
+| **OpenAI** | `gpt-4o-mini` |
+| **Gemini** | `gemini-2.5-flash` |
+| **Gemini** | `gemini-2.5-flash-lite` |
+| **Groq** | `openai/gpt-oss-120b` |
+| **Groq** | `openai/gpt-oss-20b` |
+| **Groq** | `llama-3.3-70b-versatile` |
+
+### Using Other Models
+
+Other LLM models not listed above may work with Hindsight, but they must support **at least 65,000 output tokens** to ensure reliable fact extraction. If you need support for a specific model that doesn't meet this requirement, please [open an issue](https://github.com/hindsight-ai/hindsight/issues) to request an exception.
+
+### Configuration
+
+```bash
+# Groq (recommended)
+export HINDSIGHT_API_LLM_PROVIDER=groq
+export HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx
+export HINDSIGHT_API_LLM_MODEL=openai/gpt-oss-20b
+
+# OpenAI
+export HINDSIGHT_API_LLM_PROVIDER=openai
+export HINDSIGHT_API_LLM_API_KEY=sk-xxxxxxxxxxxx
+export HINDSIGHT_API_LLM_MODEL=gpt-4o
+
+# Gemini
+export HINDSIGHT_API_LLM_PROVIDER=gemini
+export HINDSIGHT_API_LLM_API_KEY=xxxxxxxxxxxx
+export HINDSIGHT_API_LLM_MODEL=gemini-2.0-flash
+
+# Ollama (local)
+export HINDSIGHT_API_LLM_PROVIDER=ollama
+export HINDSIGHT_API_LLM_BASE_URL=http://localhost:11434/v1
+export HINDSIGHT_API_LLM_MODEL=llama3.1
+```
+
+**Note:** The LLM is the primary bottleneck for retain operations. See [Performance](./performance) for optimization strategies.
+
+---
+
 ## Embedding Model
 
 Converts text into dense vector representations for semantic similarity search.
@@ -2766,14 +2822,13 @@ Converts text into dense vector representations for semantic similarity search.
 
 **Alternatives:**
 
-| Model | Dimensions | Use Case |
-|-------|------------|----------|
-| `BAAI/bge-small-en-v1.5` | 384 | Default, fast, good quality |
-| `BAAI/bge-base-en-v1.5` | 768 | Higher accuracy, slower |
-| `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2` | 384 | Multilingual (50+ languages) |
+| Model | Use Case |
+|-------|----------|
+| `BAAI/bge-small-en-v1.5` | Default, fast, good quality |
+| `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2` | Multilingual (50+ languages) |
 
 :::warning
-All embedding models must produce 384-dimensional vectors to match the database schema.
+All embedding models must produce **384-dimensional vectors** to match the database schema.
 :::
 
 **Configuration:**
@@ -2816,47 +2871,6 @@ export HINDSIGHT_API_RERANKER_PROVIDER=tei
 export HINDSIGHT_API_RERANKER_TEI_URL=http://localhost:8081
 ```
 
----
-
-## LLM
-
-Used for fact extraction, entity resolution, opinion generation, and answer synthesis.
-
-**Supported providers:** Groq, OpenAI, Gemini, Ollama
-
-| Provider | Recommended Model | Best For |
-|----------|------------------|----------|
-| **Groq** | `openai/gpt-oss-20b` | Fast inference, high throughput (recommended) |
-| **OpenAI** | `gpt-4o` | Good quality |
-| **Gemini** | `gemini-2.0-flash` | Good quality, cost effective |
-| **Ollama** | `llama3.1` | Local deployment, privacy |
-
-**Configuration:**
-
-```bash
-# Groq (recommended)
-export HINDSIGHT_API_LLM_PROVIDER=groq
-export HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx
-export HINDSIGHT_API_LLM_MODEL=openai/gpt-oss-20b
-
-# OpenAI
-export HINDSIGHT_API_LLM_PROVIDER=openai
-export HINDSIGHT_API_LLM_API_KEY=sk-xxxxxxxxxxxx
-export HINDSIGHT_API_LLM_MODEL=gpt-4o
-
-# Gemini
-export HINDSIGHT_API_LLM_PROVIDER=gemini
-export HINDSIGHT_API_LLM_API_KEY=xxxxxxxxxxxx
-export HINDSIGHT_API_LLM_MODEL=gemini-2.0-flash
-
-# Ollama (local)
-export HINDSIGHT_API_LLM_PROVIDER=ollama
-export HINDSIGHT_API_LLM_BASE_URL=http://localhost:11434/v1
-export HINDSIGHT_API_LLM_MODEL=llama3.1
-```
-
-**Note:** The LLM is the primary bottleneck for retain operations. See [Performance](./performance) for optimization strategies.
-
 
 ---
 
@@ -4067,7 +4081,38 @@ Known Solutions:
 
 # Changelog
 
-Coming soon.
+For full release details, see [GitHub Releases](https://github.com/vectorize-io/hindsight/releases).
+
+## [0.1.5](https://github.com/vectorize-io/hindsight/releases/tag/v0.1.5)
+
+**Features**
+
+- Added LiteLLM integration so Hindsight can capture and manage memories from LiteLLM-based LLM calls. ([`dfccbf2`](https://github.com/vectorize-io/hindsight/commit/dfccbf2))
+- Added an optional graph-based retriever (MPFP) to improve recall by leveraging relationships between memories. ([`7445cef`](https://github.com/vectorize-io/hindsight/commit/7445cef))
+
+**Improvements**
+
+- Switched the embedded Postgres layer to pg0-embedded for a smoother local/standalone experience. ([`94c2b85`](https://github.com/vectorize-io/hindsight/commit/94c2b85))
+
+**Bug Fixes**
+
+- Fixed repeated retries on 400 errors from the LLM, preventing unnecessary request loops and failures. ([`70983f5`](https://github.com/vectorize-io/hindsight/commit/70983f5))
+- Fixed recall trace visualization in the control plane so search/recall debugging displays correctly. ([`922164e`](https://github.com/vectorize-io/hindsight/commit/922164e))
+- Fixed the CLI installer to make installation more reliable. ([`158a6aa`](https://github.com/vectorize-io/hindsight/commit/158a6aa))
+- Updated Next.js to patch security vulnerabilities (CVE-2025-55184, CVE-2025-55183). ([`f018cc5`](https://github.com/vectorize-io/hindsight/commit/f018cc5))
+
+## [0.1.3](https://github.com/vectorize-io/hindsight/releases/tag/v0.1.3)
+
+**Improvements**
+
+- Improved CLI and UI branding/polish, including new banner/logo assets and updated interface styling. ([`fa554b8`](https://github.com/vectorize-io/hindsight/commit/fa554b8))
+
+
+## [0.1.2](https://github.com/vectorize-io/hindsight/releases/tag/v0.1.2)
+
+**Bug Fixes**
+
+- Fixed the standalone Docker image so it builds/runs correctly. ([`1056a20`](https://github.com/vectorize-io/hindsight/commit/1056a20))
 
 
 ---

scripts/dev/generate-changelog.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -e
+
+cd "$(dirname "$0")/../.."
+
+if [ -z "$1" ]; then
+  echo "Usage: $0 VERSION [--model MODEL]"
+  echo ""
+  echo "Generate changelog entry for a release."
+  echo ""
+  echo "Examples:"
+  echo "  $0 1.0.5"
+  echo "  $0 v1.0.5"
+  echo "  $0 1.0.5 --model gpt-4o"
+  exit 1
+fi
+
+if [ -z "$OPENAI_API_KEY" ]; then
+  ENV_FILE=".env"
+  if [ -f "$ENV_FILE" ]; then
+    echo "Loading environment from $ENV_FILE"
+    set -a
+    source "$ENV_FILE"
+    set +a
+  else
+    echo "Error: OPENAI_API_KEY not set and no .env file found"
+    exit 1
+  fi
+fi
+
+cd hindsight-dev
+uv run generate-changelog "$@"

scripts/test-doc-examples.py

@@ -0,0 +1,798 @@
+#!/usr/bin/env python3
+"""
+Documentation Example Tester
+
+Tests code examples from documentation by running them directly.
+Uses deterministic transformations (no LLM) for test generation.
+LLM is only used to analyze failures and determine if they're real doc bugs.
+
+Usage:
+    python scripts/test-doc-examples.py
+
+Environment variables:
+    OPENAI_API_KEY: Required for failure analysis
+    HINDSIGHT_API_URL: URL of running Hindsight server (default: http://localhost:8888)
+"""
+
+import os
+import re
+import sys
+import site
+import json
+import glob
+import subprocess
+import tempfile
+import traceback
+import uuid
+from dataclasses import dataclass, field
+from typing import Optional
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import threading
+
+from openai import OpenAI
+
+# Thread-safe print
+print_lock = threading.Lock()
+
+def safe_print(*args, **kwargs):
+    with print_lock:
+        print(*args, **kwargs)
+        sys.stdout.flush()
+
+
+@dataclass
+class CodeExample:
+    file_path: str
+    language: str
+    code: str
+    context: str
+    line_number: int
+
+
+@dataclass
+class TestResult:
+    example: CodeExample
+    success: bool
+    output: str
+    error: Optional[str] = None
+    transformed_code: Optional[str] = None
+    skip_reason: Optional[str] = None
+
+
+@dataclass
+class TestReport:
+    total: int = 0
+    passed: int = 0
+    failed: int = 0
+    skipped: int = 0
+    results: list[TestResult] = field(default_factory=list)
+
+    def add_result(self, result: TestResult):
+        self.total += 1
+        self.results.append(result)
+        if result.skip_reason:
+            self.skipped += 1
+        elif result.success:
+            self.passed += 1
+        else:
+            self.failed += 1
+
+
+# =============================================================================
+# STEP 1: Extract code blocks from markdown
+# =============================================================================
+
+def find_markdown_files(repo_root: str) -> list[str]:
+    """Find all markdown files, excluding auto-generated docs."""
+    skip_patterns = [
+        "node_modules", ".git", "venv", "__pycache__",
+        "hindsight_client_api/docs", "hindsight-clients/typescript/docs",
+        "target/", "dist/",
+    ]
+    md_files = []
+    for pattern in ["*.md", "**/*.md"]:
+        for f in glob.glob(os.path.join(repo_root, pattern), recursive=True):
+            if os.path.islink(f):
+                continue
+            if any(skip in f for skip in skip_patterns):
+                continue
+            md_files.append(f)
+    return sorted(set(md_files))
+
+
+def extract_code_blocks(file_path: str) -> list[CodeExample]:
+    """Extract code blocks from a markdown file."""
+    with open(file_path, "r") as f:
+        content = f.read()
+
+    examples = []
+    pattern = r"```(\w+)\n(.*?)```"
+
+    for match in re.finditer(pattern, content, re.DOTALL):
+        language = match.group(1).lower()
+        code = match.group(2).strip()
+        line_number = content[:match.start()].count('\n') + 1
+
+        if language in ["python", "typescript", "javascript", "bash", "sh"]:
+            start = max(0, match.start() - 150)
+            end = min(len(content), match.end() + 150)
+            context = content[start:end]
+
+            examples.append(CodeExample(
+                file_path=file_path,
+                language=language,
+                code=code,
+                context=context,
+                line_number=line_number
+            ))
+
+    return examples
+
+
+# =============================================================================
+# STEP 2: Determine if example should be skipped (no LLM needed)
+# =============================================================================
+
+def should_skip(code: str, language: str) -> Optional[str]:
+    """Determine if example should be skipped. Returns reason or None."""
+    code_lower = code.lower().strip()
+
+    # Installation/setup commands
+    if language in ["bash", "sh"]:
+        if code_lower.startswith(("pip install", "npm install", "yarn add", "uv pip", "cargo install", "curl ", "wget ")):
+            return "Installation command"
+        if "docker" in code_lower or "docker-compose" in code_lower:
+            return "Docker command"
+        if code_lower.startswith("helm "):
+            return "Helm command"
+        if code_lower.startswith(("cargo build", "cargo test")):
+            return "Cargo command"
+        if "pytest" in code_lower:
+            return "Test suite command"
+        if code_lower.startswith("git clone"):
+            return "Git clone"
+        if "./scripts/" in code_lower:
+            return "Development script"
+        if any(x in code_lower for x in ["npm run dev", "npm run start", "npm run build", "npm run deploy"]):
+            return "NPM script"
+        if code_lower.startswith("cd ") and not code_lower.startswith("cd /tmp"):
+            return "Directory change"
+        if code_lower.startswith("export "):
+            return "Environment variable"
+
+    # Config files
+    if language in ["yaml", "toml", "json", "env"]:
+        return "Configuration file"
+
+    # Too short
+    if len(code.strip()) < 20:
+        return "Too short"
+
+    return None
+
+
+# =============================================================================
+# STEP 3: Transform code (LLM adds setup/cleanup around sacred doc code)
+# =============================================================================
+
+def transform_code(client: OpenAI, example: CodeExample, hindsight_url: str, cli_available: bool, model: str) -> tuple[str, Optional[str]]:
+    """Use LLM to add setup/cleanup around doc code. The doc code itself is not modified."""
+
+    bank_id = f"doc-test-{uuid.uuid4()}"
+
+    # Skip CLI examples if CLI not available
+    if not cli_available and example.language in ["bash", "sh"] and "hindsight " in example.code.lower():
+        return "", "CLI not available"
+
+    if example.language == "python":
+        output_format = f"""Output a Python script (.py):
+- The doc code goes inside a try block
+- Add cleanup in finally: requests.delete("{hindsight_url}/v1/default/banks/{bank_id}")
+- End with: print("TEST PASSED")
+- Do NOT use async/await - the Hindsight client is synchronous"""
+
+    elif example.language in ["typescript", "javascript"]:
+        output_format = f"""Output a JavaScript ES module (.mjs):
+- Remove TypeScript type annotations
+- Wrap in async IIFE: (async () => {{ try {{ ... }} finally {{ ... }} }})();
+- Add cleanup in finally: await fetch("{hindsight_url}/v1/default/banks/{bank_id}", {{ method: "DELETE" }})
+- End with: console.log("TEST PASSED")"""
+
+    elif example.language in ["bash", "sh"]:
+        output_format = f"""Output a Bash script:
+- Start with #!/bin/bash and set -e
+- Use trap for cleanup: curl -s -X DELETE "{hindsight_url}/v1/default/banks/{bank_id}"
+- End with: echo "TEST PASSED" """
+
+    else:
+        return "", f"Unsupported language: {example.language}"
+
+    prompt = f"""The documentation code below is the TEST CASE. Your job is to make it runnable.
+
+DOCUMENTATION CODE ({example.language}):
+```
+{example.code}
+```
+
+RULES:
+1. The doc code is SACRED - do not modify its logic, method calls, or parameters
+2. You MAY add setup BEFORE it:
+   - Import statements the code assumes exist
+   - Object instantiation (e.g., if code uses 'client.foo()', create the client first)
+   - Variable definitions
+3. You MAY add cleanup AFTER it
+4. Replace placeholder values:
+   - URLs like localhost:8888 → {hindsight_url}
+   - Bank IDs like "my-bank", "demo", <bank_id> → "{bank_id}"
+   - Placeholder IDs like <entity_id>, <document_id> → "test-id"
+
+{output_format}
+
+Output ONLY the complete runnable code, no explanation."""
+
+    is_reasoning = model.startswith(("o1", "o3"))
+    kwargs = {
+        "model": model,
+        "messages": [{"role": "user", "content": prompt}],
+    }
+    if is_reasoning:
+        kwargs["max_completion_tokens"] = 4000
+    else:
+        kwargs["temperature"] = 0
+        kwargs["max_tokens"] = 4000
+
+    try:
+        response = client.chat.completions.create(**kwargs)
+        script = response.choices[0].message.content
+
+        # Clean up markdown code blocks if present
+        script = re.sub(r'^```\w*\n', '', script)
+        script = re.sub(r'\n```$', '', script)
+        script = script.strip()
+
+        return script, None
+    except Exception as e:
+        return "", f"Transform failed: {e}"
+
+
+# =============================================================================
+# STEP 4: Run tests
+# =============================================================================
+
+def get_python_path() -> str:
+    """Get PYTHONPATH that includes all installed packages."""
+    paths = []
+
+    # Add virtual environment site-packages if in a venv
+    if hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix):
+        # We're in a virtual environment
+        venv_site = os.path.join(sys.prefix, 'lib', f'python{sys.version_info.major}.{sys.version_info.minor}', 'site-packages')
+        if os.path.exists(venv_site):
+            paths.append(venv_site)
+
+    # Add system site-packages
+    paths.extend(site.getsitepackages())
+
+    # Add user site-packages
+    user_site = site.getusersitepackages()
+    if user_site and os.path.exists(user_site):
+        paths.append(user_site)
+
+    # Add existing PYTHONPATH
+    existing = os.environ.get("PYTHONPATH", "")
+    if existing:
+        paths.append(existing)
+
+    return ":".join(paths)
+
+
+def run_python(script: str, timeout: int = 60) -> tuple[bool, str, Optional[str]]:
+    """Run Python script."""
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
+        f.write(script)
+        f.flush()
+        try:
+            pythonpath = get_python_path()
+
+            result = subprocess.run(
+                [sys.executable, f.name],
+                capture_output=True, text=True, timeout=timeout,
+                env={**os.environ, "PYTHONPATH": pythonpath}
+            )
+            output = result.stdout + result.stderr
+            if "TEST PASSED" in output:
+                return True, output, None
+            return result.returncode == 0, output, result.stderr if result.returncode != 0 else None
+        except subprocess.TimeoutExpired:
+            return False, "", "Timeout"
+        except Exception as e:
+            return False, "", str(e)
+        finally:
+            os.unlink(f.name)
+
+
+def run_javascript(script: str, timeout: int = 60) -> tuple[bool, str, Optional[str]]:
+    """Run JavaScript script."""
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.mjs', delete=False, dir='/tmp') as f:
+        f.write(script)
+        f.flush()
+        try:
+            env = {**os.environ}
+            env["NODE_PATH"] = f"/tmp/node_modules:{env.get('NODE_PATH', '')}"
+
+            result = subprocess.run(
+                ["node", f.name],
+                capture_output=True, text=True, timeout=timeout,
+                env=env, cwd="/tmp"
+            )
+            output = result.stdout + result.stderr
+            if "TEST PASSED" in output:
+                return True, output, None
+            return result.returncode == 0, output, result.stderr if result.returncode != 0 else None
+        except subprocess.TimeoutExpired:
+            return False, "", "Timeout"
+        except Exception as e:
+            return False, "", str(e)
+        finally:
+            os.unlink(f.name)
+
+
+def run_bash(script: str, timeout: int = 60) -> tuple[bool, str, Optional[str]]:
+    """Run bash script."""
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.sh', delete=False) as f:
+        f.write(script)
+        f.flush()
+        os.chmod(f.name, 0o755)
+        try:
+            result = subprocess.run(
+                ["bash", f.name],
+                capture_output=True, text=True, timeout=timeout
+            )
+            output = result.stdout + result.stderr
+            if "TEST PASSED" in output:
+                return True, output, None
+            return result.returncode == 0, output, result.stderr if result.returncode != 0 else None
+        except subprocess.TimeoutExpired:
+            return False, "", "Timeout"
+        except Exception as e:
+            return False, "", str(e)
+        finally:
+            os.unlink(f.name)
+
+
+# =============================================================================
+# STEP 5: Analyze failures with LLM
+# =============================================================================
+
+def get_source_context(example: CodeExample, repo_root: str) -> str:
+    """Get relevant source code for failure analysis."""
+    parts = []
+    code_lower = example.code.lower()
+
+    if example.language == "python":
+        if "recall" in code_lower or "weight" in code_lower:
+            try:
+                with open(os.path.join(repo_root, "hindsight-clients/python/hindsight_client_api/models/recall_result.py")) as f:
+                    parts.append("=== RecallResult Model ===\n" + f.read()[:2000])
+            except: pass
+        if "reflect" in code_lower:
+            try:
+                with open(os.path.join(repo_root, "hindsight-clients/python/hindsight_client_api/models/reflect_response.py")) as f:
+                    parts.append("=== ReflectResponse Model ===\n" + f.read()[:2000])
+            except: pass
+        try:
+            with open(os.path.join(repo_root, "hindsight-clients/python/hindsight_client/__init__.py")) as f:
+                parts.append("=== Hindsight Client ===\n" + f.read()[:3000])
+        except: pass
+
+    elif example.language in ["typescript", "javascript"]:
+        try:
+            with open(os.path.join(repo_root, "hindsight-clients/typescript/src/index.ts")) as f:
+                parts.append("=== TypeScript Client ===\n" + f.read()[:4000])
+        except: pass
+
+    elif example.language in ["bash", "sh"]:
+        try:
+            with open(os.path.join(repo_root, "hindsight-cli/src/main.rs")) as f:
+                lines = f.read().split('\n')[:350]
+                parts.append("=== CLI Commands ===\n" + '\n'.join(lines))
+        except: pass
+
+    return "\n\n".join(parts)
+
+
+def get_doc_context(example: CodeExample) -> str:
+    """Get the full documentation context around the failing code example."""
+    try:
+        with open(example.file_path, "r") as f:
+            content = f.read()
+
+        # Find the code block and get surrounding context (500 chars before/after)
+        # This gives us the explanatory text around the code
+        code_start = content.find(example.code[:50])  # Find by first 50 chars
+        if code_start == -1:
+            code_start = example.line_number * 50  # Rough estimate
+
+        start = max(0, code_start - 500)
+        end = min(len(content), code_start + len(example.code) + 500)
+
+        return content[start:end]
+    except:
+        return example.context  # Fall back to the small context we already have
+
+
+def analyze_failure(client: OpenAI, result: TestResult, repo_root: str, model: str) -> dict:
+    """Use LLM to determine if failure is a real doc bug."""
+    source = get_source_context(result.example, repo_root)
+    doc_context = get_doc_context(result.example)
+
+    prompt = f"""Analyze this documentation test failure.
+
+## Documentation File: {result.example.file_path}
+
+### Documentation Context (text around the code example)
+```markdown
+{doc_context}
+```
+
+### The Code Example Being Tested (line {result.example.line_number})
+```{result.example.language}
+{result.example.code}
+```
+
+## Error When Running
+{result.error[:800] if result.error else "Unknown"}
+
+## Transformed Test Code (what we actually ran)
+```
+{result.transformed_code[:1500] if result.transformed_code else "N/A"}
+```
+
+## Actual Source Code (ground truth - what the API really looks like)
+{source[:6000] if source else "Not available"}
+
+## Your Task
+Compare the DOCUMENTATION against the ACTUAL SOURCE CODE.
+
+1. Does the documentation show something that doesn't exist in the source code?
+   - Wrong method names?
+   - Wrong attribute names (e.g., .weight when there's no weight field)?
+   - Wrong CLI commands?
+   - Wrong parameters?
+
+2. Or is the documentation correct, but our test transformation/execution failed?
+   - Missing imports we didn't add?
+   - Environment issues?
+   - Timing/race conditions?
+
+Respond JSON:
+{{
+    "is_doc_bug": true/false,
+    "confidence": "high/medium/low",
+    "reason": "brief explanation of what's wrong",
+    "fix": "if doc bug, what should the doc say instead"
+}}"""
+
+    is_reasoning = model.startswith(("o1", "o3"))
+    kwargs = {
+        "model": model,
+        "messages": [{"role": "user", "content": prompt}],
+        "response_format": {"type": "json_object"},
+    }
+    if is_reasoning:
+        kwargs["max_completion_tokens"] = 2000
+    else:
+        kwargs["temperature"] = 0
+
+    try:
+        response = client.chat.completions.create(**kwargs)
+        return json.loads(response.choices[0].message.content)
+    except Exception as e:
+        return {"is_doc_bug": True, "confidence": "low", "reason": str(e)}
+
+
+# =============================================================================
+# Main test runner
+# =============================================================================
+
+def test_example(example: CodeExample, openai_client: OpenAI, hindsight_url: str, cli_available: bool, model: str) -> TestResult:
+    """Test a single code example."""
+
+    # Check if should skip
+    skip = should_skip(example.code, example.language)
+    if skip:
+        return TestResult(example=example, success=True, output="", skip_reason=skip)
+
+    # Transform using LLM
+    try:
+        transformed, skip = transform_code(openai_client, example, hindsight_url, cli_available, model)
+        if skip:
+            return TestResult(example=example, success=True, output="", skip_reason=skip)
+
+        if not transformed:
+            return TestResult(example=example, success=True, output="", skip_reason="Transform returned empty")
+
+        # Run based on language
+        if example.language == "python":
+            success, output, error = run_python(transformed)
+        elif example.language in ["typescript", "javascript"]:
+            success, output, error = run_javascript(transformed)
+        elif example.language in ["bash", "sh"]:
+            success, output, error = run_bash(transformed)
+        else:
+            return TestResult(example=example, success=True, output="", skip_reason=f"Unsupported: {example.language}")
+
+        return TestResult(
+            example=example,
+            success=success,
+            output=output,
+            error=error,
+            transformed_code=transformed
+        )
+    except Exception as e:
+        return TestResult(
+            example=example,
+            success=False,
+            output="",
+            error=f"Transform error: {e}\n{traceback.format_exc()}"
+        )
+
+
+def check_cli_available() -> bool:
+    """Check if hindsight CLI is available."""
+    try:
+        result = subprocess.run(["hindsight", "--version"], capture_output=True, timeout=5)
+        return result.returncode == 0
+    except:
+        return False
+
+
+def check_dependencies() -> dict[str, bool]:
+    """Check which dependencies are available for doc tests."""
+    deps = {}
+
+    # Check Python packages
+    python_packages = [
+        ("hindsight_client", "Hindsight Python client"),
+        ("hindsight_litellm", "Hindsight LiteLLM integration"),
+        ("hindsight_openai", "Hindsight OpenAI integration"),
+        ("anthropic", "Anthropic SDK"),
+        ("openai", "OpenAI SDK"),
+    ]
+
+    for module, name in python_packages:
+        try:
+            __import__(module)
+            deps[module] = True
+        except ImportError:
+            deps[module] = False
+
+    return deps
+
+
+def print_dependency_status(deps: dict[str, bool]):
+    """Print dependency availability status."""
+    print("\n=== Dependencies ===")
+    for name, available in deps.items():
+        status = "✓" if available else "✗"
+        print(f"  {status} {name}")
+
+    # Print PYTHONPATH for debugging
+    pythonpath = get_python_path()
+    print(f"\nPYTHONPATH: {pythonpath[:100]}..." if len(pythonpath) > 100 else f"\nPYTHONPATH: {pythonpath}")
+    print(f"Python: {sys.executable}")
+    print(f"Prefix: {sys.prefix}")
+    print()
+
+
+def main():
+    sys.stdout.reconfigure(line_buffering=True)
+
+    openai_key = os.environ.get("OPENAI_API_KEY")
+    if not openai_key:
+        print("ERROR: OPENAI_API_KEY required")
+        sys.exit(1)
+
+    hindsight_url = os.environ.get("HINDSIGHT_API_URL", "http://localhost:8888")
+    model = os.environ.get("DOC_TEST_MODEL", "gpt-4o")
+
+    # Find repo root - go up from script location
+    script_path = os.path.abspath(__file__)
+    repo_root = os.path.dirname(os.path.dirname(script_path))
+
+    # If running from a subdirectory (like hindsight-api), detect and fix
+    if not os.path.exists(os.path.join(repo_root, "hindsight-docs")):
+        # Try going up one more level
+        repo_root = os.path.dirname(repo_root)
+    if not os.path.exists(os.path.join(repo_root, "hindsight-docs")):
+        # Fall back to REPO_ROOT env var or cwd
+        repo_root = os.environ.get("REPO_ROOT", os.getcwd())
+
+    print(f"Repo: {repo_root}")
+    print(f"API: {hindsight_url}")
+    print(f"Model: {model}")
+
+    # Check CLI
+    cli_available = check_cli_available()
+    print(f"CLI: {'available' if cli_available else 'not available'}")
+
+    # Check and print dependencies
+    deps = check_dependencies()
+    print_dependency_status(deps)
+
+    # Warn if critical dependencies are missing
+    if not deps.get("hindsight_client"):
+        print("WARNING: hindsight_client not available - Python examples will fail")
+        print("  Install with: pip install hindsight-client or uv pip install <path-to-client>")
+
+    # Check API health
+    try:
+        import urllib.request
+        urllib.request.urlopen(f"{hindsight_url}/health", timeout=5)
+        print("API: healthy")
+    except Exception as e:
+        print(f"API: WARNING - {e}")
+
+    # Initialize OpenAI client early (needed for transforms and analysis)
+    client = OpenAI(api_key=openai_key)
+
+    # Find and extract examples
+    md_files = find_markdown_files(repo_root)
+    print(f"\nFound {len(md_files)} markdown files")
+
+    all_examples = []
+    for md_file in md_files:
+        examples = extract_code_blocks(md_file)
+        if examples:
+            all_examples.extend(examples)
+
+    print(f"Found {len(all_examples)} code examples")
+
+    # Run tests
+    report = TestReport()
+    max_workers = int(os.environ.get("MAX_WORKERS", "4"))  # Lower default since LLM calls are slower
+
+    print(f"\nRunning tests with {max_workers} workers...")
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(test_example, ex, client, hindsight_url, cli_available, model): ex for ex in all_examples}
+        for future in as_completed(futures):
+            result = future.result()
+            report.add_result(result)
+
+            status = "SKIP" if result.skip_reason else ("PASS" if result.success else "FAIL")
+            safe_print(f"  [{status}] {result.example.file_path}:{result.example.line_number}")
+
+    # Print summary
+    print("\n" + "=" * 60)
+    print(f"Total: {report.total} | Pass: {report.passed} | Fail: {report.failed} | Skip: {report.skipped}")
+    print("=" * 60)
+
+    # Analyze failures with LLM
+    failures = [r for r in report.results if not r.success and not r.skip_reason]
+
+    if failures:
+        print(f"\n=== Analyzing {len(failures)} failures (parallel) ===")
+
+        doc_bugs = []
+        test_issues = []
+        results_lock = threading.Lock()
+        completed = [0]  # Use list for mutable counter in closure
+
+        def analyze_one(result: TestResult) -> None:
+            analysis = analyze_failure(client, result, repo_root, model)
+            entry = {
+                "file": result.example.file_path,
+                "line": result.example.line_number,
+                "error": result.error[:200] if result.error else "",
+                "analysis": analysis
+            }
+
+            with results_lock:
+                completed[0] += 1
+                idx = completed[0]
+                if analysis.get("is_doc_bug", True):
+                    doc_bugs.append(entry)
+                    safe_print(f"  [{idx}/{len(failures)}] {result.example.file_path}:{result.example.line_number}")
+                    safe_print(f"       → DOC BUG: {analysis.get('reason', '')[:50]}")
+                else:
+                    test_issues.append(entry)
+                    safe_print(f"  [{idx}/{len(failures)}] {result.example.file_path}:{result.example.line_number}")
+                    safe_print(f"       → Test issue: {analysis.get('reason', '')[:50]}")
+
+        # Run analysis in parallel (limit concurrency to avoid rate limits)
+        with ThreadPoolExecutor(max_workers=10) as executor:
+            futures = [executor.submit(analyze_one, result) for result in failures]
+            for future in as_completed(futures):
+                try:
+                    future.result()
+                except Exception as e:
+                    safe_print(f"  Analysis error: {e}")
+
+        # Write summary
+        print(f"\n=== RESULTS ===")
+        print(f"Documentation bugs: {len(doc_bugs)}")
+        print(f"Test/CI issues: {len(test_issues)}")
+
+        if doc_bugs:
+            print(f"\n--- Documentation Bugs ---")
+            for bug in doc_bugs:
+                print(f"  {bug['file']}:{bug['line']}")
+                print(f"    Reason: {bug['analysis'].get('reason', 'Unknown')}")
+                if bug['analysis'].get('fix'):
+                    print(f"    Fix: {bug['analysis']['fix']}")
+
+        if test_issues:
+            print(f"\n--- Test/CI Issues (not doc bugs) ---")
+            for issue in test_issues:
+                print(f"  {issue['file']}:{issue['line']}")
+                print(f"    Reason: {issue['analysis'].get('reason', 'Unknown')}")
+
+        # Write GitHub summary (include ALL failures for visibility)
+        write_summary(report, doc_bugs, test_issues)
+
+        # Exit code based on real doc bugs only
+        sys.exit(1 if doc_bugs else 0)
+    else:
+        print("\nAll tests passed!")
+        write_summary(report, [], [])
+        sys.exit(0)
+
+
+def write_summary(report: TestReport, doc_bugs: list, test_issues: list):
+    """Write GitHub Actions summary file."""
+    with open("/tmp/doc-test-summary.md", "w") as f:
+        # Header
+        status = "❌" if doc_bugs else "✅"
+        f.write(f"# {status} Documentation Test Results\n\n")
+
+        # Summary table
+        f.write(f"| Metric | Count |\n")
+        f.write(f"|--------|-------|\n")
+        f.write(f"| Total | {report.total} |\n")
+        f.write(f"| ✅ Passed | {report.passed} |\n")
+        f.write(f"| ❌ Failed | {report.failed} |\n")
+        f.write(f"| ⏭️ Skipped | {report.skipped} |\n\n")
+
+        if doc_bugs or test_issues:
+            f.write(f"| Category | Count |\n")
+            f.write(f"|----------|-------|\n")
+            f.write(f"| 🐛 Documentation Bugs | {len(doc_bugs)} |\n")
+            f.write(f"| ⚠️ Test/CI Issues | {len(test_issues)} |\n\n")
+
+        # Documentation bugs section
+        if doc_bugs:
+            f.write(f"## 🐛 Documentation Bugs ({len(doc_bugs)})\n\n")
+            f.write("These are real issues in the documentation that need to be fixed:\n\n")
+            for bug in doc_bugs:
+                file_short = bug['file'].split('/hindsight/')[-1] if '/hindsight/' in bug['file'] else bug['file']
+                f.write(f"### `{file_short}:{bug['line']}`\n")
+                f.write(f"- **Issue**: {bug['analysis'].get('reason', 'Unknown')}\n")
+                if bug['analysis'].get('fix'):
+                    f.write(f"- **Suggested Fix**: {bug['analysis']['fix']}\n")
+                if bug.get('error'):
+                    f.write(f"- **Error**: `{bug['error'][:150]}...`\n")
+                f.write("\n")
+
+        # Test/CI issues section
+        if test_issues:
+            f.write(f"## ⚠️ Test/CI Issues ({len(test_issues)})\n\n")
+            f.write("These failures are NOT documentation bugs - they're issues with the test setup or CI environment:\n\n")
+            for issue in test_issues:
+                file_short = issue['file'].split('/hindsight/')[-1] if '/hindsight/' in issue['file'] else issue['file']
+                f.write(f"### `{file_short}:{issue['line']}`\n")
+                f.write(f"- **Reason**: {issue['analysis'].get('reason', 'Unknown')}\n")
+                if issue.get('error'):
+                    f.write(f"- **Error**: `{issue['error'][:150]}...`\n")
+                f.write("\n")
+
+        # No failures
+        if not doc_bugs and not test_issues:
+            if report.passed > 0:
+                f.write(f"All {report.passed} tests passed! ({report.skipped} skipped)\n")
+            else:
+                f.write(f"All {report.skipped} examples were skipped (install commands, docker, etc.)\n")
+
+
+if __name__ == "__main__":
+    main()

uv.lock

@@ -1141,7 +1141,7 @@ wheels = [
 
 [[package]]
 name = "hindsight-all"
-version = "0.1.4"
+version = "0.1.5"
 source = { editable = "hindsight" }
 dependencies = [
     { name = "hindsight-api" },
@@ -1165,7 +1165,7 @@ provides-extras = ["test"]
 
 [[package]]
 name = "hindsight-api"
-version = "0.1.4"
+version = "0.1.5"
 source = { editable = "hindsight-api" }
 dependencies = [
     { name = "alembic" },
@@ -1223,7 +1223,7 @@ requires-dist = [
     { name = "asyncpg", specifier = ">=0.29.0" },
     { name = "dateparser", specifier = ">=1.2.2" },
     { name = "fastapi", extras = ["standard"], specifier = ">=0.120.3" },
-    { name = "fastmcp", specifier = ">=2.0.0" },
+    { name = "fastmcp", specifier = ">=2.3.0" },
     { name = "filelock", marker = "extra == 'test'", specifier = ">=3.0.0" },
     { name = "google-genai", specifier = ">=1.0.0" },
     { name = "greenlet", specifier = ">=3.2.4" },
@@ -1267,7 +1267,7 @@ dev = [
 
 [[package]]
 name = "hindsight-client"
-version = "0.1.4"
+version = "0.1.5"
 source = { editable = "hindsight-clients/python" }
 dependencies = [
     { name = "aiohttp" },
@@ -1299,7 +1299,7 @@ provides-extras = ["test"]
 
 [[package]]
 name = "hindsight-dev"
-version = "0.1.4"
+version = "0.1.5"
 source = { editable = "hindsight-dev" }
 dependencies = [
     { name = "hindsight-api" },