ci: add upgrade tests

.github/workflows/test.yml

@@ -875,6 +875,66 @@ jobs:
         echo "=== API Server Logs ==="
         cat /tmp/api-server.log || echo "No API server log found"
 
+  test-upgrade:
+    runs-on: ubuntu-latest
+    env:
+      HINDSIGHT_API_LLM_PROVIDER: groq
+      HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
+      HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # Full history needed for git clone of tags
+
+    - name: Fetch tags
+      run: git fetch --tags
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version-file: ".python-version"
+
+    - name: Cache HuggingFace models
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/huggingface
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-huggingface-
+
+    - name: Install hindsight-dev dependencies
+      working-directory: ./hindsight-dev
+      run: uv sync --frozen --extra test --index-strategy unsafe-best-match
+
+    - name: Install current hindsight-api
+      working-directory: ./hindsight-api
+      run: uv sync --frozen --index-strategy unsafe-best-match
+
+    - name: Pre-download models
+      working-directory: ./hindsight-api
+      run: |
+        uv run python -c "
+        from sentence_transformers import SentenceTransformer, CrossEncoder
+        print('Downloading embedding model...')
+        SentenceTransformer('BAAI/bge-small-en-v1.5')
+        print('Downloading cross-encoder model...')
+        CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
+        print('Models downloaded successfully')
+        "
+
+    - name: Run upgrade tests
+      working-directory: ./hindsight-dev
+      run: uv run pytest upgrade_tests/ -v --tb=short
+
   verify-generated-files:
     runs-on: ubuntu-latest
     env:

hindsight-dev/pyproject.toml

@@ -16,8 +16,15 @@ dependencies = [
     "pydantic>=2.0.0",
 ]
 
+[project.optional-dependencies]
+test = [
+    "pytest>=8.0.0",
+    "httpx>=0.27.0",
+    "python-dotenv>=1.0.0",
+]
+
 [tool.hatch.build.targets.wheel]
-packages = ["hindsight_dev", "benchmarks"]
+packages = ["hindsight_dev", "benchmarks", "upgrade_tests"]
 
 [tool.uv.sources]
 hindsight-api = { workspace = true }

hindsight-dev/upgrade_tests/__init__.py

@@ -0,0 +1 @@
+# Upgrade and backwards compatibility tests

hindsight-dev/upgrade_tests/conftest.py

@@ -0,0 +1,110 @@
+"""
+Pytest configuration and fixtures for upgrade tests.
+"""
+
+import asyncio
+import logging
+import os
+from pathlib import Path
+
+import pytest
+from dotenv import load_dotenv
+
+# Configure logging for tests
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+
+# Reduce noise from httpx
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("httpcore").setLevel(logging.WARNING)
+
+
+def pytest_configure(config):
+    """Load environment variables before running tests."""
+    # Look for .env in the workspace root
+    env_file = Path(__file__).parent.parent.parent / ".env"
+    if env_file.exists():
+        load_dotenv(env_file)
+
+
+_pg0_instance = None
+_pg0_url = None
+
+
+def _get_or_create_pg0():
+    """Get or create the shared pg0 instance for upgrade tests."""
+    global _pg0_instance, _pg0_url
+    from hindsight_api.pg0 import EmbeddedPostgres
+
+    if _pg0_instance is None:
+        _pg0_instance = EmbeddedPostgres(name="hindsight-upgrade-test", port=5560)
+
+    loop = asyncio.new_event_loop()
+    try:
+        _pg0_url = loop.run_until_complete(_pg0_instance.ensure_running())
+    finally:
+        loop.close()
+
+    return _pg0_url
+
+
+def _clean_database(db_url: str):
+    """Drop all tables in the database to reset state for next test."""
+    from sqlalchemy import create_engine, text
+
+    engine = create_engine(db_url)
+    with engine.connect() as conn:
+        # Drop all tables in public schema (cascade to handle foreign keys)
+        tables = conn.execute(
+            text("""
+                SELECT tablename FROM pg_tables
+                WHERE schemaname = 'public'
+                AND tablename NOT LIKE 'pg_%'
+            """)
+        ).fetchall()
+        for table in tables:
+            conn.execute(text(f'DROP TABLE IF EXISTS public."{table[0]}" CASCADE'))
+        conn.commit()
+    engine.dispose()
+
+
+@pytest.fixture(scope="function")
+def db_url():
+    """
+    Provide a PostgreSQL connection URL for upgrade tests.
+
+    Uses pg0 (embedded PostgreSQL) for a clean, isolated test database.
+    The database is cleaned between tests to ensure fresh state for migrations.
+    """
+    url = _get_or_create_pg0()
+
+    # Clean database before each test
+    _clean_database(url)
+
+    yield url
+
+    # No cleanup after - database is cleaned at start of next test
+
+
+@pytest.fixture(scope="module")
+def llm_config():
+    """
+    Provide LLM configuration from environment.
+
+    Returns a dict with provider, api_key, and model.
+    """
+    return {
+        "provider": os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq"),
+        "api_key": os.getenv("HINDSIGHT_API_LLM_API_KEY") or os.getenv("GROQ_API_KEY"),
+        "model": os.getenv("HINDSIGHT_API_LLM_MODEL", "llama-3.3-70b-versatile"),
+    }
+
+
+@pytest.fixture
+def unique_bank_id():
+    """Generate a unique bank ID for each test."""
+    import uuid
+
+    return f"upgrade_test_{uuid.uuid4().hex[:8]}"

hindsight-dev/upgrade_tests/test_upgrade.py

@@ -0,0 +1,303 @@
+"""
+Upgrade and backwards compatibility tests.
+
+These tests verify that:
+1. Data stored in older versions is accessible after upgrade
+2. Database migrations run correctly
+3. API behavior remains compatible
+"""
+
+import logging
+
+import httpx
+import pytest
+
+from .version_runner import VersionRunner
+
+logger = logging.getLogger(__name__)
+
+# Version upgrade paths to test
+# Format: (old_version, new_version)
+UPGRADE_PATHS = [
+    ("v0.3.0", "HEAD"),
+]
+
+
+class TestUpgrade:
+    """Tests for version upgrades."""
+
+    @pytest.mark.parametrize("old_version,new_version", UPGRADE_PATHS)
+    def test_upgrade_preserves_memories(self, db_url, llm_config, unique_bank_id, old_version, new_version):
+        """
+        Verify memories stored in old version are accessible after upgrade.
+
+        Workflow:
+        1. Start old version
+        2. Store memories via retain
+        3. Verify recall works on old version
+        4. Stop old version
+        5. Start new version (same database - migrations run)
+        6. Verify recall returns same data
+        7. Verify reflect works
+        """
+        bank_id = unique_bank_id
+
+        # Test data to store
+        test_memories = [
+            {"content": "Alice is a software engineer at TechCorp.", "context": "team introduction"},
+            {"content": "Bob manages the infrastructure team and loves Kubernetes.", "context": "team introduction"},
+            {"content": "The project deadline is next Friday.", "context": "project planning"},
+        ]
+
+        # Phase 1: Store data with old version
+        logger.info(f"=== Phase 1: Setting up data with {old_version} ===")
+
+        with VersionRunner(
+            old_version,
+            db_url,
+            port=8891,
+            llm_provider=llm_config["provider"],
+            llm_api_key=llm_config["api_key"],
+            llm_model=llm_config["model"],
+        ) as old:
+            server = old.start()
+            client = httpx.Client(base_url=server.url, timeout=60)
+
+            # Store memories
+            resp = client.post(
+                f"/v1/default/banks/{bank_id}/memories",
+                json={"items": test_memories},
+            )
+            assert resp.status_code == 200, f"Failed to store memories: {resp.text}"
+            result = resp.json()
+            assert result["success"] is True
+            assert result["items_count"] == len(test_memories)
+
+            # Verify recall works on old version
+            resp = client.post(
+                f"/v1/default/banks/{bank_id}/memories/recall",
+                json={"query": "Who works at TechCorp?"},
+            )
+            assert resp.status_code == 200, f"Recall failed on old version: {resp.text}"
+            old_results = resp.json()["results"]
+            assert len(old_results) > 0, "No results from recall on old version"
+
+            # Get stats for comparison
+            resp = client.get(f"/v1/default/banks/{bank_id}/stats")
+            assert resp.status_code == 200
+            old_stats = resp.json()
+            logger.info(f"Old version stats: {old_stats}")
+
+            client.close()
+
+        # Phase 2: Verify data with new version
+        logger.info(f"=== Phase 2: Verifying data with {new_version} ===")
+
+        with VersionRunner(
+            new_version,
+            db_url,
+            port=8892,
+            llm_provider=llm_config["provider"],
+            llm_api_key=llm_config["api_key"],
+            llm_model=llm_config["model"],
+        ) as new:
+            server = new.start()
+            client = httpx.Client(base_url=server.url, timeout=60)
+
+            # Verify recall returns data
+            resp = client.post(
+                f"/v1/default/banks/{bank_id}/memories/recall",
+                json={"query": "Who works at TechCorp?"},
+            )
+            assert resp.status_code == 200, f"Recall failed on new version: {resp.text}"
+            new_results = resp.json()["results"]
+            assert len(new_results) > 0, f"No results from recall after upgrade. Bank: {bank_id}"
+
+            # Verify Alice is found
+            found_alice = any("Alice" in r.get("text", "") for r in new_results)
+            assert found_alice, f"Alice not found in results after upgrade: {new_results}"
+
+            # Verify reflect works
+            resp = client.post(
+                f"/v1/default/banks/{bank_id}/reflect",
+                json={"query": "Tell me about the team members"},
+            )
+            assert resp.status_code == 200, f"Reflect failed after upgrade: {resp.text}"
+            reflect_result = resp.json()
+            assert len(reflect_result.get("text", "")) > 0, "Empty reflect response after upgrade"
+
+            # Verify stats are preserved
+            resp = client.get(f"/v1/default/banks/{bank_id}/stats")
+            assert resp.status_code == 200
+            new_stats = resp.json()
+            logger.info(f"New version stats: {new_stats}")
+
+            # Stats should be similar (might have small differences due to re-indexing)
+            assert new_stats["total_nodes"] >= old_stats["total_nodes"], (
+                f"Lost nodes after upgrade: {old_stats['total_nodes']} -> {new_stats['total_nodes']}"
+            )
+
+            # Cleanup - delete test bank
+            resp = client.delete(f"/v1/default/banks/{bank_id}")
+            assert resp.status_code == 200
+
+            client.close()
+
+    @pytest.mark.parametrize("old_version,new_version", UPGRADE_PATHS)
+    def test_upgrade_preserves_documents(self, db_url, llm_config, unique_bank_id, old_version, new_version):
+        """
+        Verify documents stored in old version are accessible after upgrade.
+        """
+        bank_id = unique_bank_id
+        doc_id = "test-document-001"
+
+        # Phase 1: Store document with old version
+        logger.info(f"=== Phase 1: Storing document with {old_version} ===")
+
+        with VersionRunner(
+            old_version,
+            db_url,
+            port=8893,
+            llm_provider=llm_config["provider"],
+            llm_api_key=llm_config["api_key"],
+            llm_model=llm_config["model"],
+        ) as old:
+            server = old.start()
+            client = httpx.Client(base_url=server.url, timeout=60)
+
+            # Store memory with document
+            resp = client.post(
+                f"/v1/default/banks/{bank_id}/memories",
+                json={
+                    "items": [
+                        {
+                            "content": "The quarterly report shows 25% revenue growth.",
+                            "context": "Q1 financial review",
+                            "document_id": doc_id,
+                        }
+                    ]
+                },
+            )
+            assert resp.status_code == 200, f"Failed to store document: {resp.text}"
+
+            # Verify document exists
+            resp = client.get(f"/v1/default/banks/{bank_id}/documents")
+            assert resp.status_code == 200
+            docs = resp.json()["items"]
+            doc_ids = [d["id"] for d in docs]
+            assert doc_id in doc_ids, f"Document not found in old version: {doc_ids}"
+
+            client.close()
+
+        # Phase 2: Verify document with new version
+        logger.info(f"=== Phase 2: Verifying document with {new_version} ===")
+
+        with VersionRunner(
+            new_version,
+            db_url,
+            port=8894,
+            llm_provider=llm_config["provider"],
+            llm_api_key=llm_config["api_key"],
+            llm_model=llm_config["model"],
+        ) as new:
+            server = new.start()
+            client = httpx.Client(base_url=server.url, timeout=60)
+
+            # Verify document still exists
+            resp = client.get(f"/v1/default/banks/{bank_id}/documents")
+            assert resp.status_code == 200
+            docs = resp.json()["items"]
+            doc_ids = [d["id"] for d in docs]
+            assert doc_id in doc_ids, f"Document not found after upgrade: {doc_ids}"
+
+            # Verify document details
+            resp = client.get(f"/v1/default/banks/{bank_id}/documents/{doc_id}")
+            assert resp.status_code == 200
+            doc_info = resp.json()
+            assert doc_info["id"] == doc_id
+            assert doc_info["memory_unit_count"] > 0
+
+            # Cleanup
+            resp = client.delete(f"/v1/default/banks/{bank_id}")
+            assert resp.status_code == 200
+
+            client.close()
+
+    @pytest.mark.parametrize("old_version,new_version", UPGRADE_PATHS)
+    def test_upgrade_preserves_bank_profile(self, db_url, llm_config, unique_bank_id, old_version, new_version):
+        """
+        Verify bank profile (disposition) is preserved after upgrade.
+        """
+        bank_id = unique_bank_id
+
+        # Phase 1: Create bank with custom disposition
+        logger.info(f"=== Phase 1: Creating bank profile with {old_version} ===")
+
+        with VersionRunner(
+            old_version,
+            db_url,
+            port=8895,
+            llm_provider=llm_config["provider"],
+            llm_api_key=llm_config["api_key"],
+            llm_model=llm_config["model"],
+        ) as old:
+            server = old.start()
+            client = httpx.Client(base_url=server.url, timeout=60)
+
+            # Create bank by storing a memory
+            resp = client.post(
+                f"/v1/default/banks/{bank_id}/memories",
+                json={"items": [{"content": "Test memory", "context": "test"}]},
+            )
+            assert resp.status_code == 200
+
+            # Set custom disposition
+            resp = client.put(
+                f"/v1/default/banks/{bank_id}/profile",
+                json={
+                    "disposition": {
+                        "skepticism": 4,
+                        "literalism": 2,
+                        "empathy": 5,
+                    }
+                },
+            )
+            assert resp.status_code == 200
+
+            # Verify profile
+            resp = client.get(f"/v1/default/banks/{bank_id}/profile")
+            assert resp.status_code == 200
+            old_profile = resp.json()
+            assert old_profile["disposition"]["skepticism"] == 4
+            assert old_profile["disposition"]["literalism"] == 2
+            assert old_profile["disposition"]["empathy"] == 5
+
+            client.close()
+
+        # Phase 2: Verify profile with new version
+        logger.info(f"=== Phase 2: Verifying profile with {new_version} ===")
+
+        with VersionRunner(
+            new_version,
+            db_url,
+            port=8896,
+            llm_provider=llm_config["provider"],
+            llm_api_key=llm_config["api_key"],
+            llm_model=llm_config["model"],
+        ) as new:
+            server = new.start()
+            client = httpx.Client(base_url=server.url, timeout=60)
+
+            # Verify profile is preserved
+            resp = client.get(f"/v1/default/banks/{bank_id}/profile")
+            assert resp.status_code == 200
+            new_profile = resp.json()
+            assert new_profile["disposition"]["skepticism"] == 4, "Skepticism not preserved"
+            assert new_profile["disposition"]["literalism"] == 2, "Literalism not preserved"
+            assert new_profile["disposition"]["empathy"] == 5, "Empathy not preserved"
+
+            # Cleanup
+            resp = client.delete(f"/v1/default/banks/{bank_id}")
+            assert resp.status_code == 200
+
+            client.close()

hindsight-dev/upgrade_tests/version_runner.py

@@ -0,0 +1,275 @@
+"""
+Version runner for upgrade tests.
+
+Manages running different git versions of the Hindsight API for upgrade testing.
+Handles git checkout, venv creation, dependency installation, and server lifecycle.
+"""
+
+import logging
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ServerInfo:
+    """Information about a running server."""
+
+    url: str
+    port: int
+    version: str
+
+
+class VersionRunner:
+    """
+    Manages running a specific git version of the Hindsight API.
+
+    For "HEAD" or "current", uses the current working directory.
+    For git tags (e.g., "v0.3.0"), clones the repo at that tag to a temp directory.
+    """
+
+    def __init__(
+        self,
+        version: str,
+        db_url: str,
+        port: int = 8890,
+        llm_provider: str | None = None,
+        llm_api_key: str | None = None,
+        llm_model: str | None = None,
+    ):
+        """
+        Initialize a version runner.
+
+        Args:
+            version: Git tag (e.g., "v0.3.0") or "HEAD"/"current" for current code
+            db_url: PostgreSQL connection URL
+            port: Port to run the API on
+            llm_provider: LLM provider (defaults to env var)
+            llm_api_key: LLM API key (defaults to env var)
+            llm_model: LLM model (defaults to env var)
+        """
+        self.version = version
+        self.db_url = db_url
+        self.port = port
+        self.llm_provider = llm_provider or os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq")
+        self.llm_api_key = llm_api_key or os.getenv("HINDSIGHT_API_LLM_API_KEY") or os.getenv("GROQ_API_KEY")
+        self.llm_model = llm_model or os.getenv("HINDSIGHT_API_LLM_MODEL", "llama-3.3-70b-versatile")
+
+        self.work_dir: Path | None = None
+        self.process: subprocess.Popen | None = None
+        self._temp_dir: str | None = None
+        self._is_current = version.lower() in ("head", "current")
+
+    def _find_repo_root(self) -> Path:
+        """Find the git repository root."""
+        result = subprocess.run(
+            ["git", "rev-parse", "--show-toplevel"],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        return Path(result.stdout.strip())
+
+    def setup(self) -> None:
+        """Checkout version and install dependencies."""
+        if self._is_current:
+            # Use current working directory
+            self.work_dir = self._find_repo_root()
+            logger.info(f"Using current code at {self.work_dir}")
+            return
+
+        # Create temp dir and checkout specific version
+        self._temp_dir = tempfile.mkdtemp(prefix=f"hindsight-{self.version}-")
+        self.work_dir = Path(self._temp_dir)
+
+        repo_root = self._find_repo_root()
+        logger.info(f"Cloning {repo_root} at {self.version} to {self.work_dir}")
+
+        # Shallow clone at specific tag
+        subprocess.run(
+            ["git", "clone", "--depth", "1", "--branch", self.version, str(repo_root), str(self.work_dir)],
+            check=True,
+            capture_output=True,
+        )
+
+        # Create venv and install
+        venv_path = self.work_dir / ".venv-upgrade-test"
+        logger.info(f"Creating venv at {venv_path}")
+
+        subprocess.run(["uv", "venv", str(venv_path)], check=True, capture_output=True)
+
+        api_path = self.work_dir / "hindsight-api"
+        logger.info(f"Installing hindsight-api from {api_path}")
+
+        # Install with uv pip - use --index-strategy for pytorch
+        subprocess.run(
+            [
+                "uv",
+                "pip",
+                "install",
+                "-e",
+                str(api_path),
+                "--python",
+                str(venv_path / "bin" / "python"),
+                "--index-strategy",
+                "unsafe-best-match",
+            ],
+            check=True,
+            capture_output=True,
+            env={**os.environ, "UV_INDEX": "pytorch=https://download.pytorch.org/whl/cpu"},
+        )
+
+        logger.info(f"Version {self.version} setup complete")
+
+    def _get_venv_path(self) -> Path:
+        """Get the path to the venv for this version."""
+        if self._is_current:
+            # For current code, the venv is at the workspace root (uv workspace layout)
+            # Check both possible locations
+            workspace_venv = self.work_dir / ".venv"
+            api_venv = self.work_dir / "hindsight-api" / ".venv"
+
+            if (workspace_venv / "bin" / "hindsight-api").exists():
+                return workspace_venv
+            elif (api_venv / "bin" / "hindsight-api").exists():
+                return api_venv
+            else:
+                # Default to workspace root
+                return workspace_venv
+        return self.work_dir / ".venv-upgrade-test"
+
+    def start(self) -> ServerInfo:
+        """
+        Start the API server.
+
+        Returns:
+            ServerInfo with the URL and port
+        """
+        venv_path = self._get_venv_path()
+        hindsight_api_bin = venv_path / "bin" / "hindsight-api"
+
+        if not hindsight_api_bin.exists():
+            raise RuntimeError(f"hindsight-api binary not found at {hindsight_api_bin}")
+
+        env = os.environ.copy()
+        env.update(
+            {
+                "HINDSIGHT_API_PORT": str(self.port),
+                "HINDSIGHT_API_DATABASE_URL": self.db_url,
+                "HINDSIGHT_API_HOST": "127.0.0.1",
+                "HINDSIGHT_API_LLM_PROVIDER": self.llm_provider,
+                "HINDSIGHT_API_LLM_API_KEY": self.llm_api_key or "",
+                "HINDSIGHT_API_LLM_MODEL": self.llm_model,
+                "PYTHONUNBUFFERED": "1",
+            }
+        )
+
+        logger.info(f"Starting {self.version} API on port {self.port}")
+        logger.info(f"Database URL: {self.db_url}")
+
+        # Determine working directory
+        # For HEAD/current, use a temp directory to avoid .env file from workspace root
+        # (hindsight-api loads .env with override=True which would override our env vars)
+        if self._is_current:
+            # Create a temp directory for HEAD to avoid workspace .env
+            self._head_cwd = tempfile.mkdtemp(prefix="hindsight-head-cwd-")
+            cwd = self._head_cwd
+        else:
+            cwd = str(self.work_dir)
+            self._head_cwd = None
+
+        # Start the server
+        self.process = subprocess.Popen(
+            [str(hindsight_api_bin)],
+            env=env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            cwd=cwd,
+        )
+
+        self._wait_healthy()
+
+        url = f"http://127.0.0.1:{self.port}"
+        logger.info(f"Server {self.version} ready at {url}")
+
+        return ServerInfo(url=url, port=self.port, version=self.version)
+
+    def _wait_healthy(self, timeout: int = 120) -> None:
+        """Wait for /health endpoint to respond."""
+        url = f"http://127.0.0.1:{self.port}/health"
+        deadline = time.time() + timeout
+
+        while time.time() < deadline:
+            # Check if process is still alive
+            if self.process and self.process.poll() is not None:
+                stdout = self.process.stdout.read().decode() if self.process.stdout else ""
+                raise RuntimeError(f"Server {self.version} exited unexpectedly.\nLogs:\n{stdout}")
+
+            try:
+                resp = httpx.get(url, timeout=2)
+                if resp.status_code == 200:
+                    return
+            except httpx.RequestError:
+                pass
+
+            time.sleep(1)
+
+        # Timeout - dump logs
+        if self.process:
+            self.process.terminate()
+            try:
+                stdout, _ = self.process.communicate(timeout=5)
+                logs = stdout.decode() if stdout else ""
+            except Exception:
+                logs = "(failed to read logs)"
+            raise TimeoutError(f"Server {self.version} not healthy after {timeout}s.\nLogs:\n{logs}")
+
+    def stop(self) -> None:
+        """Stop the server and cleanup temp directory."""
+        if self.process:
+            logger.info(f"Stopping {self.version} server")
+            self.process.terminate()
+            try:
+                self.process.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                logger.warning(f"Server {self.version} did not stop gracefully, killing")
+                self.process.kill()
+                self.process.wait()
+            self.process = None
+
+        if self._temp_dir and os.path.exists(self._temp_dir):
+            logger.info(f"Cleaning up {self._temp_dir}")
+            shutil.rmtree(self._temp_dir, ignore_errors=True)
+            self._temp_dir = None
+
+        # Clean up HEAD's temp cwd
+        if hasattr(self, "_head_cwd") and self._head_cwd and os.path.exists(self._head_cwd):
+            shutil.rmtree(self._head_cwd, ignore_errors=True)
+            self._head_cwd = None
+
+    def get_logs(self) -> str:
+        """Get current server logs (if process is running)."""
+        if self.process and self.process.stdout:
+            # Non-blocking read of available output
+            import select
+
+            if hasattr(select, "select"):
+                readable, _, _ = select.select([self.process.stdout], [], [], 0)
+                if readable:
+                    return self.process.stdout.read(4096).decode()
+        return ""
+
+    def __enter__(self) -> "VersionRunner":
+        self.setup()
+        return self
+
+    def __exit__(self, *args) -> None:
+        self.stop()

scripts/run-upgrade-tests.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Run upgrade tests locally
+#
+# Usage:
+#   ./scripts/run-upgrade-tests.sh
+#
+# Environment variables:
+#   HINDSIGHT_API_LLM_PROVIDER - LLM provider (default: groq)
+#   HINDSIGHT_API_LLM_API_KEY  - LLM API key (or GROQ_API_KEY)
+#   HINDSIGHT_API_LLM_MODEL    - LLM model
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+echo "=== Running Hindsight Upgrade Tests ==="
+echo ""
+
+# Check for LLM API key
+if [ -z "${HINDSIGHT_API_LLM_API_KEY:-}" ] && [ -z "${GROQ_API_KEY:-}" ]; then
+    echo "Warning: No LLM API key found. Set HINDSIGHT_API_LLM_API_KEY or GROQ_API_KEY"
+fi
+
+# Load .env if present
+if [ -f "$ROOT_DIR/.env" ]; then
+    echo "Loading .env file..."
+    set -a
+    source "$ROOT_DIR/.env"
+    set +a
+fi
+
+cd "$ROOT_DIR/hindsight-dev"
+
+# Run tests
+echo "Running upgrade tests..."
+uv run pytest upgrade_tests/ -v "$@"

uv.lock

@@ -1492,6 +1492,13 @@ dependencies = [
     { name = "streamlit" },
 ]
 
+[package.optional-dependencies]
+test = [
+    { name = "httpx" },
+    { name = "pytest" },
+    { name = "python-dotenv" },
+]
+
 [package.dev-dependencies]
 dev = [
     { name = "ruff" },
@@ -1501,12 +1508,16 @@ dev = [
 [package.metadata]
 requires-dist = [
     { name = "hindsight-api", editable = "hindsight-api" },
+    { name = "httpx", marker = "extra == 'test'", specifier = ">=0.27.0" },
     { name = "openai", specifier = ">=1.0.0" },
     { name = "pydantic", specifier = ">=2.0.0" },
+    { name = "pytest", marker = "extra == 'test'", specifier = ">=8.0.0" },
+    { name = "python-dotenv", marker = "extra == 'test'", specifier = ">=1.0.0" },
     { name = "python-fasthtml", specifier = ">=0.12.33" },
     { name = "rich", specifier = ">=13.0.0" },
     { name = "streamlit", specifier = ">=1.51.0" },
 ]
+provides-extras = ["test"]
 
 [package.metadata.requires-dev]
 dev = [