chore: update Rust client Cargo.lock for CI verification

Run generate-clients.sh to sync Cargo.lock with current dependencies.

CLAUDE.md

@@ -84,14 +84,28 @@ PostgreSQL with pgvector. Schema managed via Alembic migrations in `hindsight-ap
 
 Key tables: `banks`, `memory_units`, `documents`, `entities`, `entity_links`
 
-## Key Conventions
-
-### Code Quality
-**Always run the lint script after making Python or TypeScript/Node changes:**
+### Database Backups (IMPORTANT)
+**Before any operation that may affect the database, run a backup:**
 ```bash
-./scripts/hooks/lint.sh
+docker exec hindsight /backups/backup.sh
 ```
-This runs the same checks as the pre-commit hook (Ruff for Python, ESLint/Prettier for TypeScript).
+
+Operations requiring backup:
+- Running database migrations
+- Modifying Alembic migration files
+- Rebuilding Docker images
+- Resetting or recreating containers
+- Any schema changes
+- Bulk data operations
+
+Backups are stored in `~/hindsight-backups/` on the host.
+
+To restore:
+```bash
+docker exec -it hindsight /backups/restore.sh <backup-file.sql.gz>
+```
+
+## Key Conventions
 
 ### Memory Banks
 - Each bank is isolated (no cross-bank data access)
@@ -113,29 +127,6 @@ This runs the same checks as the pre-commit hook (Ruff for Python, ESLint/Pretti
 - Next.js App Router for control plane
 - Tailwind CSS with shadcn/ui components
 
-### Adding New API Configuration Flags
-
-When adding a new environment variable configuration:
-
-1. **config.py** (`hindsight-api/hindsight_api/config.py`):
-   - Add `ENV_*` constant for the environment variable name
-   - Add `DEFAULT_*` constant for the default value
-   - Add field to `HindsightConfig` dataclass
-   - Add initialization in `from_env()` method
-
-2. **main.py** (`hindsight-api/hindsight_api/main.py`):
-   - Add field to the manual `HindsightConfig()` constructor call (search for "CLI override")
-
-3. **Use the config** in code:
-   ```python
-   from ...config import get_config
-   config = get_config()
-   value = config.your_new_field
-   ```
-
-4. **Documentation** (`hindsight-docs/docs/developer/configuration.md`):
-   - Add to appropriate section table with Variable, Description, Default
-
 ## Environment Setup
 
 ```bash

hindsight-api/hindsight_api/config.py

@@ -8,11 +8,6 @@ import logging
 import os
 from dataclasses import dataclass
 
-from dotenv import find_dotenv, load_dotenv
-
-# Load .env file, searching current and parent directories (overrides existing env vars)
-load_dotenv(find_dotenv(usecwd=True), override=True)
-
 logger = logging.getLogger(__name__)
 
 # Environment variable names
@@ -47,9 +42,6 @@ ENV_MCP_INSTRUCTIONS = "HINDSIGHT_API_MCP_INSTRUCTIONS"
 ENV_OBSERVATION_MIN_FACTS = "HINDSIGHT_API_OBSERVATION_MIN_FACTS"
 ENV_OBSERVATION_TOP_ENTITIES = "HINDSIGHT_API_OBSERVATION_TOP_ENTITIES"
 
-# Retain settings
-ENV_RETAIN_MAX_COMPLETION_TOKENS = "HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS"
-
 # Optimization flags
 ENV_SKIP_LLM_VERIFICATION = "HINDSIGHT_API_SKIP_LLM_VERIFICATION"
 ENV_LAZY_RERANKER = "HINDSIGHT_API_LAZY_RERANKER"
@@ -80,9 +72,6 @@ DEFAULT_MCP_LOCAL_BANK_ID = "mcp"
 DEFAULT_OBSERVATION_MIN_FACTS = 5  # Min facts required to generate entity observations
 DEFAULT_OBSERVATION_TOP_ENTITIES = 5  # Max entities to process per retain batch
 
-# Retain settings
-DEFAULT_RETAIN_MAX_COMPLETION_TOKENS = 64000  # Max tokens for fact extraction LLM call
-
 # Default MCP tool descriptions (can be customized via env vars)
 DEFAULT_MCP_RETAIN_DESCRIPTION = """Store important information to long-term memory.
 
@@ -145,9 +134,6 @@ class HindsightConfig:
     observation_min_facts: int
     observation_top_entities: int
 
-    # Retain settings
-    retain_max_completion_tokens: int
-
     # Optimization flags
     skip_llm_verification: bool
     lazy_reranker: bool
@@ -188,10 +174,6 @@ class HindsightConfig:
             observation_top_entities=int(
                 os.getenv(ENV_OBSERVATION_TOP_ENTITIES, str(DEFAULT_OBSERVATION_TOP_ENTITIES))
             ),
-            # Retain settings
-            retain_max_completion_tokens=int(
-                os.getenv(ENV_RETAIN_MAX_COMPLETION_TOKENS, str(DEFAULT_RETAIN_MAX_COMPLETION_TOKENS))
-            ),
         )
 
     def get_llm_base_url(self) -> str:
@@ -238,19 +220,6 @@ class HindsightConfig:
         logger.info(f"Graph retriever: {self.graph_retriever}")
 
 
-# Cached config instance
-_config_cache: HindsightConfig | None = None
-
-
 def get_config() -> HindsightConfig:
-    """Get the cached configuration, loading from environment on first call."""
-    global _config_cache
-    if _config_cache is None:
-        _config_cache = HindsightConfig.from_env()
-    return _config_cache
-
-
-def clear_config_cache() -> None:
-    """Clear the config cache. Useful for testing or reloading config."""
-    global _config_cache
-    _config_cache = None
+    """Get the current configuration from environment variables."""
+    return HindsightConfig.from_env()

hindsight-api/hindsight_api/engine/retain/fact_extraction.py

@@ -14,7 +14,6 @@ from typing import Literal
 
 from pydantic import BaseModel, ConfigDict, Field, field_validator
 
-from ...config import get_config
 from ..llm_wrapper import LLMConfig, OutputTooLongError
 
 
@@ -110,7 +109,7 @@ class Fact(BaseModel):
 
 
 class CausalRelation(BaseModel):
-    """Causal relationship between facts (legacy - embedded in each fact)."""
+    """Causal relationship between facts."""
 
     target_fact_index: int = Field(
         description="Index of the related fact in the facts array (0-based). "
@@ -132,36 +131,6 @@ class CausalRelation(BaseModel):
     )
 
 
-class TopLevelCausalRelation(BaseModel):
-    """
-    Causal relationship between two facts (top-level schema).
-
-    This is the preferred format - defined AFTER all facts are extracted,
-    allowing the LLM to see the full list of facts before specifying relationships.
-    """
-
-    from_fact_index: int = Field(
-        description="Index of the source fact (0-based). The fact that causes/enables/prevents."
-    )
-    to_fact_index: int = Field(
-        description="Index of the target fact (0-based). The fact that is caused/enabled/prevented."
-    )
-    relation_type: Literal["causes", "caused_by", "enables", "prevents"] = Field(
-        description="Type of causal relationship: "
-        "'causes' = source fact directly causes the target fact, "
-        "'caused_by' = source fact was caused by the target fact, "
-        "'enables' = source fact enables/allows the target fact, "
-        "'prevents' = source fact prevents/blocks the target fact"
-    )
-    strength: float = Field(
-        description="Strength of causal relationship (0.0 to 1.0). "
-        "1.0 = direct/strong causation, 0.5 = moderate, 0.3 = weak/indirect",
-        ge=0.0,
-        le=1.0,
-        default=1.0,
-    )
-
-
 class ExtractedFact(BaseModel):
     """A single extracted fact with 5 required dimensions for comprehensive capture."""
 
@@ -284,15 +253,9 @@ class ExtractedFact(BaseModel):
 
 
 class FactExtractionResponse(BaseModel):
-    """Response containing all extracted facts and their causal relationships."""
+    """Response containing all extracted facts."""
 
     facts: list[ExtractedFact] = Field(description="List of extracted factual statements")
-    causal_relationships: list[TopLevelCausalRelation] | None = Field(
-        default=None,
-        description="Causal relationships between facts. Define these AFTER listing all facts. "
-        "Each relationship specifies from_fact_index -> to_fact_index with a relation type. "
-        "Indices must be valid (0 to N-1 where N is the number of facts).",
-    )
 
 
 def chunk_text(text: str, max_chars: int) -> list[str]:
@@ -609,53 +572,7 @@ WHAT TO EXTRACT vs SKIP
 ══════════════════════════════════════════════════════════════════════════
 
 ✅ EXTRACT: User preferences (ALWAYS as separate facts!), feelings, plans, events, relationships, achievements
-❌ SKIP: Greetings, filler ("thanks", "cool"), purely structural statements
-
-══════════════════════════════════════════════════════════════════════════
-CAUSAL RELATIONSHIPS (CRITICAL - DEFINE AFTER ALL FACTS)
-══════════════════════════════════════════════════════════════════════════
-
-⚠️ IMPORTANT: Causal relationships are defined at the TOP LEVEL, AFTER listing all facts!
-
-The `causal_relationships` array goes at the root of your response (NOT inside each fact).
-This allows you to see all facts first before defining how they relate.
-
-Format:
-```json
-{{
-  "facts": [...all your extracted facts...],
-  "causal_relationships": [
-    {{"from_fact_index": 0, "to_fact_index": 1, "relation_type": "causes", "strength": 0.9}},
-    {{"from_fact_index": 1, "to_fact_index": 2, "relation_type": "enables", "strength": 0.7}}
-  ]
-}}
-```
-
-Relationship types:
-- "causes": Fact A directly causes Fact B (A → B)
-- "caused_by": Fact A was caused by Fact B (A ← B)
-- "enables": Fact A enables/allows Fact B to happen
-- "prevents": Fact A prevents/blocks Fact B from happening
-
-⚠️ INDEX VALIDATION: If you extract N facts (indices 0 to N-1), both from_fact_index and to_fact_index MUST be in range [0, N-1].
-
-Example (Event Date: March 15, 2024):
-Input: "I lost my job in January. Because of that, I couldn't pay rent. So I had to move to a cheaper apartment."
-
-Facts extracted:
-- Fact 0: "User lost their job in January due to layoffs"
-- Fact 1: "User couldn't pay rent because of job loss"
-- Fact 2: "User moved to a cheaper apartment"
-
-Causal relationships (at root level):
-```json
-"causal_relationships": [
-  {{"from_fact_index": 0, "to_fact_index": 1, "relation_type": "causes", "strength": 1.0}},
-  {{"from_fact_index": 1, "to_fact_index": 2, "relation_type": "causes", "strength": 0.9}}
-]
-```
-
-This creates a chain: Job loss (0) → Can't pay rent (1) → Moved to cheaper apartment (2)"""
+❌ SKIP: Greetings, filler ("thanks", "cool"), purely structural statements"""
 
     import logging
 
@@ -666,7 +583,6 @@ This creates a chain: Job loss (0) → Can't pay rent (1) → Moved to cheaper a
     # Retry logic for JSON validation errors
     max_retries = 2
     last_error = None
-    config = get_config()
 
     # Sanitize input text to prevent Unicode encoding errors (e.g., unpaired surrogates)
     sanitized_chunk = _sanitize_text(chunk)
@@ -692,7 +608,7 @@ Text:
                 response_format=FactExtractionResponse,
                 scope="memory_extract_facts",
                 temperature=0.1,
-                max_completion_tokens=config.retain_max_completion_tokens,
+                max_completion_tokens=65000,
                 skip_validation=True,  # Get raw JSON, we'll validate leniently
             )
 
@@ -715,9 +631,6 @@ Text:
                     return []
 
             raw_facts = extraction_response_json.get("facts", [])
-            # Get top-level causal relationships (new schema)
-            top_level_causal_relations = extraction_response_json.get("causal_relationships", [])
-
             if not raw_facts:
                 logger.debug(
                     f"LLM response missing 'facts' field or returned empty list. "
@@ -728,47 +641,6 @@ Text:
                     f"text: {chunk}"
                 )
 
-            # Build a map from fact index to causal relations (from top-level field)
-            # This converts from_fact_index -> [{target_fact_index, relation_type, strength}]
-            causal_relations_by_fact: dict[int, list[dict]] = {}
-            if top_level_causal_relations:
-                num_facts = len(raw_facts)
-                for rel in top_level_causal_relations:
-                    if not isinstance(rel, dict):
-                        continue
-                    from_idx = rel.get("from_fact_index")
-                    to_idx = rel.get("to_fact_index")
-                    relation_type = rel.get("relation_type")
-                    strength = rel.get("strength", 1.0)
-
-                    # Validate indices
-                    if from_idx is None or to_idx is None or relation_type is None:
-                        logger.warning(f"Skipping malformed top-level causal relation: {rel}")
-                        continue
-                    if from_idx < 0 or from_idx >= num_facts:
-                        logger.warning(
-                            f"Invalid from_fact_index {from_idx} in top-level causal relation "
-                            f"(valid range: 0-{num_facts - 1}). Skipping."
-                        )
-                        continue
-                    if to_idx < 0 or to_idx >= num_facts:
-                        logger.warning(
-                            f"Invalid to_fact_index {to_idx} in top-level causal relation "
-                            f"(valid range: 0-{num_facts - 1}). Skipping."
-                        )
-                        continue
-
-                    # Add to the map for the from_fact_index
-                    if from_idx not in causal_relations_by_fact:
-                        causal_relations_by_fact[from_idx] = []
-                    causal_relations_by_fact[from_idx].append(
-                        {
-                            "target_fact_index": to_idx,
-                            "relation_type": relation_type,
-                            "strength": strength,
-                        }
-                    )
-
             for i, llm_fact in enumerate(raw_facts):
                 # Skip non-dict entries but track them for retry
                 if not isinstance(llm_fact, dict):
@@ -873,40 +745,19 @@ Text:
                     if validated_entities:
                         fact_data["entities"] = validated_entities
 
-                # Add causal relations from both sources:
-                # 1. Top-level causal_relationships (preferred, new schema)
-                # 2. Per-fact causal_relations (legacy, for backward compatibility)
-                validated_relations = []
-
-                # First, add relations from top-level (already validated above)
-                if i in causal_relations_by_fact:
-                    for rel in causal_relations_by_fact[i]:
-                        try:
-                            validated_relations.append(CausalRelation.model_validate(rel))
-                        except Exception as e:
-                            logger.warning(f"Invalid top-level causal relation for fact {i}: {rel}: {e}")
-
-                # Then, add any legacy per-fact relations (with index validation)
-                legacy_causal_relations = get_value("causal_relations")
-                if legacy_causal_relations:
-                    num_facts = len(raw_facts)
-                    for rel in legacy_causal_relations:
+                # Add causal relations if present (validate as CausalRelation objects)
+                # Filter out invalid relations (missing required fields)
+                causal_relations = get_value("causal_relations")
+                if causal_relations:
+                    validated_relations = []
+                    for rel in causal_relations:
                         if isinstance(rel, dict) and "target_fact_index" in rel and "relation_type" in rel:
-                            target_idx = rel.get("target_fact_index")
-                            # Validate target index for legacy format too
-                            if target_idx is not None and 0 <= target_idx < num_facts:
-                                try:
-                                    validated_relations.append(CausalRelation.model_validate(rel))
-                                except Exception as e:
-                                    logger.warning(f"Invalid causal relation {rel}: {e}")
-                            else:
-                                logger.warning(
-                                    f"Invalid target_fact_index {target_idx} in per-fact causal relation "
-                                    f"from fact {i} (valid range: 0-{num_facts - 1}). Skipping."
-                                )
-
-                if validated_relations:
-                    fact_data["causal_relations"] = validated_relations
+                            try:
+                                validated_relations.append(CausalRelation.model_validate(rel))
+                            except Exception as e:
+                                logger.warning(f"Invalid causal relation {rel}: {e}")
+                    if validated_relations:
+                        fact_data["causal_relations"] = validated_relations
 
                 # Always set mentioned_at to the event_date (when the conversation/document occurred)
                 fact_data["mentioned_at"] = event_date.isoformat()

hindsight-api/hindsight_api/main.py

@@ -184,7 +184,6 @@ def main():
             graph_retriever=config.graph_retriever,
             observation_min_facts=config.observation_min_facts,
             observation_top_entities=config.observation_top_entities,
-            retain_max_completion_tokens=config.retain_max_completion_tokens,
             skip_llm_verification=config.skip_llm_verification,
             lazy_reranker=config.lazy_reranker,
         )

hindsight-api/tests/test_causal_relationships.py

@@ -1,222 +0,0 @@
-"""
-Test suite for causal relationship extraction.
-
-Tests that the fact extraction system correctly identifies and validates
-causal relationships between facts, with valid indices.
-"""
-
-from datetime import datetime
-
-import pytest
-
-from hindsight_api import LLMConfig
-from hindsight_api.engine.retain.fact_extraction import extract_facts_from_text
-
-
-class TestCausalRelationships:
-    """Tests for causal relationship extraction and validation."""
-
-    @pytest.mark.asyncio
-    async def test_causal_chain_extraction(self):
-        """
-        Test that a clear causal chain is extracted with valid relationships.
-
-        Story: Lost job -> couldn't pay rent -> had to move -> found new apartment
-
-        This is a 4-fact causal chain where each fact causes the next.
-        The extracted causal relations should have valid indices (0-3).
-        """
-        text = """
-I lost my job at the tech company in January because of layoffs.
-Because I lost my job, I couldn't pay my rent anymore.
-Since I couldn't afford rent, I had to move out of my apartment.
-After searching for weeks, I finally found a cheaper apartment in Brooklyn.
-"""
-
-        context = "Personal story about housing change"
-        llm_config = LLMConfig.for_memory()
-
-        facts, _ = await extract_facts_from_text(
-            text=text,
-            event_date=datetime(2024, 3, 15),
-            context=context,
-            llm_config=llm_config,
-            agent_name="TestUser"
-        )
-
-        assert len(facts) >= 3, f"Should extract at least 3 facts from the causal chain. Got {len(facts)}"
-
-        # Collect all causal relations from all facts
-        all_causal_relations = []
-        for i, fact in enumerate(facts):
-            if fact.causal_relations:
-                for rel in fact.causal_relations:
-                    all_causal_relations.append({
-                        "from_fact_index": i,
-                        "to_fact_index": rel.target_fact_index,
-                        "relation_type": rel.relation_type,
-                        "strength": rel.strength,
-                        "from_fact_text": fact.fact[:50],
-                    })
-
-        # Verify that ALL causal relation indices are valid
-        num_facts = len(facts)
-        invalid_relations = []
-        for rel in all_causal_relations:
-            if rel["to_fact_index"] < 0 or rel["to_fact_index"] >= num_facts:
-                invalid_relations.append(rel)
-
-        assert len(invalid_relations) == 0, (
-            f"Found {len(invalid_relations)} causal relations with invalid indices! "
-            f"Valid range is 0-{num_facts - 1}. "
-            f"Invalid relations: {invalid_relations}"
-        )
-
-        # Should have at least some causal relations extracted
-        assert len(all_causal_relations) >= 2, (
-            f"Should extract at least 2 causal relationships from this clear chain. "
-            f"Got {len(all_causal_relations)}: {all_causal_relations}"
-        )
-
-        # Verify relation types are valid
-        valid_types = {"causes", "caused_by", "enables", "prevents"}
-        for rel in all_causal_relations:
-            assert rel["relation_type"] in valid_types, (
-                f"Invalid relation_type '{rel['relation_type']}'. Must be one of {valid_types}"
-            )
-
-    @pytest.mark.asyncio
-    async def test_complex_causal_web(self):
-        """
-        Test a more complex scenario with multiple interconnected causes.
-
-        This tests the LLM's ability to identify multiple causal links and
-        ensure all referenced indices exist.
-        """
-        text = """
-The heavy rain caused flooding in the basement.
-The flooding damaged the electrical system.
-Because of the electrical damage, we had to call an electrician.
-The electrician found that the wiring was old and needed replacement.
-We decided to renovate the entire basement while fixing the wiring.
-The renovation took three months and cost $15,000.
-"""
-
-        context = "Home repair story"
-        llm_config = LLMConfig.for_memory()
-
-        facts, _ = await extract_facts_from_text(
-            text=text,
-            event_date=datetime(2024, 6, 1),
-            context=context,
-            llm_config=llm_config,
-            agent_name="TestUser"
-        )
-
-        assert len(facts) >= 4, f"Should extract at least 4 facts. Got {len(facts)}"
-
-        # Validate all causal relation indices
-        num_facts = len(facts)
-        for i, fact in enumerate(facts):
-            if fact.causal_relations:
-                for rel in fact.causal_relations:
-                    assert 0 <= rel.target_fact_index < num_facts, (
-                        f"Fact {i} has causal relation to invalid index {rel.target_fact_index}. "
-                        f"Valid range is 0-{num_facts - 1}. "
-                        f"Fact text: {fact.fact[:80]}..."
-                    )
-
-    @pytest.mark.asyncio
-    async def test_no_self_referencing_causal_relations(self):
-        """
-        Test that facts don't have causal relations pointing to themselves.
-        """
-        text = """
-I started learning Python because I wanted to automate my work tasks.
-Learning Python led me to discover machine learning.
-Machine learning fascinated me so much that I changed my career to data science.
-"""
-
-        context = "Career change story"
-        llm_config = LLMConfig.for_memory()
-
-        facts, _ = await extract_facts_from_text(
-            text=text,
-            event_date=datetime(2024, 1, 1),
-            context=context,
-            llm_config=llm_config,
-            agent_name="TestUser"
-        )
-
-        # Check no fact references itself
-        for i, fact in enumerate(facts):
-            if fact.causal_relations:
-                for rel in fact.causal_relations:
-                    assert rel.target_fact_index != i, (
-                        f"Fact {i} has a self-referencing causal relation! "
-                        f"Fact text: {fact.fact}"
-                    )
-
-    @pytest.mark.asyncio
-    async def test_bidirectional_causal_relationships(self):
-        """
-        Test that bidirectional causal relationships (causes and caused_by)
-        are handled correctly.
-        """
-        text = """
-My promotion at work caused me to move to New York.
-Moving to New York was caused by my promotion at work.
-The new role enabled me to lead a team of engineers.
-"""
-
-        context = "Work promotion story"
-        llm_config = LLMConfig.for_memory()
-
-        facts, _ = await extract_facts_from_text(
-            text=text,
-            event_date=datetime(2024, 2, 15),
-            context=context,
-            llm_config=llm_config,
-            agent_name="TestUser"
-        )
-
-        num_facts = len(facts)
-
-        # Validate all indices
-        for i, fact in enumerate(facts):
-            if fact.causal_relations:
-                for rel in fact.causal_relations:
-                    assert 0 <= rel.target_fact_index < num_facts, (
-                        f"Invalid target_fact_index {rel.target_fact_index} in fact {i}. "
-                        f"Valid range: 0-{num_facts - 1}"
-                    )
-
-    @pytest.mark.asyncio
-    async def test_causal_relation_strength_values(self):
-        """
-        Test that causal relation strength values are within valid range [0.0, 1.0].
-        """
-        text = """
-The stock market crash directly caused the company to lay off employees.
-The layoffs indirectly led to reduced consumer spending in the area.
-Reduced spending somewhat affected local businesses.
-"""
-
-        context = "Economic impact story"
-        llm_config = LLMConfig.for_memory()
-
-        facts, _ = await extract_facts_from_text(
-            text=text,
-            event_date=datetime(2024, 4, 1),
-            context=context,
-            llm_config=llm_config,
-            agent_name="TestUser"
-        )
-
-        for i, fact in enumerate(facts):
-            if fact.causal_relations:
-                for rel in fact.causal_relations:
-                    assert 0.0 <= rel.strength <= 1.0, (
-                        f"Causal relation strength {rel.strength} is outside valid range [0.0, 1.0]. "
-                        f"Fact {i}: {fact.fact[:50]}..."
-                    )

hindsight-docs/docs/developer/configuration.md

@@ -183,14 +183,6 @@ Controls when the system generates entity observations (summaries about entities
 | `HINDSIGHT_API_OBSERVATION_MIN_FACTS` | Minimum facts about an entity before generating observations | `5` |
 | `HINDSIGHT_API_OBSERVATION_TOP_ENTITIES` | Max entities to process per retain batch | `5` |
 
-### Retain
-
-Controls the retain (memory ingestion) pipeline.
-
-| Variable | Description | Default |
-|----------|-------------|---------|
-| `HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS` | Max completion tokens for fact extraction LLM calls | `64000` |
-
 ### Local MCP Server
 
 Configuration for the local MCP server (`hindsight-local-mcp` command).