fix test

hindsight-api/hindsight_api/engine/providers/claude_code_llm.py

@@ -291,61 +291,202 @@ class ClaudeCodeLLM(LLMInterface):
         tool_choice: str | dict[str, Any] = "auto",
     ) -> LLMToolCallResult:
         """
-        Make an LLM API call with tool/function calling support.
+        Make an LLM API call with tool/function calling support using Claude Agent SDK.
 
-        Note: This is a simplified implementation. Full tool support would require
-        integrating with Claude Agent SDK's tool system.
+        This implementation uses ClaudeSDKClient (not query()) because custom tools via
+        SDK MCP servers are only supported with the client. Tools are converted from OpenAI
+        format to SDK MCP tools, and tool names are formatted as mcp__hindsight_tools__{name}.
 
         Args:
             messages: List of message dicts. Can include tool results with role='tool'.
             tools: List of tool definitions in OpenAI format.
-            max_completion_tokens: Maximum tokens in response.
-            temperature: Sampling temperature.
+            max_completion_tokens: Maximum tokens in response (not used by Claude Agent SDK).
+            temperature: Sampling temperature (not used by Claude Agent SDK).
             scope: Scope identifier for tracking.
             max_retries: Maximum retry attempts.
             initial_backoff: Initial backoff time in seconds.
             max_backoff: Maximum backoff time in seconds.
-            tool_choice: How to choose tools - "auto", "none", "required", or specific function.
+            tool_choice: How to choose tools (not used by Claude Agent SDK).
 
         Returns:
             LLMToolCallResult with content and/or tool_calls.
         """
-        # For now, use regular call without tools
-        # Full implementation would require mapping OpenAI tool format to Claude Agent SDK tools
-        logger.warning(
-            "Claude Code provider does not fully support tool calling yet. Falling back to regular text completion."
+        from claude_agent_sdk import (
+            AssistantMessage,
+            ClaudeAgentOptions,
+            ClaudeSDKClient,
+            SdkMcpTool,
+            TextBlock,
+            ToolUseBlock,
+            create_sdk_mcp_server,
         )
 
-        result = await self.call(
-            messages=messages,
-            response_format=None,
-            max_completion_tokens=max_completion_tokens,
-            temperature=temperature,
-            scope=scope,
-            max_retries=max_retries,
-            initial_backoff=initial_backoff,
-            max_backoff=max_backoff,
-            return_usage=True,
+        start_time = time.time()
+
+        # Convert OpenAI tool format to Claude Agent SDK SdkMcpTool format
+        sdk_tools: list[SdkMcpTool] = []
+        tool_names: list[str] = []
+
+        for tool in tools:
+            func = tool.get("function", {})
+            tool_name = func.get("name", "")
+            tool_description = func.get("description", "")
+            parameters = func.get("parameters", {})
+
+            # Create a handler with proper closure to avoid transport issues
+            def make_handler(name: str):
+                async def handler(args: dict[str, Any]) -> dict[str, Any]:
+                    # Return immediately with success - tool execution happens externally
+                    return {
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": f"[Tool {name} called successfully]",
+                            }
+                        ]
+                    }
+
+                return handler
+
+            sdk_tools.append(
+                SdkMcpTool(
+                    name=tool_name,
+                    description=tool_description,
+                    input_schema=parameters,
+                    handler=make_handler(tool_name),
+                )
+            )
+            tool_names.append(tool_name)
+
+        # Create an MCP server with the tools
+        mcp_server = create_sdk_mcp_server(
+            name="hindsight_tools",
+            version="1.0.0",
+            tools=sdk_tools if sdk_tools else None,
         )
 
-        if isinstance(result, tuple):
-            text, usage = result
-            return LLMToolCallResult(
-                content=text,
-                tool_calls=[],
-                finish_reason="stop",
-                input_tokens=usage.input_tokens,
-                output_tokens=usage.output_tokens,
-            )
-        else:
-            # Fallback if return_usage didn't work as expected
-            return LLMToolCallResult(
-                content=str(result),
-                tool_calls=[],
-                finish_reason="stop",
-                input_tokens=0,
-                output_tokens=0,
-            )
+        # Build system prompt and user content from messages
+        system_prompt = ""
+        user_content = ""
+
+        for msg in messages:
+            role = msg.get("role", "user")
+            content = msg.get("content", "")
+
+            if role == "system":
+                system_prompt += ("\n\n" + content) if system_prompt else content
+            elif role == "user":
+                user_content += ("\n\n" + content) if user_content else content
+            elif role == "assistant":
+                # Include previous assistant messages as context
+                user_content += f"\n\n[Previous assistant response: {content}]"
+            elif role == "tool":
+                # Tool results are already in tool_results_map, append to user context
+                tool_call_id = msg.get("tool_call_id", "")
+                user_content += f"\n\n[Tool result for {tool_call_id}: {content}]"
+
+        # Format tool names for SDK MCP servers: mcp__{server_name}__{tool_name}
+        # This is required by the Claude Agent SDK for MCP server tools
+        allowed_tool_names = [f"mcp__hindsight_tools__{name}" for name in tool_names]
+
+        # Configure SDK options with MCP server
+        options = ClaudeAgentOptions(
+            system_prompt=system_prompt if system_prompt else None,
+            max_turns=1,  # Single-turn for API-style interactions
+            mcp_servers={"hindsight_tools": mcp_server} if sdk_tools else {},
+            allowed_tools=allowed_tool_names if allowed_tool_names else [],
+        )
+
+        # Call Claude Agent SDK with retry logic
+        last_exception = None
+        for attempt in range(max_retries + 1):
+            try:
+                full_text = ""
+                tool_calls: list[LLMToolCall] = []
+
+                # Use ClaudeSDKClient for tool calling support
+                # Note: query() does NOT support custom tools, only ClaudeSDKClient does
+                async with ClaudeSDKClient(options=options) as client:
+                    # Send the query
+                    await client.query(user_content)
+
+                    # Receive response
+                    async for message in client.receive_response():
+                        if isinstance(message, AssistantMessage):
+                            for block in message.content:
+                                if isinstance(block, TextBlock):
+                                    full_text += block.text
+                                elif isinstance(block, ToolUseBlock):
+                                    # SDK returns tool names with MCP prefix (mcp__hindsight_tools__{name})
+                                    # Strip the prefix to return original tool name expected by caller
+                                    tool_name = block.name
+                                    if tool_name.startswith("mcp__hindsight_tools__"):
+                                        tool_name = tool_name.replace("mcp__hindsight_tools__", "", 1)
+
+                                    tool_calls.append(
+                                        LLMToolCall(
+                                            id=block.id,
+                                            name=tool_name,
+                                            arguments=block.input,
+                                        )
+                                    )
+
+                # Record metrics
+                duration = time.time() - start_time
+                metrics = get_metrics_collector()
+
+                # Estimate token usage (Claude Agent SDK doesn't report exact counts)
+                estimated_input = sum(len(m.get("content", "")) for m in messages) // 4
+                estimated_output = len(full_text) // 4
+
+                metrics.record_llm_call(
+                    provider=self.provider,
+                    model=self.model,
+                    scope=scope,
+                    duration=duration,
+                    input_tokens=estimated_input,
+                    output_tokens=estimated_output,
+                    success=True,
+                )
+
+                # Log slow calls
+                if duration > 10.0:
+                    logger.info(
+                        f"slow llm call: scope={scope}, model={self.provider}/{self.model}, time={duration:.3f}s"
+                    )
+
+                return LLMToolCallResult(
+                    content=full_text if full_text else None,
+                    tool_calls=tool_calls,
+                    finish_reason="tool_calls" if tool_calls else "stop",
+                    input_tokens=estimated_input,
+                    output_tokens=estimated_output,
+                )
+
+            except Exception as e:
+                last_exception = e
+
+                # Check for authentication errors
+                error_str = str(e).lower()
+                if "auth" in error_str or "login" in error_str or "credential" in error_str:
+                    logger.error(f"Claude Code authentication error: {e}")
+                    raise RuntimeError(
+                        f"Claude Code authentication failed: {e}\n\n"
+                        "Run 'claude auth login' to authenticate with Claude Pro/Max."
+                    ) from e
+
+                if attempt < max_retries:
+                    backoff = min(initial_backoff * (2**attempt), max_backoff)
+                    logger.warning(f"Claude Code tool call error (attempt {attempt + 1}/{max_retries + 1}): {e}")
+                    await asyncio.sleep(backoff)
+                    continue
+                else:
+                    logger.error(f"Claude Code tool call error after {max_retries + 1} attempts: {e}")
+                    raise
+
+        if last_exception:
+            raise last_exception
+        raise RuntimeError("Claude Code tool call failed after all retries")
 
     async def cleanup(self) -> None:
         """Clean up resources (no HTTP client to close for Claude Agent SDK)."""

hindsight-api/hindsight_api/engine/providers/codex_llm.py

@@ -177,6 +177,9 @@ class CodexLLM(LLMInterface):
             schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}"
             system_instruction += schema_msg
 
+        # gpt-5.2-codex only supports "detailed" reasoning summary
+        reasoning_summary = "detailed" if "5.2" in self.model else self.reasoning_summary
+
         # Build Codex request payload
         payload = {
             "model": self.model,
@@ -192,7 +195,7 @@ class CodexLLM(LLMInterface):
             "tools": [],
             "tool_choice": "auto",
             "parallel_tool_calls": True,
-            "reasoning": {"summary": self.reasoning_summary},
+            "reasoning": {"summary": reasoning_summary},
             "store": False,  # Codex uses stateless mode
             "stream": True,  # SSE streaming
             "include": ["reasoning.encrypted_content"],
@@ -283,13 +286,20 @@ class CodexLLM(LLMInterface):
                         "Run 'codex auth login' to re-authenticate."
                     ) from e
 
+                # Log the actual error message from the API
+                error_detail = e.response.text[:500] if hasattr(e.response, "text") else str(e)
+
                 if attempt < max_retries:
                     backoff = min(initial_backoff * (2**attempt), max_backoff)
-                    logger.warning(f"Codex HTTP error {status_code} (attempt {attempt + 1}/{max_retries + 1})")
+                    logger.warning(
+                        f"Codex HTTP error {status_code} (attempt {attempt + 1}/{max_retries + 1}): {error_detail}"
+                    )
                     await asyncio.sleep(backoff)
                     continue
                 else:
-                    logger.error(f"Codex HTTP error after {max_retries + 1} attempts: {e}")
+                    logger.error(
+                        f"Codex HTTP error after {max_retries + 1} attempts: Status {status_code}, Detail: {error_detail}"
+                    )
                     raise
 
             except httpx.RequestError as e:
@@ -379,8 +389,22 @@ class CodexLLM(LLMInterface):
         """
         Make API call with tool calling support.
 
-        Note: This is a basic implementation. Full tool calling support for Codex
-        may require additional SSE event parsing.
+        Parses Codex SSE stream to extract tool calls from response.output_item.done events.
+        Tools are converted from OpenAI format to Codex format (flat structure at top level).
+
+        Args:
+            messages: List of message dicts. Can include tool results with role='tool'.
+            tools: List of tool definitions in OpenAI format.
+            max_completion_tokens: Maximum tokens in response.
+            temperature: Sampling temperature.
+            scope: Scope identifier for tracking.
+            max_retries: Maximum retry attempts.
+            initial_backoff: Initial backoff time in seconds.
+            max_backoff: Maximum backoff time in seconds.
+            tool_choice: How to choose tools - "auto", "none", "required", or specific function.
+
+        Returns:
+            LLMToolCallResult with content and/or tool_calls.
         """
         start_time = time.time()
 
@@ -413,20 +437,22 @@ class CodexLLM(LLMInterface):
                 )
 
         # Convert tools to Codex format
+        # Codex expects tools with type and name/description/parameters at top level
         codex_tools = []
         for tool in tools:
             func = tool.get("function", {})
             codex_tools.append(
                 {
                     "type": "function",
-                    "function": {
-                        "name": func.get("name", ""),
-                        "description": func.get("description", ""),
-                        "parameters": func.get("parameters", {}),
-                    },
+                    "name": func.get("name", ""),
+                    "description": func.get("description", ""),
+                    "parameters": func.get("parameters", {}),
                 }
             )
 
+        # gpt-5.2-codex only supports "detailed" reasoning summary
+        reasoning_summary = "detailed" if "5.2" in self.model else self.reasoning_summary
+
         payload = {
             "model": self.model,
             "instructions": system_instruction,
@@ -434,7 +460,7 @@ class CodexLLM(LLMInterface):
             "tools": codex_tools,
             "tool_choice": tool_choice,
             "parallel_tool_calls": True,
-            "reasoning": {"summary": self.reasoning_summary},
+            "reasoning": {"summary": reasoning_summary},
             "store": False,
             "stream": True,
             "include": ["reasoning.encrypted_content"],
@@ -451,8 +477,16 @@ class CodexLLM(LLMInterface):
 
         url = f"{self.base_url}/codex/responses"
 
+        # Debug logging for troubleshooting
+        logger.debug(f"Codex tool call request: url={url}, model={payload['model']}, tools={len(codex_tools)}")
+
         try:
             response = await self._client.post(url, json=payload, headers=headers, timeout=120.0)
+
+            # Log response details on error
+            if response.status_code != 200:
+                logger.error(f"Codex API error {response.status_code}: {response.text[:500]}")
+
             response.raise_for_status()
 
             # Parse SSE for tool calls and content
@@ -512,13 +546,30 @@ class CodexLLM(LLMInterface):
                     if event_type == "response.text.delta" and "delta" in data:
                         content += data["delta"]
 
-                    # Extract tool calls
-                    elif event_type == "response.function_call_arguments.delta":
-                        # Handle tool call events (implementation depends on actual Codex SSE format)
-                        pass
+                    # Extract completed tool calls from response.output_item.done
+                    elif event_type == "response.output_item.done":
+                        item = data.get("item", {})
+                        if item.get("type") == "function_call" and item.get("status") == "completed":
+                            tool_name = item.get("name", "")
+                            arguments_str = item.get("arguments", "{}")
+                            call_id = item.get("call_id", "")
 
-                except json.JSONDecodeError:
-                    pass
+                            try:
+                                arguments = json.loads(arguments_str)
+                            except json.JSONDecodeError:
+                                logger.warning(f"Failed to parse tool arguments: {arguments_str}")
+                                arguments = {}
+
+                            tool_calls.append(
+                                LLMToolCall(
+                                    id=call_id,
+                                    name=tool_name,
+                                    arguments=arguments,
+                                )
+                            )
+
+                except json.JSONDecodeError as e:
+                    logger.warning(f"Failed to parse SSE data: {e}, data_str: {data_str[:200]}")
 
         return content if content else None, tool_calls
 

hindsight-api/hindsight_api/engine/reflect/prompts.py

@@ -463,9 +463,12 @@ def build_final_prompt(
     parts.append(
         "\n## Instructions\n"
         "Provide a thoughtful answer by synthesizing and reasoning from the retrieved data above. "
-        "You can make reasonable inferences from the memories, but don't completely fabricate information."
+        "You can make reasonable inferences from the memories, but don't completely fabricate information. "
         "If the exact answer isn't stated, use what IS stated to give the best possible answer. "
-        "Only say 'I don't have information' if the retrieved data is truly unrelated to the question."
+        "Only say 'I don't have information' if the retrieved data is truly unrelated to the question.\n\n"
+        "IMPORTANT: Output ONLY the final answer. Do NOT include meta-commentary like "
+        '"I\'ll search..." or "Let me analyze...". Do NOT explain your reasoning process. '
+        "Just provide the direct synthesized answer."
     )
 
     return "\n".join(parts)
@@ -480,4 +483,10 @@ Your approach:
 - Be helpful - if you have related information, use it to give the best possible answer
 
 Only say "I don't have information" if the retrieved data is truly unrelated to the question.
-Do NOT fabricate information that has no basis in the retrieved data."""
+Do NOT fabricate information that has no basis in the retrieved data.
+
+CRITICAL: Output ONLY the final synthesized answer. Do NOT include:
+- Meta-commentary about what you're doing ("I'll search...", "Let me analyze...")
+- Explanations of your reasoning process
+- Descriptions of your approach
+Just provide the direct answer."""

hindsight-api/tests/conftest.py

@@ -220,3 +220,34 @@ async def memory(pg0_db_url, embeddings, cross_encoder, query_analyzer):
             await mem.close()
     except Exception:
         pass
+
+
+@pytest_asyncio.fixture(scope="function")
+async def memory_no_llm_verify(pg0_db_url, embeddings, cross_encoder, query_analyzer):
+    """
+    Provide a MemoryEngine instance that skips LLM connection verification.
+
+    This fixture is useful for tests that override the LLM configuration
+    after initialization (e.g., to test specific providers).
+    """
+    mem = MemoryEngine(
+        db_url=pg0_db_url,
+        memory_llm_provider="mock",  # Use mock provider as placeholder
+        memory_llm_api_key="",
+        memory_llm_model="mock",
+        embeddings=embeddings,
+        cross_encoder=cross_encoder,
+        query_analyzer=query_analyzer,
+        pool_min_size=1,
+        pool_max_size=5,
+        run_migrations=False,
+        task_backend=SyncTaskBackend(),
+        skip_llm_verification=True,  # Skip verification - will be overridden by test
+    )
+    await mem.initialize()
+    yield mem
+    try:
+        if mem._pool and not mem._pool._closing:
+            await mem.close()
+    except Exception:
+        pass

hindsight-api/tests/test_llm_provider.py

@@ -1,5 +1,10 @@
 """
-Test LLM provider with different models using actual memory operations.
+Test LLM provider with different models using actual Hindsight memory operations.
+
+Tests validate that providers work correctly with:
+1. Retain (memory ingestion with fact extraction)
+2. Reflect (memory retrieval with tool calling)
+3. Mental models (consolidated knowledge generation)
 """
 import os
 from datetime import datetime
@@ -33,6 +38,12 @@ MODEL_MATRIX = [
     # Ollama models (local)
     ("ollama", "gemma3:12b"),
     ("ollama", "gemma3:1b"),
+    # Claude Code (uses Claude Agent SDK with Claude models)
+    ("claude-code", "claude-sonnet-4-20250514"),
+    # OpenAI Codex (uses MCP with Codex-specific models)
+    ("openai-codex", "gpt-5.2-codex"),
+    # Mock provider (for testing)
+    ("mock", "mock"),
 ]
 
 
@@ -48,6 +59,165 @@ def get_api_key_for_provider(provider: str) -> str | None:
     return os.getenv(env_var) if env_var else None
 
 
+def should_skip_provider(provider: str, model: str = "") -> tuple[bool, str]:
+    """Check if provider should be skipped and return reason."""
+    # Never skip mock provider
+    if provider == "mock":
+        return False, ""
+
+    # Skip claude-code and openai-codex in CI (require local auth)
+    if os.getenv("CI") and provider in ("claude-code", "openai-codex"):
+        return True, f"{provider} not available in CI (requires local authentication)"
+
+    # Skip Ollama in CI (no models available)
+    if provider == "ollama" and os.getenv("CI"):
+        return True, "Ollama not available in CI"
+
+    # Skip Ollama gemma models (don't support tool calling)
+    if provider == "ollama" and "gemma" in model.lower():
+        return True, f"Ollama {model} does not support tool calling"
+
+    # Other providers need an API key
+    if provider not in ("ollama", "claude-code", "openai-codex", "mock"):
+        api_key = get_api_key_for_provider(provider)
+        if not api_key:
+            return True, f"No API key available (set {provider.upper()}_API_KEY)"
+
+    return False, ""
+
+
+@pytest.mark.parametrize("provider,model", MODEL_MATRIX)
+@pytest.mark.asyncio
+async def test_llm_provider_api_methods(provider: str, model: str):
+    """
+    Test all LLM API methods used by Hindsight at runtime.
+    This validates that the provider correctly implements the LLMInterface.
+
+    Tests:
+    1. verify_connection() - Connection verification
+    2. call() with plain text - Basic LLM call
+    3. call() with response_format - Structured output (used in fact extraction)
+    4. call_with_tools() - Tool calling (used in reflect agent)
+    """
+    # Skip mock provider - it's a test stub, not a real LLM implementation
+    if provider == "mock":
+        pytest.skip("Mock provider is a test stub, not a real LLM")
+
+    should_skip, reason = should_skip_provider(provider, model)
+    if should_skip:
+        pytest.skip(f"Skipping {provider}/{model}: {reason}")
+
+    api_key = get_api_key_for_provider(provider)
+
+    llm = LLMProvider(
+        provider=provider,
+        api_key=api_key or "",
+        base_url="",
+        model=model,
+    )
+
+    print(f"\n{provider}/{model} - API methods test:")
+
+    # Test 1: verify_connection()
+    try:
+        await llm.verify_connection()
+        print("  ✓ verify_connection()")
+    except Exception as e:
+        pytest.fail(f"{provider}/{model} verify_connection() failed: {e}")
+
+    # Test 2: call() with plain text
+    try:
+        response = await llm.call(
+            messages=[
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": "What is 2+2? Answer in one word."},
+            ],
+            max_completion_tokens=50,
+        )
+        assert response is not None, "call() returned None"
+        assert len(response) > 0, "call() returned empty string"
+        print(f"  ✓ call() plain text: {response[:50]}")
+    except Exception as e:
+        pytest.fail(f"{provider}/{model} call() plain text failed: {e}")
+
+    # Test 3: call() with response_format (structured output)
+    try:
+        from pydantic import BaseModel
+
+        class TestResponse(BaseModel):
+            answer: str
+            confidence: str
+
+        response = await llm.call(
+            messages=[
+                {"role": "system", "content": "You are a math assistant."},
+                {"role": "user", "content": "What is the capital of France?"},
+            ],
+            response_format=TestResponse,
+            max_completion_tokens=100,
+        )
+        assert isinstance(response, TestResponse), f"Expected TestResponse, got {type(response)}"
+        assert hasattr(response, "answer"), "Structured output missing 'answer' field"
+        assert hasattr(response, "confidence"), "Structured output missing 'confidence' field"
+        print(f"  ✓ call() structured output: answer={response.answer}, confidence={response.confidence}")
+    except Exception as e:
+        pytest.fail(f"{provider}/{model} call() structured output failed: {e}")
+
+    # Test 4: call_with_tools() (tool calling)
+    try:
+        tools = [
+            {
+                "type": "function",
+                "function": {
+                    "name": "get_weather",
+                    "description": "Get the weather for a location",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "location": {"type": "string", "description": "City name"},
+                            "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
+                        },
+                        "required": ["location"],
+                    },
+                },
+            }
+        ]
+
+        result = await llm.call_with_tools(
+            messages=[
+                {"role": "system", "content": "You are a helpful assistant with access to tools."},
+                {"role": "user", "content": "What's the weather like in Paris?"},
+            ],
+            tools=tools,
+            max_completion_tokens=200,
+        )
+
+        assert result is not None, "call_with_tools() returned None"
+        assert hasattr(result, "tool_calls"), "Result missing 'tool_calls' attribute"
+
+        # Nano models may hit token limits before making tool calls - that's acceptable
+        is_nano_model = "nano" in model.lower()
+        if is_nano_model and len(result.tool_calls) == 0:
+            # Check if it hit length limit (expected for nano models)
+            if hasattr(result, "finish_reason") and result.finish_reason == "length":
+                print(f"  ✓ call_with_tools(): nano model hit token limit (expected)")
+            else:
+                pytest.fail(f"Nano model made 0 tool calls but didn't hit length limit (finish_reason={getattr(result, 'finish_reason', 'unknown')})")
+        else:
+            assert len(result.tool_calls) > 0, f"Expected at least 1 tool call, got {len(result.tool_calls)}"
+
+            # Verify tool call structure
+            tool_call = result.tool_calls[0]
+            assert hasattr(tool_call, "name"), "Tool call missing 'name'"
+            assert hasattr(tool_call, "arguments"), "Tool call missing 'arguments'"
+            assert tool_call.name == "get_weather", f"Expected 'get_weather', got '{tool_call.name}'"
+            assert "location" in tool_call.arguments, "Tool call arguments missing 'location'"
+
+            print(f"  ✓ call_with_tools(): {tool_call.name}({tool_call.arguments})")
+    except Exception as e:
+        pytest.fail(f"{provider}/{model} call_with_tools() failed: {e}")
+
+
 @pytest.mark.parametrize("provider,model", MODEL_MATRIX)
 @pytest.mark.asyncio
 async def test_llm_provider_memory_operations(provider: str, model: str):
@@ -55,16 +225,16 @@ async def test_llm_provider_memory_operations(provider: str, model: str):
     Test LLM provider with actual memory operations: fact extraction and reflect.
     All models must pass this test.
     """
+    # Skip mock provider - it's a test stub, not designed for real operations
+    if provider == "mock":
+        pytest.skip("Mock provider is a test stub, not designed for real operations")
+
+    should_skip, reason = should_skip_provider(provider, model)
+    if should_skip:
+        pytest.skip(f"Skipping {provider}/{model}: {reason}")
+
     api_key = get_api_key_for_provider(provider)
 
-    # Skip Ollama tests in CI (no models available)
-    if provider == "ollama" and os.getenv("CI"):
-        pytest.skip(f"Skipping {provider}/{model}: Ollama not available in CI")
-
-    # Other providers need an API key
-    if provider != "ollama" and not api_key:
-        pytest.skip(f"Skipping {provider}/{model}: no API key available")
-
     llm = LLMProvider(
         provider=provider,
         api_key=api_key or "",
@@ -122,3 +292,115 @@ async def test_llm_provider_memory_operations(provider: str, model: str):
 
     assert response is not None, f"{provider}/{model} reflect returned None"
     assert len(response) > 10, f"{provider}/{model} reflect response too short"
+
+
+@pytest.mark.parametrize("provider,model", [
+    ("claude-code", "claude-sonnet-4-20250514"),
+    ("openai-codex", "gpt-5.2-codex"),
+])
+@pytest.mark.asyncio
+async def test_llm_provider_consolidation(memory_no_llm_verify, request_context, provider: str, model: str):
+    """
+    Test LLM provider with consolidation (automatic mental model generation from observations).
+    This validates that the provider can generate synthesized knowledge from raw memories.
+
+    This test is limited to claude-code and codex since they're the critical providers
+    that needed tool calling fixes for reflect and consolidation operations.
+    """
+    should_skip, reason = should_skip_provider(provider, model)
+    if should_skip:
+        pytest.skip(f"Skipping {provider}/{model}: {reason}")
+
+    # Use provider-specific LLM for this test
+    api_key = get_api_key_for_provider(provider)
+    memory_no_llm_verify._consolidation_llm = LLMProvider(
+        provider=provider,
+        api_key=api_key or "",
+        base_url="",
+        model=model,
+    )
+    # Also need retain LLM for ingesting data
+    memory_no_llm_verify._retain_llm = memory_no_llm_verify._consolidation_llm
+
+    test_bank_id = f"llm_test_consolidation_{provider}_{model}_{datetime.now().timestamp()}"
+
+    # Enable observations for this bank
+    from hindsight_api.config import get_config
+    config = get_config()
+    original_value = config.enable_observations
+    config.enable_observations = True
+
+    try:
+        # Retain memories to consolidate
+        test_content = """
+        Bob prefers functional programming with Rust and Haskell.
+        He emphasizes immutability and pure functions in code reviews.
+        Bob advocates for type safety and compile-time guarantees.
+        He avoids mutable state and prefers declarative code patterns.
+        """
+
+        await memory_no_llm_verify.retain_async(
+            bank_id=test_bank_id,
+            content=test_content,
+            context="Team coding preferences",
+            event_date=datetime(2024, 12, 1),
+            request_context=request_context,
+        )
+
+        print(f"\n{provider}/{model} - Consolidation test:")
+
+        # Run consolidation to generate observations (mental models)
+        from hindsight_api.engine.consolidation.consolidator import run_consolidation_job
+
+        result = await run_consolidation_job(
+            memory_engine=memory_no_llm_verify,
+            bank_id=test_bank_id,
+            request_context=request_context,
+        )
+
+        print(f"  Processed: {result.get('memories_processed', 0)} memories")
+        print(f"  Created: {result.get('observations_created', 0)} observations")
+        print(f"  Updated: {result.get('observations_updated', 0)} observations")
+
+        # Verify consolidation ran successfully
+        assert result["status"] in ["success", "no_new_memories"], f"{provider}/{model} consolidation failed"
+
+        # If observations were created, verify they contain relevant content
+        if result.get("observations_created", 0) > 0:
+            observations = await memory_no_llm_verify.list_mental_models_consolidated(
+                bank_id=test_bank_id,
+                request_context=request_context,
+            )
+
+            assert len(observations) > 0, f"{provider}/{model} consolidation created 0 observations"
+
+            # Check first observation contains relevant information
+            obs_content = observations[0].get("content", "").lower()
+            relevant_terms = ["bob", "functional", "rust", "immutab", "type"]
+            matches = [term for term in relevant_terms if term in obs_content]
+
+            print(f"  Observation preview: {observations[0].get('content', '')[:200]}...")
+            print(f"  Found {len(matches)} relevant terms: {matches}")
+
+            assert len(matches) >= 2, (
+                f"{provider}/{model} consolidated observation doesn't contain relevant info. "
+                f"Expected at least 2 of {relevant_terms}, found {len(matches)}: {matches}"
+            )
+
+    finally:
+        # Restore original config
+        config.enable_observations = original_value
+
+
+# NOTE: The tests above validate the critical Hindsight operations:
+#
+# test_llm_provider_memory_operations (ALL providers):
+#   - Fact extraction (retain): tests structured output generation
+#   - Reflect: tests memory retrieval and reasoning (uses tool calling for claude-code/codex)
+#
+# test_llm_provider_consolidation (claude-code and codex only):
+#   - Consolidation: tests automatic mental model generation from observations
+#   - Requires MemoryEngine fixture with working LLM (from .env or env vars)
+#   - Run your local LLM server OR set HINDSIGHT_API_LLM_PROVIDER/API_KEY/MODEL env vars
+#
+# For full end-to-end integration tests using the HTTP API, see tests/test_http_api_integration.py