fix: improve tei client parameters

.gitignore

@@ -27,6 +27,9 @@ docker-compose.override.yml
 # NLTK data (will be downloaded automatically)
 nltk_data/
 
+# Monitoring stack (Prometheus/Grafana binaries and data)
+.monitoring/
+
 # Large benchmark datasets (will be downloaded automatically)
 **/longmemeval_s_cleaned.json
 

hindsight-api/hindsight_api/config.py

@@ -49,6 +49,8 @@ ENV_RERANKER_COHERE_MODEL = "HINDSIGHT_API_RERANKER_COHERE_MODEL"
 ENV_RERANKER_PROVIDER = "HINDSIGHT_API_RERANKER_PROVIDER"
 ENV_RERANKER_LOCAL_MODEL = "HINDSIGHT_API_RERANKER_LOCAL_MODEL"
 ENV_RERANKER_TEI_URL = "HINDSIGHT_API_RERANKER_TEI_URL"
+ENV_RERANKER_TEI_BATCH_SIZE = "HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE"
+ENV_RERANKER_TEI_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT"
 
 ENV_HOST = "HINDSIGHT_API_HOST"
 ENV_PORT = "HINDSIGHT_API_PORT"
@@ -98,6 +100,8 @@ DEFAULT_EMBEDDING_DIMENSION = 384
 
 DEFAULT_RERANKER_PROVIDER = "local"
 DEFAULT_RERANKER_LOCAL_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"
+DEFAULT_RERANKER_TEI_BATCH_SIZE = 128
+DEFAULT_RERANKER_TEI_MAX_CONCURRENT = 8
 
 DEFAULT_EMBEDDINGS_COHERE_MODEL = "embed-english-v3.0"
 DEFAULT_RERANKER_COHERE_MODEL = "rerank-english-v3.0"
@@ -190,6 +194,8 @@ class HindsightConfig:
     reranker_provider: str
     reranker_local_model: str
     reranker_tei_url: str | None
+    reranker_tei_batch_size: int
+    reranker_tei_max_concurrent: int
 
     # Server
     host: str
@@ -256,6 +262,10 @@ class HindsightConfig:
             reranker_provider=os.getenv(ENV_RERANKER_PROVIDER, DEFAULT_RERANKER_PROVIDER),
             reranker_local_model=os.getenv(ENV_RERANKER_LOCAL_MODEL, DEFAULT_RERANKER_LOCAL_MODEL),
             reranker_tei_url=os.getenv(ENV_RERANKER_TEI_URL),
+            reranker_tei_batch_size=int(os.getenv(ENV_RERANKER_TEI_BATCH_SIZE, str(DEFAULT_RERANKER_TEI_BATCH_SIZE))),
+            reranker_tei_max_concurrent=int(
+                os.getenv(ENV_RERANKER_TEI_MAX_CONCURRENT, str(DEFAULT_RERANKER_TEI_MAX_CONCURRENT))
+            ),
             # Server
             host=os.getenv(ENV_HOST, DEFAULT_HOST),
             port=int(os.getenv(ENV_PORT, DEFAULT_PORT)),

hindsight-api/hindsight_api/engine/cross_encoder.py

@@ -6,6 +6,7 @@ Provides an interface for reranking with different backends.
 Configuration via environment variables - see hindsight_api.config for all env var names.
 """
 
+import asyncio
 import logging
 import os
 from abc import ABC, abstractmethod
@@ -16,10 +17,14 @@ from ..config import (
     DEFAULT_RERANKER_COHERE_MODEL,
     DEFAULT_RERANKER_LOCAL_MODEL,
     DEFAULT_RERANKER_PROVIDER,
+    DEFAULT_RERANKER_TEI_BATCH_SIZE,
+    DEFAULT_RERANKER_TEI_MAX_CONCURRENT,
     ENV_COHERE_API_KEY,
     ENV_RERANKER_COHERE_MODEL,
     ENV_RERANKER_LOCAL_MODEL,
     ENV_RERANKER_PROVIDER,
+    ENV_RERANKER_TEI_BATCH_SIZE,
+    ENV_RERANKER_TEI_MAX_CONCURRENT,
     ENV_RERANKER_TEI_URL,
 )
 
@@ -50,7 +55,7 @@ class CrossEncoderModel(ABC):
         pass
 
     @abstractmethod
-    def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
         """
         Score query-document pairs for relevance.
 
@@ -107,7 +112,7 @@ class LocalSTCrossEncoder(CrossEncoderModel):
         self._model = CrossEncoder(self.model_name)
         logger.info("Reranker: local provider initialized")
 
-    def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
         """
         Score query-document pairs for relevance.
 
@@ -119,7 +124,10 @@ class LocalSTCrossEncoder(CrossEncoderModel):
         """
         if self._model is None:
             raise RuntimeError("Reranker not initialized. Call initialize() first.")
-        scores = self._model.predict(pairs, show_progress_bar=False)
+
+        # Run CPU-bound inference in thread pool to avoid blocking event loop
+        loop = asyncio.get_event_loop()
+        scores = await loop.run_in_executor(None, lambda: self._model.predict(pairs, show_progress_bar=False))
         return scores.tolist() if hasattr(scores, "tolist") else list(scores)
 
 
@@ -131,13 +139,16 @@ class RemoteTEICrossEncoder(CrossEncoderModel):
     See: https://github.com/huggingface/text-embeddings-inference
 
     Note: The TEI server must be running a cross-encoder/reranker model.
+
+    Requests are made in parallel with configurable batch size and max concurrency (backpressure).
     """
 
     def __init__(
         self,
         base_url: str,
         timeout: float = 30.0,
-        batch_size: int = 32,
+        batch_size: int = DEFAULT_RERANKER_TEI_BATCH_SIZE,
+        max_concurrent: int = DEFAULT_RERANKER_TEI_MAX_CONCURRENT,
         max_retries: int = 3,
         retry_delay: float = 0.5,
     ):
@@ -147,138 +158,178 @@ class RemoteTEICrossEncoder(CrossEncoderModel):
         Args:
             base_url: Base URL of the TEI server (e.g., "http://localhost:8080")
             timeout: Request timeout in seconds (default: 30.0)
-            batch_size: Maximum batch size for rerank requests (default: 32)
+            batch_size: Maximum batch size for rerank requests (default: 128)
+            max_concurrent: Maximum concurrent requests for backpressure (default: 8)
             max_retries: Maximum number of retries for failed requests (default: 3)
             retry_delay: Initial delay between retries in seconds, doubles each retry (default: 0.5)
         """
         self.base_url = base_url.rstrip("/")
         self.timeout = timeout
         self.batch_size = batch_size
+        self.max_concurrent = max_concurrent
         self.max_retries = max_retries
         self.retry_delay = retry_delay
-        self._client: httpx.Client | None = None
+        self._async_client: httpx.AsyncClient | None = None
         self._model_id: str | None = None
 
     @property
     def provider_name(self) -> str:
         return "tei"
 
-    def _request_with_retry(self, method: str, url: str, **kwargs) -> httpx.Response:
-        """Make an HTTP request with automatic retries on transient errors."""
-        import time
-
+    async def _async_request_with_retry(
+        self,
+        client: httpx.AsyncClient,
+        semaphore: asyncio.Semaphore,
+        method: str,
+        url: str,
+        **kwargs,
+    ) -> httpx.Response:
+        """Make an async HTTP request with automatic retries on transient errors and semaphore for backpressure."""
         last_error = None
         delay = self.retry_delay
 
-        for attempt in range(self.max_retries + 1):
-            try:
-                if method == "GET":
-                    response = self._client.get(url, **kwargs)
-                else:
-                    response = self._client.post(url, **kwargs)
-                response.raise_for_status()
-                return response
-            except (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout) as e:
-                last_error = e
-                if attempt < self.max_retries:
-                    logger.warning(
-                        f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. Retrying in {delay}s..."
-                    )
-                    time.sleep(delay)
-                    delay *= 2  # Exponential backoff
-            except httpx.HTTPStatusError as e:
-                # Retry on 5xx server errors
-                if e.response.status_code >= 500 and attempt < self.max_retries:
+        async with semaphore:
+            for attempt in range(self.max_retries + 1):
+                try:
+                    if method == "GET":
+                        response = await client.get(url, **kwargs)
+                    else:
+                        response = await client.post(url, **kwargs)
+                    response.raise_for_status()
+                    return response
+                except (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout) as e:
                     last_error = e
-                    logger.warning(
-                        f"TEI server error (attempt {attempt + 1}/{self.max_retries + 1}): {e}. Retrying in {delay}s..."
-                    )
-                    time.sleep(delay)
-                    delay *= 2
-                else:
-                    raise
+                    if attempt < self.max_retries:
+                        logger.warning(
+                            f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. "
+                            f"Retrying in {delay}s..."
+                        )
+                        await asyncio.sleep(delay)
+                        delay *= 2  # Exponential backoff
+                except httpx.HTTPStatusError as e:
+                    # Retry on 5xx server errors
+                    if e.response.status_code >= 500 and attempt < self.max_retries:
+                        last_error = e
+                        logger.warning(
+                            f"TEI server error (attempt {attempt + 1}/{self.max_retries + 1}): {e}. "
+                            f"Retrying in {delay}s..."
+                        )
+                        await asyncio.sleep(delay)
+                        delay *= 2
+                    else:
+                        raise
 
         raise last_error
 
     async def initialize(self) -> None:
         """Initialize the HTTP client and verify server connectivity."""
-        if self._client is not None:
+        if self._async_client is not None:
             return
 
-        logger.info(f"Reranker: initializing TEI provider at {self.base_url}")
-        self._client = httpx.Client(timeout=self.timeout)
+        logger.info(
+            f"Reranker: initializing TEI provider at {self.base_url} "
+            f"(batch_size={self.batch_size}, max_concurrent={self.max_concurrent})"
+        )
+        self._async_client = httpx.AsyncClient(timeout=self.timeout)
 
         # Verify server is reachable and get model info
+        # Use a temporary semaphore for initialization
+        init_semaphore = asyncio.Semaphore(1)
         try:
-            response = self._request_with_retry("GET", f"{self.base_url}/info")
+            response = await self._async_request_with_retry(
+                self._async_client, init_semaphore, "GET", f"{self.base_url}/info"
+            )
             info = response.json()
             self._model_id = info.get("model_id", "unknown")
             logger.info(f"Reranker: TEI provider initialized (model: {self._model_id})")
         except httpx.HTTPError as e:
+            self._async_client = None
             raise RuntimeError(f"Failed to connect to TEI server at {self.base_url}: {e}")
 
-    def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+    async def _rerank_query_group(
+        self,
+        client: httpx.AsyncClient,
+        semaphore: asyncio.Semaphore,
+        query: str,
+        texts: list[str],
+    ) -> list[tuple[int, float]]:
+        """Rerank a single query group and return list of (original_index, score) tuples."""
+        try:
+            response = await self._async_request_with_retry(
+                client,
+                semaphore,
+                "POST",
+                f"{self.base_url}/rerank",
+                json={
+                    "query": query,
+                    "texts": texts,
+                    "return_text": False,
+                },
+            )
+            results = response.json()
+            # TEI returns results sorted by score descending, with original index
+            return [(result["index"], result["score"]) for result in results]
+        except httpx.HTTPError as e:
+            raise RuntimeError(f"TEI rerank request failed: {e}")
+
+    async def _predict_async(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """Async implementation of predict that runs requests in parallel with backpressure."""
+        if not pairs:
+            return []
+
+        # Group all pairs by query
+        query_groups: dict[str, list[tuple[int, str]]] = {}
+        for idx, (query, text) in enumerate(pairs):
+            if query not in query_groups:
+                query_groups[query] = []
+            query_groups[query].append((idx, text))
+
+        # Split each query group into batches
+        tasks_info: list[tuple[str, list[int], list[str]]] = []  # (query, indices, texts)
+        for query, indexed_texts in query_groups.items():
+            indices = [idx for idx, _ in indexed_texts]
+            texts = [text for _, text in indexed_texts]
+
+            # Split into batches
+            for i in range(0, len(texts), self.batch_size):
+                batch_indices = indices[i : i + self.batch_size]
+                batch_texts = texts[i : i + self.batch_size]
+                tasks_info.append((query, batch_indices, batch_texts))
+
+        # Run all requests in parallel with semaphore for backpressure
+        all_scores = [0.0] * len(pairs)
+        semaphore = asyncio.Semaphore(self.max_concurrent)
+
+        tasks = [
+            self._rerank_query_group(self._async_client, semaphore, query, texts)
+            for query, _, texts in tasks_info
+        ]
+        results = await asyncio.gather(*tasks)
+
+        # Map scores back to original positions
+        for (_, indices, _), result_scores in zip(tasks_info, results):
+            for original_idx_in_batch, score in result_scores:
+                global_idx = indices[original_idx_in_batch]
+                all_scores[global_idx] = score
+
+        return all_scores
+
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
         """
         Score query-document pairs using the remote TEI reranker.
 
+        Requests are made in parallel with configurable backpressure.
+
         Args:
             pairs: List of (query, document) tuples to score
 
         Returns:
             List of relevance scores
         """
-        if self._client is None:
+        if self._async_client is None:
             raise RuntimeError("Reranker not initialized. Call initialize() first.")
 
-        if not pairs:
-            return []
-
-        all_scores = []
-
-        # Process in batches
-        for i in range(0, len(pairs), self.batch_size):
-            batch = pairs[i : i + self.batch_size]
-
-            # TEI rerank endpoint expects query and texts separately
-            # All pairs in a batch should have the same query for optimal performance
-            # but we handle mixed queries by making separate requests per unique query
-            query_groups: dict[str, list[tuple[int, str]]] = {}
-            for idx, (query, text) in enumerate(batch):
-                if query not in query_groups:
-                    query_groups[query] = []
-                query_groups[query].append((idx, text))
-
-            batch_scores = [0.0] * len(batch)
-
-            for query, indexed_texts in query_groups.items():
-                texts = [text for _, text in indexed_texts]
-                indices = [idx for idx, _ in indexed_texts]
-
-                try:
-                    response = self._request_with_retry(
-                        "POST",
-                        f"{self.base_url}/rerank",
-                        json={
-                            "query": query,
-                            "texts": texts,
-                            "return_text": False,
-                        },
-                    )
-                    results = response.json()
-
-                    # TEI returns results sorted by score descending, with original index
-                    for result in results:
-                        original_idx = result["index"]
-                        score = result["score"]
-                        # Map back to batch position
-                        batch_scores[indices[original_idx]] = score
-
-                except httpx.HTTPError as e:
-                    raise RuntimeError(f"TEI rerank request failed: {e}")
-
-            all_scores.extend(batch_scores)
-
-        return all_scores
+        return await self._predict_async(pairs)
 
 
 class CohereCrossEncoder(CrossEncoderModel):
@@ -325,7 +376,7 @@ class CohereCrossEncoder(CrossEncoderModel):
         self._client = cohere.Client(api_key=self.api_key, timeout=self.timeout)
         logger.info("Reranker: Cohere provider initialized")
 
-    def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
+    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
         """
         Score query-document pairs using the Cohere Rerank API.
 
@@ -341,6 +392,12 @@ class CohereCrossEncoder(CrossEncoderModel):
         if not pairs:
             return []
 
+        # Run sync Cohere API calls in thread pool
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, self._predict_sync, pairs)
+
+    def _predict_sync(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """Synchronous predict implementation for Cohere API."""
         # Group pairs by query for efficient batching
         # Cohere rerank expects one query with multiple documents
         query_groups: dict[str, list[tuple[int, str]]] = {}
@@ -386,7 +443,9 @@ def create_cross_encoder_from_env() -> CrossEncoderModel:
         url = os.environ.get(ENV_RERANKER_TEI_URL)
         if not url:
             raise ValueError(f"{ENV_RERANKER_TEI_URL} is required when {ENV_RERANKER_PROVIDER} is 'tei'")
-        return RemoteTEICrossEncoder(base_url=url)
+        batch_size = int(os.environ.get(ENV_RERANKER_TEI_BATCH_SIZE, str(DEFAULT_RERANKER_TEI_BATCH_SIZE)))
+        max_concurrent = int(os.environ.get(ENV_RERANKER_TEI_MAX_CONCURRENT, str(DEFAULT_RERANKER_TEI_MAX_CONCURRENT)))
+        return RemoteTEICrossEncoder(base_url=url, batch_size=batch_size, max_concurrent=max_concurrent)
     elif provider == "local":
         model = os.environ.get(ENV_RERANKER_LOCAL_MODEL)
         model_name = model or DEFAULT_RERANKER_LOCAL_MODEL

hindsight-api/hindsight_api/engine/memory_engine.py

@@ -1778,7 +1778,7 @@ class MemoryEngine(MemoryEngineInterface):
             await reranker_instance.ensure_initialized()
 
             # Rerank using cross-encoder
-            scored_results = reranker_instance.rerank(query, merged_candidates)
+            scored_results = await reranker_instance.rerank(query, merged_candidates)
 
             step_duration = time.time() - step_start
             log_buffer.append(f"  [4] Reranking: {len(scored_results)} candidates scored in {step_duration:.3f}s")

hindsight-api/hindsight_api/engine/search/mpfp_retrieval.py

@@ -288,7 +288,6 @@ class MPFPGraphRetriever(GraphRetriever):
             config: Algorithm configuration (uses defaults if None)
         """
         self.config = config or MPFPConfig()
-        self._adjacency_cache: dict[str, TypedAdjacency] = {}
 
     @property
     def name(self) -> str:

hindsight-api/hindsight_api/engine/search/reranking.py

@@ -44,7 +44,7 @@ class CrossEncoderReranker:
             await cross_encoder.initialize()
         self._initialized = True
 
-    def rerank(self, query: str, candidates: list[MergedCandidate]) -> list[ScoredResult]:
+    async def rerank(self, query: str, candidates: list[MergedCandidate]) -> list[ScoredResult]:
         """
         Rerank candidates using cross-encoder scores.
 
@@ -85,7 +85,7 @@ class CrossEncoderReranker:
             pairs.append([query, doc_text])
 
         # Get cross-encoder scores
-        scores = self.cross_encoder.predict(pairs)
+        scores = await self.cross_encoder.predict(pairs)
 
         # Normalize scores using sigmoid to [0, 1] range
         # Cross-encoder returns logits which can be negative

hindsight-api/hindsight_api/main.py

@@ -185,6 +185,8 @@ def main():
             reranker_provider=config.reranker_provider,
             reranker_local_model=config.reranker_local_model,
             reranker_tei_url=config.reranker_tei_url,
+            reranker_tei_batch_size=config.reranker_tei_batch_size,
+            reranker_tei_max_concurrent=config.reranker_tei_max_concurrent,
             host=args.host,
             port=args.port,
             log_level=args.log_level,

hindsight-api/tests/test_custom_embedding_dimension.py

@@ -514,14 +514,15 @@ class TestCohereCrossEncoder:
         """Test that Cohere cross-encoder initializes correctly."""
         assert cohere_cross_encoder.provider_name == "cohere"
 
-    def test_cohere_cross_encoder_predict(self, cohere_cross_encoder):
+    @pytest.mark.asyncio
+    async def test_cohere_cross_encoder_predict(self, cohere_cross_encoder):
         """Test that Cohere cross-encoder can score pairs."""
         pairs = [
             ("What is the capital of France?", "Paris is the capital of France."),
             ("What is the capital of France?", "The Eiffel Tower is in Paris."),
             ("What is the capital of France?", "Python is a programming language."),
         ]
-        scores = cohere_cross_encoder.predict(pairs)
+        scores = await cohere_cross_encoder.predict(pairs)
 
         assert len(scores) == 3
         assert all(isinstance(s, float) for s in scores)

hindsight-api/tests/test_tei_cross_encoder.py

@@ -0,0 +1,546 @@
+"""
+Tests for RemoteTEICrossEncoder (TEI reranker client).
+
+Tests cover:
+- Initialization and server connectivity
+- Basic predict functionality
+- Batch splitting
+- Parallel request handling
+- Backpressure/semaphore behavior
+- Retry logic on transient errors
+- Multiple queries handling
+"""
+
+import asyncio
+import time
+from unittest.mock import MagicMock, patch
+
+import httpx
+import pytest
+
+from hindsight_api.engine.cross_encoder import RemoteTEICrossEncoder
+
+
+class TestRemoteTEICrossEncoderInitialization:
+    """Tests for TEI cross-encoder initialization."""
+
+    @pytest.mark.asyncio
+    async def test_initialize_success(self):
+        """Test successful initialization with valid TEI server."""
+
+        async def mock_handler(request: httpx.Request) -> httpx.Response:
+            if request.url.path == "/info":
+                return httpx.Response(
+                    200,
+                    json={"model_id": "BAAI/bge-reranker-base", "version": "1.0"},
+                )
+            return httpx.Response(404)
+
+        transport = httpx.MockTransport(mock_handler)
+
+        with patch.object(httpx, "AsyncClient", return_value=httpx.AsyncClient(transport=transport)):
+            encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080")
+            await encoder.initialize()
+
+            assert encoder._model_id == "BAAI/bge-reranker-base"
+            assert encoder._async_client is not None
+
+    @pytest.mark.asyncio
+    async def test_initialize_server_unreachable(self):
+        """Test initialization fails when server is unreachable."""
+
+        async def mock_handler(request: httpx.Request) -> httpx.Response:
+            raise httpx.ConnectError("Connection refused")
+
+        transport = httpx.MockTransport(mock_handler)
+
+        with patch.object(httpx, "AsyncClient", return_value=httpx.AsyncClient(transport=transport)):
+            encoder = RemoteTEICrossEncoder(
+                base_url="http://localhost:8080",
+                max_retries=1,
+                retry_delay=0.01,
+            )
+
+            with pytest.raises(RuntimeError, match="Failed to connect to TEI server"):
+                await encoder.initialize()
+
+    @pytest.mark.asyncio
+    async def test_initialize_idempotent(self):
+        """Test that initialize() is idempotent."""
+        call_count = 0
+
+        async def mock_handler(request: httpx.Request) -> httpx.Response:
+            nonlocal call_count
+            if request.url.path == "/info":
+                call_count += 1
+                return httpx.Response(200, json={"model_id": "test-model"})
+            return httpx.Response(404)
+
+        transport = httpx.MockTransport(mock_handler)
+
+        with patch.object(httpx, "AsyncClient", return_value=httpx.AsyncClient(transport=transport)):
+            encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080")
+            await encoder.initialize()
+            await encoder.initialize()
+            await encoder.initialize()
+
+            assert call_count == 1
+
+
+def create_mock_async_client(handler):
+    """Create a mock AsyncClient that uses the given handler for requests."""
+
+    class MockAsyncClient:
+        def __init__(self, **kwargs):
+            self.timeout = kwargs.get("timeout", 30.0)
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *args):
+            pass
+
+        async def post(self, url, **kwargs):
+            return await handler("POST", url, **kwargs)
+
+        async def get(self, url, **kwargs):
+            return await handler("GET", url, **kwargs)
+
+    return MockAsyncClient()
+
+
+class TestRemoteTEICrossEncoderPredict:
+    """Tests for TEI cross-encoder predict functionality."""
+
+    @pytest.mark.asyncio
+    async def test_predict_not_initialized(self):
+        """Test predict raises error when not initialized."""
+        encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080")
+
+        with pytest.raises(RuntimeError, match="Reranker not initialized"):
+            await encoder.predict([("query", "doc")])
+
+    @pytest.mark.asyncio
+    async def test_predict_empty_pairs(self):
+        """Test predict returns empty list for empty input."""
+        encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080")
+        encoder._async_client = httpx.AsyncClient()
+        encoder._model_id = "test-model"
+
+        result = await encoder.predict([])
+        assert result == []
+
+    @pytest.mark.asyncio
+    async def test_predict_single_query(self):
+        """Test predict with single query and multiple documents."""
+        rerank_calls = []
+
+        async def mock_handler(method, url, **kwargs):
+            if "/rerank" in url:
+                body = kwargs.get("json", {})
+                rerank_calls.append(body)
+                texts = body["texts"]
+                # Return scores in descending order with original indices
+                results = [{"index": i, "score": 1.0 - (i * 0.1)} for i in range(len(texts))]
+                response = MagicMock()
+                response.status_code = 200
+                response.raise_for_status = MagicMock()
+                response.json = MagicMock(return_value=results)
+                return response
+            raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock())
+
+        encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080")
+        encoder._async_client = create_mock_async_client(mock_handler)
+        encoder._model_id = "test-model"
+
+        pairs = [
+            ("What is Python?", "Python is a programming language."),
+            ("What is Python?", "Python is a snake."),
+            ("What is Python?", "Java is also a language."),
+        ]
+
+        scores = await encoder.predict(pairs)
+
+        assert len(scores) == 3
+        assert len(rerank_calls) == 1
+        assert rerank_calls[0]["query"] == "What is Python?"
+        assert len(rerank_calls[0]["texts"]) == 3
+        # Scores should be mapped back correctly
+        assert scores[0] == 1.0
+        assert scores[1] == 0.9
+        assert scores[2] == pytest.approx(0.8, rel=0.01)
+
+    @pytest.mark.asyncio
+    async def test_predict_multiple_queries(self):
+        """Test predict with multiple different queries."""
+        rerank_calls = []
+
+        async def mock_handler(method, url, **kwargs):
+            if "/rerank" in url:
+                body = kwargs.get("json", {})
+                rerank_calls.append(body)
+                texts = body["texts"]
+                results = [{"index": i, "score": 0.5 + (i * 0.1)} for i in range(len(texts))]
+                response = MagicMock()
+                response.status_code = 200
+                response.raise_for_status = MagicMock()
+                response.json = MagicMock(return_value=results)
+                return response
+            raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock())
+
+        encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080")
+        encoder._async_client = create_mock_async_client(mock_handler)
+        encoder._model_id = "test-model"
+
+        pairs = [
+            ("Query A", "Doc A1"),
+            ("Query B", "Doc B1"),
+            ("Query A", "Doc A2"),
+            ("Query B", "Doc B2"),
+        ]
+
+        scores = await encoder.predict(pairs)
+
+        assert len(scores) == 4
+        # Two queries = two rerank calls (run in parallel)
+        assert len(rerank_calls) == 2
+
+
+class TestRemoteTEICrossEncoderBatching:
+    """Tests for batch splitting behavior."""
+
+    @pytest.mark.asyncio
+    async def test_batch_splitting(self):
+        """Test that large inputs are split into batches."""
+        rerank_calls = []
+
+        async def mock_handler(method, url, **kwargs):
+            if "/rerank" in url:
+                body = kwargs.get("json", {})
+                rerank_calls.append(body)
+                texts = body["texts"]
+                results = [{"index": i, "score": 0.5} for i in range(len(texts))]
+                response = MagicMock()
+                response.status_code = 200
+                response.raise_for_status = MagicMock()
+                response.json = MagicMock(return_value=results)
+                return response
+            raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock())
+
+        encoder = RemoteTEICrossEncoder(
+            base_url="http://localhost:8080",
+            batch_size=3,  # Small batch for testing
+        )
+        encoder._async_client = create_mock_async_client(mock_handler)
+        encoder._model_id = "test-model"
+
+        # 7 documents with same query, batch_size=3 -> 3 batches (3+3+1)
+        pairs = [("Query", f"Doc {i}") for i in range(7)]
+
+        scores = await encoder.predict(pairs)
+
+        assert len(scores) == 7
+        assert len(rerank_calls) == 3
+        # Check batch sizes
+        batch_sizes = sorted([len(call["texts"]) for call in rerank_calls])
+        assert batch_sizes == [1, 3, 3]
+
+    @pytest.mark.asyncio
+    async def test_score_mapping_across_batches(self):
+        """Test that scores are correctly mapped back across batches."""
+        call_counter = [0]
+
+        async def mock_handler(method, url, **kwargs):
+            if "/rerank" in url:
+                body = kwargs.get("json", {})
+                batch_num = call_counter[0]
+                call_counter[0] += 1
+                texts = body["texts"]
+                # Each batch returns different scores to verify mapping
+                base_score = batch_num * 10
+                results = [{"index": i, "score": float(base_score + i)} for i in range(len(texts))]
+                response = MagicMock()
+                response.status_code = 200
+                response.raise_for_status = MagicMock()
+                response.json = MagicMock(return_value=results)
+                return response
+            raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock())
+
+        encoder = RemoteTEICrossEncoder(
+            base_url="http://localhost:8080",
+            batch_size=3,
+        )
+        encoder._async_client = create_mock_async_client(mock_handler)
+        encoder._model_id = "test-model"
+
+        pairs = [("Query", f"Doc {i}") for i in range(7)]
+
+        scores = await encoder.predict(pairs)
+
+        assert len(scores) == 7
+        # All scores should be present (exact values depend on batch ordering)
+        assert all(isinstance(s, (int, float)) for s in scores)
+
+
+class TestRemoteTEICrossEncoderParallelism:
+    """Tests for parallel request handling and backpressure."""
+
+    @pytest.mark.asyncio
+    async def test_parallel_requests(self):
+        """Test that requests are made in parallel."""
+        concurrent_count = [0]
+        max_concurrent_observed = [0]
+
+        async def mock_handler(method, url, **kwargs):
+            if "/rerank" in url:
+                concurrent_count[0] += 1
+                max_concurrent_observed[0] = max(max_concurrent_observed[0], concurrent_count[0])
+
+                await asyncio.sleep(0.03)  # Simulate latency
+
+                concurrent_count[0] -= 1
+                body = kwargs.get("json", {})
+                texts = body["texts"]
+                results = [{"index": i, "score": 0.5} for i in range(len(texts))]
+                response = MagicMock()
+                response.status_code = 200
+                response.raise_for_status = MagicMock()
+                response.json = MagicMock(return_value=results)
+                return response
+            raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock())
+
+        encoder = RemoteTEICrossEncoder(
+            base_url="http://localhost:8080",
+            batch_size=2,
+            max_concurrent=10,  # High limit to allow parallelism
+        )
+        encoder._async_client = create_mock_async_client(mock_handler)
+        encoder._model_id = "test-model"
+
+        # 6 docs = 3 batches, should run in parallel
+        pairs = [("Query", f"Doc {i}") for i in range(6)]
+
+        start = time.time()
+        scores = await encoder.predict(pairs)
+        elapsed = time.time() - start
+
+        assert len(scores) == 6
+        # If parallel, 3 batches with 30ms each should take ~30ms, not 90ms
+        assert elapsed < 0.08, f"Requests should run in parallel, took {elapsed}s"
+        assert max_concurrent_observed[0] > 1, "Multiple requests should run concurrently"
+
+    @pytest.mark.asyncio
+    async def test_backpressure_semaphore(self):
+        """Test that semaphore limits concurrent requests."""
+        concurrent_count = [0]
+        max_concurrent_observed = [0]
+
+        async def mock_handler(method, url, **kwargs):
+            if "/rerank" in url:
+                concurrent_count[0] += 1
+                max_concurrent_observed[0] = max(max_concurrent_observed[0], concurrent_count[0])
+
+                await asyncio.sleep(0.01)  # Simulate latency
+
+                concurrent_count[0] -= 1
+                body = kwargs.get("json", {})
+                texts = body["texts"]
+                results = [{"index": i, "score": 0.5} for i in range(len(texts))]
+                response = MagicMock()
+                response.status_code = 200
+                response.raise_for_status = MagicMock()
+                response.json = MagicMock(return_value=results)
+                return response
+            raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock())
+
+        max_concurrent_limit = 2
+        encoder = RemoteTEICrossEncoder(
+            base_url="http://localhost:8080",
+            batch_size=1,  # 1 doc per batch to maximize requests
+            max_concurrent=max_concurrent_limit,
+        )
+        encoder._async_client = create_mock_async_client(mock_handler)
+        encoder._model_id = "test-model"
+
+        # 10 docs = 10 batches, but only 2 should run at a time
+        pairs = [("Query", f"Doc {i}") for i in range(10)]
+
+        scores = await encoder.predict(pairs)
+
+        assert len(scores) == 10
+        assert max_concurrent_observed[0] <= max_concurrent_limit, (
+            f"Semaphore should limit to {max_concurrent_limit}, observed {max_concurrent_observed[0]}"
+        )
+
+
+class TestRemoteTEICrossEncoderRetry:
+    """Tests for retry logic on transient errors."""
+
+    @pytest.mark.asyncio
+    async def test_retry_on_connect_error(self):
+        """Test that connect errors trigger retries."""
+        attempt_count = [0]
+
+        async def mock_handler(method, url, **kwargs):
+            if "/rerank" in url:
+                attempt_count[0] += 1
+                if attempt_count[0] < 3:
+                    raise httpx.ConnectError("Connection refused")
+                body = kwargs.get("json", {})
+                texts = body["texts"]
+                results = [{"index": i, "score": 0.5} for i in range(len(texts))]
+                response = MagicMock()
+                response.status_code = 200
+                response.raise_for_status = MagicMock()
+                response.json = MagicMock(return_value=results)
+                return response
+            raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock())
+
+        encoder = RemoteTEICrossEncoder(
+            base_url="http://localhost:8080",
+            max_retries=3,
+            retry_delay=0.01,
+        )
+        encoder._async_client = create_mock_async_client(mock_handler)
+        encoder._model_id = "test-model"
+
+        pairs = [("Query", "Doc 1")]
+        scores = await encoder.predict(pairs)
+
+        assert len(scores) == 1
+        assert attempt_count[0] == 3  # 2 failures + 1 success
+
+    @pytest.mark.asyncio
+    async def test_retry_on_server_error(self):
+        """Test that 5xx errors trigger retries."""
+        attempt_count = [0]
+
+        async def mock_handler(method, url, **kwargs):
+            if "/rerank" in url:
+                attempt_count[0] += 1
+                if attempt_count[0] < 2:
+                    response = MagicMock()
+                    response.status_code = 503
+
+                    def raise_for_status():
+                        raise httpx.HTTPStatusError(
+                            "Service unavailable",
+                            request=MagicMock(),
+                            response=response,
+                        )
+
+                    response.raise_for_status = raise_for_status
+                    return response
+
+                body = kwargs.get("json", {})
+                texts = body["texts"]
+                results = [{"index": i, "score": 0.5} for i in range(len(texts))]
+                response = MagicMock()
+                response.status_code = 200
+                response.raise_for_status = MagicMock()
+                response.json = MagicMock(return_value=results)
+                return response
+            raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock())
+
+        encoder = RemoteTEICrossEncoder(
+            base_url="http://localhost:8080",
+            max_retries=3,
+            retry_delay=0.01,
+        )
+        encoder._async_client = create_mock_async_client(mock_handler)
+        encoder._model_id = "test-model"
+
+        pairs = [("Query", "Doc 1")]
+        scores = await encoder.predict(pairs)
+
+        assert len(scores) == 1
+        assert attempt_count[0] == 2
+
+    @pytest.mark.asyncio
+    async def test_no_retry_on_client_error(self):
+        """Test that 4xx errors do not trigger retries."""
+        attempt_count = [0]
+
+        async def mock_handler(method, url, **kwargs):
+            if "/rerank" in url:
+                attempt_count[0] += 1
+                response = MagicMock()
+                response.status_code = 400
+
+                def raise_for_status():
+                    raise httpx.HTTPStatusError(
+                        "Bad request",
+                        request=MagicMock(),
+                        response=response,
+                    )
+
+                response.raise_for_status = raise_for_status
+                return response
+            raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock())
+
+        encoder = RemoteTEICrossEncoder(
+            base_url="http://localhost:8080",
+            max_retries=3,
+            retry_delay=0.01,
+        )
+        encoder._async_client = create_mock_async_client(mock_handler)
+        encoder._model_id = "test-model"
+
+        pairs = [("Query", "Doc 1")]
+
+        with pytest.raises(RuntimeError, match="TEI rerank request failed"):
+            await encoder.predict(pairs)
+
+        assert attempt_count[0] == 1  # No retries for 4xx
+
+
+class TestRemoteTEICrossEncoderConfig:
+    """Tests for configuration from environment variables."""
+
+    def test_default_values(self):
+        """Test default configuration values."""
+        encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080")
+
+        assert encoder.batch_size == 128
+        assert encoder.max_concurrent == 8
+        assert encoder.timeout == 30.0
+        assert encoder.max_retries == 3
+
+    def test_custom_values(self):
+        """Test custom configuration values."""
+        encoder = RemoteTEICrossEncoder(
+            base_url="http://localhost:8080",
+            batch_size=64,
+            max_concurrent=4,
+            timeout=60.0,
+            max_retries=5,
+            retry_delay=1.0,
+        )
+
+        assert encoder.batch_size == 64
+        assert encoder.max_concurrent == 4
+        assert encoder.timeout == 60.0
+        assert encoder.max_retries == 5
+        assert encoder.retry_delay == 1.0
+
+    def test_create_from_env(self):
+        """Test creating encoder from environment variables."""
+        import os
+
+        from hindsight_api.engine.cross_encoder import create_cross_encoder_from_env
+
+        with patch.dict(
+            os.environ,
+            {
+                "HINDSIGHT_API_RERANKER_PROVIDER": "tei",
+                "HINDSIGHT_API_RERANKER_TEI_URL": "http://test:9000",
+                "HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE": "256",
+                "HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT": "16",
+            },
+        ):
+            encoder = create_cross_encoder_from_env()
+
+            assert isinstance(encoder, RemoteTEICrossEncoder)
+            assert encoder.base_url == "http://test:9000"
+            assert encoder.batch_size == 256
+            assert encoder.max_concurrent == 16

hindsight-control-plane/src/app/api/health/route.ts

@@ -1,5 +1,7 @@
 import { NextResponse } from "next/server";
-import { sdk, lowLevelClient } from "@/lib/hindsight-client";
+import { createClient, createConfig, sdk } from "@vectorize-io/hindsight-client";
+
+const HEALTH_CHECK_TIMEOUT_MS = 3000;
 
 export async function GET() {
   const status: {
@@ -15,19 +17,37 @@ export async function GET() {
     service: "hindsight-control-plane",
   };
 
-  // Check dataplane connectivity
+  // Check dataplane connectivity with a short timeout
   const dataplaneUrl = process.env.HINDSIGHT_CP_DATAPLANE_API_URL || "http://localhost:8888";
   try {
-    await sdk.listBanks({ client: lowLevelClient });
-    status.dataplane = {
-      status: "connected",
-      url: dataplaneUrl,
-    };
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), HEALTH_CHECK_TIMEOUT_MS);
+
+    const healthClient = createClient(
+      createConfig({
+        baseUrl: dataplaneUrl,
+        signal: controller.signal,
+      })
+    );
+
+    try {
+      await sdk.listBanks({ client: healthClient });
+      status.dataplane = {
+        status: "connected",
+        url: dataplaneUrl,
+      };
+    } finally {
+      clearTimeout(timeoutId);
+    }
   } catch (error) {
+    let errorMessage = error instanceof Error ? error.message : String(error);
+    if (error instanceof Error && error.name === "AbortError") {
+      errorMessage = `Request timed out after ${HEALTH_CHECK_TIMEOUT_MS}ms`;
+    }
     status.dataplane = {
       status: "disconnected",
       url: dataplaneUrl,
-      error: error instanceof Error ? error.message : String(error),
+      error: errorMessage,
     };
   }
 

scripts/dev/monitoring/grafana/dashboards/hindsight-llm.json

@@ -0,0 +1,541 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum(hindsight_llm_calls_total)",
+          "refId": "A"
+        }
+      ],
+      "title": "Total LLM Calls",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum(hindsight_llm_tokens_input_tokens_total) + sum(hindsight_llm_tokens_output_tokens_total)",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Tokens Used",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "blue",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum(hindsight_llm_tokens_input_tokens_total)",
+          "refId": "A"
+        }
+      ],
+      "title": "Input Tokens",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "green",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum(hindsight_llm_tokens_output_tokens_total)",
+          "refId": "A"
+        }
+      ],
+      "title": "Output Tokens",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum by (scope) (rate(hindsight_llm_calls_total[1m]))",
+          "legendFormat": "{{scope}}",
+          "refId": "A"
+        }
+      ],
+      "title": "LLM Calls per Second by Scope",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "Input" },
+            "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }]
+          },
+          {
+            "matcher": { "id": "byName", "options": "Output" },
+            "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }]
+          }
+        ]
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum(rate(hindsight_llm_tokens_input_tokens_total[1m]))",
+          "legendFormat": "Input",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(rate(hindsight_llm_tokens_output_tokens_total[1m]))",
+          "legendFormat": "Output",
+          "refId": "B"
+        }
+      ],
+      "title": "Token Usage Rate (tokens/sec)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum by (scope, le) (rate(hindsight_llm_duration_seconds_bucket[5m])))",
+          "legendFormat": "{{scope}}",
+          "refId": "A"
+        }
+      ],
+      "title": "LLM Call Latency p95 by Scope",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 20 },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum by (scope) (rate(hindsight_llm_tokens_input_tokens_total[1m]))",
+          "legendFormat": "{{scope}} (input)",
+          "refId": "A"
+        },
+        {
+          "expr": "sum by (scope) (rate(hindsight_llm_tokens_output_tokens_total[1m]))",
+          "legendFormat": "{{scope}} (output)",
+          "refId": "B"
+        }
+      ],
+      "title": "Token Usage by Scope (tokens/sec)",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 38,
+  "tags": ["hindsight", "llm"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Hindsight LLM Metrics",
+  "uid": "hindsight-llm",
+  "version": 1,
+  "weekStart": ""
+}

scripts/dev/monitoring/grafana/dashboards/hindsight-operations.json

@@ -0,0 +1,604 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum(hindsight_operation_operations_total)",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Operations",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum(rate(hindsight_operation_operations_total[1m]))",
+          "refId": "A"
+        }
+      ],
+      "title": "Operations/sec",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "green",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum(rate(hindsight_operation_operations_total{operation=\"retain\"}[1m]))",
+          "refId": "A"
+        }
+      ],
+      "title": "Retain/sec",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "blue",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum(rate(hindsight_operation_operations_total{operation=\"recall\"}[1m]))",
+          "refId": "A"
+        }
+      ],
+      "title": "Recall/sec",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "orange",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum(rate(hindsight_operation_operations_total{operation=\"reflect\"}[1m]))",
+          "refId": "A"
+        }
+      ],
+      "title": "Reflect/sec",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "ops"
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "retain" },
+            "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }]
+          },
+          {
+            "matcher": { "id": "byName", "options": "recall" },
+            "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }]
+          },
+          {
+            "matcher": { "id": "byName", "options": "reflect" },
+            "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }]
+          }
+        ]
+      },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum by (operation) (rate(hindsight_operation_operations_total[1m]))",
+          "legendFormat": "{{operation}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Operations per Second by Type",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "p50" },
+            "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }]
+          },
+          {
+            "matcher": { "id": "byName", "options": "p95" },
+            "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }]
+          },
+          {
+            "matcher": { "id": "byName", "options": "p99" },
+            "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }]
+          }
+        ]
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))",
+          "legendFormat": "p50",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))",
+          "legendFormat": "p95",
+          "refId": "B"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))",
+          "legendFormat": "p99",
+          "refId": "C"
+        }
+      ],
+      "title": "Recall Latency Percentiles",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum by (operation, le) (rate(hindsight_operation_duration_seconds_bucket[5m])))",
+          "legendFormat": "{{operation}}",
+          "refId": "A"
+        }
+      ],
+      "title": "p95 Latency by Operation Type",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 20 },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "expr": "sum by (bank_id) (rate(hindsight_operation_operations_total[1m]))",
+          "legendFormat": "{{bank_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Operations per Second by Bank",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 38,
+  "tags": ["hindsight"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Hindsight Operations",
+  "uid": "hindsight-operations",
+  "version": 1,
+  "weekStart": ""
+}

scripts/dev/monitoring/start.sh

@@ -0,0 +1,221 @@
+#!/bin/bash
+set -e
+
+# Script to start Prometheus and Grafana for Hindsight metrics
+# This provides a single command for the full monitoring stack
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+MONITORING_DATA_DIR="$PROJECT_ROOT/.monitoring"
+API_PORT="${API_PORT:-8888}"
+PROMETHEUS_PORT="${PROMETHEUS_PORT:-8889}"
+GRAFANA_PORT="${GRAFANA_PORT:-8890}"
+
+# Versions
+PROMETHEUS_VERSION="2.48.0"
+GRAFANA_VERSION="10.2.2"
+
+# Detect OS and architecture
+OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m)
+
+case "$OS" in
+    darwin) OS_NAME="darwin" ;;
+    linux) OS_NAME="linux" ;;
+    *) echo "Unsupported OS: $OS"; exit 1 ;;
+esac
+
+case "$ARCH" in
+    x86_64) ARCH_NAME="amd64" ;;
+    arm64|aarch64) ARCH_NAME="arm64" ;;
+    *) echo "Unsupported architecture: $ARCH"; exit 1 ;;
+esac
+
+# Prometheus paths
+PROMETHEUS_DIR="$MONITORING_DATA_DIR/prometheus"
+PROMETHEUS_ARCHIVE="prometheus-${PROMETHEUS_VERSION}.${OS_NAME}-${ARCH_NAME}.tar.gz"
+PROMETHEUS_URL="https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/${PROMETHEUS_ARCHIVE}"
+PROMETHEUS_BIN="$PROMETHEUS_DIR/prometheus-${PROMETHEUS_VERSION}.${OS_NAME}-${ARCH_NAME}/prometheus"
+
+# Grafana paths
+GRAFANA_DIR="$MONITORING_DATA_DIR/grafana"
+GRAFANA_ARCHIVE="grafana-${GRAFANA_VERSION}.${OS_NAME}-${ARCH_NAME}.tar.gz"
+GRAFANA_URL="https://dl.grafana.com/oss/release/${GRAFANA_ARCHIVE}"
+GRAFANA_HOME="$GRAFANA_DIR/grafana-v${GRAFANA_VERSION}"
+GRAFANA_BIN="$GRAFANA_HOME/bin/grafana"
+
+# Cleanup function
+cleanup() {
+    echo ""
+    echo "Shutting down monitoring stack..."
+
+    if [ -n "$PROM_PID" ] && kill -0 "$PROM_PID" 2>/dev/null; then
+        kill "$PROM_PID" 2>/dev/null || true
+    fi
+
+    if [ -n "$GRAFANA_PID" ] && kill -0 "$GRAFANA_PID" 2>/dev/null; then
+        kill "$GRAFANA_PID" 2>/dev/null || true
+    fi
+
+    echo "Monitoring stack stopped"
+    exit 0
+}
+
+trap cleanup SIGINT SIGTERM
+
+# Download Prometheus if needed
+if [ ! -f "$PROMETHEUS_BIN" ]; then
+    echo "Downloading Prometheus ${PROMETHEUS_VERSION}..."
+    mkdir -p "$PROMETHEUS_DIR"
+    cd "$PROMETHEUS_DIR"
+    curl -sL -o "$PROMETHEUS_ARCHIVE" "$PROMETHEUS_URL"
+    tar xzf "$PROMETHEUS_ARCHIVE"
+    rm "$PROMETHEUS_ARCHIVE"
+    echo "Prometheus ready"
+fi
+
+# Download Grafana if needed
+if [ ! -f "$GRAFANA_BIN" ]; then
+    echo "Downloading Grafana ${GRAFANA_VERSION}..."
+    mkdir -p "$GRAFANA_DIR"
+    cd "$GRAFANA_DIR"
+    curl -sL -o "$GRAFANA_ARCHIVE" "$GRAFANA_URL"
+    tar xzf "$GRAFANA_ARCHIVE"
+    rm "$GRAFANA_ARCHIVE"
+    echo "Grafana ready"
+fi
+
+# Create Prometheus config
+mkdir -p "$PROMETHEUS_DIR"
+cat > "$PROMETHEUS_DIR/prometheus.yml" <<EOF
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+
+scrape_configs:
+  - job_name: 'hindsight-api'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['localhost:$API_PORT']
+    metrics_path: '/metrics'
+EOF
+
+# Create Grafana provisioning directories
+GRAFANA_PROV_DIR="$GRAFANA_DIR/provisioning"
+mkdir -p "$GRAFANA_PROV_DIR/datasources"
+mkdir -p "$GRAFANA_PROV_DIR/dashboards"
+mkdir -p "$GRAFANA_DIR/dashboards"
+mkdir -p "$GRAFANA_DIR/data"
+
+# Copy dashboards
+cp "$SCRIPT_DIR/grafana/dashboards/"*.json "$GRAFANA_DIR/dashboards/"
+
+# Create Grafana datasource config
+cat > "$GRAFANA_PROV_DIR/datasources/prometheus.yaml" <<EOF
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://localhost:$PROMETHEUS_PORT
+    isDefault: true
+    editable: false
+    uid: prometheus
+EOF
+
+# Create Grafana dashboard provisioning config
+cat > "$GRAFANA_PROV_DIR/dashboards/dashboards.yaml" <<EOF
+apiVersion: 1
+providers:
+  - name: 'Hindsight'
+    orgId: 1
+    folder: 'Hindsight'
+    folderUid: 'hindsight'
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: $GRAFANA_DIR/dashboards
+EOF
+
+# Create Grafana config
+cat > "$GRAFANA_DIR/grafana.ini" <<EOF
+[server]
+http_port = $GRAFANA_PORT
+root_url = http://localhost:$GRAFANA_PORT
+
+[security]
+admin_user = admin
+admin_password = admin
+disable_initial_admin_creation = false
+
+[auth.anonymous]
+enabled = true
+org_name = Main Org.
+org_role = Viewer
+
+[paths]
+data = $GRAFANA_DIR/data
+logs = $GRAFANA_DIR/logs
+plugins = $GRAFANA_DIR/plugins
+provisioning = $GRAFANA_PROV_DIR
+
+[log]
+mode = console
+level = warn
+
+[dashboards]
+default_home_dashboard_path = $GRAFANA_DIR/dashboards/hindsight-operations.json
+EOF
+
+echo ""
+echo "=================================="
+echo "  Hindsight Monitoring Stack"
+echo "=================================="
+echo ""
+echo "  Grafana:     http://localhost:$GRAFANA_PORT"
+echo "  Prometheus:  http://localhost:$PROMETHEUS_PORT"
+echo "  API Metrics: http://localhost:$API_PORT/metrics"
+echo ""
+echo "  Dashboards:"
+echo "    - Hindsight Operations"
+echo "    - Hindsight LLM Metrics"
+echo ""
+echo "=================================="
+echo ""
+
+# Check if API is running
+if ! curl -s "http://localhost:$API_PORT/metrics" > /dev/null 2>&1; then
+    echo "WARNING: Hindsight API not detected at localhost:$API_PORT"
+    echo "         Start the API first: ./scripts/dev/start-api.sh"
+    echo ""
+fi
+
+# Start Prometheus in background
+cd "$(dirname "$PROMETHEUS_BIN")"
+"$PROMETHEUS_BIN" \
+    --config.file="$PROMETHEUS_DIR/prometheus.yml" \
+    --storage.tsdb.path="$PROMETHEUS_DIR/data" \
+    --web.console.templates="$(dirname "$PROMETHEUS_BIN")/consoles" \
+    --web.console.libraries="$(dirname "$PROMETHEUS_BIN")/console_libraries" \
+    --web.listen-address="0.0.0.0:$PROMETHEUS_PORT" \
+    --web.enable-lifecycle \
+    --log.level=warn &
+PROM_PID=$!
+
+# Start Grafana in background
+cd "$GRAFANA_HOME"
+"$GRAFANA_BIN" server \
+    --homepath="$GRAFANA_HOME" \
+    --config="$GRAFANA_DIR/grafana.ini" &
+GRAFANA_PID=$!
+
+echo "Monitoring stack running. Press Ctrl+C to stop."
+echo ""
+
+# Wait for processes
+wait "$PROM_PID" "$GRAFANA_PID" 2>/dev/null || true
+
+# If we get here, clean up
+cleanup

scripts/dev/start-monitoring.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+# Convenience wrapper to start the monitoring stack
+exec "$(dirname "${BASH_SOURCE[0]}")/monitoring/start.sh" "$@"

scripts/prometheus-dashboard.html

@@ -1,384 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Hindsight Metrics Dashboard</title>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/chart.umd.min.js"></script>
-    <style>
-        * {
-            margin: 0;
-            padding: 0;
-            box-sizing: border-box;
-        }
-
-        body {
-            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
-            background: #0f1419;
-            color: #e6e6e6;
-            padding: 20px;
-        }
-
-        .header {
-            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
-            padding: 30px;
-            border-radius: 10px;
-            margin-bottom: 30px;
-        }
-
-        h1 {
-            font-size: 32px;
-            font-weight: 600;
-        }
-
-        .subtitle {
-            opacity: 0.9;
-            margin-top: 5px;
-        }
-
-        .grid {
-            display: grid;
-            grid-template-columns: repeat(auto-fit, minmax(500px, 1fr));
-            gap: 20px;
-            margin-bottom: 20px;
-        }
-
-        .card {
-            background: #1a1f2e;
-            border-radius: 10px;
-            padding: 20px;
-            border: 1px solid #2d3548;
-        }
-
-        .card-title {
-            font-size: 18px;
-            font-weight: 600;
-            margin-bottom: 15px;
-            color: #a0aec0;
-        }
-
-        .chart-container {
-            position: relative;
-            height: 300px;
-        }
-
-        .stats {
-            display: grid;
-            grid-template-columns: repeat(3, 1fr);
-            gap: 15px;
-            margin-bottom: 20px;
-        }
-
-        .stat {
-            background: #1a1f2e;
-            padding: 20px;
-            border-radius: 10px;
-            border: 1px solid #2d3548;
-        }
-
-        .stat-label {
-            font-size: 12px;
-            color: #a0aec0;
-            text-transform: uppercase;
-            letter-spacing: 0.5px;
-            margin-bottom: 8px;
-        }
-
-        .stat-value {
-            font-size: 28px;
-            font-weight: 600;
-            color: #667eea;
-        }
-
-        .stat-unit {
-            font-size: 14px;
-            color: #a0aec0;
-            margin-left: 4px;
-        }
-
-        .error {
-            background: #2d1b1b;
-            border: 1px solid #7d2828;
-            color: #f56565;
-            padding: 15px;
-            border-radius: 8px;
-            margin: 20px 0;
-        }
-
-        .loading {
-            text-align: center;
-            padding: 40px;
-            color: #a0aec0;
-        }
-    </style>
-</head>
-<body>
-    <div class="header">
-        <h1>🧠 Hindsight Metrics Dashboard</h1>
-        <div class="subtitle">Real-time performance monitoring • Updates every 15s</div>
-    </div>
-
-    <div id="error-container"></div>
-    <div id="loading" class="loading">Loading metrics...</div>
-
-    <div class="stats" id="stats" style="display: none;"></div>
-
-    <div class="grid" id="charts" style="display: none;">
-        <div class="card">
-            <div class="card-title">📊 Recall Latency Percentiles</div>
-            <div class="chart-container">
-                <canvas id="latencyChart"></canvas>
-            </div>
-        </div>
-
-        <div class="card">
-            <div class="card-title">⚡ Operations per Second</div>
-            <div class="chart-container">
-                <canvas id="opsChart"></canvas>
-            </div>
-        </div>
-
-        <div class="card">
-            <div class="card-title">🎯 Latency by Bank</div>
-            <div class="chart-container">
-                <canvas id="bankLatencyChart"></canvas>
-            </div>
-        </div>
-
-        <div class="card">
-            <div class="card-title">💰 Token Usage Rate</div>
-            <div class="chart-container">
-                <canvas id="tokenChart"></canvas>
-            </div>
-        </div>
-    </div>
-
-    <script>
-        const PROMETHEUS_URL = 'http://localhost:9090';
-        const UPDATE_INTERVAL = 15000; // 15 seconds
-
-        // Chart configurations
-        const chartConfig = {
-            type: 'line',
-            options: {
-                responsive: true,
-                maintainAspectRatio: false,
-                plugins: {
-                    legend: {
-                        labels: { color: '#e6e6e6' }
-                    }
-                },
-                scales: {
-                    x: {
-                        ticks: { color: '#a0aec0' },
-                        grid: { color: '#2d3548' }
-                    },
-                    y: {
-                        ticks: { color: '#a0aec0' },
-                        grid: { color: '#2d3548' }
-                    }
-                }
-            }
-        };
-
-        let charts = {};
-
-        async function queryPrometheus(query) {
-            const url = `${PROMETHEUS_URL}/api/v1/query?query=${encodeURIComponent(query)}`;
-            const response = await fetch(url);
-            if (!response.ok) throw new Error(`HTTP ${response.status}`);
-            const data = await response.json();
-            if (data.status !== 'success') throw new Error('Query failed');
-            return data.data.result;
-        }
-
-        async function queryRange(query, minutes = 30) {
-            const end = Math.floor(Date.now() / 1000);
-            const start = end - (minutes * 60);
-            const url = `${PROMETHEUS_URL}/api/v1/query_range?query=${encodeURIComponent(query)}&start=${start}&end=${end}&step=30`;
-            const response = await fetch(url);
-            if (!response.ok) throw new Error(`HTTP ${response.status}`);
-            const data = await response.json();
-            if (data.status !== 'success') throw new Error('Query failed');
-            return data.data.result;
-        }
-
-        function showError(message) {
-            const container = document.getElementById('error-container');
-            container.innerHTML = `
-                <div class="error">
-                    <strong>⚠️ Error:</strong> ${message}<br>
-                    <small>Make sure Prometheus is running at ${PROMETHEUS_URL}</small>
-                </div>
-            `;
-        }
-
-        function hideLoading() {
-            document.getElementById('loading').style.display = 'none';
-            document.getElementById('stats').style.display = 'grid';
-            document.getElementById('charts').style.display = 'grid';
-        }
-
-        async function updateStats() {
-            try {
-                // Query current stats
-                const totalOps = await queryPrometheus('sum(hindsight_operation_total)');
-                const successRate = await queryPrometheus('sum(hindsight_operation_total{success="true"}) / sum(hindsight_operation_total)');
-                const opsRate = await queryPrometheus('sum(rate(hindsight_operation_total[5m]))');
-
-                const stats = [
-                    {
-                        label: 'Total Operations',
-                        value: totalOps[0]?.value[1] || '0',
-                        unit: ''
-                    },
-                    {
-                        label: 'Success Rate',
-                        value: ((parseFloat(successRate[0]?.value[1] || 1) * 100).toFixed(1)),
-                        unit: '%'
-                    },
-                    {
-                        label: 'Ops/sec',
-                        value: (parseFloat(opsRate[0]?.value[1] || 0).toFixed(2)),
-                        unit: 'ops/s'
-                    }
-                ];
-
-                document.getElementById('stats').innerHTML = stats.map(stat => `
-                    <div class="stat">
-                        <div class="stat-label">${stat.label}</div>
-                        <div class="stat-value">
-                            ${stat.value}
-                            <span class="stat-unit">${stat.unit}</span>
-                        </div>
-                    </div>
-                `).join('');
-
-            } catch (error) {
-                console.error('Error updating stats:', error);
-            }
-        }
-
-        async function updateCharts() {
-            try {
-                // Latency percentiles over time
-                const p50Data = await queryRange('histogram_quantile(0.50, rate(hindsight_operation_duration_seconds_bucket{operation="recall"}[5m]))');
-                const p95Data = await queryRange('histogram_quantile(0.95, rate(hindsight_operation_duration_seconds_bucket{operation="recall"}[5m]))');
-                const p99Data = await queryRange('histogram_quantile(0.99, rate(hindsight_operation_duration_seconds_bucket{operation="recall"}[5m]))');
-
-                if (p50Data.length > 0) {
-                    const timestamps = p50Data[0].values.map(v => new Date(v[0] * 1000).toLocaleTimeString());
-
-                    if (!charts.latency) {
-                        charts.latency = new Chart(document.getElementById('latencyChart'), {
-                            ...chartConfig,
-                            data: {
-                                labels: timestamps,
-                                datasets: [
-                                    { label: 'p50', data: p50Data[0].values.map(v => parseFloat(v[1])), borderColor: '#48bb78', tension: 0.4 },
-                                    { label: 'p95', data: p95Data[0].values.map(v => parseFloat(v[1])), borderColor: '#ed8936', tension: 0.4 },
-                                    { label: 'p99', data: p99Data[0].values.map(v => parseFloat(v[1])), borderColor: '#f56565', tension: 0.4 }
-                                ]
-                            }
-                        });
-                    } else {
-                        charts.latency.data.labels = timestamps;
-                        charts.latency.data.datasets[0].data = p50Data[0].values.map(v => parseFloat(v[1]));
-                        charts.latency.data.datasets[1].data = p95Data[0].values.map(v => parseFloat(v[1]));
-                        charts.latency.data.datasets[2].data = p99Data[0].values.map(v => parseFloat(v[1]));
-                        charts.latency.update();
-                    }
-                }
-
-                // Operations per second
-                const opsData = await queryRange('sum by (operation) (rate(hindsight_operation_total[5m]))');
-                if (opsData.length > 0) {
-                    const timestamps = opsData[0].values.map(v => new Date(v[0] * 1000).toLocaleTimeString());
-                    const datasets = opsData.map((series, i) => ({
-                        label: series.metric.operation,
-                        data: series.values.map(v => parseFloat(v[1])),
-                        borderColor: ['#667eea', '#48bb78', '#ed8936'][i % 3],
-                        tension: 0.4
-                    }));
-
-                    if (!charts.ops) {
-                        charts.ops = new Chart(document.getElementById('opsChart'), {
-                            ...chartConfig,
-                            data: { labels: timestamps, datasets }
-                        });
-                    } else {
-                        charts.ops.data.labels = timestamps;
-                        charts.ops.data.datasets = datasets;
-                        charts.ops.update();
-                    }
-                }
-
-                // Latency by bank
-                const bankLatency = await queryRange('histogram_quantile(0.95, sum by (bank_id, le) (rate(hindsight_operation_duration_seconds_bucket{operation="recall"}[5m])))');
-                if (bankLatency.length > 0) {
-                    const timestamps = bankLatency[0].values.map(v => new Date(v[0] * 1000).toLocaleTimeString());
-                    const datasets = bankLatency.map((series, i) => ({
-                        label: series.metric.bank_id,
-                        data: series.values.map(v => parseFloat(v[1])),
-                        borderColor: ['#667eea', '#48bb78', '#ed8936', '#f56565'][i % 4],
-                        tension: 0.4
-                    }));
-
-                    if (!charts.bankLatency) {
-                        charts.bankLatency = new Chart(document.getElementById('bankLatencyChart'), {
-                            ...chartConfig,
-                            data: { labels: timestamps, datasets }
-                        });
-                    } else {
-                        charts.bankLatency.data.labels = timestamps;
-                        charts.bankLatency.data.datasets = datasets;
-                        charts.bankLatency.update();
-                    }
-                }
-
-                // Token usage
-                const tokenInput = await queryRange('sum(rate(hindsight_tokens_input_total[5m]))');
-                const tokenOutput = await queryRange('sum(rate(hindsight_tokens_output_total[5m]))');
-                if (tokenInput.length > 0) {
-                    const timestamps = tokenInput[0].values.map(v => new Date(v[0] * 1000).toLocaleTimeString());
-
-                    if (!charts.tokens) {
-                        charts.tokens = new Chart(document.getElementById('tokenChart'), {
-                            ...chartConfig,
-                            data: {
-                                labels: timestamps,
-                                datasets: [
-                                    { label: 'Input', data: tokenInput[0].values.map(v => parseFloat(v[1])), borderColor: '#667eea', tension: 0.4 },
-                                    { label: 'Output', data: tokenOutput[0]?.values.map(v => parseFloat(v[1])) || [], borderColor: '#48bb78', tension: 0.4 }
-                                ]
-                            }
-                        });
-                    } else {
-                        charts.tokens.data.labels = timestamps;
-                        charts.tokens.data.datasets[0].data = tokenInput[0].values.map(v => parseFloat(v[1]));
-                        charts.tokens.data.datasets[1].data = tokenOutput[0]?.values.map(v => parseFloat(v[1])) || [];
-                        charts.tokens.update();
-                    }
-                }
-
-                hideLoading();
-
-            } catch (error) {
-                console.error('Error updating charts:', error);
-                showError(error.message);
-            }
-        }
-
-        async function update() {
-            await updateStats();
-            await updateCharts();
-        }
-
-        // Initial update
-        update();
-
-        // Update every 15 seconds
-        setInterval(update, UPDATE_INTERVAL);
-    </script>
-</body>
-</html>

scripts/start-prometheus.sh

@@ -1,138 +0,0 @@
-#!/bin/bash
-set -e
-
-# Script to download and start Prometheus for Hindsight metrics
-# This creates a local Prometheus instance that scrapes metrics from the Hindsight API
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
-PROMETHEUS_DIR="$PROJECT_ROOT/.prometheus"
-PROMETHEUS_VERSION="2.48.0"
-
-# Detect OS and architecture
-OS=$(uname -s | tr '[:upper:]' '[:lower:]')
-ARCH=$(uname -m)
-
-case "$OS" in
-    darwin)
-        OS_NAME="darwin"
-        ;;
-    linux)
-        OS_NAME="linux"
-        ;;
-    *)
-        echo "Unsupported OS: $OS"
-        exit 1
-        ;;
-esac
-
-case "$ARCH" in
-    x86_64)
-        ARCH_NAME="amd64"
-        ;;
-    arm64|aarch64)
-        ARCH_NAME="arm64"
-        ;;
-    *)
-        echo "Unsupported architecture: $ARCH"
-        exit 1
-        ;;
-esac
-
-PROMETHEUS_ARCHIVE="prometheus-${PROMETHEUS_VERSION}.${OS_NAME}-${ARCH_NAME}.tar.gz"
-PROMETHEUS_URL="https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/${PROMETHEUS_ARCHIVE}"
-PROMETHEUS_BIN="$PROMETHEUS_DIR/prometheus-${PROMETHEUS_VERSION}.${OS_NAME}-${ARCH_NAME}/prometheus"
-
-echo "🔧 Setting up Prometheus for Hindsight metrics..."
-echo ""
-
-# Create prometheus directory
-mkdir -p "$PROMETHEUS_DIR"
-cd "$PROMETHEUS_DIR"
-
-# Download Prometheus if not exists
-if [ ! -f "$PROMETHEUS_BIN" ]; then
-    echo "📥 Downloading Prometheus ${PROMETHEUS_VERSION} for ${OS_NAME}-${ARCH_NAME}..."
-    curl -L -o "$PROMETHEUS_ARCHIVE" "$PROMETHEUS_URL"
-
-    echo "📦 Extracting..."
-    tar xzf "$PROMETHEUS_ARCHIVE"
-
-    echo "✅ Prometheus downloaded successfully"
-    echo ""
-else
-    echo "✅ Prometheus already downloaded"
-    echo ""
-fi
-
-# Create prometheus.yml configuration
-echo "📝 Creating Prometheus configuration..."
-cat > "$PROMETHEUS_DIR/prometheus.yml" <<EOF
-# Prometheus configuration for Hindsight API metrics
-global:
-  scrape_interval: 15s      # Scrape metrics every 15 seconds
-  evaluation_interval: 15s  # Evaluate rules every 15 seconds
-
-# Scrape configuration
-scrape_configs:
-  - job_name: 'hindsight-api'
-    scrape_interval: 15s
-    static_configs:
-      - targets: ['localhost:8000']  # Hindsight API endpoint
-    metrics_path: '/metrics'         # Metrics endpoint path
-
-    # Optional: Add labels to all metrics from this job
-    # relabeling_configs:
-    #   - source_labels: [__address__]
-    #     target_label: instance
-    #     replacement: 'hindsight-api'
-EOF
-
-echo "✅ Configuration created at $PROMETHEUS_DIR/prometheus.yml"
-echo ""
-
-# Check if Hindsight API is running
-echo "🔍 Checking if Hindsight API is running..."
-if curl -s http://localhost:8000/metrics > /dev/null 2>&1; then
-    echo "✅ Hindsight API is running and serving metrics"
-    echo ""
-else
-    echo "⚠️  WARNING: Hindsight API is not reachable at http://localhost:8000/metrics"
-    echo "   Make sure to start the API before Prometheus can scrape metrics"
-    echo ""
-fi
-
-# Start Prometheus
-echo "🚀 Starting Prometheus..."
-echo ""
-echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-echo "  Prometheus UI:  http://localhost:9090"
-echo "  Metrics source: http://localhost:8000/metrics"
-echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-echo ""
-echo "📊 Example queries to try in the UI:"
-echo ""
-echo "  p95 latency (all operations):"
-echo "    histogram_quantile(0.95, rate(hindsight_operation_duration_seconds_bucket[5m]))"
-echo ""
-echo "  p95 latency by bank:"
-echo "    histogram_quantile(0.95, sum by (bank_id, le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))"
-echo ""
-echo "  Operations per second:"
-echo "    rate(hindsight_operation_total[5m])"
-echo ""
-echo "  Token usage rate:"
-echo "    rate(hindsight_tokens_input_total[5m]) + rate(hindsight_tokens_output_total[5m])"
-echo ""
-echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-echo ""
-echo "Press Ctrl+C to stop Prometheus"
-echo ""
-
-# Start Prometheus with config
-cd "$(dirname "$PROMETHEUS_BIN")"
-exec "$PROMETHEUS_BIN" \
-    --config.file="$PROMETHEUS_DIR/prometheus.yml" \
-    --storage.tsdb.path="$PROMETHEUS_DIR/data" \
-    --web.console.templates="$PROMETHEUS_DIR/prometheus-${PROMETHEUS_VERSION}.${OS_NAME}-${ARCH_NAME}/consoles" \
-    --web.console.libraries="$PROMETHEUS_DIR/prometheus-${PROMETHEUS_VERSION}.${OS_NAME}-${ARCH_NAME}/console_libraries"