refactor deamin

hindsight-api/hindsight_api/config.py

@@ -33,6 +33,10 @@ ENV_GRAPH_RETRIEVER = "HINDSIGHT_API_GRAPH_RETRIEVER"
 ENV_MCP_LOCAL_BANK_ID = "HINDSIGHT_API_MCP_LOCAL_BANK_ID"
 ENV_MCP_INSTRUCTIONS = "HINDSIGHT_API_MCP_INSTRUCTIONS"
 
+# Optimization flags
+ENV_SKIP_LLM_VERIFICATION = "HINDSIGHT_API_SKIP_LLM_VERIFICATION"
+ENV_LAZY_RERANKER = "HINDSIGHT_API_LAZY_RERANKER"
+
 # Default values
 DEFAULT_DATABASE_URL = "pg0"
 DEFAULT_LLM_PROVIDER = "openai"
@@ -107,6 +111,10 @@ class HindsightConfig:
     # Recall
     graph_retriever: str
 
+    # Optimization flags
+    skip_llm_verification: bool
+    lazy_reranker: bool
+
     @classmethod
     def from_env(cls) -> "HindsightConfig":
         """Create configuration from environment variables."""
@@ -133,6 +141,9 @@ class HindsightConfig:
             mcp_enabled=os.getenv(ENV_MCP_ENABLED, str(DEFAULT_MCP_ENABLED)).lower() == "true",
             # Recall
             graph_retriever=os.getenv(ENV_GRAPH_RETRIEVER, DEFAULT_GRAPH_RETRIEVER),
+            # Optimization flags
+            skip_llm_verification=os.getenv(ENV_SKIP_LLM_VERIFICATION, "false").lower() == "true",
+            lazy_reranker=os.getenv(ENV_LAZY_RERANKER, "false").lower() == "true",
         )
 
     def get_llm_base_url(self) -> str:

hindsight-api/hindsight_api/daemon.py

@@ -0,0 +1,204 @@
+"""
+Daemon mode support for Hindsight API.
+
+Provides idle timeout and lockfile management for running as a background daemon.
+"""
+
+import asyncio
+import fcntl
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Default daemon configuration
+DEFAULT_DAEMON_PORT = 8889
+DEFAULT_IDLE_TIMEOUT = 0  # 0 = no auto-exit (hindsight-embed passes its own timeout)
+LOCKFILE_PATH = Path.home() / ".hindsight" / "daemon.lock"
+DAEMON_LOG_PATH = Path.home() / ".hindsight" / "daemon.log"
+
+
+class IdleTimeoutMiddleware:
+    """ASGI middleware that tracks activity and exits after idle timeout."""
+
+    def __init__(self, app, idle_timeout: int = DEFAULT_IDLE_TIMEOUT):
+        self.app = app
+        self.idle_timeout = idle_timeout
+        self.last_activity = time.time()
+        self._checker_task = None
+
+    async def __call__(self, scope, receive, send):
+        # Update activity timestamp on each request
+        self.last_activity = time.time()
+        await self.app(scope, receive, send)
+
+    def start_idle_checker(self):
+        """Start the background task that checks for idle timeout."""
+        self._checker_task = asyncio.create_task(self._check_idle())
+
+    async def _check_idle(self):
+        """Background task that exits the process after idle timeout."""
+        # If idle_timeout is 0, don't auto-exit
+        if self.idle_timeout <= 0:
+            return
+
+        while True:
+            await asyncio.sleep(30)  # Check every 30 seconds
+            idle_time = time.time() - self.last_activity
+            if idle_time > self.idle_timeout:
+                logger.info(f"Idle timeout reached ({self.idle_timeout}s), shutting down daemon")
+                # Give a moment for any in-flight requests
+                await asyncio.sleep(1)
+                os._exit(0)
+
+
+class DaemonLock:
+    """
+    File-based lock to prevent multiple daemon instances.
+
+    Uses fcntl.flock for atomic locking on Unix systems.
+    """
+
+    def __init__(self, lockfile: Path = LOCKFILE_PATH):
+        self.lockfile = lockfile
+        self._fd = None
+
+    def acquire(self) -> bool:
+        """
+        Try to acquire the daemon lock.
+
+        Returns True if lock acquired, False if another daemon is running.
+        """
+        self.lockfile.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            self._fd = open(self.lockfile, "w")
+            fcntl.flock(self._fd.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+            # Write PID for debugging
+            self._fd.write(str(os.getpid()))
+            self._fd.flush()
+            return True
+        except (IOError, OSError):
+            # Lock is held by another process
+            if self._fd:
+                self._fd.close()
+                self._fd = None
+            return False
+
+    def release(self):
+        """Release the daemon lock."""
+        if self._fd:
+            try:
+                fcntl.flock(self._fd.fileno(), fcntl.LOCK_UN)
+                self._fd.close()
+            except Exception:
+                pass
+            finally:
+                self._fd = None
+                # Remove lockfile
+                try:
+                    self.lockfile.unlink()
+                except Exception:
+                    pass
+
+    def is_locked(self) -> bool:
+        """Check if the lock is held by another process."""
+        if not self.lockfile.exists():
+            return False
+
+        try:
+            fd = open(self.lockfile, "r")
+            fcntl.flock(fd.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+            # We got the lock, so no one else has it
+            fcntl.flock(fd.fileno(), fcntl.LOCK_UN)
+            fd.close()
+            return False
+        except (IOError, OSError):
+            return True
+
+    def get_pid(self) -> int | None:
+        """Get the PID of the daemon holding the lock."""
+        if not self.lockfile.exists():
+            return None
+        try:
+            with open(self.lockfile, "r") as f:
+                return int(f.read().strip())
+        except (ValueError, IOError):
+            return None
+
+
+def daemonize():
+    """
+    Fork the current process into a background daemon.
+
+    Uses double-fork technique to properly detach from terminal.
+    """
+    # First fork
+    pid = os.fork()
+    if pid > 0:
+        # Parent exits
+        sys.exit(0)
+
+    # Create new session
+    os.setsid()
+
+    # Second fork to prevent zombie processes
+    pid = os.fork()
+    if pid > 0:
+        sys.exit(0)
+
+    # Redirect standard file descriptors to log file
+    DAEMON_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
+
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    # Redirect stdin to /dev/null
+    with open("/dev/null", "r") as devnull:
+        os.dup2(devnull.fileno(), sys.stdin.fileno())
+
+    # Redirect stdout/stderr to log file
+    log_fd = open(DAEMON_LOG_PATH, "a")
+    os.dup2(log_fd.fileno(), sys.stdout.fileno())
+    os.dup2(log_fd.fileno(), sys.stderr.fileno())
+
+
+def check_daemon_running(port: int = DEFAULT_DAEMON_PORT) -> bool:
+    """Check if a daemon is running and responsive on the given port."""
+    import socket
+
+    try:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.settimeout(1)
+        result = sock.connect_ex(("127.0.0.1", port))
+        sock.close()
+        return result == 0
+    except Exception:
+        return False
+
+
+def stop_daemon(port: int = DEFAULT_DAEMON_PORT) -> bool:
+    """Stop a running daemon by sending SIGTERM to the process."""
+    lock = DaemonLock()
+    pid = lock.get_pid()
+
+    if pid is None:
+        return False
+
+    try:
+        import signal
+
+        os.kill(pid, signal.SIGTERM)
+        # Wait for process to exit
+        for _ in range(50):  # Wait up to 5 seconds
+            time.sleep(0.1)
+            try:
+                os.kill(pid, 0)  # Check if process exists
+            except OSError:
+                return True  # Process exited
+        return False
+    except OSError:
+        return False

hindsight-api/hindsight_api/engine/memory_engine.py

@@ -202,6 +202,8 @@ class MemoryEngine(MemoryEngineInterface):
         run_migrations: bool = True,
         operation_validator: "OperationValidatorExtension | None" = None,
         tenant_extension: "TenantExtension | None" = None,
+        skip_llm_verification: bool | None = None,
+        lazy_reranker: bool | None = None,
     ):
         """
         Initialize the temporal + semantic memory system.
@@ -227,12 +229,23 @@ class MemoryEngine(MemoryEngineInterface):
                                 If provided, retain/recall/reflect operations will be validated.
             tenant_extension: Optional extension for multi-tenancy and API key authentication.
                              If provided, operations require a RequestContext for authentication.
+            skip_llm_verification: Skip LLM connection verification during initialization.
+                                  Defaults to HINDSIGHT_API_SKIP_LLM_VERIFICATION env var or False.
+            lazy_reranker: Delay reranker initialization until first use. Useful for retain-only
+                          operations that don't need the cross-encoder. Defaults to
+                          HINDSIGHT_API_LAZY_RERANKER env var or False.
         """
         # Load config from environment for any missing parameters
         from ..config import get_config
 
         config = get_config()
 
+        # Apply optimization flags from config if not explicitly provided
+        self._skip_llm_verification = (
+            skip_llm_verification if skip_llm_verification is not None else config.skip_llm_verification
+        )
+        self._lazy_reranker = lazy_reranker if lazy_reranker is not None else config.lazy_reranker
+
         # Apply defaults from config
         db_url = db_url or config.database_url
         memory_llm_provider = memory_llm_provider or config.llm_provider
@@ -594,6 +607,8 @@ class MemoryEngine(MemoryEngineInterface):
                 await loop.run_in_executor(None, lambda: asyncio.run(cross_encoder.initialize()))
             else:
                 await cross_encoder.initialize()
+            # Mark reranker as initialized
+            self._cross_encoder_reranker._initialized = True
 
         async def init_query_analyzer():
             """Initialize query analyzer model."""
@@ -602,16 +617,26 @@ class MemoryEngine(MemoryEngineInterface):
 
         async def verify_llm():
             """Verify LLM connection is working."""
-            await self._llm_config.verify_connection()
+            if not self._skip_llm_verification:
+                await self._llm_config.verify_connection()
 
-        # Run pg0 and all model initializations in parallel
-        await asyncio.gather(
+        # Build list of initialization tasks
+        init_tasks = [
             start_pg0(),
             init_embeddings(),
-            init_cross_encoder(),
             init_query_analyzer(),
-            verify_llm(),
-        )
+        ]
+
+        # Only init cross-encoder eagerly if not using lazy initialization
+        if not self._lazy_reranker:
+            init_tasks.append(init_cross_encoder())
+
+        # Only verify LLM if not skipping
+        if not self._skip_llm_verification:
+            init_tasks.append(verify_llm())
+
+        # Run pg0 and selected model initializations in parallel
+        await asyncio.gather(*init_tasks)
 
         # Run database migrations if enabled
         if self._run_migrations:
@@ -1641,6 +1666,9 @@ class MemoryEngine(MemoryEngineInterface):
             step_start = time.time()
             reranker_instance = self._cross_encoder_reranker
 
+            # Ensure reranker is initialized (for lazy initialization mode)
+            await reranker_instance.ensure_initialized()
+
             # Rerank using cross-encoder
             scored_results = reranker_instance.rerank(query, merged_candidates)
 

hindsight-api/hindsight_api/engine/search/reranking.py

@@ -26,6 +26,23 @@ class CrossEncoderReranker:
 
             cross_encoder = create_cross_encoder_from_env()
         self.cross_encoder = cross_encoder
+        self._initialized = False
+
+    async def ensure_initialized(self):
+        """Ensure the cross-encoder model is initialized (for lazy initialization)."""
+        if self._initialized:
+            return
+
+        import asyncio
+
+        cross_encoder = self.cross_encoder
+        # For local providers, run in thread pool to avoid blocking event loop
+        if cross_encoder.provider_name == "local":
+            loop = asyncio.get_event_loop()
+            await loop.run_in_executor(None, lambda: asyncio.run(cross_encoder.initialize()))
+        else:
+            await cross_encoder.initialize()
+        self._initialized = True
 
     def rerank(self, query: str, candidates: list[MergedCandidate]) -> list[ScoredResult]:
         """

hindsight-api/hindsight_api/main.py

@@ -4,6 +4,9 @@ Command-line interface for Hindsight API.
 Run the server with:
     hindsight-api
 
+Run as background daemon:
+    hindsight-api --daemon
+
 Stop with Ctrl+C.
 """
 
@@ -21,9 +24,13 @@ from . import MemoryEngine
 from .api import create_app
 from .banner import print_banner
 from .config import HindsightConfig, get_config
-
-print()
-print_banner()
+from .daemon import (
+    DEFAULT_DAEMON_PORT,
+    DEFAULT_IDLE_TIMEOUT,
+    DaemonLock,
+    IdleTimeoutMiddleware,
+    daemonize,
+)
 
 # Filter deprecation warnings from third-party libraries
 warnings.filterwarnings("ignore", message="websockets.legacy is deprecated")
@@ -106,8 +113,52 @@ def main():
     parser.add_argument("--ssl-keyfile", default=None, help="SSL key file")
     parser.add_argument("--ssl-certfile", default=None, help="SSL certificate file")
 
+    # Daemon mode options
+    parser.add_argument(
+        "--daemon",
+        action="store_true",
+        help=f"Run as background daemon (uses port {DEFAULT_DAEMON_PORT}, auto-exits after idle)",
+    )
+    parser.add_argument(
+        "--idle-timeout",
+        type=int,
+        default=DEFAULT_IDLE_TIMEOUT,
+        help=f"Idle timeout in seconds before auto-exit in daemon mode (default: {DEFAULT_IDLE_TIMEOUT})",
+    )
+
     args = parser.parse_args()
 
+    # Daemon mode handling
+    if args.daemon:
+        # Use fixed daemon port
+        args.port = DEFAULT_DAEMON_PORT
+        args.host = "127.0.0.1"  # Only bind to localhost for security
+
+        # Check if another daemon is already running
+        daemon_lock = DaemonLock()
+        if not daemon_lock.acquire():
+            print(f"Daemon already running (PID: {daemon_lock.get_pid()})", file=sys.stderr)
+            sys.exit(1)
+
+        # Fork into background
+        daemonize()
+
+        # Re-acquire lock in child process
+        daemon_lock = DaemonLock()
+        if not daemon_lock.acquire():
+            sys.exit(1)
+
+        # Register cleanup to release lock
+        def release_lock():
+            daemon_lock.release()
+
+        atexit.register(release_lock)
+
+    # Print banner (not in daemon mode)
+    if not args.daemon:
+        print()
+        print_banner()
+
     # Configure Python logging based on log level
     # Update config with CLI override if provided
     if args.log_level != config.log_level:
@@ -128,9 +179,12 @@ def main():
             log_level=args.log_level,
             mcp_enabled=config.mcp_enabled,
             graph_retriever=config.graph_retriever,
+            skip_llm_verification=config.skip_llm_verification,
+            lazy_reranker=config.lazy_reranker,
         )
     config.configure_logging()
-    config.log_config()
+    if not args.daemon:
+        config.log_config()
 
     # Register cleanup handlers
     atexit.register(_cleanup)
@@ -149,6 +203,12 @@ def main():
         initialize_memory=True,
     )
 
+    # Wrap with idle timeout middleware in daemon mode
+    idle_middleware = None
+    if args.daemon:
+        idle_middleware = IdleTimeoutMiddleware(app, idle_timeout=args.idle_timeout)
+        app = idle_middleware
+
     # Prepare uvicorn config
     uvicorn_config = {
         "app": app,
@@ -172,18 +232,38 @@ def main():
     if args.ssl_certfile:
         uvicorn_config["ssl_certfile"] = args.ssl_certfile
 
-    from .banner import print_startup_info
+    # Print startup info (not in daemon mode)
+    if not args.daemon:
+        from .banner import print_startup_info
 
-    print_startup_info(
-        host=args.host,
-        port=args.port,
-        database_url=config.database_url,
-        llm_provider=config.llm_provider,
-        llm_model=config.llm_model,
-        embeddings_provider=config.embeddings_provider,
-        reranker_provider=config.reranker_provider,
-        mcp_enabled=config.mcp_enabled,
-    )
+        print_startup_info(
+            host=args.host,
+            port=args.port,
+            database_url=config.database_url,
+            llm_provider=config.llm_provider,
+            llm_model=config.llm_model,
+            embeddings_provider=config.embeddings_provider,
+            reranker_provider=config.reranker_provider,
+            mcp_enabled=config.mcp_enabled,
+        )
+
+    # Start idle checker in daemon mode
+    if idle_middleware is not None:
+        # Start the idle checker in a background thread with its own event loop
+        import threading
+
+        def run_idle_checker():
+            import time
+
+            time.sleep(2)  # Wait for uvicorn to start
+            try:
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+                loop.run_until_complete(idle_middleware._check_idle())
+            except Exception:
+                pass
+
+        threading.Thread(target=run_idle_checker, daemon=True).start()
 
     uvicorn.run(**uvicorn_config)  # type: ignore[invalid-argument-type] - dict kwargs
 

hindsight-clients/python/hindsight_client/hindsight_client.py

@@ -85,9 +85,21 @@ class Hindsight:
         self.close()
 
     def close(self):
-        """Close the API client."""
+        """Close the API client (sync version - use aclose() in async code)."""
         if self._api_client:
-            _run_async(self._api_client.close())
+            try:
+                loop = asyncio.get_running_loop()
+                # We're in an async context - schedule but don't wait
+                # The caller should use aclose() instead
+                loop.create_task(self._api_client.close())
+            except RuntimeError:
+                # No running loop - safe to run synchronously
+                _run_async(self._api_client.close())
+
+    async def aclose(self):
+        """Close the API client (async version)."""
+        if self._api_client:
+            await self._api_client.close()
 
     # Simplified methods for main operations
 

hindsight-docs/static/get-skill

@@ -76,24 +76,45 @@ You have access to persistent memory via the `hindsight-embed` CLI. Use it to re
 
 Run: `uvx hindsight-embed configure`
 
+This will configure your LLM provider and start a local daemon that manages your memory bank.
+
 ## Commands
 
+The CLI uses a bank ID to organize memories. Use `default` for general memories or create project-specific banks.
+
 ### Store a memory
 
-Use `retain` to store important facts, preferences, decisions, or context:
+Use `memory retain` to store important facts, preferences, decisions, or context:
 
 ```bash
-uvx hindsight-embed retain "User prefers dark mode for all UIs"
-uvx hindsight-embed retain "Project uses Python 3.11 with FastAPI" --context work
+uvx hindsight-embed memory retain default "User prefers dark mode for all UIs"
+uvx hindsight-embed memory retain default "Project uses Python 3.11 with FastAPI" --context work
+uvx hindsight-embed memory retain myproject "API uses JWT authentication"
 ```
 
 ### Recall memories
 
-Use `recall` to search for relevant memories before starting tasks:
+Use `memory recall` to search for relevant memories before starting tasks:
 
 ```bash
-uvx hindsight-embed recall "What are the user'"'"'s UI preferences?"
-uvx hindsight-embed recall "What tech stack does this project use?"
+uvx hindsight-embed memory recall default "What are the user'"'"'s UI preferences?"
+uvx hindsight-embed memory recall default "What tech stack does this project use?"
+```
+
+### Reflect on memories
+
+Use `memory reflect` for contextual answers that synthesize multiple memories:
+
+```bash
+uvx hindsight-embed memory reflect default "How should I set up the dev environment?"
+```
+
+### Other commands
+
+```bash
+uvx hindsight-embed bank list              # List all memory banks
+uvx hindsight-embed daemon status          # Check daemon status
+uvx hindsight-embed --help                 # Full CLI help
 ```
 
 ## When to Use
@@ -114,6 +135,7 @@ uvx hindsight-embed recall "What tech stack does this project use?"
 1. **Be specific**: Store "User prefers 2-space indentation" not "User has preferences"
 2. **Recall first**: Before starting tasks, recall relevant context
 3. **Use context tags**: Organize with `--context` (work, personal, preferences)
+4. **Use project banks**: Create separate banks for different projects
 '
 
 # Get skills directory for app (bash 3.x compatible)
@@ -235,8 +257,8 @@ echo ""
 echo -e "  The Hindsight skill is now available in ${BOLD}$APP_NAME${NC}."
 echo ""
 echo -e "  ${DIM}Test the CLI:${NC}"
-echo -e "    ${CYAN}uvx hindsight-embed retain \"Test memory\"${NC}"
-echo -e "    ${CYAN}uvx hindsight-embed recall \"test\"${NC}"
+echo -e "    ${CYAN}uvx hindsight-embed memory retain default \"Test memory\"${NC}"
+echo -e "    ${CYAN}uvx hindsight-embed memory recall default \"test\"${NC}"
 echo ""
 echo -e "  ${DIM}$APP_NAME will automatically use the skill when relevant.${NC}"
 echo ""

hindsight-embed/README.md

@@ -1,8 +1,18 @@
 # hindsight-embed
 
-Hindsight embedded CLI - local memory operations without a server.
+Hindsight embedded CLI - local memory operations with automatic daemon management.
 
-This package provides a simple CLI for storing and recalling memories using Hindsight's memory engine with an embedded PostgreSQL database (pg0). No external server or database setup required.
+This package provides a simple CLI for storing and recalling memories using Hindsight's memory engine. It automatically manages a background daemon for fast operations - no manual server setup required.
+
+## How It Works
+
+`hindsight-embed` uses a background daemon architecture for optimal performance:
+
+1. **First command**: Automatically starts a local daemon (first run downloads dependencies and loads ML models - can take 1-3 minutes)
+2. **Subsequent commands**: Near-instant responses (~1-2s) since daemon is already running
+3. **Auto-shutdown**: Daemon automatically exits after 5 minutes of inactivity
+
+The daemon runs on `localhost:8889` and uses an embedded PostgreSQL database (pg0) - everything stays local on your machine.
 
 ## Installation
 
@@ -15,49 +25,115 @@ uvx hindsight-embed --help
 ## Quick Start
 
 ```bash
-# Set your LLM API key
+# Interactive setup (recommended)
+hindsight-embed configure
+
+# Or set your LLM API key manually
 export OPENAI_API_KEY=sk-...
 
-# Store a memory
-hindsight-embed retain "User prefers dark mode"
+# Store a memory (bank_id = "default")
+hindsight-embed memory retain default "User prefers dark mode"
 
 # Recall memories
-hindsight-embed recall "What are user preferences?"
+hindsight-embed memory recall default "What are user preferences?"
 ```
 
 ## Commands
 
-### retain
+### configure
+
+Interactive setup wizard:
+
+```bash
+hindsight-embed configure
+```
+
+This will:
+- Let you choose an LLM provider (OpenAI, Groq, Google, Ollama)
+- Configure your API key
+- Set the model and memory bank ID
+- Start the daemon with your configuration
+
+### memory retain
 
 Store a memory:
 
 ```bash
-hindsight-embed retain "User prefers dark mode"
-hindsight-embed retain "Meeting on Monday" --context work
+hindsight-embed memory retain default "User prefers dark mode"
+hindsight-embed memory retain default "Meeting on Monday" --context work
+hindsight-embed memory retain myproject "API uses JWT authentication"
 ```
 
-### recall
+### memory recall
 
 Search memories:
 
 ```bash
-hindsight-embed recall "user preferences"
-hindsight-embed recall "upcoming events" --budget high
-hindsight-embed recall "project details" -v  # verbose output
+hindsight-embed memory recall default "user preferences"
+hindsight-embed memory recall default "upcoming events"
 ```
 
-## Environment Variables
+Use `-o json` for JSON output:
+```bash
+hindsight-embed memory recall default "user preferences" -o json
+```
+
+### memory reflect
+
+Get contextual answers that synthesize multiple memories:
+
+```bash
+hindsight-embed memory reflect default "How should I set up the dev environment?"
+```
+
+### bank list
+
+List all memory banks:
+
+```bash
+hindsight-embed bank list
+```
+
+### daemon
+
+Manage the background daemon:
+
+```bash
+hindsight-embed daemon status    # Check if daemon is running
+hindsight-embed daemon start     # Start the daemon
+hindsight-embed daemon stop      # Stop the daemon
+hindsight-embed daemon logs      # View last 50 lines of logs
+hindsight-embed daemon logs -f   # Follow logs in real-time
+hindsight-embed daemon logs -n 100  # View last 100 lines
+```
+
+## Configuration
+
+### Interactive Setup
+
+Run `hindsight-embed configure` for a guided setup that saves to `~/.hindsight/embed`.
+
+### Environment Variables
 
 | Variable | Description | Default |
 |----------|-------------|---------|
 | `HINDSIGHT_EMBED_LLM_API_KEY` | LLM API key (or use `OPENAI_API_KEY`) | Required |
-| `HINDSIGHT_EMBED_LLM_PROVIDER` | LLM provider (`openai`, `anthropic`, `google`, `ollama`) | `openai` |
+| `HINDSIGHT_EMBED_LLM_PROVIDER` | LLM provider (`openai`, `groq`, `google`, `ollama`) | `openai` |
 | `HINDSIGHT_EMBED_LLM_MODEL` | LLM model | `gpt-4o-mini` |
 | `HINDSIGHT_EMBED_BANK_ID` | Memory bank ID | `default` |
 
+### Files
+
+| Path | Description |
+|------|-------------|
+| `~/.hindsight/embed` | Configuration file |
+| `~/.hindsight/config.env` | Alternative config file location |
+| `~/.hindsight/daemon.log` | Daemon logs |
+| `~/.hindsight/daemon.lock` | Daemon lock file (PID) |
+
 ## Use with AI Coding Assistants
 
-This CLI is designed to work with AI coding assistants like Claude Code, OpenCode, and Codex CLI. Install the Hindsight skill:
+This CLI is designed to work with AI coding assistants like Claude Code, Cursor, and Windsurf. Install the Hindsight skill:
 
 ```bash
 curl -fsSL https://hindsight.vectorize.io/get-skill | bash
@@ -65,6 +141,27 @@ curl -fsSL https://hindsight.vectorize.io/get-skill | bash
 
 This will configure the LLM provider and install the skill to your assistant's skills directory.
 
+## Troubleshooting
+
+**Daemon won't start:**
+```bash
+# Check logs for errors
+hindsight-embed daemon logs
+
+# Stop any stuck daemon and restart
+hindsight-embed daemon stop
+hindsight-embed daemon start
+```
+
+**Slow first command:**
+This is expected - the first command needs to download dependencies, start the daemon, and load ML models. First run can take 1-3 minutes depending on network speed. Subsequent commands will be fast (~1-2s).
+
+**Change configuration:**
+```bash
+# Re-run configure (automatically restarts daemon)
+hindsight-embed configure
+```
+
 ## License
 
 Apache 2.0

hindsight-embed/hindsight_embed/cli.py

@@ -1,24 +1,23 @@
 """
 Hindsight Embedded CLI.
 
-A simple CLI for local memory operations using embedded PostgreSQL (pg0).
-No external server required - runs everything locally.
+A wrapper CLI that manages a local daemon and forwards commands to hindsight-cli.
+No external server required - runs everything locally with automatic daemon management.
 
 Usage:
     hindsight-embed configure              # Interactive setup
     hindsight-embed retain "User prefers dark mode"
     hindsight-embed recall "What are user preferences?"
+    hindsight-embed daemon status          # Check daemon status
 
 Environment variables:
     HINDSIGHT_EMBED_LLM_API_KEY: Required. API key for LLM provider.
     HINDSIGHT_EMBED_LLM_PROVIDER: Optional. LLM provider (default: "openai").
     HINDSIGHT_EMBED_LLM_MODEL: Optional. LLM model (default: "gpt-4o-mini").
     HINDSIGHT_EMBED_BANK_ID: Optional. Memory bank ID (default: "default").
-    HINDSIGHT_EMBED_LOG_LEVEL: Optional. Log level (default: "warning").
 """
 
 import argparse
-import asyncio
 import logging
 import os
 import sys
@@ -26,11 +25,12 @@ from pathlib import Path
 
 CONFIG_DIR = Path.home() / ".hindsight"
 CONFIG_FILE = CONFIG_DIR / "embed"
+CONFIG_FILE_ALT = CONFIG_DIR / "config.env"  # Alternative config file location
 
 
 def setup_logging(verbose: bool = False):
     """Configure logging."""
-    level_str = os.environ.get("HINDSIGHT_EMBED_LOG_LEVEL", "warning").lower()
+    level_str = os.environ.get("HINDSIGHT_EMBED_LOG_LEVEL", "info").lower()
     if verbose:
         level_str = "debug"
 
@@ -40,7 +40,7 @@ def setup_logging(verbose: bool = False):
         "warning": logging.WARNING,
         "error": logging.ERROR,
     }
-    level = level_map.get(level_str, logging.WARNING)
+    level = level_map.get(level_str, logging.INFO)
 
     logging.basicConfig(
         level=level,
@@ -52,17 +52,20 @@ def setup_logging(verbose: bool = False):
 
 def load_config_file():
     """Load configuration from file if it exists."""
-    if CONFIG_FILE.exists():
-        with open(CONFIG_FILE) as f:
-            for line in f:
-                line = line.strip()
-                if line and not line.startswith("#") and "=" in line:
-                    # Handle 'export VAR=value' format
-                    if line.startswith("export "):
-                        line = line[7:]
-                    key, value = line.split("=", 1)
-                    if key not in os.environ:  # Don't override env vars
-                        os.environ[key] = value
+    # Check both config file locations
+    config_files = [CONFIG_FILE, CONFIG_FILE_ALT]
+    for config_path in config_files:
+        if config_path.exists():
+            with open(config_path) as f:
+                for line in f:
+                    line = line.strip()
+                    if line and not line.startswith("#") and "=" in line:
+                        # Handle 'export VAR=value' format
+                        if line.startswith("export "):
+                            line = line[7:]
+                        key, value = line.split("=", 1)
+                        if key not in os.environ:  # Don't override env vars
+                            os.environ[key] = value
 
 
 def get_config():
@@ -85,6 +88,16 @@ def do_configure(args):
     import questionary
     from questionary import Style
 
+    # If stdin is not a terminal (e.g., running via curl | bash),
+    # reopen stdin from /dev/tty for interactive prompts
+    if not sys.stdin.isatty():
+        try:
+            sys.stdin = open('/dev/tty', 'r')
+        except OSError:
+            print("Error: Cannot run interactive configuration without a terminal.", file=sys.stderr)
+            print("Run directly: uvx hindsight-embed configure", file=sys.stderr)
+            return 1
+
     # Custom style for the prompts
     custom_style = Style([
         ('qmark', 'fg:cyan bold'),
@@ -197,6 +210,25 @@ def do_configure(args):
 
     CONFIG_FILE.chmod(0o600)
 
+    # Stop existing daemon if running (it needs to pick up new config)
+    from . import daemon_client
+
+    if daemon_client._is_daemon_running():
+        print("\n  \033[2mRestarting daemon with new configuration...\033[0m")
+        daemon_client.stop_daemon()
+
+    # Start daemon with new config
+    new_config = {
+        "llm_api_key": api_key,
+        "llm_provider": provider,
+        "llm_model": model,
+        "bank_id": bank_id,
+    }
+    if daemon_client.ensure_daemon_running(new_config):
+        print("  \033[32m✓ Daemon started\033[0m")
+    else:
+        print("  \033[33m⚠ Failed to start daemon (will start on first command)\033[0m")
+
     print()
     print("\033[32m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\033[0m")
     print("\033[32m  ✓ Configuration saved!\033[0m")
@@ -212,211 +244,173 @@ def do_configure(args):
     return 0
 
 
-async def _create_engine(config: dict, logger):
-    """Create and initialize the memory engine."""
-    logger.debug("Setting up environment variables...")
+def do_daemon(args, config: dict, logger):
+    """Handle daemon subcommands."""
+    from pathlib import Path
+    from . import daemon_client
 
-    # Set hindsight-api environment variables from our config
-    if config["llm_api_key"]:
-        os.environ["HINDSIGHT_API_LLM_API_KEY"] = config["llm_api_key"]
-    if config["llm_provider"]:
-        os.environ["HINDSIGHT_API_LLM_PROVIDER"] = config["llm_provider"]
-    if config["llm_model"]:
-        os.environ["HINDSIGHT_API_LLM_MODEL"] = config["llm_model"]
+    daemon_log_path = Path.home() / ".hindsight" / "daemon.log"
 
-    logger.debug("Importing MemoryEngine...")
+    if args.daemon_command == "start":
+        if daemon_client._is_daemon_running():
+            print("Daemon is already running")
+            return 0
 
-    # Import after setting env vars
-    from hindsight_api import MemoryEngine
-    from hindsight_api.engine.task_backend import SyncTaskBackend
-
-    # Use pg0 embedded database
-    db_name = f"hindsight-embed-{config['bank_id']}"
-    logger.debug(f"Creating MemoryEngine with pg0://{db_name}")
-
-    # Use SyncTaskBackend to avoid background workers that prevent clean exit
-    memory = MemoryEngine(
-        db_url=f"pg0://{db_name}",
-        task_backend=SyncTaskBackend(),
-    )
-
-    logger.debug("Initializing engine...")
-    await memory.initialize()
-
-    logger.debug("Engine initialized")
-    return memory
-
-
-async def do_retain(args, config: dict, logger):
-    """Execute retain command."""
-    from hindsight_api.models import RequestContext
-
-    logger.info(f"Retaining memory: {args.content[:50]}...")
-
-    memory = await _create_engine(config, logger)
-
-    try:
-        logger.debug("Calling retain_batch_async...")
-        await memory.retain_batch_async(
-            bank_id=config["bank_id"],
-            contents=[{
-                "content": args.content,
-                "context": args.context or "general",
-            }],
-            request_context=RequestContext(),
-        )
-        msg = f"Stored memory: {args.content[:50]}..." if len(args.content) > 50 else f"Stored memory: {args.content}"
-        print(msg, flush=True)
-        return 0
-    except Exception as e:
-        logger.error(f"Retain failed: {e}", exc_info=True)
-        print(f"Error: {e}", file=sys.stderr)
-        return 1
-
-
-async def do_recall(args, config: dict, logger):
-    """Execute recall command."""
-    from hindsight_api.engine.memory_engine import Budget
-    from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES
-    from hindsight_api.models import RequestContext
-
-    logger.info(f"Recalling with query: {args.query}")
-
-    memory = await _create_engine(config, logger)
-
-    try:
-        budget_map = {"low": Budget.LOW, "mid": Budget.MID, "high": Budget.HIGH}
-        budget_enum = budget_map.get(args.budget.lower(), Budget.LOW)
-
-        logger.debug(f"Calling recall_async with budget={budget_enum}...")
-        result = await memory.recall_async(
-            bank_id=config["bank_id"],
-            query=args.query,
-            fact_type=list(VALID_RECALL_FACT_TYPES),
-            budget=budget_enum,
-            max_tokens=args.max_tokens,
-            request_context=RequestContext(),
-        )
-
-        logger.debug(f"Recall returned {len(result.results)} results")
-
-        if result.results:
-            print("Memories found:", flush=True)
-            print("-" * 40, flush=True)
-            for fact in result.results:
-                print(f"- {fact.text}", flush=True)
-                if args.verbose and fact.occurred_start:
-                    print(f"  (Date: {fact.occurred_start})", flush=True)
-            print("-" * 40, flush=True)
-            print(f"Total: {len(result.results)} memories", flush=True)
+        print("Starting daemon...")
+        if daemon_client.ensure_daemon_running(config):
+            print("Daemon started successfully")
+            print(f"  Port: {daemon_client.DAEMON_PORT}")
+            print(f"  Logs: {daemon_log_path}")
+            return 0
         else:
-            print("No relevant memories found.", flush=True)
+            print("Failed to start daemon", file=sys.stderr)
+            return 1
 
-        return 0
-    except Exception as e:
-        logger.error(f"Recall failed: {e}", exc_info=True)
-        print(f"Error: {e}", file=sys.stderr)
+    elif args.daemon_command == "stop":
+        if not daemon_client._is_daemon_running():
+            print("Daemon is not running")
+            return 0
+
+        print("Stopping daemon...")
+        if daemon_client.stop_daemon():
+            print("Daemon stopped")
+            return 0
+        else:
+            print("Failed to stop daemon", file=sys.stderr)
+            return 1
+
+    elif args.daemon_command == "status":
+        if daemon_client._is_daemon_running():
+            # Get PID from lockfile
+            lockfile = Path.home() / ".hindsight" / "daemon.lock"
+            pid = "unknown"
+            if lockfile.exists():
+                try:
+                    pid = lockfile.read_text().strip()
+                except Exception:
+                    pass
+            print(f"Daemon is running (PID: {pid})")
+            print(f"  URL: http://127.0.0.1:{daemon_client.DAEMON_PORT}")
+            print(f"  Logs: {daemon_log_path}")
+            return 0
+        else:
+            print("Daemon is not running")
+            return 1
+
+    elif args.daemon_command == "logs":
+        if not daemon_log_path.exists():
+            print("No daemon logs found", file=sys.stderr)
+            print(f"  Expected at: {daemon_log_path}")
+            return 1
+
+        if args.follow:
+            # Follow mode - like tail -f
+            import subprocess
+            try:
+                subprocess.run(["tail", "-f", str(daemon_log_path)])
+            except KeyboardInterrupt:
+                pass
+            return 0
+        else:
+            # Show last N lines
+            try:
+                with open(daemon_log_path) as f:
+                    lines = f.readlines()
+                    for line in lines[-args.lines:]:
+                        print(line, end="")
+                return 0
+            except Exception as e:
+                print(f"Error reading logs: {e}", file=sys.stderr)
+                return 1
+
+    else:
+        print("Usage: hindsight-embed daemon {start|stop|status|logs}", file=sys.stderr)
         return 1
 
 
 def main():
     """Main entry point."""
-    parser = argparse.ArgumentParser(
-        description="Hindsight Embedded CLI - local memory operations without a server",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Examples:
-    hindsight-embed configure                              # Interactive setup
-    hindsight-embed retain "User prefers dark mode"
-    hindsight-embed retain "Meeting on Monday" -c work
-    hindsight-embed recall "user preferences"
-    hindsight-embed recall "meetings" --budget high
-        """
-    )
+    # Check for built-in commands first (before argparse)
+    # This allows us to forward unknown commands to hindsight-cli
+    if len(sys.argv) > 1:
+        command = sys.argv[1]
 
-    parser.add_argument(
-        "--verbose", "-v",
-        action="store_true",
-        help="Enable verbose/debug logging"
-    )
+        # Handle configure
+        if command == "configure":
+            logger = setup_logging(False)
+            exit_code = do_configure(None)
+            sys.exit(exit_code)
 
-    subparsers = parser.add_subparsers(dest="command", help="Commands")
+        # Handle daemon subcommands
+        if command == "daemon":
+            # Parse daemon subcommand
+            parser = argparse.ArgumentParser(prog="hindsight-embed daemon")
+            subparsers = parser.add_subparsers(dest="daemon_command")
+            subparsers.add_parser("start", help="Start the daemon")
+            subparsers.add_parser("stop", help="Stop the daemon")
+            subparsers.add_parser("status", help="Check daemon status")
+            logs_parser = subparsers.add_parser("logs", help="View daemon logs")
+            logs_parser.add_argument("--follow", "-f", action="store_true")
+            logs_parser.add_argument("--lines", "-n", type=int, default=50)
 
-    # Configure command
-    subparsers.add_parser("configure", help="Interactive configuration setup")
+            args = parser.parse_args(sys.argv[2:])
+            logger = setup_logging(False)
+            config = get_config()
+            exit_code = do_daemon(args, config, logger)
+            sys.exit(exit_code)
 
-    # Retain command
-    retain_parser = subparsers.add_parser("retain", help="Store a memory")
-    retain_parser.add_argument("content", help="The memory content to store")
-    retain_parser.add_argument(
-        "--context", "-c",
-        help="Category for the memory (e.g., 'preferences', 'work')",
-        default="general"
-    )
+        # Handle --help / -h
+        if command in ("--help", "-h"):
+            print_help()
+            sys.exit(0)
 
-    # Recall command
-    recall_parser = subparsers.add_parser("recall", help="Search memories")
-    recall_parser.add_argument("query", help="Search query")
-    recall_parser.add_argument(
-        "--budget", "-b",
-        choices=["low", "mid", "high"],
-        default="low",
-        help="Search budget level (default: low)"
-    )
-    recall_parser.add_argument(
-        "--max-tokens", "-m",
-        type=int,
-        default=4096,
-        help="Maximum tokens in results (default: 4096)"
-    )
-    recall_parser.add_argument(
-        "--verbose", "-v",
-        action="store_true",
-        help="Show additional details"
-    )
+        # Forward all other commands to hindsight-cli
+        config = get_config()
 
-    args = parser.parse_args()
+        # Check for LLM API key
+        if not config["llm_api_key"]:
+            print("Error: LLM API key is required.", file=sys.stderr)
+            print("Run 'hindsight-embed configure' to set up.", file=sys.stderr)
+            sys.exit(1)
 
-    # Setup logging
-    verbose = getattr(args, 'verbose', False)
-    logger = setup_logging(verbose)
+        from . import daemon_client
 
-    if not args.command:
-        parser.print_help()
-        sys.exit(1)
-
-    # Handle configure separately (no config needed)
-    if args.command == "configure":
-        exit_code = do_configure(args)
+        # Forward to hindsight-cli (handles daemon startup and CLI installation)
+        exit_code = daemon_client.run_cli(sys.argv[1:], config)
         sys.exit(exit_code)
 
-    config = get_config()
+    # No command - show help
+    print_help()
+    sys.exit(1)
 
-    # Check for LLM API key
-    if not config["llm_api_key"]:
-        print("Error: LLM API key is required.", file=sys.stderr)
-        print("Run 'hindsight-embed configure' to set up.", file=sys.stderr)
-        sys.exit(1)
 
-    # Run the appropriate command
-    exit_code = 1
-    try:
-        if args.command == "retain":
-            exit_code = asyncio.run(do_retain(args, config, logger))
-        elif args.command == "recall":
-            exit_code = asyncio.run(do_recall(args, config, logger))
-        else:
-            parser.print_help()
-            exit_code = 1
-    except KeyboardInterrupt:
-        logger.debug("Interrupted")
-        exit_code = 130
-    except Exception as e:
-        logger.error(f"Unexpected error: {e}", exc_info=True)
-        print(f"Error: {e}", file=sys.stderr)
-        exit_code = 1
+def print_help():
+    """Print help message."""
+    print("""Hindsight Embedded CLI - local memory operations with automatic daemon management.
 
-    sys.exit(exit_code)
+Usage: hindsight-embed <command> [options]
+
+Built-in commands:
+    configure              Interactive configuration setup
+    daemon start           Start the background daemon
+    daemon stop            Stop the daemon
+    daemon status          Check daemon status
+    daemon logs [-f] [-n]  View daemon logs
+
+CLI commands (forwarded to hindsight-cli):
+    memory retain <bank> <content>   Store a memory
+    memory recall <bank> <query>     Search memories
+    memory reflect <bank> <query>    Generate contextual answer
+    bank list                        List memory banks
+    ...                              Run 'hindsight --help' for all commands
+
+Examples:
+    hindsight-embed configure
+    hindsight-embed memory retain default "User prefers dark mode"
+    hindsight-embed memory recall default "user preferences"
+    hindsight-embed daemon status
+    hindsight-embed daemon logs -f
+""")
 
 
 if __name__ == "__main__":

hindsight-embed/hindsight_embed/daemon_client.py

@@ -0,0 +1,298 @@
+"""
+Client for communicating with the Hindsight daemon.
+
+Handles daemon lifecycle (start if needed) and API requests via the Python client.
+"""
+
+import logging
+import os
+import subprocess
+import time
+from pathlib import Path
+
+import httpx  # Used only for health check
+
+logger = logging.getLogger(__name__)
+
+DAEMON_PORT = 8889
+DAEMON_URL = f"http://127.0.0.1:{DAEMON_PORT}"
+DAEMON_STARTUP_TIMEOUT = 180  # seconds - needs to be long for first run (downloads dependencies)
+DAEMON_IDLE_TIMEOUT = 300  # 5 minutes - auto-exit after idle
+
+# CLI paths - check multiple locations
+CLI_INSTALL_DIRS = [
+    Path.home() / ".local" / "bin",  # Standard location from get-cli installer
+    Path.home() / ".hindsight" / "bin",  # Alternative location
+]
+CLI_INSTALLER_URL = "https://hindsight.vectorize.io/get-cli"
+
+
+def _find_hindsight_api_command() -> list[str]:
+    """Find the command to run hindsight-api."""
+    # Check if we're in development mode (local hindsight-api available)
+    # Path: daemon_client.py -> hindsight_embed/ -> hindsight-embed/ -> memory-poc/
+    dev_api_path = Path(__file__).parent.parent.parent / "hindsight-api"
+    if dev_api_path.exists() and (dev_api_path / "pyproject.toml").exists():
+        # Use uv run with the local project
+        return ["uv", "run", "--project", str(dev_api_path), "hindsight-api"]
+
+    # Fall back to uvx for installed version
+    return ["uvx", "hindsight-api"]
+
+
+def _is_daemon_running() -> bool:
+    """Check if daemon is running and responsive."""
+    try:
+        with httpx.Client(timeout=2) as client:
+            response = client.get(f"{DAEMON_URL}/health")
+            return response.status_code == 200
+    except Exception:
+        return False
+
+
+def _start_daemon(config: dict) -> bool:
+    """
+    Start the daemon in background.
+
+    Returns True if daemon started successfully.
+    """
+    import sys
+
+    logger.info("Starting daemon...")
+
+    # Build environment with LLM config
+    env = os.environ.copy()
+    if config.get("llm_api_key"):
+        env["HINDSIGHT_API_LLM_API_KEY"] = config["llm_api_key"]
+    if config.get("llm_provider"):
+        env["HINDSIGHT_API_LLM_PROVIDER"] = config["llm_provider"]
+    if config.get("llm_model"):
+        env["HINDSIGHT_API_LLM_MODEL"] = config["llm_model"]
+
+    # Use pg0 database specific to bank
+    bank_id = config.get("bank_id", "default")
+    env["HINDSIGHT_API_DATABASE_URL"] = f"pg0://hindsight-embed-{bank_id}"
+
+    # Optimization flags for faster startup
+    env["HINDSIGHT_API_SKIP_LLM_VERIFICATION"] = "true"
+    env["HINDSIGHT_API_LOG_LEVEL"] = "warning"
+
+    cmd = _find_hindsight_api_command() + ["--daemon", "--idle-timeout", str(DAEMON_IDLE_TIMEOUT)]
+
+    # Create log directory
+    log_dir = Path.home() / ".hindsight"
+    log_dir.mkdir(parents=True, exist_ok=True)
+    daemon_log = log_dir / "daemon.log"
+    daemon_stderr = log_dir / "daemon.stderr"
+
+    print(f"Starting daemon with command: {' '.join(cmd)}", file=sys.stderr)
+    print(f"  Log file: {daemon_log}", file=sys.stderr)
+
+    try:
+        # Start daemon in background, but capture initial stderr for debugging
+        with open(daemon_stderr, "w") as stderr_file:
+            process = subprocess.Popen(
+                cmd,
+                env=env,
+                stdout=subprocess.DEVNULL,
+                stderr=stderr_file,
+                start_new_session=True,
+            )
+
+        # Wait for daemon to be ready
+        # Note: With --daemon flag, the parent process forks and exits immediately (code 0).
+        # The child process (actual daemon) continues running. So we can't rely on process.poll()
+        # to detect failures - we must use the health check.
+        start_time = time.time()
+        last_check_time = start_time
+        while time.time() - start_time < DAEMON_STARTUP_TIMEOUT:
+            if _is_daemon_running():
+                logger.info("Daemon started successfully")
+                return True
+
+            # Periodically log progress
+            if time.time() - last_check_time > 5:
+                elapsed = int(time.time() - start_time)
+                print(f"  Still waiting for daemon... ({elapsed}s elapsed)", file=sys.stderr)
+                last_check_time = time.time()
+
+            time.sleep(0.5)
+
+        logger.error("Daemon failed to start within timeout")
+        # Show logs on timeout
+        if daemon_log.exists():
+            log_content = daemon_log.read_text()
+            if log_content:
+                print(f"Daemon log:\n{log_content[-2000:]}", file=sys.stderr)  # Last 2000 chars
+        if daemon_stderr.exists():
+            stderr_content = daemon_stderr.read_text()
+            if stderr_content:
+                print(f"Daemon stderr:\n{stderr_content}", file=sys.stderr)
+        return False
+
+    except FileNotFoundError as e:
+        print(f"Command not found: {cmd[0]}", file=sys.stderr)
+        print(f"  Full command: {' '.join(cmd)}", file=sys.stderr)
+        logger.error("hindsight-api command not found. Install with: pip install hindsight-api")
+        return False
+    except Exception as e:
+        print(f"Failed to start daemon: {e}", file=sys.stderr)
+        logger.error(f"Failed to start daemon: {e}")
+        return False
+
+
+def ensure_daemon_running(config: dict) -> bool:
+    """
+    Ensure daemon is running, starting it if needed.
+
+    Returns True if daemon is running.
+    """
+    if _is_daemon_running():
+        logger.debug("Daemon already running")
+        return True
+
+    return _start_daemon(config)
+
+
+def stop_daemon() -> bool:
+    """Stop the running daemon."""
+    # Try to kill by PID from lockfile
+    lockfile = Path.home() / ".hindsight" / "daemon.lock"
+    if lockfile.exists():
+        try:
+            pid = int(lockfile.read_text().strip())
+            os.kill(pid, 15)  # SIGTERM
+            # Wait for process to exit
+            for _ in range(50):
+                time.sleep(0.1)
+                try:
+                    os.kill(pid, 0)
+                except OSError:
+                    return True
+        except (ValueError, OSError):
+            pass
+
+    return not _is_daemon_running()
+
+
+def find_cli_binary() -> Path | None:
+    """Find the hindsight CLI binary in known locations or PATH."""
+    import shutil
+
+    # Check standard install locations
+    for install_dir in CLI_INSTALL_DIRS:
+        binary = install_dir / "hindsight"
+        if binary.exists() and os.access(binary, os.X_OK):
+            return binary
+
+    # Check PATH
+    path_binary = shutil.which("hindsight")
+    if path_binary:
+        return Path(path_binary)
+
+    return None
+
+
+def is_cli_installed() -> bool:
+    """Check if the hindsight CLI is installed."""
+    return find_cli_binary() is not None
+
+
+def install_cli() -> bool:
+    """
+    Install the hindsight CLI using the official installer.
+
+    Returns True if installation succeeded.
+    """
+    import subprocess
+    import sys
+
+    print("Installing hindsight CLI...")
+    print(f"  Installer URL: {CLI_INSTALLER_URL}")
+
+    try:
+        # Download and run installer
+        result = subprocess.run(
+            ["bash", "-c", f"curl -fsSL {CLI_INSTALLER_URL} | bash"],
+            capture_output=True,
+            text=True,
+        )
+
+        if result.returncode != 0:
+            print(f"CLI installation failed (exit code {result.returncode}):", file=sys.stderr)
+            if result.stdout:
+                print(f"  stdout: {result.stdout}", file=sys.stderr)
+            if result.stderr:
+                print(f"  stderr: {result.stderr}", file=sys.stderr)
+            return False
+
+        cli_binary = find_cli_binary()
+        if cli_binary:
+            print(f"CLI installed to {cli_binary}")
+            return True
+        else:
+            print("CLI installation completed but binary not found", file=sys.stderr)
+            print(f"  stdout: {result.stdout}", file=sys.stderr)
+            print(f"  stderr: {result.stderr}", file=sys.stderr)
+            # Check known locations
+            for install_dir in CLI_INSTALL_DIRS:
+                binary = install_dir / "hindsight"
+                print(f"  Checking {binary}: exists={binary.exists()}", file=sys.stderr)
+            return False
+
+    except Exception as e:
+        print(f"CLI installation failed: {e}", file=sys.stderr)
+        return False
+
+
+def ensure_cli_installed() -> bool:
+    """Ensure CLI is installed, installing if needed."""
+    if is_cli_installed():
+        return True
+    return install_cli()
+
+
+def run_cli(args: list[str], config: dict) -> int:
+    """
+    Run the hindsight CLI with the given arguments.
+
+    Ensures daemon is running and passes the API URL.
+
+    Args:
+        args: CLI arguments (e.g., ["memory", "retain", "bank", "content"])
+        config: Configuration dict with llm settings
+
+    Returns:
+        Exit code from CLI
+    """
+    import subprocess
+    import sys
+
+    # Ensure CLI is installed
+    if not ensure_cli_installed():
+        return 1
+
+    cli_binary = find_cli_binary()
+    if not cli_binary:
+        print("Error: hindsight CLI not found", file=sys.stderr)
+        return 1
+
+    # Ensure daemon is running
+    if not ensure_daemon_running(config):
+        print("Error: Failed to start daemon", file=sys.stderr)
+        return 1
+
+    # Build environment with API URL pointing to daemon
+    env = os.environ.copy()
+    env["HINDSIGHT_API_URL"] = DAEMON_URL
+
+    # Run CLI
+    try:
+        result = subprocess.run(
+            [str(cli_binary)] + args,
+            env=env,
+        )
+        return result.returncode
+    except Exception as e:
+        print(f"Error running CLI: {e}", file=sys.stderr)
+        return 1

hindsight-embed/pyproject.toml

@@ -9,8 +9,8 @@ description = "Hindsight embedded CLI - local memory operations without a server
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "hindsight-api>=0.1.11",
     "questionary>=2.0.0",
+    "httpx>=0.27.0",
 ]
 
 [project.scripts]
@@ -18,6 +18,3 @@ hindsight-embed = "hindsight_embed.cli:main"
 
 [tool.hatch.build.targets.wheel]
 packages = ["hindsight_embed"]
-
-[tool.uv.sources]
-hindsight-api = { workspace = true }

hindsight-embed/test.sh

@@ -1,68 +1,236 @@
 #!/bin/bash
 #
-# Simple smoke test for hindsight-embed CLI
-# Tests retain and recall operations with embedded PostgreSQL
+# Smoke test for hindsight-embed CLI with daemon mode
+# Tests retain and recall operations via the background daemon
 #
 
 set -e
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+API_DIR="$(cd "$SCRIPT_DIR/../hindsight-api" && pwd)"
 
-echo "=== Hindsight Embed Smoke Test ==="
+echo "=== Hindsight Embed Smoke Test (Daemon Mode) ==="
 
-# Check required environment
-if [ -z "$HINDSIGHT_EMBED_LLM_API_KEY" ]; then
-    echo "Error: HINDSIGHT_EMBED_LLM_API_KEY is required"
+# Check required environment (load from config if not set)
+if [ -f ~/.hindsight/config.env ]; then
+    source ~/.hindsight/config.env
+fi
+
+if [ -z "$HINDSIGHT_EMBED_LLM_API_KEY" ] && [ -z "$OPENAI_API_KEY" ]; then
+    echo "Error: HINDSIGHT_EMBED_LLM_API_KEY or OPENAI_API_KEY is required"
     exit 1
 fi
 
 # Use a unique bank ID for this test run
-export HINDSIGHT_EMBED_BANK_ID="test-$$-$(date +%s)"
-echo "Using bank ID: $HINDSIGHT_EMBED_BANK_ID"
+BANK_ID="test-$$-$(date +%s)"
+echo "Using bank ID: $BANK_ID"
+echo "Script dir: $SCRIPT_DIR"
+echo "API dir: $API_DIR"
 
-# Test 1: Retain a memory
+# Debug: Check if hindsight CLI is available
 echo ""
-echo "Test 1: Retaining a memory..."
-OUTPUT=$(uv run --project "$SCRIPT_DIR" hindsight-embed retain "The user's favorite color is blue" 2>&1)
+echo "Checking hindsight CLI availability..."
+if command -v hindsight &> /dev/null; then
+    echo "  hindsight CLI found at: $(which hindsight)"
+else
+    echo "  hindsight CLI not in PATH"
+    if [ -f ~/.local/bin/hindsight ]; then
+        echo "  Found at ~/.local/bin/hindsight"
+    else
+        echo "  Not found at ~/.local/bin/hindsight - attempting installation..."
+        # Try to install the CLI
+        if curl -fsSL https://hindsight.vectorize.io/get-cli | bash; then
+            echo "  CLI installation completed"
+            if [ -f ~/.local/bin/hindsight ]; then
+                echo "  CLI now available at ~/.local/bin/hindsight"
+                export PATH="$HOME/.local/bin:$PATH"
+            else
+                echo "  WARNING: CLI still not found after installation"
+                ls -la ~/.local/bin/ 2>/dev/null || echo "  ~/.local/bin does not exist"
+            fi
+        else
+            echo "  WARNING: CLI installation failed"
+        fi
+    fi
+fi
+
+# Show environment info for debugging
+echo ""
+echo "Environment:"
+echo "  HINDSIGHT_EMBED_LLM_PROVIDER: ${HINDSIGHT_EMBED_LLM_PROVIDER:-not set}"
+echo "  HINDSIGHT_EMBED_LLM_MODEL: ${HINDSIGHT_EMBED_LLM_MODEL:-not set}"
+echo "  HINDSIGHT_EMBED_LLM_API_KEY: ${HINDSIGHT_EMBED_LLM_API_KEY:+set (hidden)}"
+echo "  PATH includes ~/.local/bin: $(echo $PATH | grep -q "$HOME/.local/bin" && echo yes || echo no)"
+
+# Final check that CLI is available before proceeding
+echo ""
+echo "Final CLI check before tests..."
+CLI_PATH=""
+if [ -f ~/.local/bin/hindsight ]; then
+    CLI_PATH="$HOME/.local/bin/hindsight"
+elif command -v hindsight &> /dev/null; then
+    CLI_PATH="$(which hindsight)"
+fi
+
+if [ -n "$CLI_PATH" ]; then
+    echo "  CLI found at: $CLI_PATH"
+    echo "  CLI version: $($CLI_PATH --version 2>&1 || echo 'unknown')"
+else
+    echo "  ERROR: hindsight CLI not found. Tests cannot proceed."
+    echo "  The hindsight-embed package forwards commands to the hindsight CLI."
+    echo "  Please ensure the CLI is installed or check the get-cli installer output above."
+    exit 1
+fi
+
+# Stop any existing daemon
+echo ""
+echo "Stopping any existing daemon..."
+uv run --project "$SCRIPT_DIR" hindsight-embed daemon stop 2>/dev/null || true
+sleep 1
+
+# Test 1: Retain (this should start the daemon)
+echo ""
+echo "Test 1: Retaining a memory (first call - daemon will start)..."
+START_TIME=$(python3 -c "import time; print(time.time())")
+set +e  # Temporarily disable exit on error to capture output
+OUTPUT=$(uv run --project "$SCRIPT_DIR" hindsight-embed memory retain "$BANK_ID" "The user's favorite color is blue" 2>&1)
+EXIT_CODE=$?
+set -e
+END_TIME=$(python3 -c "import time; print(time.time())")
+DURATION=$(python3 -c "print(f'{$END_TIME - $START_TIME:.2f}')")
 echo "$OUTPUT"
-if ! echo "$OUTPUT" | grep -q "Stored memory"; then
-    echo "FAIL: Expected 'Stored memory' in output"
+echo "Duration: ${DURATION}s"
+echo "Exit code: $EXIT_CODE"
+
+if [ $EXIT_CODE -ne 0 ]; then
+    echo "FAIL: Command exited with code $EXIT_CODE"
+    echo ""
+    echo "Checking daemon logs..."
+    if [ -f ~/.hindsight/daemon.log ]; then
+        echo "=== daemon.log ==="
+        tail -50 ~/.hindsight/daemon.log
+    else
+        echo "No daemon.log found"
+    fi
+    if [ -f ~/.hindsight/daemon.stderr ]; then
+        echo ""
+        echo "=== daemon.stderr ==="
+        cat ~/.hindsight/daemon.stderr
+    else
+        echo "No daemon.stderr found"
+    fi
+    echo ""
+    echo "Checking for hindsight-api..."
+    which hindsight-api 2>/dev/null || echo "hindsight-api not in PATH"
+    which uvx 2>/dev/null || echo "uvx not in PATH"
+    which uv 2>/dev/null || echo "uv not in PATH"
+    exit 1
+fi
+
+if ! echo "$OUTPUT" | grep -qi "retained"; then
+    echo "FAIL: Expected 'retained' in output"
     exit 1
 fi
 echo "PASS: Memory retained successfully"
 
-# Test 2: Recall the memory
+# Test 2: Recall (daemon already running - should be faster)
 echo ""
-echo "Test 2: Recalling memories..."
-OUTPUT=$(uv run --project "$SCRIPT_DIR" hindsight-embed recall "What is the user's favorite color?" 2>&1)
+echo "Test 2: Recalling memories (daemon already running)..."
+START_TIME=$(python3 -c "import time; print(time.time())")
+set +e
+OUTPUT=$(uv run --project "$SCRIPT_DIR" hindsight-embed memory recall "$BANK_ID" "What is the user's favorite color?" 2>&1)
+EXIT_CODE=$?
+set -e
+END_TIME=$(python3 -c "import time; print(time.time())")
+DURATION=$(python3 -c "print(f'{$END_TIME - $START_TIME:.2f}')")
 echo "$OUTPUT"
+echo "Duration: ${DURATION}s"
+echo "Exit code: $EXIT_CODE"
+if [ $EXIT_CODE -ne 0 ]; then
+    echo "FAIL: Command exited with code $EXIT_CODE"
+    exit 1
+fi
 if ! echo "$OUTPUT" | grep -qi "blue"; then
     echo "FAIL: Expected 'blue' in recall output"
     exit 1
 fi
 echo "PASS: Memory recalled successfully"
 
-# Test 3: Retain with context
+# Test 3: Retain with context (daemon should still be running)
 echo ""
 echo "Test 3: Retaining memory with context..."
-OUTPUT=$(uv run --project "$SCRIPT_DIR" hindsight-embed retain "User prefers Python over JavaScript" --context work 2>&1)
+START_TIME=$(python3 -c "import time; print(time.time())")
+set +e
+OUTPUT=$(uv run --project "$SCRIPT_DIR" hindsight-embed memory retain "$BANK_ID" "User prefers Python over JavaScript" --context work 2>&1)
+EXIT_CODE=$?
+set -e
+END_TIME=$(python3 -c "import time; print(time.time())")
+DURATION=$(python3 -c "print(f'{$END_TIME - $START_TIME:.2f}')")
 echo "$OUTPUT"
-if ! echo "$OUTPUT" | grep -q "Stored memory"; then
-    echo "FAIL: Expected 'Stored memory' in output"
+echo "Duration: ${DURATION}s"
+echo "Exit code: $EXIT_CODE"
+if [ $EXIT_CODE -ne 0 ]; then
+    echo "FAIL: Command exited with code $EXIT_CODE"
+    exit 1
+fi
+if ! echo "$OUTPUT" | grep -qi "retained"; then
+    echo "FAIL: Expected 'retained' in output"
     exit 1
 fi
 echo "PASS: Memory with context retained successfully"
 
-# Test 4: Recall with budget
+# Test 4: Recall with JSON output
 echo ""
-echo "Test 4: Recalling with budget..."
-OUTPUT=$(uv run --project "$SCRIPT_DIR" hindsight-embed recall "programming preferences" --budget mid 2>&1)
-echo "$OUTPUT"
-if ! echo "$OUTPUT" | grep -qi "python"; then
+echo "Test 4: Recalling with JSON output..."
+START_TIME=$(python3 -c "import time; print(time.time())")
+set +e
+JSON_OUTPUT=$(uv run --project "$SCRIPT_DIR" hindsight-embed memory recall "$BANK_ID" "programming preferences" -o json 2>&1)
+EXIT_CODE=$?
+set -e
+END_TIME=$(python3 -c "import time; print(time.time())")
+DURATION=$(python3 -c "print(f'{$END_TIME - $START_TIME:.2f}')")
+echo "$JSON_OUTPUT"
+echo "Duration: ${DURATION}s"
+echo "Exit code: $EXIT_CODE"
+if [ $EXIT_CODE -ne 0 ]; then
+    echo "FAIL: Command exited with code $EXIT_CODE"
+    exit 1
+fi
+if ! echo "$JSON_OUTPUT" | grep -qi "python"; then
     echo "FAIL: Expected 'Python' in recall output"
     exit 1
 fi
-echo "PASS: Memory recalled with budget successfully"
+if ! echo "$JSON_OUTPUT" | python3 -c "import sys, json; json.load(sys.stdin)" 2>/dev/null; then
+    echo "FAIL: Expected valid JSON output"
+    exit 1
+fi
+echo "PASS: Memory recalled with JSON format successfully"
+
+# Test 5: Check daemon is running
+echo ""
+echo "Test 5: Verifying daemon is running..."
+if curl -s http://127.0.0.1:8889/health | grep -q "healthy"; then
+    echo "PASS: Daemon is running and healthy"
+else
+    echo "FAIL: Daemon is not running"
+    exit 1
+fi
+
+# Test 6: Daemon status command
+echo ""
+echo "Test 6: Testing daemon status command..."
+STATUS_OUTPUT=$(uv run --project "$SCRIPT_DIR" hindsight-embed daemon status 2>&1)
+echo "$STATUS_OUTPUT"
+if ! echo "$STATUS_OUTPUT" | grep -qi "running"; then
+    echo "FAIL: Expected 'running' in daemon status output"
+    exit 1
+fi
+echo "PASS: Daemon status command works"
+
+# Cleanup: Stop daemon
+echo ""
+echo "Stopping daemon..."
+uv run --project "$SCRIPT_DIR" hindsight-embed daemon stop 2>/dev/null || true
 
 echo ""
 echo "=== All tests passed! ==="

uv.lock

@@ -1292,7 +1292,7 @@ wheels = [
 
 [[package]]
 name = "hindsight-all"
-version = "0.1.11"
+version = "0.1.12"
 source = { editable = "hindsight" }
 dependencies = [
     { name = "hindsight-api" },
@@ -1316,7 +1316,7 @@ provides-extras = ["test"]
 
 [[package]]
 name = "hindsight-api"
-version = "0.1.11"
+version = "0.1.12"
 source = { editable = "hindsight-api" }
 dependencies = [
     { name = "alembic" },
@@ -1422,7 +1422,7 @@ dev = [
 
 [[package]]
 name = "hindsight-client"
-version = "0.1.11"
+version = "0.1.12"
 source = { editable = "hindsight-clients/python" }
 dependencies = [
     { name = "aiohttp" },
@@ -1456,7 +1456,7 @@ provides-extras = ["test"]
 
 [[package]]
 name = "hindsight-dev"
-version = "0.1.11"
+version = "0.1.12"
 source = { editable = "hindsight-dev" }
 dependencies = [
     { name = "hindsight-api" },
@@ -1494,13 +1494,13 @@ name = "hindsight-embed"
 version = "0.1.0"
 source = { editable = "hindsight-embed" }
 dependencies = [
-    { name = "hindsight-api" },
+    { name = "httpx" },
     { name = "questionary" },
 ]
 
 [package.metadata]
 requires-dist = [
-    { name = "hindsight-api", editable = "hindsight-api" },
+    { name = "httpx", specifier = ">=0.27.0" },
     { name = "questionary", specifier = ">=2.0.0" },
 ]