fix: clients don't respect timeout setting

* changelog for 0.4.12

* changelog for 0.4.12

.env.example

@@ -41,6 +41,12 @@ HINDSIGHT_API_LOG_LEVEL=info
 # HINDSIGHT_API_DATABASE_URL=postgresql://user:pass@host:5432/db
 # HINDSIGHT_API_DATABASE_SCHEMA=public  # PostgreSQL schema name (default: public)
 
+# Vector Extension (Optional - uses pgvector by default)
+# Options: "pgvector" (default), "vchord", "pgvectorscale" (DiskANN)
+# HINDSIGHT_API_VECTOR_EXTENSION=pgvector
+# For Azure PostgreSQL with DiskANN:
+# HINDSIGHT_API_VECTOR_EXTENSION=pgvectorscale  # Auto-detects pg_diskann on Azure
+
 # Embeddings Configuration (Optional - uses local by default)
 # Provider: "local" (default) or "tei" (HuggingFace Text Embeddings Inference)
 # HINDSIGHT_API_EMBEDDINGS_PROVIDER=local

.github/workflows/test.yml

@@ -712,6 +712,10 @@ jobs:
           sleep 1
         done
 
+    - name: Build Go client
+      working-directory: ./hindsight-clients/go
+      run: go build ./...
+
     - name: Run Go client tests
       working-directory: ./hindsight-clients/go
       run: go test -v -tags=integration
@@ -722,6 +726,107 @@ jobs:
         echo "=== API Server Logs ==="
         cat /tmp/api-server.log || echo "No API server log found"
 
+  test-openclaw-integration:
+    runs-on: ubuntu-latest
+    env:
+      HINDSIGHT_API_LLM_PROVIDER: groq
+      HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
+      HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
+      HINDSIGHT_API_URL: http://localhost:8888
+      HINDSIGHT_EMBED_PACKAGE_PATH: ${{ github.workspace }}/hindsight-embed
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version-file: ".python-version"
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '22'
+
+    - name: Build API
+      working-directory: ./hindsight-api
+      run: uv build
+
+    - name: Install API dependencies
+      working-directory: ./hindsight-api
+      run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
+
+    - name: Install embed dependencies
+      working-directory: ./hindsight-embed
+      run: uv sync --frozen --index-strategy unsafe-best-match
+
+    - name: Cache HuggingFace models
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/huggingface
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-huggingface-
+
+    - name: Pre-download models
+      working-directory: ./hindsight-api
+      run: |
+        uv run python -c "
+        from sentence_transformers import SentenceTransformer, CrossEncoder
+        print('Downloading embedding model...')
+        SentenceTransformer('BAAI/bge-small-en-v1.5')
+        print('Downloading cross-encoder model...')
+        CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
+        print('Models downloaded successfully')
+        "
+
+    - name: Install openclaw integration dependencies
+      working-directory: ./hindsight-integrations/openclaw
+      run: npm ci
+
+    - name: Create .env file
+      run: |
+        cat > .env << EOF
+        HINDSIGHT_API_LLM_PROVIDER=${{ env.HINDSIGHT_API_LLM_PROVIDER }}
+        HINDSIGHT_API_LLM_API_KEY=${{ env.HINDSIGHT_API_LLM_API_KEY }}
+        HINDSIGHT_API_LLM_MODEL=${{ env.HINDSIGHT_API_LLM_MODEL }}
+        EOF
+
+    - name: Start API server
+      run: |
+        ./scripts/dev/start-api.sh > /tmp/api-server.log 2>&1 &
+        echo "Waiting for API server to be ready..."
+        for i in {1..60}; do
+          if curl -sf http://localhost:8888/health > /dev/null 2>&1; then
+            echo "API server is ready after ${i}s"
+            break
+          fi
+          if [ $i -eq 60 ]; then
+            echo "API server failed to start after 60s"
+            cat /tmp/api-server.log
+            exit 1
+          fi
+          sleep 1
+        done
+
+    - name: Run openclaw integration tests
+      working-directory: ./hindsight-integrations/openclaw
+      run: npm run test:integration
+
+    - name: Show API server logs
+      if: always()
+      run: |
+        echo "=== API Server Logs ==="
+        cat /tmp/api-server.log || echo "No API server log found"
+
   test-integration:
     runs-on: ubuntu-latest
     env:

docker/docker-compose/s3-file-storage/docker-compose.yaml

@@ -0,0 +1,83 @@
+# Docker Compose file for Hindsight with S3 file storage (SeaweedFS)
+#
+# SeaweedFS (Apache 2.0) provides an S3-compatible object storage backend
+# for storing uploaded files instead of PostgreSQL BYTEA storage.
+#
+# Make sure to set the required environment variables before running:
+# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
+# - Configure LLM provider variables as needed (see below in the hindsight service)
+#
+# Usage:
+# docker compose up -d
+#
+# Optional environment variables with defaults:
+# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
+# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
+# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
+# - HINDSIGHT_DB_VERSION: PostgreSQL version (default: 18)
+# - SEAWEEDFS_S3_ACCESS_KEY: S3 access key (default: hindsight_s3_key)
+# - SEAWEEDFS_S3_SECRET_KEY: S3 secret key (default: hindsight_s3_secret)
+
+services:
+  db:
+    image: pgvector/pgvector:pg${HINDSIGHT_DB_VERSION:-18}
+    container_name: hindsight-db
+    restart: always
+    environment:
+      POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
+      POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}
+      POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
+    volumes:
+      - pg_data:/var/lib/postgresql/${HINDSIGHT_DB_VERSION:-18}/docker
+    networks:
+      - hindsight-net
+
+  seaweedfs:
+    image: chrislusf/seaweedfs:latest
+    container_name: hindsight-seaweedfs
+    restart: always
+    # Single-node mode: master + volume + filer + S3 gateway all in one process
+    command: >
+      server
+      -s3
+      -s3.port=8333
+      -s3.config=/etc/seaweedfs/s3.json
+      -ip.bind=0.0.0.0
+    volumes:
+      - seaweedfs_data:/data
+      - ./s3.json:/etc/seaweedfs/s3.json:ro
+    # Expose S3 API port (uncomment to access from host)
+    # ports:
+    #   - "8333:8333"
+    networks:
+      - hindsight-net
+
+  hindsight:
+    image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
+    container_name: hindsight-app
+    ports:
+      - "8888:8888"
+      - "9999:9999"
+    environment:
+      - HINDSIGHT_API_LLM_API_KEY=${OPENAI_API_KEY?Please set the OPENAI_API_KEY env variable}
+      - HINDSIGHT_API_DATABASE_URL=postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
+      # S3 file storage configuration (SeaweedFS)
+      - HINDSIGHT_API_FILE_STORAGE_TYPE=s3
+      - HINDSIGHT_API_FILE_STORAGE_S3_BUCKET=hindsight
+      - HINDSIGHT_API_FILE_STORAGE_S3_ENDPOINT=http://seaweedfs:8333
+      - HINDSIGHT_API_FILE_STORAGE_S3_REGION=us-east-1
+      - HINDSIGHT_API_FILE_STORAGE_S3_ACCESS_KEY_ID=${SEAWEEDFS_S3_ACCESS_KEY:-hindsight_s3_key}
+      - HINDSIGHT_API_FILE_STORAGE_S3_SECRET_ACCESS_KEY=${SEAWEEDFS_S3_SECRET_KEY:-hindsight_s3_secret}
+    depends_on:
+      - db
+      - seaweedfs
+    networks:
+      - hindsight-net
+
+networks:
+  hindsight-net:
+    driver: bridge
+
+volumes:
+  pg_data:
+  seaweedfs_data:

docker/docker-compose/s3-file-storage/s3.json

@@ -0,0 +1,19 @@
+{
+  "identities": [
+    {
+      "name": "hindsight",
+      "credentials": [
+        {
+          "accessKey": "hindsight_s3_key",
+          "secretKey": "hindsight_s3_secret"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "Write",
+        "List"
+      ]
+    }
+  ]
+}

helm/hindsight/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: hindsight
 description: Hindsight helm chart
 type: application
-version: 0.4.11
-appVersion: "0.4.11"
+version: 0.4.12
+appVersion: "0.4.12"
 keywords:
   - ai
   - memory

hindsight-api/hindsight_api/__init__.py

@@ -46,4 +46,4 @@ __all__ = [
     "RemoteTEICrossEncoder",
     "LLMConfig",
 ]
-__version__ = "0.4.11"
+__version__ = "0.4.12"

hindsight-api/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py

@@ -32,18 +32,26 @@ def _detect_vector_extension() -> str:
 
     # Validate configured extension is installed
     if vector_extension == "pgvectorscale":
-        # pgvectorscale requires pgvector
+        # pgvectorscale/DiskANN requires pgvector
         pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
         if not pgvector_check:
             raise RuntimeError(
-                "pgvectorscale requires pgvector. Install with: CREATE EXTENSION vector; CREATE EXTENSION vectorscale CASCADE;"
+                "DiskANN requires pgvector. Install with: CREATE EXTENSION vector; then vectorscale or pg_diskann CASCADE;"
             )
+        # Check for either vectorscale (open source) or pg_diskann (Azure)
         vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar()
-        if not vectorscale_check:
+        pg_diskann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_diskann'")).scalar()
+
+        if vectorscale_check:
+            return "pgvectorscale"
+        elif pg_diskann_check:
+            return "pg_diskann"
+        else:
             raise RuntimeError(
-                "Configured vector extension 'pgvectorscale' not found. Install it with: CREATE EXTENSION vectorscale CASCADE;"
+                "Configured vector extension 'pgvectorscale' not found. Install either:\n"
+                "  - pgvectorscale: CREATE EXTENSION vectorscale CASCADE;\n"
+                "  - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
             )
-        return "pgvectorscale"
     elif vector_extension == "vchord":
         vchord_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord'")).scalar()
         if not vchord_check:
@@ -311,6 +319,13 @@ def upgrade() -> None:
             USING diskann (embedding vector_cosine_ops)
             WITH (num_neighbors = 50)
         """)
+    elif vector_ext == "pg_diskann":
+        # Use DiskANN index for pg_diskann (Azure)
+        op.execute("""
+            CREATE INDEX idx_memory_units_embedding ON memory_units
+            USING diskann (embedding vector_cosine_ops)
+            WITH (max_neighbors = 50)
+        """)
     elif vector_ext == "vchord":
         # Use vchordrq index for vchord (supports high-dimensional embeddings)
         op.execute("""

hindsight-api/hindsight_api/alembic/versions/a1b2c3d4e5f6_add_file_storage_table.py

@@ -0,0 +1,70 @@
+"""Add file_storage table for BYTEA-based file storage
+
+Revision ID: a1b2c3d4e5f6
+Revises: y0t1u2v3w4x5
+Create Date: 2026-02-16
+
+Creates a dedicated table for storing uploaded files using BYTEA.
+This provides zero-config file storage that "just works" for development
+and small deployments. For production/scale, use S3-compatible storage.
+
+Files are stored in a separate table to avoid bloating the documents table.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "a1b2c3d4e5f6"
+down_revision: str | Sequence[str] | None = "y0t1u2v3w4x5"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Create file_storage table for BYTEA storage."""
+    schema = _get_schema_prefix()
+
+    # Create file_storage table (minimal: just key + data)
+    op.execute(
+        f"""
+        CREATE TABLE {schema}file_storage (
+            storage_key TEXT PRIMARY KEY,
+            data BYTEA NOT NULL
+        )
+    """
+    )
+
+    # Add file tracking columns to documents table
+    op.execute(
+        f"""
+        ALTER TABLE {schema}documents
+        ADD COLUMN IF NOT EXISTS file_storage_key TEXT,
+        ADD COLUMN IF NOT EXISTS file_original_name TEXT,
+        ADD COLUMN IF NOT EXISTS file_content_type TEXT
+    """
+    )
+
+
+def downgrade() -> None:
+    """Remove file_storage table and related columns."""
+    schema = _get_schema_prefix()
+
+    # Drop columns from documents table
+    op.execute(
+        f"""
+        ALTER TABLE {schema}documents
+        DROP COLUMN IF EXISTS file_storage_key,
+        DROP COLUMN IF EXISTS file_original_name,
+        DROP COLUMN IF EXISTS file_content_type
+    """
+    )
+
+    # Drop file_storage table
+    op.execute(f"DROP TABLE IF EXISTS {schema}file_storage")

hindsight-api/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py

@@ -39,18 +39,26 @@ def _detect_vector_extension() -> str:
 
     # Validate configured extension is installed
     if vector_extension == "pgvectorscale":
-        # pgvectorscale requires pgvector
+        # pgvectorscale/DiskANN requires pgvector
         pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
         if not pgvector_check:
             raise RuntimeError(
-                "pgvectorscale requires pgvector. Install with: CREATE EXTENSION vector; CREATE EXTENSION vectorscale CASCADE;"
+                "DiskANN requires pgvector. Install with: CREATE EXTENSION vector; then vectorscale or pg_diskann CASCADE;"
             )
+        # Check for either vectorscale (open source) or pg_diskann (Azure)
         vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar()
-        if not vectorscale_check:
+        pg_diskann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_diskann'")).scalar()
+
+        if vectorscale_check:
+            return "pgvectorscale"
+        elif pg_diskann_check:
+            return "pg_diskann"
+        else:
             raise RuntimeError(
-                "Configured vector extension 'pgvectorscale' not found. Install it with: CREATE EXTENSION vectorscale CASCADE;"
+                "Configured vector extension 'pgvectorscale' not found. Install either:\n"
+                "  - pgvectorscale: CREATE EXTENSION vectorscale CASCADE;\n"
+                "  - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
             )
-        return "pgvectorscale"
     elif vector_extension == "vchord":
         vchord_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord'")).scalar()
         if not vchord_check:
@@ -155,6 +163,12 @@ def upgrade() -> None:
             USING diskann (embedding vector_cosine_ops)
             WITH (num_neighbors = 50)
         """)
+    elif vector_ext == "pg_diskann":
+        op.execute(f"""
+            CREATE INDEX idx_learnings_embedding ON {schema}learnings
+            USING diskann (embedding vector_cosine_ops)
+            WITH (max_neighbors = 50)
+        """)
     elif vector_ext == "vchord":
         op.execute(f"""
             CREATE INDEX idx_learnings_embedding ON {schema}learnings
@@ -228,6 +242,12 @@ def upgrade() -> None:
             USING diskann (embedding vector_cosine_ops)
             WITH (num_neighbors = 50)
         """)
+    elif vector_ext == "pg_diskann":
+        op.execute(f"""
+            CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
+            USING diskann (embedding vector_cosine_ops)
+            WITH (max_neighbors = 50)
+        """)
     elif vector_ext == "vchord":
         op.execute(f"""
             CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections

hindsight-api/hindsight_api/api/http.py

@@ -13,7 +13,7 @@ from contextlib import asynccontextmanager
 from datetime import datetime
 from typing import Any, Literal
 
-from fastapi import Depends, FastAPI, Header, HTTPException, Query
+from fastapi import Depends, FastAPI, File, Form, Header, HTTPException, Query, UploadFile
 
 from hindsight_api.extensions import AuthenticationError
 
@@ -430,6 +430,36 @@ class RetainRequest(BaseModel):
     )
 
 
+class FileRetainMetadata(BaseModel):
+    """Metadata for a single file in file retain request."""
+
+    document_id: str | None = Field(default=None, description="Document ID (auto-generated if not provided)")
+    context: str | None = Field(default=None, description="Context for the file")
+    metadata: dict[str, Any] | None = Field(default=None, description="Additional metadata")
+    tags: list[str] | None = Field(default=None, description="Tags for this file")
+    timestamp: str | None = Field(default=None, description="ISO timestamp")
+
+
+class FileRetainRequest(BaseModel):
+    """Request model for file retain endpoint."""
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "files_metadata": [
+                    {"document_id": "report_2024", "tags": ["quarterly"]},
+                    {"context": "meeting notes"},
+                ],
+            }
+        }
+    )
+
+    files_metadata: list[FileRetainMetadata] | None = Field(
+        default=None,
+        description="Metadata for each file (optional, must match number of files if provided)",
+    )
+
+
 class RetainResponse(BaseModel):
     """Response model for retain endpoint."""
 
@@ -454,7 +484,7 @@ class RetainResponse(BaseModel):
     )
     operation_id: str | None = Field(
         default=None,
-        description="Operation ID for tracking async operations. Use GET /v1/default/banks/{bank_id}/operations to list operations and find this ID. Only present when async=true.",
+        description="Operation ID for tracking async operations. Use GET /v1/default/banks/{bank_id}/operations to list operations. Only present when async=true.",
     )
     usage: TokenUsage | None = Field(
         default=None,
@@ -462,6 +492,26 @@ class RetainResponse(BaseModel):
     )
 
 
+class FileRetainResponse(BaseModel):
+    """Response model for file upload endpoint."""
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "operation_ids": [
+                    "550e8400-e29b-41d4-a716-446655440000",
+                    "550e8400-e29b-41d4-a716-446655440001",
+                    "550e8400-e29b-41d4-a716-446655440002",
+                ],
+            }
+        },
+    )
+
+    operation_ids: list[str] = Field(
+        description="Operation IDs for tracking file conversion operations. Use GET /v1/default/banks/{bank_id}/operations to list operations."
+    )
+
+
 class FactsIncludeOptions(BaseModel):
     """Options for including facts (based_on) in reflect results."""
 
@@ -1423,6 +1473,7 @@ class FeaturesInfo(BaseModel):
     mcp: bool = Field(description="Whether MCP (Model Context Protocol) server is enabled")
     worker: bool = Field(description="Whether the background worker is enabled")
     bank_config_api: bool = Field(description="Whether per-bank configuration API is enabled")
+    file_upload_api: bool = Field(description="Whether file upload/conversion API is enabled")
 
 
 class VersionResponse(BaseModel):
@@ -1437,6 +1488,7 @@ class VersionResponse(BaseModel):
                     "mcp": True,
                     "worker": True,
                     "bank_config_api": False,
+                    "file_upload_api": True,
                 },
             }
         }
@@ -1731,6 +1783,7 @@ def _register_routes(app: FastAPI):
                 mcp=config.mcp_enabled,
                 worker=config.worker_enabled,
                 bank_config_api=config.enable_bank_config_api,
+                file_upload_api=config.enable_file_upload_api,
             ),
         )
 
@@ -3632,6 +3685,147 @@ def _register_routes(app: FastAPI):
             logger.error(f"Error in /v1/default/banks/{bank_id}/memories (retain): {error_detail}")
             raise HTTPException(status_code=500, detail=str(e))
 
+    @app.post(
+        "/v1/default/banks/{bank_id}/files/retain",
+        response_model=FileRetainResponse,
+        summary="Convert files to memories",
+        description="Upload files (PDF, DOCX, etc.), convert them to markdown, and retain as memories.\n\n"
+        "This endpoint handles file upload, conversion, and memory creation in a single operation.\n\n"
+        "**Features:**\n"
+        "- Supports PDF, DOCX, PPTX, XLSX, images (with OCR), audio (with transcription)\n"
+        "- Automatic file-to-markdown conversion using pluggable parsers\n"
+        "- Files stored in object storage (PostgreSQL by default, S3 for production)\n"
+        "- Each file becomes a separate document with optional metadata/tags\n"
+        "- Always processes asynchronously — returns operation IDs immediately\n\n"
+        "**The system automatically:**\n"
+        "1. Stores uploaded files in object storage\n"
+        "2. Converts files to markdown\n"
+        "3. Creates document records with file metadata\n"
+        "4. Extracts facts and creates memory units (same as regular retain)\n\n"
+        "Use the operations endpoint to monitor progress.\n\n"
+        "**Request format:** multipart/form-data with:\n"
+        "- `files`: One or more files to upload\n"
+        "- `request`: JSON string with FileRetainRequest model (files_metadata)\n\n"
+        "**Note:** File parser is configured server-side via `HINDSIGHT_API_FILE_PARSER` (default: markitdown).",
+        operation_id="file_retain",
+        tags=["Files"],
+    )
+    async def api_file_retain(
+        bank_id: str,
+        files: list[UploadFile] = File(..., description="Files to upload and convert"),
+        request: str = Form(..., description="JSON string with FileRetainRequest model"),
+        request_context: RequestContext = Depends(get_request_context),
+    ):
+        """Upload and convert files to memories."""
+        from hindsight_api.config import get_config
+
+        config = get_config()
+
+        # Check if file upload API is enabled
+        if not config.enable_file_upload_api:
+            raise HTTPException(
+                status_code=404,
+                detail="File upload API is disabled. Set HINDSIGHT_API_ENABLE_FILE_UPLOAD_API=true to enable.",
+            )
+
+        try:
+            # Parse request JSON
+            try:
+                request_data = FileRetainRequest.model_validate_json(request)
+            except Exception as e:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Invalid request JSON: {str(e)}",
+                )
+
+            # Validate file count
+            if len(files) > config.file_conversion_max_batch_size:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Too many files. Maximum {config.file_conversion_max_batch_size} files per request.",
+                )
+
+            # Validate files_metadata count matches files count if provided
+            if request_data.files_metadata and len(request_data.files_metadata) != len(files):
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"files_metadata count ({len(request_data.files_metadata)}) must match files count ({len(files)})",
+                )
+
+            # Prepare file items and calculate total batch size
+            file_items = []
+            total_batch_size = 0
+
+            for i, file in enumerate(files):
+                # Read file content to check size
+                file_content = await file.read()
+                size = len(file_content)
+                total_batch_size += size
+
+                # Create a temporary file-like object from the bytes
+                import io
+
+                file_obj = io.BytesIO(file_content)
+
+                # Create a mock UploadFile with the necessary attributes
+                class FileWrapper:
+                    def __init__(self, content, filename, content_type):
+                        self._content = content
+                        self.filename = filename
+                        self.content_type = content_type
+                        self._buffer = io.BytesIO(content)
+
+                    async def read(self):
+                        return self._content
+
+                wrapped_file = FileWrapper(file_content, file.filename, file.content_type)
+
+                # Get per-file metadata
+                file_meta = request_data.files_metadata[i] if request_data.files_metadata else FileRetainMetadata()
+                doc_id = file_meta.document_id or f"file_{uuid.uuid4()}"
+
+                item = {
+                    "file": wrapped_file,
+                    "document_id": doc_id,
+                    "context": file_meta.context,
+                    "metadata": file_meta.metadata or {},
+                    "tags": file_meta.tags or [],
+                    "timestamp": file_meta.timestamp,
+                }
+                file_items.append(item)
+
+            # Check total batch size after processing all files
+            if total_batch_size > config.file_conversion_max_batch_size_bytes:
+                total_mb = total_batch_size / (1024 * 1024)
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Total batch size ({total_mb:.1f}MB) exceeds maximum of {config.file_conversion_max_batch_size_mb}MB",
+                )
+
+            result = await app.state.memory.submit_async_file_retain(
+                bank_id=bank_id,
+                file_items=file_items,
+                parser=config.file_parser,
+                document_tags=None,
+                request_context=request_context,
+            )
+            return FileRetainResponse.model_validate(
+                {
+                    "operation_ids": result["operation_ids"],
+                }
+            )
+
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in /v1/default/banks/{bank_id}/files/retain: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
     @app.delete(
         "/v1/default/banks/{bank_id}/memories",
         response_model=DeleteResponse,

hindsight-api/hindsight_api/config.py

@@ -259,6 +259,26 @@ ENV_RETAIN_BATCH_TOKENS = "HINDSIGHT_API_RETAIN_BATCH_TOKENS"
 ENV_RETAIN_BATCH_ENABLED = "HINDSIGHT_API_RETAIN_BATCH_ENABLED"
 ENV_RETAIN_BATCH_POLL_INTERVAL_SECONDS = "HINDSIGHT_API_RETAIN_BATCH_POLL_INTERVAL_SECONDS"
 
+# File storage configuration
+ENV_FILE_STORAGE_TYPE = "HINDSIGHT_API_FILE_STORAGE_TYPE"
+ENV_FILE_STORAGE_S3_BUCKET = "HINDSIGHT_API_FILE_STORAGE_S3_BUCKET"
+ENV_FILE_STORAGE_S3_REGION = "HINDSIGHT_API_FILE_STORAGE_S3_REGION"
+ENV_FILE_STORAGE_S3_ENDPOINT = "HINDSIGHT_API_FILE_STORAGE_S3_ENDPOINT"
+ENV_FILE_STORAGE_S3_ACCESS_KEY_ID = "HINDSIGHT_API_FILE_STORAGE_S3_ACCESS_KEY_ID"
+ENV_FILE_STORAGE_S3_SECRET_ACCESS_KEY = "HINDSIGHT_API_FILE_STORAGE_S3_SECRET_ACCESS_KEY"
+ENV_FILE_STORAGE_GCS_BUCKET = "HINDSIGHT_API_FILE_STORAGE_GCS_BUCKET"
+ENV_FILE_STORAGE_GCS_SERVICE_ACCOUNT_KEY = "HINDSIGHT_API_FILE_STORAGE_GCS_SERVICE_ACCOUNT_KEY"
+ENV_FILE_STORAGE_AZURE_CONTAINER = "HINDSIGHT_API_FILE_STORAGE_AZURE_CONTAINER"
+ENV_FILE_STORAGE_AZURE_ACCOUNT_NAME = "HINDSIGHT_API_FILE_STORAGE_AZURE_ACCOUNT_NAME"
+ENV_FILE_STORAGE_AZURE_ACCOUNT_KEY = "HINDSIGHT_API_FILE_STORAGE_AZURE_ACCOUNT_KEY"
+ENV_FILE_PARSER = "HINDSIGHT_API_FILE_PARSER"
+ENV_FILE_PARSER_IRIS_TOKEN = "HINDSIGHT_API_FILE_PARSER_IRIS_TOKEN"
+ENV_FILE_PARSER_IRIS_ORG_ID = "HINDSIGHT_API_FILE_PARSER_IRIS_ORG_ID"
+ENV_FILE_CONVERSION_MAX_BATCH_SIZE_MB = "HINDSIGHT_API_FILE_CONVERSION_MAX_BATCH_SIZE_MB"
+ENV_FILE_CONVERSION_MAX_BATCH_SIZE = "HINDSIGHT_API_FILE_CONVERSION_MAX_BATCH_SIZE"
+ENV_ENABLE_FILE_UPLOAD_API = "HINDSIGHT_API_ENABLE_FILE_UPLOAD_API"
+ENV_FILE_DELETE_AFTER_RETAIN = "HINDSIGHT_API_FILE_DELETE_AFTER_RETAIN"
+
 # Observations settings (consolidated knowledge from facts)
 ENV_ENABLE_OBSERVATIONS = "HINDSIGHT_API_ENABLE_OBSERVATIONS"
 ENV_CONSOLIDATION_BATCH_SIZE = "HINDSIGHT_API_CONSOLIDATION_BATCH_SIZE"
@@ -383,6 +403,14 @@ DEFAULT_RETAIN_BATCH_TOKENS = 10_000  # ~40KB of text  # Max chars per sub-batch
 DEFAULT_RETAIN_BATCH_ENABLED = False  # Use LLM Batch API for fact extraction (only when async=True)
 DEFAULT_RETAIN_BATCH_POLL_INTERVAL_SECONDS = 60  # Batch API polling interval in seconds
 
+# File storage defaults
+DEFAULT_FILE_STORAGE_TYPE = "native"  # PostgreSQL BYTEA storage
+DEFAULT_FILE_PARSER = "markitdown"  # File parser to use (markitdown is the only supported parser)
+DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE_MB = 100  # Max total batch size in MB (all files combined)
+DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE = 10  # Max files per batch upload
+DEFAULT_ENABLE_FILE_UPLOAD_API = True  # Enable file upload endpoint
+DEFAULT_FILE_DELETE_AFTER_RETAIN = True  # Delete file bytes after retain (saves storage)
+
 # Observations defaults (consolidated knowledge from facts)
 DEFAULT_ENABLE_OBSERVATIONS = True  # Observations enabled by default
 DEFAULT_CONSOLIDATION_BATCH_SIZE = 50  # Memories to load per batch (internal memory optimization)
@@ -607,6 +635,26 @@ class HindsightConfig:
     retain_batch_enabled: bool
     retain_batch_poll_interval_seconds: int
 
+    # File storage (static - server-level only)
+    file_storage_type: str  # "native" (PostgreSQL) or "s3" (S3-compatible)
+    file_storage_s3_bucket: str | None  # S3 bucket name (required for s3 storage)
+    file_storage_s3_region: str | None  # S3 region (optional, uses SDK default)
+    file_storage_s3_endpoint: str | None  # S3 endpoint URL (for MinIO, R2, etc.)
+    file_storage_s3_access_key_id: str | None  # S3 access key (optional, uses env/IAM)
+    file_storage_s3_secret_access_key: str | None  # S3 secret key (optional, uses env/IAM)
+    file_storage_gcs_bucket: str | None  # GCS bucket name (required for gcs storage)
+    file_storage_gcs_service_account_key: str | None  # GCS service account key JSON (optional, uses ADC)
+    file_storage_azure_container: str | None  # Azure container name (required for azure storage)
+    file_storage_azure_account_name: str | None  # Azure storage account name
+    file_storage_azure_account_key: str | None  # Azure storage account key
+    file_parser: str  # File parser to use (e.g., "markitdown", "iris")
+    file_parser_iris_token: str | None  # Vectorize API token for iris parser (VECTORIZE_TOKEN)
+    file_parser_iris_org_id: str | None  # Vectorize org ID for iris parser (VECTORIZE_ORG_ID)
+    file_conversion_max_batch_size_mb: int  # Max total batch size in MB (all files combined)
+    file_conversion_max_batch_size: int  # Max files per request
+    enable_file_upload_api: bool
+    file_delete_after_retain: bool
+
     # Observations settings (consolidated knowledge from facts)
     enable_observations: bool
     consolidation_batch_size: int
@@ -663,6 +711,13 @@ class HindsightConfig:
         "reranker_cohere_base_url",
         # Service Account Keys
         "llm_vertexai_service_account_key",
+        # File storage credentials
+        "file_storage_s3_access_key_id",
+        "file_storage_s3_secret_access_key",
+        "file_storage_gcs_service_account_key",
+        "file_storage_azure_account_key",
+        # File parser credentials
+        "file_parser_iris_token",
     }
 
     # CONFIGURABLE_FIELDS: Safe behavioral settings that can be customized per-tenant/bank
@@ -677,6 +732,11 @@ class HindsightConfig:
         "enable_observations",
     }
 
+    @property
+    def file_conversion_max_batch_size_bytes(self) -> int:
+        """Get maximum total batch size in bytes."""
+        return self.file_conversion_max_batch_size_mb * 1024 * 1024
+
     @classmethod
     def get_configurable_fields(cls) -> set[str]:
         """
@@ -963,6 +1023,33 @@ class HindsightConfig:
             retain_batch_poll_interval_seconds=int(
                 os.getenv(ENV_RETAIN_BATCH_POLL_INTERVAL_SECONDS, str(DEFAULT_RETAIN_BATCH_POLL_INTERVAL_SECONDS))
             ),
+            # File storage
+            file_storage_type=os.getenv(ENV_FILE_STORAGE_TYPE, DEFAULT_FILE_STORAGE_TYPE),
+            file_storage_s3_bucket=os.getenv(ENV_FILE_STORAGE_S3_BUCKET) or None,
+            file_storage_s3_region=os.getenv(ENV_FILE_STORAGE_S3_REGION) or None,
+            file_storage_s3_endpoint=os.getenv(ENV_FILE_STORAGE_S3_ENDPOINT) or None,
+            file_storage_s3_access_key_id=os.getenv(ENV_FILE_STORAGE_S3_ACCESS_KEY_ID) or None,
+            file_storage_s3_secret_access_key=os.getenv(ENV_FILE_STORAGE_S3_SECRET_ACCESS_KEY) or None,
+            file_storage_gcs_bucket=os.getenv(ENV_FILE_STORAGE_GCS_BUCKET) or None,
+            file_storage_gcs_service_account_key=os.getenv(ENV_FILE_STORAGE_GCS_SERVICE_ACCOUNT_KEY) or None,
+            file_storage_azure_container=os.getenv(ENV_FILE_STORAGE_AZURE_CONTAINER) or None,
+            file_storage_azure_account_name=os.getenv(ENV_FILE_STORAGE_AZURE_ACCOUNT_NAME) or None,
+            file_storage_azure_account_key=os.getenv(ENV_FILE_STORAGE_AZURE_ACCOUNT_KEY) or None,
+            file_parser=os.getenv(ENV_FILE_PARSER, DEFAULT_FILE_PARSER),
+            file_parser_iris_token=os.getenv(ENV_FILE_PARSER_IRIS_TOKEN) or None,
+            file_parser_iris_org_id=os.getenv(ENV_FILE_PARSER_IRIS_ORG_ID) or None,
+            file_conversion_max_batch_size_mb=int(
+                os.getenv(ENV_FILE_CONVERSION_MAX_BATCH_SIZE_MB, str(DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE_MB))
+            ),
+            file_conversion_max_batch_size=int(
+                os.getenv(ENV_FILE_CONVERSION_MAX_BATCH_SIZE, str(DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE))
+            ),
+            enable_file_upload_api=os.getenv(ENV_ENABLE_FILE_UPLOAD_API, str(DEFAULT_ENABLE_FILE_UPLOAD_API)).lower()
+            == "true",
+            file_delete_after_retain=os.getenv(
+                ENV_FILE_DELETE_AFTER_RETAIN, str(DEFAULT_FILE_DELETE_AFTER_RETAIN)
+            ).lower()
+            == "true",
             # Observations settings (consolidated knowledge from facts)
             enable_observations=os.getenv(ENV_ENABLE_OBSERVATIONS, str(DEFAULT_ENABLE_OBSERVATIONS)).lower() == "true",
             consolidation_batch_size=int(

hindsight-api/hindsight_api/engine/memory_engine.py

@@ -79,6 +79,7 @@ _PROTECTED_TABLES = frozenset(
         "documents",
         "chunks",
         "async_operations",
+        "file_storage",
     ]
 )
 
@@ -585,8 +586,165 @@ class MemoryEngine(MemoryEngineInterface):
             operation_id=operation_id,
         )
 
+        # If this retain was triggered by file conversion, update document with file metadata
+        file_metadata = task_dict.get("_file_metadata")
+        if file_metadata and len(contents) == 1:
+            doc_id = contents[0].get("document_id")
+            if doc_id:
+                pool = await self._get_pool()
+                async with acquire_with_retry(pool) as conn:
+                    await conn.execute(
+                        f"""
+                        UPDATE {fq_table("documents")}
+                        SET file_storage_key = $3,
+                            file_original_name = $4,
+                            file_content_type = $5,
+                            updated_at = NOW()
+                        WHERE id = $1 AND bank_id = $2
+                        """,
+                        doc_id,
+                        bank_id,
+                        file_metadata["file_storage_key"],
+                        file_metadata["file_original_name"],
+                        file_metadata["file_content_type"],
+                    )
+
         logger.info(f"[BATCH_RETAIN_TASK] Completed background batch retain for bank_id={bank_id}")
 
+    async def _handle_file_convert_retain(self, task_dict: dict[str, Any]):
+        """
+        Handler for file conversion tasks.
+
+        Converts a file to markdown, then submits a separate async retain operation
+        and marks this conversion as completed — all in a single transaction.
+        This avoids holding a worker slot during the expensive retain pipeline.
+
+        Args:
+            task_dict: Dict with 'bank_id', 'storage_key', 'parser', etc.
+
+        Raises:
+            ValueError: If required fields are missing
+            Exception: Any exception from conversion (includes filename in error)
+        """
+        bank_id = task_dict.get("bank_id")
+        storage_key = task_dict.get("storage_key")
+        document_id = task_dict.get("document_id")
+        operation_id = task_dict.get("operation_id")
+        filename = task_dict.get("original_filename", "unknown")
+
+        if not all([bank_id, storage_key, document_id]):
+            raise ValueError("bank_id, storage_key, and document_id are required for file_convert_retain task")
+
+        logger.info(f"[FILE_CONVERT_RETAIN] Starting for bank_id={bank_id}, document_id={document_id}, file={filename}")
+
+        try:
+            # Retrieve file from storage
+            file_data = await self._file_storage.retrieve(storage_key)
+
+            # Convert to markdown
+            parser = self._parser_registry.get_parser(
+                name=task_dict.get("parser"),
+                filename=filename,
+                content_type=task_dict.get("content_type"),
+            )
+            markdown_content = await parser.convert(file_data, filename)
+        except Exception as e:
+            # Re-raise with filename context for better error reporting
+            error_msg = f"Failed to parse file '{filename}': {str(e)}"
+            logger.error(f"[FILE_CONVERT_RETAIN] {error_msg}")
+            raise RuntimeError(error_msg) from e
+
+        logger.info(
+            f"[FILE_CONVERT_RETAIN] Converted file for bank_id={bank_id}, "
+            f"document_id={document_id}, {len(markdown_content)} chars. Submitting retain task."
+        )
+
+        # Build retain task payload
+        retain_contents = [
+            {
+                "content": markdown_content,
+                "document_id": document_id,
+                "context": task_dict.get("context"),
+                "metadata": task_dict.get("metadata", {}),
+                "tags": task_dict.get("tags", []),
+                "timestamp": task_dict.get("timestamp"),
+            }
+        ]
+        document_tags = task_dict.get("document_tags")
+
+        retain_task_payload: dict[str, Any] = {"contents": retain_contents}
+        if document_tags:
+            retain_task_payload["document_tags"] = document_tags
+
+        # Pass tenant/api_key context through to retain task
+        if task_dict.get("_tenant_id"):
+            retain_task_payload["_tenant_id"] = task_dict["_tenant_id"]
+        if task_dict.get("_api_key_id"):
+            retain_task_payload["_api_key_id"] = task_dict["_api_key_id"]
+
+        # File metadata to attach after retain creates the document
+        retain_task_payload["_file_metadata"] = {
+            "file_storage_key": storage_key,
+            "file_original_name": task_dict["original_filename"],
+            "file_content_type": task_dict["content_type"],
+        }
+
+        # In one transaction: create the retain async operation AND mark this conversion as completed
+        retain_operation_id = uuid.uuid4()
+        pool = await self._get_pool()
+        async with acquire_with_retry(pool) as conn:
+            async with conn.transaction():
+                # Create the retain operation record
+                await conn.execute(
+                    f"""
+                    INSERT INTO {fq_table("async_operations")}
+                    (operation_id, bank_id, operation_type, result_metadata, status)
+                    VALUES ($1, $2, $3, $4, $5)
+                    """,
+                    retain_operation_id,
+                    bank_id,
+                    "retain",
+                    json.dumps({}),
+                    "pending",
+                )
+
+                # Mark this file_convert_retain operation as completed
+                if operation_id:
+                    await conn.execute(
+                        f"""
+                        UPDATE {fq_table("async_operations")}
+                        SET status = 'completed', updated_at = NOW(), completed_at = NOW()
+                        WHERE operation_id = $1
+                        """,
+                        uuid.UUID(operation_id),
+                    )
+
+        # Submit the retain task to the task backend (outside the transaction)
+        full_retain_payload = {
+            "type": "batch_retain",
+            "operation_id": str(retain_operation_id),
+            "bank_id": bank_id,
+            **retain_task_payload,
+        }
+        await self._task_backend.submit_task(full_retain_payload)
+
+        logger.info(
+            f"[FILE_CONVERT_RETAIN] Completed conversion for bank_id={bank_id}, "
+            f"document_id={document_id}. Retain task submitted as operation {retain_operation_id}"
+        )
+
+        # Delete file bytes from storage if configured (saves storage costs)
+        from ..config import get_config
+
+        config = get_config()
+        if config.file_delete_after_retain:
+            try:
+                await self._file_storage.delete(storage_key)
+                logger.info(f"[FILE_CONVERT_RETAIN] Deleted file bytes for {storage_key} (conversion completed)")
+            except Exception as e:
+                # Non-fatal - log and continue
+                logger.warning(f"[FILE_CONVERT_RETAIN] Failed to delete file {storage_key}: {e}")
+
     async def _handle_consolidation(self, task_dict: dict[str, Any]):
         """
         Handler for consolidation tasks.
@@ -798,6 +956,8 @@ class MemoryEngine(MemoryEngineInterface):
         try:
             if task_type == "batch_retain":
                 await self._handle_batch_retain(task_dict)
+            elif task_type == "file_convert_retain":
+                await self._handle_file_convert_retain(task_dict)
             elif task_type == "consolidation":
                 await self._handle_consolidation(task_dict)
             elif task_type == "refresh_mental_model":
@@ -810,7 +970,8 @@ class MemoryEngine(MemoryEngineInterface):
                 return
 
             # Task succeeded - mark operation as completed
-            if operation_id:
+            # file_convert_retain marks itself as completed in a transaction, skip double-marking
+            if operation_id and task_type != "file_convert_retain":
                 await self._mark_operation_completed(operation_id)
 
         except Exception as e:
@@ -823,14 +984,19 @@ class MemoryEngine(MemoryEngineInterface):
             error_traceback = traceback.format_exc()
             traceback.print_exc()
 
-            if retry_count < max_retries:
+            # Don't retry file conversion - if conversion fails, it won't succeed on retry
+            # (missing OCR, corrupted file, unsupported format, etc.)
+            should_retry = retry_count < max_retries and task_type != "file_convert_retain"
+
+            if should_retry:
                 # Reschedule with incremented retry count
                 task_dict["retry_count"] = retry_count + 1
                 logger.info(f"Rescheduling task {task_type} (retry {retry_count + 1}/{max_retries})")
                 await self._task_backend.submit_task(task_dict)
             else:
-                # Max retries exceeded - mark operation as failed
-                logger.error(f"Max retries exceeded for task {task_type}, marking as failed")
+                # Max retries exceeded or non-retryable task - mark operation as failed
+                reason = "non-retryable task type" if task_type == "file_convert_retain" else "max retries exceeded"
+                logger.error(f"Not retrying task {task_type} ({reason}), marking as failed")
                 if operation_id:
                     await self._mark_operation_failed(operation_id, str(e), error_traceback)
 
@@ -1196,6 +1362,34 @@ class MemoryEngine(MemoryEngineInterface):
         self._config_resolver = ConfigResolver(pool=self._pool, tenant_extension=self._tenant_extension)
         logger.debug("Config resolver initialized for hierarchical configuration")
 
+        # Initialize file storage
+        from .storage import create_file_storage
+
+        config = get_config()
+        self._file_storage = create_file_storage(
+            storage_type=config.file_storage_type,
+            pool_getter=lambda: self._pool,
+            schema=get_current_schema() if get_current_schema() != config.database_schema else None,
+        )
+        logger.debug(f"File storage initialized ({config.file_storage_type})")
+
+        # Initialize parser registry
+        from .parsers import FileParserRegistry, IrisParser, MarkitdownParser
+
+        self._parser_registry = FileParserRegistry()
+        try:
+            self._parser_registry.register(MarkitdownParser())
+            logger.debug("Registered markitdown parser")
+        except ImportError:
+            logger.warning("markitdown not available - file parsing disabled")
+        iris_token = config.file_parser_iris_token
+        iris_org_id = config.file_parser_iris_org_id
+        if iris_token and iris_org_id:
+            self._parser_registry.register(IrisParser(token=iris_token, org_id=iris_org_id))
+            logger.debug("Registered iris parser")
+        else:
+            logger.debug("Iris parser not registered (VECTORIZE_TOKEN or VECTORIZE_ORG_ID not set)")
+
         # Set executor for task backend and initialize
         self._task_backend.set_executor(self.execute_task)
         await self._task_backend.initialize()
@@ -2684,15 +2878,44 @@ class MemoryEngine(MemoryEngineInterface):
                     )
                 top_results_dicts.append(result_dict)
 
+            # Get entities for each fact if include_entities is requested
+            fact_entity_map = {}  # unit_id -> list of (entity_id, entity_name)
+            if include_entities and top_scored:
+                unit_ids = [uuid.UUID(sr.id) for sr in top_scored]
+                if unit_ids:
+                    async with acquire_with_retry(pool) as entity_conn:
+                        entity_rows = await entity_conn.fetch(
+                            f"""
+                            SELECT ue.unit_id, e.id as entity_id, e.canonical_name
+                            FROM {fq_table("unit_entities")} ue
+                            JOIN {fq_table("entities")} e ON ue.entity_id = e.id
+                            WHERE ue.unit_id = ANY($1::uuid[])
+                            """,
+                            unit_ids,
+                        )
+                        for row in entity_rows:
+                            unit_id = str(row["unit_id"])
+                            if unit_id not in fact_entity_map:
+                                fact_entity_map[unit_id] = []
+                            fact_entity_map[unit_id].append(
+                                {"entity_id": str(row["entity_id"]), "canonical_name": row["canonical_name"]}
+                            )
+
             # Convert results to MemoryFact objects
             memory_facts = []
             for result_dict in top_results_dicts:
+                result_id = str(result_dict.get("id"))
+                # Get entity names for this fact
+                entity_names = None
+                if include_entities and result_id in fact_entity_map:
+                    entity_names = [e["canonical_name"] for e in fact_entity_map[result_id]]
+
                 memory_facts.append(
                     MemoryFact(
-                        id=str(result_dict.get("id")),
+                        id=result_id,
                         text=result_dict.get("text"),
                         fact_type=result_dict.get("fact_type", "world"),
-                        entities=None,  # Entity observations removed
+                        entities=entity_names,
                         context=result_dict.get("context"),
                         occurred_start=result_dict.get("occurred_start"),
                         occurred_end=result_dict.get("occurred_end"),
@@ -2703,8 +2926,32 @@ class MemoryEngine(MemoryEngineInterface):
                     )
                 )
 
-            # Entity observations removed - always set to None
+            # Fetch entity observations if requested
             entities_dict = None
+            total_entity_tokens = 0
+            if include_entities and fact_entity_map:
+                # Collect unique entities in order of fact relevance (preserving order from top_scored)
+                entities_ordered = []  # list of (entity_id, entity_name) tuples
+                seen_entity_ids = set()
+
+                for sr in top_scored:
+                    unit_id = sr.id
+                    if unit_id in fact_entity_map:
+                        for entity in fact_entity_map[unit_id]:
+                            entity_id = entity["entity_id"]
+                            entity_name = entity["canonical_name"]
+                            if entity_id not in seen_entity_ids:
+                                entities_ordered.append((entity_id, entity_name))
+                                seen_entity_ids.add(entity_id)
+
+                # Return entities with empty observations (summaries now live in mental models)
+                entities_dict = {}
+                for entity_id, entity_name in entities_ordered:
+                    entities_dict[entity_name] = EntityState(
+                        entity_id=entity_id,
+                        canonical_name=entity_name,
+                        observations=[],  # Mental models provide this now
+                    )
 
             # Finalize trace if enabled
             trace_dict = None
@@ -2715,6 +2962,7 @@ class MemoryEngine(MemoryEngineInterface):
             # Log final recall stats
             total_time = time.time() - recall_start
             num_chunks = len(chunks_dict) if chunks_dict else 0
+            num_entities = len(entities_dict) if entities_dict else 0
             # Include wait times in log if significant
             wait_parts = []
             if semaphore_wait > 0.01:
@@ -2723,7 +2971,7 @@ class MemoryEngine(MemoryEngineInterface):
                 wait_parts.append(f"conn={max_conn_wait:.3f}s")
             wait_info = f" | waits: {', '.join(wait_parts)}" if wait_parts else ""
             log_buffer.append(
-                f"[RECALL {recall_id}] Complete: {len(top_scored)} facts ({total_tokens} tok), {num_chunks} chunks ({total_chunk_tokens} tok) | {fact_type_summary} | {total_time:.3f}s{wait_info}"
+                f"[RECALL {recall_id}] Complete: {len(top_scored)} facts ({total_tokens} tok), {num_chunks} chunks ({total_chunk_tokens} tok), {num_entities} entities ({total_entity_tokens} tok) | {fact_type_summary} | {total_time:.3f}s{wait_info}"
             )
             if not quiet:
                 logger.info("\n" + "\n".join(log_buffer))
@@ -5938,13 +6186,14 @@ class MemoryEngine(MemoryEngineInterface):
         async with acquire_with_retry(pool) as conn:
             await conn.execute(
                 f"""
-                INSERT INTO {fq_table("async_operations")} (operation_id, bank_id, operation_type, result_metadata)
-                VALUES ($1, $2, $3, $4)
+                INSERT INTO {fq_table("async_operations")} (operation_id, bank_id, operation_type, result_metadata, status)
+                VALUES ($1, $2, $3, $4, $5)
                 """,
                 operation_id,
                 bank_id,
                 operation_type,
                 json.dumps(result_metadata or {}),
+                "pending",
             )
 
         # Build and submit task payload
@@ -6092,6 +6341,116 @@ class MemoryEngine(MemoryEngineInterface):
             "items_count": len(contents),
         }
 
+    async def submit_async_file_retain(
+        self,
+        bank_id: str,
+        file_items: list[dict[str, Any]],
+        parser: str,
+        document_tags: list[str] | None,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        Submit batch file conversion + retain operation.
+
+        Each file is converted to markdown and then retained as a memory.
+        Files are stored in object storage and conversion happens asynchronously.
+
+        Args:
+            bank_id: Bank ID
+            file_items: List of file items, each containing:
+                - file: UploadFile object (FastAPI)
+                - document_id: Document ID
+                - context: Optional context
+                - metadata: Optional metadata dict
+                - tags: Optional tags list
+                - timestamp: Optional timestamp
+            parser: Parser name (e.g., "markitdown")
+            document_tags: Tags applied to all documents
+            request_context: Request context for authentication
+
+        Returns:
+            dict with operation_id and files_count
+        """
+        await self._authenticate_tenant(request_context)
+
+        config = get_config()
+
+        # Validate file count
+        if len(file_items) > config.file_conversion_max_batch_size:
+            raise ValueError(f"Too many files. Maximum {config.file_conversion_max_batch_size} files per request.")
+
+        # Read all files and validate total batch size
+        files_data = []
+        total_batch_size = 0
+
+        for item in file_items:
+            file = item["file"]
+            file_data = await file.read()
+            total_batch_size += len(file_data)
+            files_data.append((item, file, file_data))
+
+        # Validate total batch size
+        if total_batch_size > config.file_conversion_max_batch_size_bytes:
+            total_mb = total_batch_size / (1024 * 1024)
+            raise ValueError(
+                f"Total batch size ({total_mb:.1f}MB) exceeds maximum of {config.file_conversion_max_batch_size_mb}MB"
+            )
+
+        # Submit individual operation for each file
+        operation_ids = []
+        for item, file, file_data in files_data:
+            # Generate storage key
+            storage_key = f"banks/{bank_id}/files/{item['document_id']}/{file.filename}"
+
+            # Store file in object storage
+            await self._file_storage.store(
+                file_data=file_data,
+                key=storage_key,
+                metadata={
+                    "content_type": file.content_type or "application/octet-stream",
+                    "original_filename": file.filename,
+                    "bank_id": bank_id,
+                    "document_id": item["document_id"],
+                },
+            )
+
+            # Create individual operation and submit task
+            task_payload: dict[str, Any] = {
+                "document_id": item["document_id"],
+                "storage_key": storage_key,
+                "original_filename": file.filename,
+                "content_type": file.content_type or "application/octet-stream",
+                "parser": parser,
+                "context": item.get("context"),
+                "metadata": item.get("metadata", {}),
+                "tags": item.get("tags", []),
+                "document_tags": document_tags or [],
+                "timestamp": item.get("timestamp"),
+            }
+
+            # Pass tenant_id and api_key_id through task payload
+            if request_context.tenant_id:
+                task_payload["_tenant_id"] = request_context.tenant_id
+            if request_context.api_key_id:
+                task_payload["_api_key_id"] = request_context.api_key_id
+
+            result = await self._submit_async_operation(
+                bank_id=bank_id,
+                operation_type="file_convert_retain",
+                task_type="file_convert_retain",
+                task_payload=task_payload,
+                result_metadata={
+                    "original_filename": file.filename,
+                },
+                dedupe_by_bank=False,
+            )
+            operation_ids.append(result["operation_id"])
+
+        return {
+            "operation_ids": operation_ids,
+            "files_count": len(file_items),
+        }
+
     async def submit_async_consolidation(
         self,
         bank_id: str,

hindsight-api/hindsight_api/engine/parsers/__init__.py

@@ -0,0 +1,62 @@
+"""File parser implementations."""
+
+from .base import FileParser, UnsupportedFileTypeError
+from .iris import IrisParser
+from .markitdown import MarkitdownParser
+
+__all__ = ["FileParser", "UnsupportedFileTypeError", "IrisParser", "MarkitdownParser", "FileParserRegistry"]
+
+
+class FileParserRegistry:
+    """Registry for file parsers with auto-detection."""
+
+    def __init__(self):
+        """Initialize empty parser registry."""
+        self._parsers: dict[str, FileParser] = {}
+
+    def register(self, parser: FileParser):
+        """
+        Register a parser.
+
+        Args:
+            parser: FileParser instance
+        """
+        self._parsers[parser.name()] = parser
+
+    def get_parser(
+        self,
+        name: str | None,
+        filename: str,
+        content_type: str | None = None,
+    ) -> FileParser:
+        """
+        Get parser by name or auto-detect.
+
+        Args:
+            name: Parser name (e.g., "markitdown") or None for auto-detect
+            filename: File name for auto-detection
+            content_type: MIME type (optional)
+
+        Returns:
+            FileParser instance
+
+        Raises:
+            ValueError: If no suitable parser found
+        """
+        if name:
+            # Explicit parser requested — return it directly, let the parser
+            # raise UnsupportedFileTypeError from convert() if needed
+            if name not in self._parsers:
+                raise ValueError(f"Parser '{name}' not found. Available: {list(self._parsers.keys())}")
+            return self._parsers[name]
+
+        # Auto-detect parser
+        for parser in self._parsers.values():
+            if parser.supports(filename, content_type):
+                return parser
+
+        raise ValueError(f"No parser found for {filename}. Available parsers: {list(self._parsers.keys())}")
+
+    def list_parsers(self) -> list[str]:
+        """Get list of registered parser names."""
+        return list(self._parsers.keys())

hindsight-api/hindsight_api/engine/parsers/base.py

@@ -0,0 +1,58 @@
+"""Abstract base class for file parsers."""
+
+from abc import ABC, abstractmethod
+
+
+class UnsupportedFileTypeError(Exception):
+    """Raised by a parser when it does not support the given file type."""
+
+    pass
+
+
+class FileParser(ABC):
+    """Abstract base for file to markdown parsers."""
+
+    @abstractmethod
+    async def convert(self, file_data: bytes, filename: str) -> str:
+        """
+        Parse file to markdown.
+
+        Args:
+            file_data: Raw file bytes
+            filename: Original filename (used for format detection)
+
+        Returns:
+            Markdown content as string
+
+        Raises:
+            UnsupportedFileTypeError: If the file type is not supported by this parser
+            RuntimeError: If parsing fails for another reason
+        """
+        pass
+
+    def supports(self, filename: str, content_type: str | None = None) -> bool:
+        """
+        Check if parser supports this file type.
+
+        Override this for local/static extension-based filtering.
+        Parsers that delegate to a remote service should leave this as True
+        and raise UnsupportedFileTypeError from convert() instead.
+
+        Args:
+            filename: File name (used for extension check)
+            content_type: MIME type (optional)
+
+        Returns:
+            True if this parser can handle the file (default: True)
+        """
+        return True
+
+    @abstractmethod
+    def name(self) -> str:
+        """
+        Get parser name.
+
+        Returns:
+            Parser name (e.g., "markitdown")
+        """
+        pass

hindsight-api/hindsight_api/engine/parsers/iris.py

@@ -0,0 +1,137 @@
+"""Iris parser implementation using the Vectorize Iris HTTP API."""
+
+import asyncio
+import logging
+import mimetypes
+import time
+
+import httpx
+
+from .base import FileParser, UnsupportedFileTypeError
+
+logger = logging.getLogger(__name__)
+
+_IRIS_BASE_URL = "https://api.vectorize.io/v1"
+_DEFAULT_POLL_INTERVAL = 2.0  # seconds
+_DEFAULT_TIMEOUT = 300.0  # seconds
+
+
+class IrisParser(FileParser):
+    """
+    Iris file parser using the Vectorize Iris cloud extraction service.
+
+    Uploads files to the Vectorize Iris API, starts an extraction job,
+    and polls until the text is ready. The API determines which file types
+    are supported — UnsupportedFileTypeError is raised if the file is rejected.
+
+    Authentication:
+        Requires HINDSIGHT_API_FILE_PARSER_IRIS_TOKEN and
+        HINDSIGHT_API_FILE_PARSER_IRIS_ORG_ID environment variables,
+        or pass them explicitly via the constructor.
+    """
+
+    def __init__(
+        self,
+        token: str,
+        org_id: str,
+        poll_interval: float = _DEFAULT_POLL_INTERVAL,
+        timeout: float = _DEFAULT_TIMEOUT,
+    ):
+        """
+        Initialize iris parser.
+
+        Args:
+            token: Vectorize API token
+            org_id: Vectorize organization ID
+            poll_interval: Seconds between status poll requests (default: 2)
+            timeout: Maximum seconds to wait for extraction (default: 300)
+        """
+        self._token = token
+        self._org_id = org_id
+        self._poll_interval = poll_interval
+        self._timeout = timeout
+        self._auth_headers = {"Authorization": f"Bearer {token}"}
+
+    async def convert(self, file_data: bytes, filename: str) -> str:
+        """
+        Parse file to text using the Vectorize Iris API.
+
+        Raises:
+            UnsupportedFileTypeError: If the Iris API rejects the file type (4xx)
+            RuntimeError: If extraction fails for another reason
+        """
+        content_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
+
+        async with httpx.AsyncClient() as client:
+            # Step 1: Request a presigned upload URL
+            init_resp = await client.post(
+                f"{_IRIS_BASE_URL}/org/{self._org_id}/files",
+                headers=self._auth_headers,
+                json={"name": filename, "contentType": content_type},
+            )
+            _raise_for_status(init_resp, filename, "file upload init")
+            init_data = init_resp.json()
+            file_id: str = init_data["fileId"]
+            upload_url: str = init_data["uploadUrl"]
+
+            # Step 2: Upload the file bytes to the presigned URL (no auth header)
+            upload_resp = await client.put(
+                upload_url,
+                content=file_data,
+                headers={"Content-Type": content_type},
+            )
+            _raise_for_status(upload_resp, filename, "file upload")
+
+            # Step 3: Start extraction
+            extract_resp = await client.post(
+                f"{_IRIS_BASE_URL}/org/{self._org_id}/extraction",
+                headers=self._auth_headers,
+                json={"fileId": file_id},
+            )
+            _raise_for_status(extract_resp, filename, "start extraction")
+            extraction_id: str = extract_resp.json()["extractionId"]
+
+            # Step 4: Poll until ready or timeout
+            deadline = time.monotonic() + self._timeout
+            while True:
+                status_resp = await client.get(
+                    f"{_IRIS_BASE_URL}/org/{self._org_id}/extraction/{extraction_id}",
+                    headers=self._auth_headers,
+                )
+                _raise_for_status(status_resp, filename, "poll extraction status")
+                status_data = status_resp.json()
+
+                if status_data.get("ready"):
+                    data = status_data.get("data", {})
+                    if not data.get("success"):
+                        error = data.get("error", "unknown error")
+                        raise RuntimeError(f"Iris extraction failed for '{filename}': {error}")
+                    text = data.get("text")
+                    if not text:
+                        raise RuntimeError(f"No content extracted from '{filename}'")
+                    return text
+
+                if time.monotonic() >= deadline:
+                    raise RuntimeError(f"Iris extraction timed out after {self._timeout}s for '{filename}'")
+
+                await asyncio.sleep(self._poll_interval)
+
+    def name(self) -> str:
+        """Get parser name."""
+        return "iris"
+
+
+def _raise_for_status(response: httpx.Response, filename: str, step: str) -> None:
+    """
+    Raise an appropriate error including the response body on HTTP errors.
+
+    Raises UnsupportedFileTypeError for 4xx responses (file rejected by the API),
+    RuntimeError for other HTTP errors.
+    """
+    if not response.is_error:
+        return
+    body = response.text or "<empty>"
+    msg = f"Iris API error during {step} for '{filename}': {response.status_code} {response.reason_phrase} — {body}"
+    if response.is_client_error:
+        raise UnsupportedFileTypeError(msg)
+    raise RuntimeError(msg)

hindsight-api/hindsight_api/engine/parsers/markitdown.py

@@ -0,0 +1,109 @@
+"""Markitdown parser implementation."""
+
+import asyncio
+import logging
+import tempfile
+from pathlib import Path
+
+from .base import FileParser
+
+logger = logging.getLogger(__name__)
+
+
+class MarkitdownParser(FileParser):
+    """
+    Markitdown file parser.
+
+    Uses Microsoft's markitdown library to convert various file formats
+    to markdown including PDF, Office docs, images (via OCR), audio, HTML.
+
+    Supported formats:
+    - PDF (.pdf)
+    - Word (.docx, .doc)
+    - PowerPoint (.pptx, .ppt)
+    - Excel (.xlsx, .xls)
+    - Images (.jpg, .jpeg, .png) - with OCR
+    - HTML (.html, .htm)
+    - Text (.txt, .md)
+    - Audio (.mp3, .wav) - with transcription
+    """
+
+    def __init__(self):
+        """Initialize markitdown parser."""
+        # Lazy import to avoid requiring markitdown for all users
+        try:
+            from markitdown import MarkItDown
+
+            self._markitdown = MarkItDown()
+        except ImportError as e:
+            raise ImportError(
+                "markitdown package is required for file parsing. Install with: pip install markitdown"
+            ) from e
+
+    async def convert(self, file_data: bytes, filename: str) -> str:
+        """Parse file to markdown using markitdown."""
+        # markitdown is synchronous, so we run it in executor to avoid blocking
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, self._convert_sync, file_data, filename)
+
+    def _convert_sync(self, file_data: bytes, filename: str) -> str:
+        """Synchronous parsing (runs in thread pool)."""
+        # Write to temp file (markitdown requires file path)
+        with tempfile.NamedTemporaryFile(suffix=Path(filename).suffix, delete=False) as tmp:
+            tmp.write(file_data)
+            tmp_path = tmp.name
+
+        try:
+            # Parse using markitdown
+            result = self._markitdown.convert(tmp_path)
+
+            if not result or not result.text_content:
+                raise RuntimeError(f"No content extracted from '{filename}'")
+
+            return result.text_content
+
+        except Exception as e:
+            logger.error(f"Markitdown parsing failed for {filename}: {e}")
+            raise RuntimeError(f"Failed to parse '{filename}': {e}") from e
+
+        finally:
+            # Clean up temp file
+            try:
+                Path(tmp_path).unlink()
+            except Exception:
+                pass
+
+    def supports(self, filename: str, content_type: str | None = None) -> bool:
+        """Check if markitdown supports this file type."""
+        # Supported extensions (from markitdown docs)
+        supported_extensions = {
+            # Documents
+            ".pdf",
+            ".docx",
+            ".doc",
+            ".pptx",
+            ".ppt",
+            ".xlsx",
+            ".xls",
+            # Images (with OCR)
+            ".jpg",
+            ".jpeg",
+            ".png",
+            # Web
+            ".html",
+            ".htm",
+            # Text
+            ".txt",
+            ".md",
+            ".csv",
+            # Audio (with transcription)
+            ".mp3",
+            ".wav",
+        }
+
+        ext = Path(filename).suffix.lower()
+        return ext in supported_extensions
+
+    def name(self) -> str:
+        """Get parser name."""
+        return "markitdown"

hindsight-api/hindsight_api/engine/storage/__init__.py

@@ -0,0 +1,77 @@
+"""File storage backends for uploaded files."""
+
+from collections.abc import Callable
+
+from .base import FileStorage
+from .postgresql import PostgreSQLFileStorage
+
+__all__ = ["FileStorage", "PostgreSQLFileStorage", "create_file_storage"]
+
+
+def create_file_storage(
+    storage_type: str,
+    pool_getter: Callable | None = None,
+    schema: str | None = None,
+    **kwargs,
+) -> FileStorage:
+    """
+    Create file storage backend based on configuration.
+
+    Args:
+        storage_type: "native" (PostgreSQL BYTEA) or "s3" (S3-compatible object storage)
+        pool_getter: Database pool getter (required for native)
+        schema: Database schema (for native multi-tenant)
+        **kwargs: Additional args passed to storage backend
+
+    Returns:
+        FileStorage instance
+
+    Raises:
+        ValueError: If storage_type is unknown or required args are missing
+    """
+    if storage_type == "native":
+        if not pool_getter:
+            raise ValueError("pool_getter required for native (PostgreSQL) storage")
+        return PostgreSQLFileStorage(pool_getter=pool_getter, schema=schema)
+    elif storage_type == "s3":
+        from ...config import get_config
+        from .s3 import S3FileStorage
+
+        config = get_config()
+        bucket = config.file_storage_s3_bucket
+        if not bucket:
+            raise ValueError("HINDSIGHT_API_FILE_STORAGE_S3_BUCKET is required for S3 storage")
+        return S3FileStorage(
+            bucket=bucket,
+            region=config.file_storage_s3_region,
+            endpoint=config.file_storage_s3_endpoint,
+            access_key_id=config.file_storage_s3_access_key_id,
+            secret_access_key=config.file_storage_s3_secret_access_key,
+        )
+    elif storage_type == "gcs":
+        from ...config import get_config
+        from .gcs import GCSFileStorage
+
+        config = get_config()
+        bucket = config.file_storage_gcs_bucket
+        if not bucket:
+            raise ValueError("HINDSIGHT_API_FILE_STORAGE_GCS_BUCKET is required for GCS storage")
+        return GCSFileStorage(
+            bucket=bucket,
+            service_account_key=config.file_storage_gcs_service_account_key,
+        )
+    elif storage_type == "azure":
+        from ...config import get_config
+        from .azure import AzureFileStorage
+
+        config = get_config()
+        container = config.file_storage_azure_container
+        if not container:
+            raise ValueError("HINDSIGHT_API_FILE_STORAGE_AZURE_CONTAINER is required for Azure storage")
+        return AzureFileStorage(
+            container_name=container,
+            account_name=config.file_storage_azure_account_name,
+            account_key=config.file_storage_azure_account_key,
+        )
+    else:
+        raise ValueError(f"Unknown storage type: {storage_type}. Supported: 'native', 's3', 'gcs', 'azure'.")

hindsight-api/hindsight_api/engine/storage/azure.py

@@ -0,0 +1,62 @@
+"""Azure Blob Storage backend using obstore."""
+
+import logging
+from datetime import timedelta
+
+import obstore as obs
+from obstore.store import AzureStore
+
+from .base import FileStorage
+
+logger = logging.getLogger(__name__)
+
+
+class AzureFileStorage(FileStorage):
+    """
+    Azure Blob Storage backend.
+
+    Uses obstore (Rust-backed) for high-throughput async access to Azure Blob Storage.
+    Supports account key, SAS token, and default Azure credentials.
+    """
+
+    def __init__(
+        self,
+        container_name: str,
+        account_name: str | None = None,
+        account_key: str | None = None,
+    ):
+        kwargs: dict = {}
+        if account_name:
+            kwargs["account_name"] = account_name
+        if account_key:
+            kwargs["account_key"] = account_key
+
+        self._store = AzureStore(container_name, **kwargs)
+        logger.info(f"Initialized Azure file storage: container={container_name}, account={account_name}")
+
+    async def store(self, file_data: bytes, key: str, metadata: dict[str, str] | None = None) -> str:
+        await obs.put_async(self._store, key, file_data)
+        logger.debug(f"Stored file {key} ({len(file_data)} bytes) in Azure")
+        return key
+
+    async def retrieve(self, key: str) -> bytes:
+        try:
+            response = await obs.get_async(self._store, key)
+            return await response.bytes_async()
+        except Exception as e:
+            if "not found" in str(e).lower() or "BlobNotFound" in str(e):
+                raise FileNotFoundError(f"File not found: {key}") from e
+            raise
+
+    async def delete(self, key: str) -> None:
+        await obs.delete_async(self._store, key)
+
+    async def exists(self, key: str) -> bool:
+        try:
+            await obs.head_async(self._store, key)
+            return True
+        except Exception:
+            return False
+
+    async def get_download_url(self, key: str, expires_in: int = 3600) -> str:
+        return await obs.sign_async(self._store, "GET", key, timedelta(seconds=expires_in))

hindsight-api/hindsight_api/engine/storage/base.py

@@ -0,0 +1,83 @@
+"""Abstract base class for file storage backends."""
+
+from abc import ABC, abstractmethod
+
+
+class FileStorage(ABC):
+    """Abstract base for file storage backends."""
+
+    @abstractmethod
+    async def store(
+        self,
+        file_data: bytes,
+        key: str,
+        metadata: dict[str, str] | None = None,
+    ) -> str:
+        """
+        Store file and return storage key.
+
+        Args:
+            file_data: Raw file bytes
+            key: Storage key (e.g., "banks/{bank_id}/files/{file_id}.pdf")
+            metadata: Optional metadata to store with file
+
+        Returns:
+            Storage key that can be used to retrieve the file
+        """
+        pass
+
+    @abstractmethod
+    async def retrieve(self, key: str) -> bytes:
+        """
+        Retrieve file by storage key.
+
+        Args:
+            key: Storage key
+
+        Returns:
+            File data as bytes
+
+        Raises:
+            FileNotFoundError: If file does not exist
+        """
+        pass
+
+    @abstractmethod
+    async def delete(self, key: str) -> None:
+        """
+        Delete file by storage key.
+
+        Args:
+            key: Storage key
+        """
+        pass
+
+    @abstractmethod
+    async def exists(self, key: str) -> bool:
+        """
+        Check if file exists.
+
+        Args:
+            key: Storage key
+
+        Returns:
+            True if file exists, False otherwise
+        """
+        pass
+
+    @abstractmethod
+    async def get_download_url(self, key: str, expires_in: int = 3600) -> str:
+        """
+        Get a URL for downloading the file.
+
+        For PostgreSQL storage, this might be a relative API path.
+        For S3, this would be a pre-signed URL.
+
+        Args:
+            key: Storage key
+            expires_in: Expiration time in seconds (may be ignored for some backends)
+
+        Returns:
+            Download URL or path
+        """
+        pass

hindsight-api/hindsight_api/engine/storage/gcs.py

@@ -0,0 +1,59 @@
+"""Google Cloud Storage backend using obstore."""
+
+import logging
+from datetime import timedelta
+
+import obstore as obs
+from obstore.store import GCSStore
+
+from .base import FileStorage
+
+logger = logging.getLogger(__name__)
+
+
+class GCSFileStorage(FileStorage):
+    """
+    Google Cloud Storage backend.
+
+    Uses obstore (Rust-backed) for high-throughput async access to GCS.
+    Supports Application Default Credentials, service account keys, and explicit credentials.
+    """
+
+    def __init__(
+        self,
+        bucket: str,
+        service_account_key: str | None = None,
+    ):
+        kwargs: dict = {}
+        if service_account_key:
+            kwargs["service_account_key"] = service_account_key
+
+        self._store = GCSStore(bucket, **kwargs)
+        logger.info(f"Initialized GCS file storage: bucket={bucket}")
+
+    async def store(self, file_data: bytes, key: str, metadata: dict[str, str] | None = None) -> str:
+        await obs.put_async(self._store, key, file_data)
+        logger.debug(f"Stored file {key} ({len(file_data)} bytes) in GCS")
+        return key
+
+    async def retrieve(self, key: str) -> bytes:
+        try:
+            response = await obs.get_async(self._store, key)
+            return await response.bytes_async()
+        except Exception as e:
+            if "not found" in str(e).lower():
+                raise FileNotFoundError(f"File not found: {key}") from e
+            raise
+
+    async def delete(self, key: str) -> None:
+        await obs.delete_async(self._store, key)
+
+    async def exists(self, key: str) -> bool:
+        try:
+            await obs.head_async(self._store, key)
+            return True
+        except Exception:
+            return False
+
+    async def get_download_url(self, key: str, expires_in: int = 3600) -> str:
+        return await obs.sign_async(self._store, "GET", key, timedelta(seconds=expires_in))

hindsight-api/hindsight_api/engine/storage/postgresql.py

@@ -0,0 +1,139 @@
+"""PostgreSQL BYTEA-based file storage (default, zero-config)."""
+
+import logging
+from collections.abc import Callable
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import asyncpg
+
+from .base import FileStorage
+
+logger = logging.getLogger(__name__)
+
+
+def fq_table(table: str, schema: str | None = None) -> str:
+    """Get fully-qualified table name with optional schema prefix."""
+    if schema:
+        return f'"{schema}".{table}'
+    return table
+
+
+class PostgreSQLFileStorage(FileStorage):
+    """
+    PostgreSQL BYTEA-based file storage.
+
+    Stores files directly in PostgreSQL using BYTEA columns.
+    This is the default storage backend - zero configuration required!
+
+    Pros:
+    - Works out of the box (no external dependencies)
+    - Transactional consistency with database
+    - Simple backups (included in pg_dump)
+    - Good performance for <10MB files
+
+    Cons:
+    - Database bloat for large/many files
+    - Not ideal for distributed deployments
+    - Higher cost than object storage at scale
+
+    For production/scale, consider S3FileStorage instead.
+    """
+
+    def __init__(self, pool_getter: Callable[[], "asyncpg.Pool"], schema: str | None = None):
+        """
+        Initialize PostgreSQL file storage.
+
+        Args:
+            pool_getter: Function that returns asyncpg connection pool
+            schema: Database schema (for multi-tenant support)
+        """
+        self._pool_getter = pool_getter
+        self._schema = schema
+
+    async def store(
+        self,
+        file_data: bytes,
+        key: str,
+        metadata: dict[str, str] | None = None,
+    ) -> str:
+        """Store file in PostgreSQL."""
+        pool = self._pool_getter()
+
+        async with pool.acquire() as conn:
+            await conn.execute(
+                f"""
+                INSERT INTO {fq_table("file_storage", self._schema)}
+                (storage_key, data)
+                VALUES ($1, $2)
+                ON CONFLICT (storage_key) DO UPDATE SET
+                    data = EXCLUDED.data
+                """,
+                key,
+                file_data,
+            )
+
+        logger.debug(f"Stored file {key} ({len(file_data)} bytes) in PostgreSQL")
+        return key
+
+    async def retrieve(self, key: str) -> bytes:
+        """Retrieve file from PostgreSQL."""
+        pool = self._pool_getter()
+
+        async with pool.acquire() as conn:
+            row = await conn.fetchrow(
+                f"""
+                SELECT data FROM {fq_table("file_storage", self._schema)}
+                WHERE storage_key = $1
+                """,
+                key,
+            )
+
+            if not row:
+                raise FileNotFoundError(f"File not found: {key}")
+
+            return bytes(row["data"])
+
+    async def delete(self, key: str) -> None:
+        """Delete file from PostgreSQL."""
+        pool = self._pool_getter()
+
+        async with pool.acquire() as conn:
+            result = await conn.execute(
+                f"""
+                DELETE FROM {fq_table("file_storage", self._schema)}
+                WHERE storage_key = $1
+                """,
+                key,
+            )
+
+            # Check if anything was deleted
+            if result == "DELETE 0":
+                logger.warning(f"Attempted to delete non-existent file: {key}")
+
+    async def exists(self, key: str) -> bool:
+        """Check if file exists in PostgreSQL."""
+        pool = self._pool_getter()
+
+        async with pool.acquire() as conn:
+            row = await conn.fetchrow(
+                f"""
+                SELECT 1 FROM {fq_table("file_storage", self._schema)}
+                WHERE storage_key = $1
+                """,
+                key,
+            )
+
+            return row is not None
+
+    async def get_download_url(self, key: str, expires_in: int = 3600) -> str:
+        """
+        Get download URL for PostgreSQL-stored file.
+
+        Returns an API endpoint path (not a pre-signed URL since the file
+        is stored in the database). The expires_in parameter is ignored
+        for PostgreSQL storage.
+        """
+        # Return API path for download endpoint
+        # (expires_in ignored for database storage - auth handled at API level)
+        return f"/v1/default/files/download/{key}"

hindsight-api/hindsight_api/engine/storage/s3.py

@@ -0,0 +1,71 @@
+"""S3 object storage backend using obstore."""
+
+import logging
+from datetime import timedelta
+
+import obstore as obs
+from obstore.store import S3Store
+
+from .base import FileStorage
+
+logger = logging.getLogger(__name__)
+
+
+class S3FileStorage(FileStorage):
+    """
+    S3-compatible object storage backend.
+
+    Uses obstore (Rust-backed) for high-throughput async access to
+    Amazon S3, MinIO, Cloudflare R2, and other S3-compliant APIs.
+    """
+
+    def __init__(
+        self,
+        bucket: str,
+        region: str | None = None,
+        endpoint: str | None = None,
+        access_key_id: str | None = None,
+        secret_access_key: str | None = None,
+    ):
+        kwargs: dict = {}
+        if region:
+            kwargs["region"] = region
+        if endpoint:
+            kwargs["endpoint"] = endpoint
+            # Allow plain HTTP for local S3-compatible services (MinIO, LocalStack, etc.)
+            if endpoint.startswith("http://"):
+                kwargs["allow_http"] = True
+        if access_key_id:
+            kwargs["access_key_id"] = access_key_id
+        if secret_access_key:
+            kwargs["secret_access_key"] = secret_access_key
+
+        self._store = S3Store(bucket, **kwargs)
+        logger.info(f"Initialized S3 file storage: bucket={bucket}, region={region}, endpoint={endpoint}")
+
+    async def store(self, file_data: bytes, key: str, metadata: dict[str, str] | None = None) -> str:
+        await obs.put_async(self._store, key, file_data)
+        logger.debug(f"Stored file {key} ({len(file_data)} bytes) in S3")
+        return key
+
+    async def retrieve(self, key: str) -> bytes:
+        try:
+            response = await obs.get_async(self._store, key)
+            return await response.bytes_async()
+        except Exception as e:
+            if "not found" in str(e).lower() or "NoSuchKey" in str(e):
+                raise FileNotFoundError(f"File not found: {key}") from e
+            raise
+
+    async def delete(self, key: str) -> None:
+        await obs.delete_async(self._store, key)
+
+    async def exists(self, key: str) -> bool:
+        try:
+            await obs.head_async(self._store, key)
+            return True
+        except Exception:
+            return False
+
+    async def get_download_url(self, key: str, expires_in: int = 3600) -> str:
+        return await obs.sign_async(self._store, "GET", key, timedelta(seconds=expires_in))

hindsight-api/hindsight_api/main.py

@@ -250,6 +250,24 @@ def main():
             retain_batch_tokens=config.retain_batch_tokens,
             retain_batch_enabled=config.retain_batch_enabled,
             retain_batch_poll_interval_seconds=config.retain_batch_poll_interval_seconds,
+            file_storage_type=config.file_storage_type,
+            file_storage_s3_bucket=config.file_storage_s3_bucket,
+            file_storage_s3_region=config.file_storage_s3_region,
+            file_storage_s3_endpoint=config.file_storage_s3_endpoint,
+            file_storage_s3_access_key_id=config.file_storage_s3_access_key_id,
+            file_storage_s3_secret_access_key=config.file_storage_s3_secret_access_key,
+            file_storage_gcs_bucket=config.file_storage_gcs_bucket,
+            file_storage_gcs_service_account_key=config.file_storage_gcs_service_account_key,
+            file_storage_azure_container=config.file_storage_azure_container,
+            file_storage_azure_account_name=config.file_storage_azure_account_name,
+            file_storage_azure_account_key=config.file_storage_azure_account_key,
+            file_parser=config.file_parser,
+            file_parser_iris_token=config.file_parser_iris_token,
+            file_parser_iris_org_id=config.file_parser_iris_org_id,
+            file_conversion_max_batch_size_mb=config.file_conversion_max_batch_size_mb,
+            file_conversion_max_batch_size=config.file_conversion_max_batch_size,
+            enable_file_upload_api=config.enable_file_upload_api,
+            file_delete_after_retain=config.file_delete_after_retain,
             enable_observations=config.enable_observations,
             consolidation_batch_size=config.consolidation_batch_size,
             consolidation_max_tokens=config.consolidation_max_tokens,
@@ -351,6 +369,7 @@ def main():
         "proxy_headers": args.proxy_headers,
         "ws": "wsproto",  # Use wsproto instead of websockets to avoid deprecation warnings
         "loop": loop_impl,  # Explicitly set event loop implementation
+        "timeout_keep_alive": 30,  # Exceed aiohttp's 15s client timeout so the client always closes first
     }
 
     # Add optional parameters if provided

hindsight-api/hindsight_api/migrations.py

@@ -42,30 +42,38 @@ def _detect_vector_extension(conn, vector_extension: str = "pgvector") -> str:
         vector_extension: Configured extension ("pgvector", "vchord", or "pgvectorscale")
 
     Returns:
-        "pgvector", "vchord", or "pgvectorscale"
+        "pgvector", "vchord", "pgvectorscale", or "pg_diskann"
 
     Raises:
         RuntimeError: If configured extension is not installed
     """
     # Verify the configured extension is installed
     if vector_extension == "pgvectorscale":
-        # pgvectorscale requires pgvector to be installed first
+        # pgvectorscale/DiskANN requires pgvector to be installed first
         pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar()
         if not pgvector_check:
             raise RuntimeError(
-                "pgvectorscale requires pgvector to be installed. "
-                "Install it with: CREATE EXTENSION vector; CREATE EXTENSION vectorscale CASCADE;"
+                "DiskANN (pgvectorscale/pg_diskann) requires pgvector to be installed. "
+                "Install it with: CREATE EXTENSION vector; then CREATE EXTENSION vectorscale CASCADE; (or pg_diskann on Azure)"
             )
 
-        # Check for vectorscale extension
+        # Check for either vectorscale (open source) or pg_diskann (Azure)
         vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar()
-        if not vectorscale_check:
+        pg_diskann_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_diskann'")).scalar()
+
+        if vectorscale_check:
+            logger.debug("Using vector extension: pgvectorscale (DiskANN)")
+            return "pgvectorscale"
+        elif pg_diskann_check:
+            logger.debug("Using vector extension: pg_diskann (Azure DiskANN)")
+            return "pg_diskann"  # Return distinct name for parameter handling
+        else:
             raise RuntimeError(
                 "Configured vector extension 'pgvectorscale' not found. "
-                "Install it with: CREATE EXTENSION vectorscale CASCADE;"
+                "Install either:\n"
+                "  - pgvectorscale (open source): CREATE EXTENSION vectorscale CASCADE;\n"
+                "  - pg_diskann (Azure): CREATE EXTENSION pg_diskann CASCADE;"
             )
-        logger.debug("Using configured vector extension: pgvectorscale (DiskANN)")
-        return "pgvectorscale"
     elif vector_extension == "vchord":
         vchord_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord'")).scalar()
         if not vchord_check:
@@ -609,7 +617,7 @@ def ensure_vector_extension(
         ]
 
         # Determine target index type
-        if target_ext == "pgvectorscale":
+        if target_ext in ("pgvectorscale", "pg_diskann"):
             target_index_type = "diskann"
         elif target_ext == "vchord":
             target_index_type = "vchordrq"
@@ -713,7 +721,7 @@ def ensure_vector_extension(
 
             # Create new index with appropriate type
             if target_ext == "pgvectorscale":
-                logger.info(f"Creating DiskANN index on {table_name}")
+                logger.info(f"Creating DiskANN index on {table_name} (pgvectorscale)")
                 conn.execute(
                     text(f"""
                         CREATE INDEX IF NOT EXISTS {index_name}
@@ -722,6 +730,16 @@ def ensure_vector_extension(
                         WITH (num_neighbors = 50)
                     """)
                 )
+            elif target_ext == "pg_diskann":
+                logger.info(f"Creating DiskANN index on {table_name} (pg_diskann/Azure)")
+                conn.execute(
+                    text(f"""
+                        CREATE INDEX IF NOT EXISTS {index_name}
+                        ON {schema_name}.{table_name}
+                        USING diskann (embedding vector_cosine_ops)
+                        WITH (max_neighbors = 50)
+                    """)
+                )
             elif target_ext == "vchord":
                 logger.info(f"Creating vchordrq index on {table_name}")
                 conn.execute(

hindsight-api/hindsight_api/worker/poller.py

@@ -376,7 +376,12 @@ class WorkerPoller:
                         del self._in_flight_by_type[operation_type]
 
     async def _execute_task_inner(self, task: ClaimedTask):
-        """Inner task execution with error handling."""
+        """Inner task execution with error handling.
+
+        Note: The executor (MemoryEngine.execute_task) handles status marking internally
+        (marking operations as completed/failed and handling retries). This method should
+        NOT override those status updates.
+        """
         task_type = task.task_dict.get("type", "unknown")
         bank_id = task.task_dict.get("bank_id", "unknown")
 
@@ -386,12 +391,12 @@ class WorkerPoller:
             if task.schema:
                 task.task_dict["_schema"] = task.schema
             await self._executor(task.task_dict)
-            await self._mark_completed(task.operation_id, task.schema)
-            logger.debug(f"Task {task.operation_id} completed successfully")
+            logger.debug(f"Task {task.operation_id} execution finished")
         except Exception as e:
-            error_msg = f"{type(e).__name__}: {e}\n{traceback.format_exc()}"
-            logger.error(f"Task {task.operation_id} failed: {e}")
-            await self._retry_or_fail(task.operation_id, error_msg, task.schema)
+            # The executor should handle its own errors, but if an unexpected exception
+            # propagates (e.g., from schema setup), log it as a warning
+            logger.error(f"Task {task.operation_id} raised unexpected exception: {e}")
+            traceback.print_exc()
 
     async def recover_own_tasks(self) -> int:
         """

hindsight-api/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-api"
-version = "0.4.11"
+version = "0.4.12"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -43,6 +43,8 @@ dependencies = [
     "cohere>=5.0.0",
     "flashrank>=0.2.0",
     "litellm>=1.0.0",
+    "markitdown[pdf,docx,pptx,xlsx,xls]>=0.1.4",  # File to markdown conversion
+    "obstore>=0.4.0",  # S3/GCS/Azure object storage client (Rust-backed)
     # Local ML models for embeddings/reranking - can be excluded in Docker with INCLUDE_LOCAL_MODELS=false
     "sentence-transformers>=3.3.0",
     "transformers>=4.53.0", # Security fixes for ReDoS vulnerabilities
@@ -65,6 +67,7 @@ test = [
     "pytest-timeout>=2.4.0",
     "pytest-xdist>=3.0.0",
     "filelock>=3.20.1",  # TOCTOU race condition fix
+    "testcontainers>=4.0.0",
 ]
 
 [project.scripts]
@@ -114,6 +117,7 @@ dev = [
     "filelock>=3.20.1",  # TOCTOU race condition fix
     "ruff>=0.8.0",
     "ty>=0.0.1",
+    "testcontainers>=4.0.0",
 ]
 
 [tool.ruff]

hindsight-api/tests/test_document_tracking.py

@@ -135,3 +135,179 @@ async def test_memory_without_document(memory, request_context):
 
     finally:
         await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_document_persisted_with_zero_facts(memory, request_context):
+    """
+    Test that documents are persisted even when zero facts are extracted.
+
+    This is a regression test for issue #324 where documents with no extractable
+    facts were reported as disappearing from the system.
+    """
+    bank_id = f"test_zero_facts_{datetime.now(timezone.utc).timestamp()}"
+
+    try:
+        document_id = "doc-zero-facts"
+
+        # Retain content that produces zero facts (gibberish/random characters)
+        units = await memory.retain_async(
+            bank_id=bank_id,
+            content="xyzabc123 !!!### @@@ $$$",  # Random characters unlikely to produce facts
+            context="Test zero facts",
+            document_id=document_id,
+            request_context=request_context,
+        )
+
+        # Should return empty unit list (no facts extracted)
+        assert len(units) == 0, "Should extract zero facts from gibberish content"
+
+        # But document should still be persisted and retrievable
+        doc = await memory.get_document(document_id, bank_id, request_context=request_context)
+        assert doc is not None, "Document should be persisted even with zero facts"
+        assert doc["id"] == document_id
+        assert doc["bank_id"] == bank_id
+        assert doc["memory_unit_count"] == 0, "Should have zero memory units"
+        assert len(doc["original_text"]) > 0, "Should have non-zero text length"
+        assert "xyzabc123" in doc["original_text"], "Should contain original content"
+
+        # Document should also appear in list
+        docs_list = await memory.list_documents(
+            bank_id=bank_id,
+            search_query=None,
+            limit=100,
+            offset=0,
+            request_context=request_context,
+        )
+        assert docs_list["total"] == 1, "Document should appear in list"
+        assert any(d["id"] == document_id for d in docs_list["items"]), "Document should be in items"
+
+        listed_doc = next(d for d in docs_list["items"] if d["id"] == document_id)
+        assert listed_doc["memory_unit_count"] == 0, "Listed document should show zero memory units"
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_document_persisted_with_zero_facts_batch(memory, request_context):
+    """
+    Test that documents are persisted with zero facts in batch retain operations.
+
+    This tests the async batch code path to ensure it also handles zero facts correctly.
+    """
+    bank_id = f"test_zero_facts_batch_{datetime.now(timezone.utc).timestamp()}"
+
+    try:
+        # Mix of content: some produces facts, some produces zero facts
+        contents = [
+            {
+                "content": "Alice works at Google",
+                "document_id": "doc-with-facts",
+            },
+            {
+                "content": "!@# $$$ %%% ^^^ &&& ***",  # Gibberish - zero facts expected
+                "document_id": "doc-zero-facts",
+            },
+        ]
+
+        unit_ids = await memory.retain_batch_async(
+            bank_id=bank_id,
+            contents=contents,
+            request_context=request_context,
+        )
+
+        # First content should produce facts, second should not
+        assert len(unit_ids[0]) > 0, "First content should produce facts"
+        assert len(unit_ids[1]) == 0, "Second content should produce zero facts"
+
+        # Both documents should be persisted
+        doc_with_facts = await memory.get_document("doc-with-facts", bank_id, request_context=request_context)
+        assert doc_with_facts is not None
+        assert doc_with_facts["memory_unit_count"] > 0
+
+        doc_zero_facts = await memory.get_document("doc-zero-facts", bank_id, request_context=request_context)
+        assert doc_zero_facts is not None, "Document with zero facts should be persisted"
+        assert doc_zero_facts["memory_unit_count"] == 0, "Should have zero memory units"
+        assert "!@#" in doc_zero_facts["original_text"]
+
+        # Both should appear in list
+        docs_list = await memory.list_documents(
+            bank_id=bank_id,
+            search_query=None,
+            limit=100,
+            offset=0,
+            request_context=request_context,
+        )
+        assert docs_list["total"] == 2, "Both documents should appear in list"
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)
+
+
+@pytest.mark.asyncio
+async def test_document_persisted_with_zero_facts_async_submit(memory, request_context):
+    """
+    Test that documents are persisted with zero facts in fire-and-forget async retain.
+
+    This tests the submit_async_retain (background task) code path to ensure it also
+    handles zero facts correctly.
+    """
+    import asyncio
+
+    bank_id = f"test_zero_facts_async_{datetime.now(timezone.utc).timestamp()}"
+
+    try:
+        # Submit async retain with gibberish content
+        result = await memory.submit_async_retain(
+            bank_id=bank_id,
+            contents=[
+                {
+                    "content": "!@# $$$ %%% ^^^ &&& ***",  # Gibberish - zero facts expected
+                    "document_id": "doc-async-zero-facts",
+                }
+            ],
+            request_context=request_context,
+        )
+
+        operation_id = result["operation_id"]
+        assert operation_id is not None, "Should return operation_id"
+
+        # Wait for background task to complete
+        max_wait = 60  # 60 seconds max
+        wait_interval = 0.5
+        elapsed = 0
+
+        while elapsed < max_wait:
+            await asyncio.sleep(wait_interval)
+            elapsed += wait_interval
+
+            # Check if document exists
+            doc = await memory.get_document(
+                "doc-async-zero-facts", bank_id, request_context=request_context
+            )
+            if doc is not None:
+                break
+
+        # Document should be persisted even with zero facts
+        assert doc is not None, "Document should be persisted after async task completes"
+        assert doc["id"] == "doc-async-zero-facts"
+        assert doc["memory_unit_count"] == 0, "Should have zero memory units"
+        assert "!@#" in doc["original_text"]
+
+        # Document should appear in list
+        docs_list = await memory.list_documents(
+            bank_id=bank_id,
+            search_query=None,
+            limit=100,
+            offset=0,
+            request_context=request_context,
+        )
+        assert docs_list["total"] == 1, "Document should appear in list"
+        assert any(d["id"] == "doc-async-zero-facts" for d in docs_list["items"])
+
+        listed_doc = next(d for d in docs_list["items"] if d["id"] == "doc-async-zero-facts")
+        assert listed_doc["memory_unit_count"] == 0, "Listed document should show zero memory units"
+
+    finally:
+        await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api/tests/test_file_retain.py

@@ -0,0 +1,553 @@
+"""
+End-to-end tests for file retain (upload, convert, retain) functionality.
+"""
+
+import io
+import json
+
+import pytest
+from httpx import ASGITransport, AsyncClient
+
+
+@pytest.fixture
+def sample_pdf_content():
+    """Create a simple PDF-like content for testing."""
+    # This is a minimal PDF that markitdown can parse
+    return b"""%PDF-1.4
+1 0 obj
+<<
+/Type /Catalog
+/Pages 2 0 R
+>>
+endobj
+2 0 obj
+<<
+/Type /Pages
+/Kids [3 0 R]
+/Count 1
+>>
+endobj
+3 0 obj
+<<
+/Type /Page
+/Parent 2 0 R
+/MediaBox [0 0 612 792]
+/Contents 4 0 R
+/Resources <<
+/Font <<
+/F1 <<
+/Type /Font
+/Subtype /Type1
+/BaseFont /Helvetica
+>>
+>>
+>>
+>>
+endobj
+4 0 obj
+<<
+/Length 44
+>>
+stream
+BT
+/F1 12 Tf
+100 700 Td
+(Test Document) Tj
+ET
+endstream
+endobj
+xref
+0 5
+0000000000 65535 f
+0000000009 00000 n
+0000000058 00000 n
+0000000115 00000 n
+0000000317 00000 n
+trailer
+<<
+/Size 5
+/Root 1 0 R
+>>
+startxref
+410
+%%EOF
+"""
+
+
+@pytest.fixture
+def sample_txt_content():
+    """Create simple text content."""
+    return b"This is a test document.\nIt contains some important information.\nAlice works at Google."
+
+
+@pytest.mark.asyncio
+async def test_file_retain_basic(memory_no_llm_verify, sample_txt_content):
+    """Test basic file upload and conversion."""
+    from hindsight_api.api.http import create_app
+
+    app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # Create a bank first
+        bank_response = await client.put("/v1/default/banks/test-file-bank", json={"name": "Test File Bank"})
+        assert bank_response.status_code in (200, 201)
+
+        # Upload file
+        request_data = {
+            "document_tags": ["test"],
+            "async": True,
+        }
+
+        files = {"files": ("test.txt", sample_txt_content, "text/plain")}
+        data = {"request": json.dumps(request_data)}
+
+        response = await client.post(
+            "/v1/default/banks/test-file-bank/files/retain",
+            files=files,
+            data=data,
+        )
+
+        assert response.status_code == 200
+        result = response.json()
+        assert "operation_ids" in result
+        assert len(result["operation_ids"]) == 1
+
+
+@pytest.mark.asyncio
+async def test_file_retain_with_metadata(memory_no_llm_verify, sample_txt_content):
+    """Test file upload with per-file metadata."""
+    from hindsight_api.api.http import create_app
+
+    app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # Create bank
+        bank_response = await client.put("/v1/default/banks/test-file-meta-bank", json={"name": "Test Meta Bank"})
+        assert bank_response.status_code in (200, 201)
+
+        # Upload file with metadata
+        request_data = {
+            "document_tags": ["work", "reports"],
+            "async": True,
+            "files_metadata": [
+                {
+                    "document_id": "test_doc_123",
+                    "context": "quarterly report",
+                    "metadata": {"author": "Alice", "year": "2024"},
+                    "tags": ["Q1"],
+                }
+            ],
+        }
+
+        files = {"files": ("report.txt", sample_txt_content, "text/plain")}
+        data = {"request": json.dumps(request_data)}
+
+        response = await client.post(
+            "/v1/default/banks/test-file-meta-bank/files/retain",
+            files=files,
+            data=data,
+        )
+
+        assert response.status_code == 200
+        result = response.json()
+        assert "operation_ids" in result
+        assert len(result["operation_ids"]) == 1
+
+
+@pytest.mark.asyncio
+async def test_file_retain_multiple_files(memory_no_llm_verify, sample_txt_content):
+    """Test uploading multiple files at once."""
+    from hindsight_api.api.http import create_app
+
+    app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # Create bank
+        bank_response = await client.put("/v1/default/banks/test-multi-file-bank", json={"name": "Test Multi Bank"})
+        assert bank_response.status_code in (200, 201)
+
+        # Upload multiple files
+        request_data = {
+            "async": True,
+            "files_metadata": [
+                {"document_id": "doc1", "tags": ["file1"]},
+                {"document_id": "doc2", "tags": ["file2"]},
+            ],
+        }
+
+        content1 = b"First document content"
+        content2 = b"Second document content"
+
+        files = [
+            ("files", ("file1.txt", content1, "text/plain")),
+            ("files", ("file2.txt", content2, "text/plain")),
+        ]
+        data = {"request": json.dumps(request_data)}
+
+        response = await client.post(
+            "/v1/default/banks/test-multi-file-bank/files/retain",
+            files=files,
+            data=data,
+        )
+
+        assert response.status_code == 200
+        result = response.json()
+        assert "operation_ids" in result
+        assert len(result["operation_ids"]) == 2
+
+
+@pytest.mark.asyncio
+async def test_file_retain_validation_errors(memory_no_llm_verify):
+    """Test validation errors."""
+    from hindsight_api.api.http import create_app
+
+    app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # Create bank
+        bank_response = await client.put("/v1/default/banks/test-validation-bank", json={"name": "Test Validation Bank"})
+        assert bank_response.status_code in (200, 201)
+
+        # Test: metadata count mismatch
+        request_data = {
+            "async": True,
+            "files_metadata": [
+                {"document_id": "doc1"},
+                {"document_id": "doc2"},  # 2 metadata entries
+            ],
+        }
+
+        files = {"files": ("file1.txt", b"content", "text/plain")}  # But only 1 file
+        data = {"request": json.dumps(request_data)}
+
+        response = await client.post(
+            "/v1/default/banks/test-validation-bank/files/retain",
+            files=files,
+            data=data,
+        )
+
+        assert response.status_code == 400
+        assert "files_metadata count" in response.json()["detail"]
+
+
+@pytest.mark.asyncio
+async def test_file_retain_no_files(memory_no_llm_verify):
+    """Test error when no files provided."""
+    from hindsight_api.api.http import create_app
+
+    app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # Create bank
+        bank_response = await client.put("/v1/default/banks/test-no-files-bank", json={"name": "Test No Files Bank"})
+        assert bank_response.status_code in (200, 201)
+
+        request_data = {
+            "async": True,
+        }
+
+        # No files provided
+        data = {"request": json.dumps(request_data)}
+
+        response = await client.post(
+            "/v1/default/banks/test-no-files-bank/files/retain",
+            data=data,
+        )
+
+        # FastAPI will return 422 for missing required field
+        assert response.status_code == 422
+
+
+@pytest.mark.asyncio
+async def test_file_retain_sync_not_supported(memory_no_llm_verify, sample_txt_content):
+    """Test that file retain is always async (sync is not supported)."""
+    from hindsight_api.api.http import create_app
+
+    app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # Create bank
+        bank_response = await client.put("/v1/default/banks/test-sync-bank", json={"name": "Test Sync Bank"})
+        assert bank_response.status_code in (200, 201)
+
+        # File retain is always async - just verify it succeeds and returns operation_ids
+        files = {"files": ("test.txt", sample_txt_content, "text/plain")}
+        data = {"request": json.dumps({})}
+
+        response = await client.post(
+            "/v1/default/banks/test-sync-bank/files/retain",
+            files=files,
+            data=data,
+        )
+
+        assert response.status_code == 200
+        result = response.json()
+        assert "operation_ids" in result
+
+
+@pytest.mark.asyncio
+async def test_file_storage_postgresql(memory_no_llm_verify, sample_txt_content):
+    """Test file storage in PostgreSQL."""
+    # Test that files are stored and retrieved correctly
+    storage = memory_no_llm_verify._file_storage
+
+    # Store a file
+    key = "test/file1.txt"
+    stored_key = await storage.store(
+        file_data=sample_txt_content,
+        key=key,
+        metadata={"content_type": "text/plain"},
+    )
+
+    assert stored_key == key
+
+    # Retrieve the file
+    retrieved = await storage.retrieve(key)
+    assert retrieved == sample_txt_content
+
+    # Check if file exists
+    exists = await storage.exists(key)
+    assert exists is True
+
+    # Delete the file
+    await storage.delete(key)
+
+    # Check file no longer exists
+    exists_after = await storage.exists(key)
+    assert exists_after is False
+
+
+@pytest.mark.asyncio
+async def test_markitdown_converter():
+    """Test markitdown parser."""
+    from hindsight_api.engine.parsers import MarkitdownParser
+
+    parser = MarkitdownParser()
+
+    # Test simple text file
+    text_content = b"This is a test document.\nWith multiple lines."
+    result = await parser.convert(text_content, "test.txt")
+
+    assert isinstance(result, str)
+    assert len(result) > 0
+    assert "test document" in result.lower() or "multiple lines" in result.lower()
+
+
+@pytest.mark.asyncio
+async def test_converter_registry():
+    """Test file parser registry."""
+    from hindsight_api.engine.parsers import FileParserRegistry, MarkitdownParser
+
+    registry = FileParserRegistry()
+    parser = MarkitdownParser()
+    registry.register(parser)
+
+    # Test get by name
+    retrieved = registry.get_parser("markitdown", "test.txt")
+    assert retrieved is parser
+
+    # Test auto-detection
+    auto = registry.get_parser(None, "test.pdf")
+    assert auto is parser
+
+    # Test unsupported format
+    with pytest.raises(ValueError, match="No parser found"):
+        registry.get_parser(None, "test.xyz")
+
+
+@pytest.mark.asyncio
+async def test_file_conversion_creates_separate_retain_operation(memory_no_llm_verify, sample_txt_content):
+    """Test that file conversion and retain are two separate async operations.
+
+    The file_convert_retain task should:
+    1. Convert the file to markdown
+    2. In a single transaction: create a separate 'retain' operation AND mark itself as 'completed'
+    3. Free the worker slot immediately after conversion
+
+    The retain then runs as its own task. This prevents deadlocks where file conversion
+    tasks hold worker slots while waiting for inline retain to finish.
+    """
+    from hindsight_api.models import RequestContext
+
+    bank_id = "test_file_two_phase_bank"
+
+    context = RequestContext(internal=True)
+    await memory_no_llm_verify.get_bank_profile(bank_id, request_context=context)
+
+    class MockFile:
+        def __init__(self, content, filename, content_type):
+            self.content = content
+            self.filename = filename
+            self.content_type = content_type
+
+        async def read(self):
+            return self.content
+
+    mock_file = MockFile(sample_txt_content, "test.txt", "text/plain")
+
+    file_items = [
+        {
+            "file": mock_file,
+            "document_id": "test_doc_two_phase",
+            "context": "test context",
+            "metadata": {"source": "test"},
+            "tags": ["test_tag"],
+            "timestamp": None,
+        }
+    ]
+
+    result = await memory_no_llm_verify.submit_async_file_retain(
+        bank_id=bank_id,
+        file_items=file_items,
+        parser="markitdown",
+        document_tags=["two_phase_test"],
+        request_context=context,
+    )
+
+    assert "operation_ids" in result
+    assert len(result["operation_ids"]) == 1
+    convert_operation_id = result["operation_ids"][0]
+
+    import asyncio
+
+    await asyncio.sleep(0.1)
+
+    pool = await memory_no_llm_verify._get_pool()
+    from hindsight_api.engine.memory_engine import get_current_schema
+
+    schema = get_current_schema()
+
+    async with pool.acquire() as conn:
+        # 1. The file_convert_retain operation must be completed
+        convert_op = await conn.fetchrow(
+            f"SELECT status, operation_type FROM {schema}.async_operations WHERE operation_id = $1",
+            convert_operation_id,
+        )
+        assert convert_op is not None
+        assert convert_op["operation_type"] == "file_convert_retain"
+        assert convert_op["status"] == "completed", (
+            f"file_convert_retain should be 'completed' after conversion, got '{convert_op['status']}'"
+        )
+
+        # 2. A separate retain operation must have been created
+        retain_op = await conn.fetchrow(
+            f"""
+            SELECT status, operation_type
+            FROM {schema}.async_operations
+            WHERE bank_id = $1 AND operation_type = 'retain' AND operation_id != $2
+            """,
+            bank_id,
+            convert_operation_id,
+        )
+        assert retain_op is not None, "A separate 'retain' operation should have been created by file conversion"
+        # With SyncTaskBackend the retain runs immediately, so it should be completed
+        assert retain_op["status"] == "completed"
+
+        # 3. The document should exist with file metadata and retained content
+        doc = await conn.fetchrow(
+            f"""
+            SELECT id, original_text, file_original_name, file_content_type
+            FROM {schema}.documents
+            WHERE id = $1 AND bank_id = $2
+            """,
+            "test_doc_two_phase",
+            bank_id,
+        )
+
+    assert doc is not None
+    assert doc["file_original_name"] == "test.txt"
+    assert doc["file_content_type"] == "text/plain"
+    assert doc["original_text"] is not None
+    assert len(doc["original_text"]) > 0
+
+
+@pytest.mark.asyncio
+async def test_file_conversion_failure_sets_status_to_failed(memory_no_llm_verify, sample_txt_content):
+    """Test that when file conversion fails, the operation status is set to 'failed' not 'completed'."""
+    from hindsight_api.engine.parsers.base import FileParser
+    from hindsight_api.models import RequestContext
+
+    bank_id = "test_file_failure_bank"
+
+    # Create a mock parser that always fails
+    class FailingParser(FileParser):
+        """Mock parser that raises an error."""
+
+        async def convert(self, file_data: bytes, filename: str) -> str:
+            # Simulate conversion failure
+            raise RuntimeError(f"Failed to convert '{filename}': Mock conversion error")
+
+        def supports(self, filename: str, content_type: str | None = None) -> bool:
+            return filename.endswith(".fail")
+
+        def name(self) -> str:
+            return "failing_converter"
+
+    # Register the failing parser
+    failing_converter = FailingParser()
+    memory_no_llm_verify._parser_registry.register(failing_converter)
+
+    # Create bank
+    context = RequestContext(internal=True)
+    await memory_no_llm_verify.get_bank_profile(bank_id, request_context=context)
+
+    # Create mock file
+    class MockFile:
+        def __init__(self, content, filename, content_type):
+            self.content = content
+            self.filename = filename
+            self.content_type = content_type
+
+        async def read(self):
+            return self.content
+
+    mock_file = MockFile(sample_txt_content, "test.fail", "application/octet-stream")
+
+    file_items = [
+        {
+            "file": mock_file,
+            "document_id": "test_doc_fail",
+            "context": None,
+            "metadata": {},
+            "tags": [],
+            "timestamp": None,
+        }
+    ]
+
+    # Submit async file retain with failing parser
+    result = await memory_no_llm_verify.submit_async_file_retain(
+        bank_id=bank_id,
+        file_items=file_items,
+        parser="failing_converter",
+        document_tags=None,
+        request_context=context,
+    )
+
+    assert "operation_ids" in result
+    assert len(result["operation_ids"]) == 1
+    operation_id = result["operation_ids"][0]
+
+    # Wait for async processing (with SyncTaskBackend, this is immediate)
+    import asyncio
+
+    await asyncio.sleep(0.2)
+
+    # Check operation status - should be 'failed' not 'completed'
+    pool = await memory_no_llm_verify._get_pool()
+    from hindsight_api.engine.memory_engine import get_current_schema
+
+    async with pool.acquire() as conn:
+        operation = await conn.fetchrow(
+            f"""
+            SELECT status, error_message
+            FROM {get_current_schema()}.async_operations
+            WHERE operation_id = $1
+            """,
+            operation_id,
+        )
+
+    assert operation is not None, f"Operation {operation_id} not found"
+    assert operation["status"] == "failed", f"Expected status 'failed' but got '{operation['status']}'"
+    assert operation["error_message"] is not None
+    assert "Mock conversion error" in operation["error_message"]
+    assert "test.fail" in operation["error_message"]

hindsight-api/tests/test_file_storage_s3.py

@@ -0,0 +1,252 @@
+"""
+Integration tests for S3FileStorage against a SeaweedFS Docker container.
+
+SeaweedFS (Apache 2.0) provides an S3-compatible API via `weed server -s3`.
+Requires Docker to be running. Tests are skipped automatically if Docker is unavailable.
+"""
+
+import json
+import logging
+import subprocess
+import tempfile
+import time
+import uuid
+
+import httpx
+import pytest
+from httpx import ASGITransport, AsyncClient
+
+logger = logging.getLogger(__name__)
+
+try:
+    from testcontainers.core.container import DockerContainer
+
+    _has_testcontainers = True
+except ImportError:
+    _has_testcontainers = False
+
+pytestmark = [
+    pytest.mark.skipif(not _has_testcontainers, reason="testcontainers not installed"),
+]
+
+SEAWEEDFS_S3_PORT = 8333
+TEST_BUCKET = "hindsight-test"
+ACCESS_KEY = "test_access_key"
+SECRET_KEY = "test_secret_key"
+
+# SeaweedFS S3 IAM config granting full access to our test credentials
+_S3_CONFIG = {
+    "identities": [
+        {
+            "name": "test-user",
+            "credentials": [{"accessKey": ACCESS_KEY, "secretKey": SECRET_KEY}],
+            "actions": ["Admin", "Read", "Write", "List"],
+        }
+    ]
+}
+
+
+def _docker_available() -> bool:
+    """Check if Docker daemon is running."""
+    try:
+        result = subprocess.run(
+            ["docker", "info"],
+            capture_output=True,
+            timeout=5,
+        )
+        return result.returncode == 0
+    except (FileNotFoundError, subprocess.TimeoutExpired):
+        return False
+
+
+def _wait_for_seaweedfs(endpoint: str, timeout: int = 30) -> None:
+    """Poll SeaweedFS S3 endpoint until ready."""
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        try:
+            resp = httpx.get(endpoint, timeout=2)
+            # 200 = no auth, 403 = auth enabled but gateway is up — either means ready
+            if resp.status_code in (200, 403):
+                logger.info("SeaweedFS S3 is ready at %s", endpoint)
+                return
+        except httpx.HTTPError:
+            pass
+        time.sleep(0.5)
+    raise TimeoutError(f"SeaweedFS did not become ready at {endpoint} within {timeout}s")
+
+
+@pytest.fixture(scope="module")
+def seaweedfs_container():
+    """Start a SeaweedFS container for the test module, shared across all tests.
+
+    Mounts an s3.json config file to set up S3 credentials for the test user.
+    """
+    if not _docker_available():
+        pytest.skip("Docker is not available")
+
+    # Write S3 IAM config to a temp file that persists for the module scope
+    s3_config_file = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False)
+    json.dump(_S3_CONFIG, s3_config_file)
+    s3_config_file.flush()
+
+    container = (
+        DockerContainer(image="chrislusf/seaweedfs:latest")
+        .with_exposed_ports(SEAWEEDFS_S3_PORT)
+        .with_volume_mapping(s3_config_file.name, "/etc/seaweedfs/s3.json", "ro")
+        .with_command(
+            f"server -s3 -s3.port={SEAWEEDFS_S3_PORT} -s3.config=/etc/seaweedfs/s3.json -ip.bind=0.0.0.0"
+        )
+    )
+
+    container.start()
+
+    try:
+        host = container.get_container_host_ip()
+        port = container.get_exposed_port(SEAWEEDFS_S3_PORT)
+        endpoint = f"http://{host}:{port}"
+
+        _wait_for_seaweedfs(endpoint)
+
+        # Create test bucket using obstore (proper SigV4 signing)
+        import obstore as obs
+        from obstore.store import S3Store
+
+        admin_store = S3Store(
+            TEST_BUCKET,
+            endpoint=endpoint,
+            region="us-east-1",
+            access_key_id=ACCESS_KEY,
+            secret_access_key=SECRET_KEY,
+            allow_http=True,
+        )
+        # SeaweedFS auto-creates buckets on first write
+        obs.put(admin_store, ".bucket-init", b"")
+        obs.delete(admin_store, ".bucket-init")
+        logger.info("Test bucket '%s' is ready", TEST_BUCKET)
+
+        yield {
+            "endpoint": endpoint,
+            "access_key": ACCESS_KEY,
+            "secret_key": SECRET_KEY,
+            "bucket": TEST_BUCKET,
+        }
+    finally:
+        container.stop()
+        import os
+
+        os.unlink(s3_config_file.name)
+
+
+@pytest.fixture
+def s3_storage(seaweedfs_container):
+    """Create an S3FileStorage instance pointing at the SeaweedFS container."""
+    from hindsight_api.engine.storage.s3 import S3FileStorage
+
+    return S3FileStorage(
+        bucket=seaweedfs_container["bucket"],
+        region="us-east-1",
+        endpoint=seaweedfs_container["endpoint"],
+        access_key_id=seaweedfs_container["access_key"],
+        secret_access_key=seaweedfs_container["secret_key"],
+    )
+
+
+@pytest.mark.asyncio
+async def test_s3_storage_store_and_retrieve(s3_storage):
+    """Store a file, retrieve it, verify bytes match."""
+    content = b"Hello, SeaweedFS! This is a test file."
+    key = f"test/{uuid.uuid4()}.txt"
+
+    stored_key = await s3_storage.store(
+        file_data=content,
+        key=key,
+        metadata={"content_type": "text/plain"},
+    )
+    assert stored_key == key
+
+    retrieved = await s3_storage.retrieve(key)
+    assert retrieved == content
+
+
+@pytest.mark.asyncio
+async def test_s3_storage_exists_and_delete(s3_storage):
+    """Store, check exists=True, delete, check exists=False."""
+    content = b"File to be deleted."
+    key = f"test/{uuid.uuid4()}.txt"
+
+    await s3_storage.store(file_data=content, key=key)
+
+    assert await s3_storage.exists(key) is True
+
+    await s3_storage.delete(key)
+
+    assert await s3_storage.exists(key) is False
+
+
+@pytest.mark.asyncio
+async def test_s3_storage_file_not_found(s3_storage):
+    """Retrieve a non-existent key, expect FileNotFoundError."""
+    with pytest.raises(FileNotFoundError):
+        await s3_storage.retrieve(f"nonexistent/{uuid.uuid4()}.txt")
+
+
+@pytest.mark.asyncio
+async def test_s3_storage_get_download_url(s3_storage):
+    """Store a file, get a presigned URL, verify it's a valid URL string."""
+    content = b"Presigned URL test content."
+    key = f"test/{uuid.uuid4()}.txt"
+
+    await s3_storage.store(file_data=content, key=key)
+
+    url = await s3_storage.get_download_url(key, expires_in=300)
+    assert isinstance(url, str)
+    assert url.startswith("http")
+    assert key in url
+
+
+@pytest.mark.asyncio
+async def test_s3_file_retain_api_end_to_end(seaweedfs_container, memory_no_llm_verify):
+    """Full HTTP API flow: upload file via /files/retain with S3 storage backend."""
+    from hindsight_api.api.http import create_app
+    from hindsight_api.engine.storage.s3 import S3FileStorage
+
+    # Swap the engine's file storage to use the SeaweedFS-backed S3 storage
+    original_storage = memory_no_llm_verify._file_storage
+    s3_storage = S3FileStorage(
+        bucket=seaweedfs_container["bucket"],
+        region="us-east-1",
+        endpoint=seaweedfs_container["endpoint"],
+        access_key_id=seaweedfs_container["access_key"],
+        secret_access_key=seaweedfs_container["secret_key"],
+    )
+    memory_no_llm_verify._file_storage = s3_storage
+
+    try:
+        app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+        async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+            bank_id = f"test-s3-bank-{uuid.uuid4().hex[:8]}"
+            bank_response = await client.put(f"/v1/default/banks/{bank_id}", json={"name": "S3 Test Bank"})
+            assert bank_response.status_code in (200, 201)
+
+            txt_content = b"Alice works at Acme Corp. She joined in 2024."
+            request_data = {
+                "document_tags": ["s3-test"],
+                "async": True,
+            }
+
+            files = {"files": ("notes.txt", txt_content, "text/plain")}
+            data = {"request": json.dumps(request_data)}
+
+            response = await client.post(
+                f"/v1/default/banks/{bank_id}/files/retain",
+                files=files,
+                data=data,
+            )
+
+            assert response.status_code == 200
+            result = response.json()
+            assert "operation_ids" in result
+            assert len(result["operation_ids"]) == 1
+    finally:
+        memory_no_llm_verify._file_storage = original_storage

hindsight-api/tests/test_iris_parser.py

@@ -0,0 +1,72 @@
+"""
+Integration tests for the Iris file parser.
+
+Tests are skipped automatically if HINDSIGHT_API_FILE_PARSER_IRIS_TOKEN
+and HINDSIGHT_API_FILE_PARSER_IRIS_ORG_ID are not set in the environment.
+"""
+
+import os
+
+import pytest
+
+from hindsight_api.config import ENV_FILE_PARSER_IRIS_ORG_ID, ENV_FILE_PARSER_IRIS_TOKEN
+from hindsight_api.engine.parsers.iris import IrisParser
+
+_token = os.getenv(ENV_FILE_PARSER_IRIS_TOKEN)
+_org_id = os.getenv(ENV_FILE_PARSER_IRIS_ORG_ID)
+
+pytestmark = pytest.mark.skipif(
+    not (_token and _org_id),
+    reason="HINDSIGHT_API_FILE_PARSER_IRIS_TOKEN and HINDSIGHT_API_FILE_PARSER_IRIS_ORG_ID not set",
+)
+
+# Minimal valid PDF with the text "Hello from Hindsight"
+_SAMPLE_PDF = b"""%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+3 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792]
+   /Contents 4 0 R /Resources << /Font << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >> >>
+endobj
+4 0 obj
+<< /Length 44 >>
+stream
+BT /F1 12 Tf 100 700 Td (Hello from Hindsight) Tj ET
+endstream
+endobj
+xref
+0 5
+0000000000 65535 f
+0000000009 00000 n
+0000000058 00000 n
+0000000115 00000 n
+0000000274 00000 n
+trailer << /Size 5 /Root 1 0 R >>
+startxref
+369
+%%EOF"""
+
+
+@pytest.fixture
+def iris_parser() -> IrisParser:
+    return IrisParser(token=_token, org_id=_org_id)
+
+
+@pytest.mark.asyncio
+async def test_iris_parser_converts_pdf(iris_parser: IrisParser):
+    """IrisParser should extract text from a valid PDF."""
+    result = await iris_parser.convert(_SAMPLE_PDF, "sample.pdf")
+    assert isinstance(result, str)
+    assert len(result) > 0
+
+
+@pytest.mark.asyncio
+async def test_iris_parser_name(iris_parser: IrisParser):
+    """IrisParser.name() should return 'iris'."""
+    assert iris_parser.name() == "iris"
+
+

hindsight-api/tests/test_main_module.py

@@ -352,6 +352,47 @@ class TestMainModuleExtensionLoading:
             "main.py should use import string when workers > 1"
         assert uvicorn_calls[0]["workers"] == 2
 
+    def test_main_sets_keepalive_timeout(self, monkeypatch):
+        """
+        Verify that uvicorn is configured with timeout_keep_alive > aiohttp's
+        default client keepalive timeout (15s), so the server never closes
+        connections before the client does.
+        """
+        monkeypatch.setenv("HINDSIGHT_API_WORKERS", "1")
+        monkeypatch.delenv("HINDSIGHT_API_TENANT_EXTENSION", raising=False)
+
+        uvicorn_calls = []
+
+        def capture_uvicorn_run(**kwargs):
+            uvicorn_calls.append(kwargs)
+
+        with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
+             patch("hindsight_api.main.create_app") as mock_create_app, \
+             patch("hindsight_api.main._get_raw_config") as mock_get_config, \
+             patch("hindsight_api.main.print_banner"), \
+             patch("uvicorn.run", side_effect=capture_uvicorn_run):
+
+            mock_config = MagicMock()
+            mock_config.host = "0.0.0.0"
+            mock_config.port = 8888
+            mock_config.log_level = "info"
+            mock_config.mcp_enabled = False
+            mock_config.run_migrations_on_startup = False
+            mock_config.database_url = "postgresql://test:test@localhost/test"
+            mock_get_config.return_value = mock_config
+            mock_engine.return_value = MagicMock()
+            mock_create_app.return_value = MagicMock()
+
+            with patch.object(sys, 'argv', ['hindsight-api']):
+                from hindsight_api.main import main
+                main()
+
+        assert len(uvicorn_calls) == 1
+        assert "timeout_keep_alive" in uvicorn_calls[0], \
+            "uvicorn config must set timeout_keep_alive"
+        assert uvicorn_calls[0]["timeout_keep_alive"] > 15, \
+            "timeout_keep_alive must exceed aiohttp's 15s client default"
+
 
 # Mock extensions for testing
 from hindsight_api.extensions import (

hindsight-api/tests/test_worker.py

@@ -206,8 +206,12 @@ class TestWorkerPoller:
         assert len(claimed) == 3
 
     @pytest.mark.asyncio
-    async def test_execute_task_marks_completed(self, pool, clean_operations):
-        """Test that successful task execution marks task as completed."""
+    async def test_execute_task_executor_marks_completed(self, pool, clean_operations):
+        """Test that executor's status marking is preserved by the poller.
+
+        The executor (MemoryEngine.execute_task) handles marking operations as completed/failed.
+        The poller should NOT override those status updates.
+        """
         from hindsight_api.worker import WorkerPoller
         from hindsight_api.worker.poller import ClaimedTask
 
@@ -228,7 +232,16 @@ class TestWorkerPoller:
         executed = []
 
         async def mock_executor(task_dict):
+            """Executor that marks its own status as completed (like MemoryEngine.execute_task)."""
             executed.append(task_dict)
+            await pool.execute(
+                """
+                UPDATE async_operations
+                SET status = 'completed', completed_at = now(), updated_at = now()
+                WHERE operation_id = $1
+                """,
+                op_id,
+            )
 
         poller = WorkerPoller(
             pool=pool,
@@ -246,7 +259,7 @@ class TestWorkerPoller:
         assert completed, "Task did not complete within timeout"
         assert len(executed) == 1
 
-        # Verify task is marked as completed
+        # Verify task is marked as completed (by executor, not overridden by poller)
         row = await pool.fetchrow(
             "SELECT status, completed_at FROM async_operations WHERE operation_id = $1",
             op_id,
@@ -255,19 +268,24 @@ class TestWorkerPoller:
         assert row["completed_at"] is not None
 
     @pytest.mark.asyncio
-    async def test_execute_task_retries_on_failure(self, pool, clean_operations):
-        """Test that failed task execution triggers retry mechanism."""
+    async def test_executor_exception_does_not_crash_poller(self, pool, clean_operations):
+        """Test that unexpected exceptions from executor are caught and don't crash the poller.
+
+        If the executor raises an unexpected exception (which MemoryEngine.execute_task should NOT do,
+        but could happen from schema setup or other infrastructure issues), the poller should catch it
+        gracefully. Status remains 'processing' since neither executor nor poller handled it.
+        """
         from hindsight_api.worker import WorkerPoller
         from hindsight_api.worker.poller import ClaimedTask
 
-        # Create a pending task with retry_count=0
+        # Create a pending task
         bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
         op_id = uuid.uuid4()
         payload = json.dumps({"type": "test_task", "operation_id": str(op_id), "bank_id": bank_id})
         await pool.execute(
             """
-            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id, retry_count)
-            VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'test-worker-1', 0)
+            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id)
+            VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'test-worker-1')
             """,
             op_id,
             bank_id,
@@ -275,16 +293,15 @@ class TestWorkerPoller:
         )
 
         async def failing_executor(task_dict):
-            raise ValueError("Simulated failure")
+            raise ValueError("Unexpected infrastructure failure")
 
         poller = WorkerPoller(
             pool=pool,
             worker_id="test-worker-1",
             executor=failing_executor,
-            max_retries=3,
         )
 
-        # Execute (should fail and retry) - fire-and-forget
+        # Execute - should catch exception without crashing
         task_dict = json.loads(payload)
         claimed_task = ClaimedTask(operation_id=str(op_id), task_dict=task_dict, schema=None)
         await poller.execute_task(claimed_task)
@@ -293,61 +310,84 @@ class TestWorkerPoller:
         completed = await poller.wait_for_active_tasks(timeout=5.0)
         assert completed, "Task did not complete within timeout"
 
-        # Verify task is back to pending with incremented retry_count
+        # Status stays 'processing' since the poller no longer manages status
         row = await pool.fetchrow(
-            "SELECT status, retry_count, worker_id FROM async_operations WHERE operation_id = $1",
+            "SELECT status FROM async_operations WHERE operation_id = $1",
             op_id,
         )
-        assert row["status"] == "pending"
-        assert row["retry_count"] == 1
-        assert row["worker_id"] is None  # Worker ID cleared for retry
+        assert row["status"] == "processing"
 
     @pytest.mark.asyncio
-    async def test_execute_task_fails_after_max_retries(self, pool, clean_operations):
-        """Test that task is marked failed after exceeding max retries."""
+    async def test_executor_failed_status_not_overridden(self, pool, clean_operations):
+        """REGRESSION TEST: Verify poller does NOT overwrite executor's 'failed' status to 'completed'.
+
+        This test catches the bug where the poller always called _mark_completed() after executor
+        returned, overwriting the 'failed' status that the executor had already set.
+
+        Scenario:
+        1. Executor catches an internal error and marks the operation as 'failed' in the DB
+        2. Executor returns normally (does NOT re-raise) - this is how MemoryEngine.execute_task works
+        3. The poller must NOT overwrite the 'failed' status to 'completed'
+
+        With the old buggy code, this test would FAIL (status would be 'completed').
+        """
         from hindsight_api.worker import WorkerPoller
         from hindsight_api.worker.poller import ClaimedTask
 
-        # Create a task that has already used all retries
         bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
         op_id = uuid.uuid4()
         payload = json.dumps({"type": "test_task", "operation_id": str(op_id), "bank_id": bank_id})
         await pool.execute(
             """
-            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id, retry_count)
-            VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'test-worker-1', 3)
+            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id)
+            VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'test-worker-1')
             """,
             op_id,
             bank_id,
             payload,
         )
 
-        async def failing_executor(task_dict):
-            raise ValueError("Simulated failure")
+        async def executor_that_marks_failed(task_dict):
+            """Simulates MemoryEngine.execute_task behavior on internal error.
+
+            The executor catches the error, marks the operation as 'failed',
+            and returns normally (does NOT re-raise the exception).
+            """
+            # Simulate internal failure handling (like MemoryEngine._mark_operation_failed)
+            await pool.execute(
+                """
+                UPDATE async_operations
+                SET status = 'failed', error_message = $2, completed_at = now(), updated_at = now()
+                WHERE operation_id = $1
+                """,
+                op_id,
+                "Simulated conversion error: file format not supported",
+            )
+            # Returns normally - this is the key: executor does NOT re-raise
 
         poller = WorkerPoller(
             pool=pool,
             worker_id="test-worker-1",
-            executor=failing_executor,
-            max_retries=3,
+            executor=executor_that_marks_failed,
         )
 
-        # Execute (should fail permanently) - fire-and-forget
         task_dict = json.loads(payload)
         claimed_task = ClaimedTask(operation_id=str(op_id), task_dict=task_dict, schema=None)
         await poller.execute_task(claimed_task)
 
-        # Wait for background task to complete
         completed = await poller.wait_for_active_tasks(timeout=5.0)
         assert completed, "Task did not complete within timeout"
 
-        # Verify task is marked as failed
+        # THE KEY ASSERTION: Status must be 'failed', NOT 'completed'
         row = await pool.fetchrow(
             "SELECT status, error_message FROM async_operations WHERE operation_id = $1",
             op_id,
         )
-        assert row["status"] == "failed"
-        assert "Max retries" in row["error_message"]
+        assert row["status"] == "failed", (
+            f"REGRESSION: Poller overwrote executor's 'failed' status to '{row['status']}'. "
+            "The poller must not override status set by the executor."
+        )
+        assert "Simulated conversion error" in row["error_message"]
 
     @pytest.mark.asyncio
     async def test_claim_batch_skips_consolidation_when_same_bank_processing(self, pool, clean_operations):

hindsight-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hindsight-cli"
-version = "0.4.11"
+version = "0.4.12"
 edition = "2021"
 authors = ["Hindsight Team"]
 description = "A beautiful CLI for Hindsight - semantic memory system"
@@ -20,8 +20,8 @@ clap = { version = "4.5", features = ["derive", "env"] }
 # Async runtime
 tokio = { version = "1", features = ["full"] }
 
-# HTTP client (for timeout configuration)
-reqwest = "0.12"
+# HTTP client (for timeout configuration and multipart file uploads)
+reqwest = { version = "0.12", features = ["multipart"] }
 
 # Serialization (for config and output formatting)
 serde = { version = "1.0", features = ["derive"] }

hindsight-cli/src/api.rs

@@ -58,9 +58,16 @@ pub struct MemoryPutResult {
     pub operation_id: Option<String>,
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct FileRetainResult {
+    pub operation_ids: Vec<String>,
+}
+
 #[derive(Clone)]
 pub struct ApiClient {
     client: AsyncClient,
+    http_client: reqwest::Client,
+    base_url: String,
     runtime: std::sync::Arc<tokio::runtime::Runtime>,
 }
 
@@ -84,8 +91,8 @@ impl ApiClient {
 
         let http_client = client_builder.build()?;
 
-        let client = AsyncClient::new_with_client(&base_url, http_client);
-        Ok(ApiClient { client, runtime })
+        let client = AsyncClient::new_with_client(&base_url, http_client.clone());
+        Ok(ApiClient { client, http_client, base_url, runtime })
     }
 
     pub fn list_agents(&self, _verbose: bool) -> Result<Vec<types::BankListItem>> {
@@ -168,6 +175,67 @@ impl ApiClient {
         })
     }
 
+    /// Upload files to the file retain endpoint (multipart/form-data).
+    /// Returns a list of operation IDs for tracking. Always async server-side.
+    pub fn file_retain(
+        &self,
+        bank_id: &str,
+        files: Vec<(String, Vec<u8>)>,
+        context: Option<String>,
+        verbose: bool,
+    ) -> Result<FileRetainResult> {
+        self.runtime.block_on(async {
+            let url = format!("{}/v1/default/banks/{}/files/retain", self.base_url, bank_id);
+
+            let files_metadata: Vec<serde_json::Value> = files
+                .iter()
+                .map(|(name, _)| {
+                    let mut meta = serde_json::json!({});
+                    if let Some(ctx) = &context {
+                        meta["context"] = serde_json::Value::String(ctx.clone());
+                    }
+                    // Use filename stem as document_id for deduplication
+                    if let Some(stem) = std::path::Path::new(name)
+                        .file_stem()
+                        .and_then(|s| s.to_str())
+                    {
+                        meta["document_id"] = serde_json::Value::String(stem.to_string());
+                    }
+                    meta
+                })
+                .collect();
+
+            let request_json = serde_json::json!({
+                "files_metadata": files_metadata,
+            });
+
+            let mut form = reqwest::multipart::Form::new()
+                .text("request", request_json.to_string());
+
+            for (filename, content) in files {
+                let part = reqwest::multipart::Part::bytes(content)
+                    .file_name(filename)
+                    .mime_str("application/octet-stream")?;
+                form = form.part("files", part);
+            }
+
+            if verbose {
+                eprintln!("POST {}", url);
+            }
+
+            let response = self.http_client.post(&url).multipart(form).send().await?;
+
+            if !response.status().is_success() {
+                let status = response.status();
+                let text = response.text().await.unwrap_or_default();
+                anyhow::bail!("File retain failed ({}): {}", status, text);
+            }
+
+            let result: FileRetainResult = response.json().await?;
+            Ok(result)
+        })
+    }
+
     /// Poll an operation until it completes or fails.
     /// Returns Ok(true) if completed successfully, Ok(false) if failed, Err if polling error.
     pub fn poll_operation(&self, agent_id: &str, operation_id: &str, verbose: bool) -> Result<(bool, Option<String>)> {

hindsight-cli/src/commands/memory.rs

@@ -220,14 +220,23 @@ pub fn get(
     }
 }
 
-// Helper function to check if a file has a text-based extension
-fn is_text_file(path: &std::path::Path) -> bool {
-    const TEXT_EXTENSIONS: &[&str] = &[
-        "txt", "md", "json", "yaml", "yml", "toml", "xml", "csv", "log", "rst", "adoc",
+// Helper function to check if a file is supported by the file converter (markitdown)
+fn is_supported_file(path: &std::path::Path) -> bool {
+    const SUPPORTED_EXTENSIONS: &[&str] = &[
+        // Documents
+        "pdf", "docx", "doc", "pptx", "ppt", "xlsx", "xls",
+        // Images (OCR)
+        "jpg", "jpeg", "png", "gif", "bmp", "webp", "tiff",
+        // Web / markup
+        "html", "htm",
+        // Text / data
+        "txt", "md", "csv", "json", "yaml", "yml", "toml", "xml", "rst", "adoc", "log",
+        // Audio (transcription)
+        "mp3", "wav", "ogg", "flac",
     ];
     path.extension()
         .and_then(|ext| ext.to_str())
-        .map(|ext| TEXT_EXTENSIONS.contains(&ext.to_lowercase().as_str()))
+        .map(|ext| SUPPORTED_EXTENSIONS.contains(&ext.to_lowercase().as_str()))
         .unwrap_or(false)
 }
 
@@ -427,10 +436,10 @@ pub fn retain_files(
         anyhow::bail!("Path does not exist: {}", path.display());
     }
 
-    let mut files = Vec::new();
+    let mut file_paths = Vec::new();
 
     if path.is_file() {
-        files.push(path);
+        file_paths.push(path);
     } else if path.is_dir() {
         if recursive {
             for entry in WalkDir::new(&path)
@@ -438,133 +447,110 @@ pub fn retain_files(
                 .filter_map(|e| e.ok())
                 .filter(|e| e.file_type().is_file())
             {
-                let path = entry.path();
-                if is_text_file(&path) {
-                    files.push(path.to_path_buf());
+                let file_path = entry.path();
+                if is_supported_file(file_path) {
+                    file_paths.push(file_path.to_path_buf());
                 }
             }
         } else {
             for entry in fs::read_dir(&path)? {
                 let entry = entry?;
-                let path = entry.path();
-                if path.is_file() && is_text_file(&path) {
-                    files.push(path);
+                let file_path = entry.path();
+                if file_path.is_file() && is_supported_file(&file_path) {
+                    file_paths.push(file_path);
                 }
             }
         }
     }
 
-    if files.is_empty() {
-        ui::print_warning("No text files found (supported: txt, md, json, yaml, yml, toml, xml, csv, log, rst, adoc)");
+    if file_paths.is_empty() {
+        ui::print_warning("No supported files found. Supported formats: pdf, docx, pptx, xlsx, jpg, png, html, txt, md, csv, mp3, wav, and more.");
         return Ok(());
     }
 
-    ui::print_info(&format!("Found {} files to import", files.len()));
+    ui::print_info(&format!("Found {} file(s) to import", file_paths.len()));
 
-    let pb = ui::create_progress_bar(files.len() as u64, "Processing files");
+    // Batch files (max 10 per request)
+    const BATCH_SIZE: usize = 10;
+    let batches: Vec<&[PathBuf]> = file_paths.chunks(BATCH_SIZE).collect();
+    let mut all_operation_ids: Vec<String> = Vec::new();
 
-    let mut items = Vec::new();
+    let pb = ui::create_progress_bar(file_paths.len() as u64, "Uploading files");
 
-    for file_path in &files {
-        let content = fs::read_to_string(file_path)
-            .with_context(|| format!("Failed to read file: {}", file_path.display()))?;
+    for batch in &batches {
+        let mut file_data: Vec<(String, Vec<u8>)> = Vec::new();
+        for file_path in *batch {
+            let filename = file_path
+                .file_name()
+                .and_then(|n| n.to_str())
+                .map(|s| s.to_string())
+                .unwrap_or_else(|| "file".to_string());
+            let content = fs::read(file_path)
+                .with_context(|| format!("Failed to read file: {}", file_path.display()))?;
+            file_data.push((filename, content));
+            pb.inc(1);
+        }
 
-        let doc_id = file_path
-            .file_stem()
-            .and_then(|s| s.to_str())
-            .map(|s| s.to_string())
-            .unwrap_or_else(config::generate_doc_id);
-
-        items.push(MemoryItem {
-            content,
-            context: context.clone(),
-            metadata: None,
-            timestamp: None,
-            document_id: Some(doc_id),
-            entities: None,
-            tags: None,
-        });
-
-        pb.inc(1);
+        let result = client.file_retain(agent_id, file_data, context.clone(), verbose)?;
+        all_operation_ids.extend(result.operation_ids);
     }
 
-    pb.finish_with_message("Files processed");
+    pb.finish_with_message("Files uploaded");
 
-    // Always use async mode for the API call
-    let request = RetainRequest {
-        items,
-        async_: true,
-        document_tags: None,
-    };
-
-    let spinner = if output_format == OutputFormat::Pretty {
-        Some(ui::create_spinner("Submitting retain request..."))
+    if r#async {
+        if output_format == OutputFormat::Pretty {
+            ui::print_success("Files queued for processing");
+            println!("  Files: {}", file_paths.len());
+            for op_id in &all_operation_ids {
+                println!("  Operation ID: {}", op_id);
+            }
+        } else {
+            let result = serde_json::json!({ "operation_ids": all_operation_ids });
+            output::print_output(&result, output_format)?;
+        }
     } else {
-        None
-    };
+        // Poll all operations until they complete
+        let poll_spinner = if output_format == OutputFormat::Pretty {
+            Some(ui::create_spinner("Processing files..."))
+        } else {
+            None
+        };
 
-    let response = client.retain(agent_id, &request, true, verbose);
+        let mut failed = Vec::new();
+        for op_id in &all_operation_ids {
+            let (success, error_msg) = client.poll_operation(agent_id, op_id, verbose)?;
+            if !success {
+                failed.push(error_msg.unwrap_or_else(|| "Unknown error".to_string()));
+            }
+        }
 
-    if let Some(mut sp) = spinner {
-        sp.finish();
-    }
+        if let Some(mut sp) = poll_spinner {
+            sp.finish();
+        }
 
-    match response {
-        Ok(result) => {
-            if r#async {
-                // User requested async mode - return immediately
-                if output_format == OutputFormat::Pretty {
-                    ui::print_success("Files queued for processing");
-                    println!("  Items: {}", result.items_count);
-                    if let Some(op_id) = &result.operation_id {
-                        println!("  Operation ID: {}", op_id);
-                    }
-                } else {
-                    output::print_output(&result, output_format)?;
-                }
+        if failed.is_empty() {
+            if output_format == OutputFormat::Pretty {
+                ui::print_success("Files retained successfully");
+                println!("  Files processed: {}", file_paths.len());
             } else {
-                // Poll until completion
-                if let Some(operation_id) = &result.operation_id {
-                    let poll_spinner = if output_format == OutputFormat::Pretty {
-                        Some(ui::create_spinner("Processing memories..."))
-                    } else {
-                        None
-                    };
-
-                    let (success, error_msg) = client.poll_operation(agent_id, operation_id, verbose)?;
-
-                    if let Some(mut sp) = poll_spinner {
-                        sp.finish();
-                    }
-
-                    if success {
-                        if output_format == OutputFormat::Pretty {
-                            ui::print_success("Files retained successfully");
-                            println!("  Items processed: {}", result.items_count);
-                        } else {
-                            output::print_output(&result, output_format)?;
-                        }
-                    } else {
-                        let msg = error_msg.unwrap_or_else(|| "Unknown error".to_string());
-                        if output_format == OutputFormat::Pretty {
-                            ui::print_error(&format!("Retain operation failed: {}", msg));
-                        }
-                        anyhow::bail!("Retain operation failed: {}", msg);
-                    }
-                } else {
-                    // No operation ID returned, shouldn't happen with async=true
-                    if output_format == OutputFormat::Pretty {
-                        ui::print_success("Files retained successfully");
-                        println!("  Items processed: {}", result.items_count);
-                    } else {
-                        output::print_output(&result, output_format)?;
-                    }
+                let result = serde_json::json!({
+                    "success": true,
+                    "files_count": file_paths.len(),
+                    "operation_ids": all_operation_ids,
+                });
+                output::print_output(&result, output_format)?;
+            }
+        } else {
+            for msg in &failed {
+                if output_format == OutputFormat::Pretty {
+                    ui::print_error(&format!("Retain operation failed: {}", msg));
                 }
             }
-            Ok(())
+            anyhow::bail!("{} operation(s) failed", failed.len());
         }
-        Err(e) => Err(e)
     }
+
+    Ok(())
 }
 
 pub fn delete(
@@ -679,55 +665,71 @@ mod tests {
     use std::path::Path;
 
     #[test]
-    fn test_is_text_file_supported_extensions() {
+    fn test_is_supported_file_text_extensions() {
         let supported = [
             "file.txt", "file.md", "file.json", "file.yaml", "file.yml",
             "file.toml", "file.xml", "file.csv", "file.log", "file.rst", "file.adoc",
         ];
         for filename in supported {
             assert!(
-                is_text_file(Path::new(filename)),
-                "{} should be recognized as a text file",
+                is_supported_file(Path::new(filename)),
+                "{} should be recognized as a supported file",
                 filename
             );
         }
     }
 
     #[test]
-    fn test_is_text_file_case_insensitive() {
-        assert!(is_text_file(Path::new("file.JSON")));
-        assert!(is_text_file(Path::new("file.TXT")));
-        assert!(is_text_file(Path::new("file.Md")));
-        assert!(is_text_file(Path::new("file.YAML")));
+    fn test_is_supported_file_binary_extensions() {
+        let supported = [
+            "file.pdf", "file.docx", "file.pptx", "file.xlsx",
+            "file.png", "file.jpg", "file.jpeg", "file.gif",
+            "file.mp3", "file.wav",
+        ];
+        for filename in supported {
+            assert!(
+                is_supported_file(Path::new(filename)),
+                "{} should be recognized as a supported file",
+                filename
+            );
+        }
     }
 
     #[test]
-    fn test_is_text_file_unsupported_extensions() {
+    fn test_is_supported_file_case_insensitive() {
+        assert!(is_supported_file(Path::new("file.JSON")));
+        assert!(is_supported_file(Path::new("file.TXT")));
+        assert!(is_supported_file(Path::new("file.Md")));
+        assert!(is_supported_file(Path::new("file.YAML")));
+        assert!(is_supported_file(Path::new("file.PDF")));
+    }
+
+    #[test]
+    fn test_is_supported_file_unsupported_extensions() {
         let unsupported = [
-            "file.pdf", "file.doc", "file.docx", "file.png", "file.jpg",
             "file.exe", "file.bin", "file.zip", "file.tar", "file.gz",
         ];
         for filename in unsupported {
             assert!(
-                !is_text_file(Path::new(filename)),
-                "{} should NOT be recognized as a text file",
+                !is_supported_file(Path::new(filename)),
+                "{} should NOT be recognized as a supported file",
                 filename
             );
         }
     }
 
     #[test]
-    fn test_is_text_file_no_extension() {
-        assert!(!is_text_file(Path::new("README")));
-        assert!(!is_text_file(Path::new("Makefile")));
-        assert!(!is_text_file(Path::new(".gitignore")));
+    fn test_is_supported_file_no_extension() {
+        assert!(!is_supported_file(Path::new("README")));
+        assert!(!is_supported_file(Path::new("Makefile")));
+        assert!(!is_supported_file(Path::new(".gitignore")));
     }
 
     #[test]
-    fn test_is_text_file_with_path() {
-        assert!(is_text_file(Path::new("/some/path/to/file.json")));
-        assert!(is_text_file(Path::new("../relative/path/file.md")));
-        assert!(!is_text_file(Path::new("/path/to/image.png")));
+    fn test_is_supported_file_with_path() {
+        assert!(is_supported_file(Path::new("/some/path/to/file.json")));
+        assert!(is_supported_file(Path::new("../relative/path/file.md")));
+        assert!(is_supported_file(Path::new("/path/to/image.png")));
     }
 
     #[test]

hindsight-clients/go/README.md

@@ -22,7 +22,7 @@ go get golang.org/x/net/context
 Put the package under your project folder and add the following in import:
 
 ```go
-import hindsight "github.com/vectorize-io/hindsight-client-go"
+import hindsight "github.com/vectorize-io/hindsight/hindsight-clients/go"
 ```
 
 To use a proxy, set the environment variable `HTTP_PROXY`:

hindsight-clients/go/api/openapi.yaml

@@ -7,7 +7,7 @@ info:
     name: Apache 2.0
     url: https://www.apache.org/licenses/LICENSE-2.0.html
   title: Hindsight HTTP API
-  version: 0.4.11
+  version: 0.4.12
 servers:
 - url: /
 paths:
@@ -2124,6 +2124,73 @@ paths:
       summary: Retain memories
       tags:
       - Memory
+  /v1/default/banks/{bank_id}/files/retain:
+    post:
+      description: |-
+        Upload files (PDF, DOCX, etc.), convert them to markdown, and retain as memories.
+
+        This endpoint handles file upload, conversion, and memory creation in a single operation.
+
+        **Features:**
+        - Supports PDF, DOCX, PPTX, XLSX, images (with OCR), audio (with transcription)
+        - Automatic file-to-markdown conversion using pluggable parsers
+        - Files stored in object storage (PostgreSQL by default, S3 for production)
+        - Each file becomes a separate document with optional metadata/tags
+        - Always processes asynchronously — returns operation IDs immediately
+
+        **The system automatically:**
+        1. Stores uploaded files in object storage
+        2. Converts files to markdown
+        3. Creates document records with file metadata
+        4. Extracts facts and creates memory units (same as regular retain)
+
+        Use the operations endpoint to monitor progress.
+
+        **Request format:** multipart/form-data with:
+        - `files`: One or more files to upload
+        - `request`: JSON string with FileRetainRequest model (files_metadata)
+
+        **Note:** File parser is configured server-side via `HINDSIGHT_API_FILE_PARSER` (default: markitdown).
+      operationId: file_retain
+      parameters:
+      - explode: false
+        in: path
+        name: bank_id
+        required: true
+        schema:
+          title: Bank Id
+          type: string
+        style: simple
+      - explode: false
+        in: header
+        name: authorization
+        required: false
+        schema:
+          nullable: true
+          type: string
+        style: simple
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/Body_file_retain'
+        required: true
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/FileRetainResponse'
+          description: Successful Response
+        "422":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      summary: Convert files to memories
+      tags:
+      - Files
 components:
   schemas:
     AddBackgroundRequest:
@@ -2394,6 +2461,22 @@ components:
       - total_links
       - total_nodes
       title: BankStatsResponse
+    Body_file_retain:
+      properties:
+        files:
+          description: Files to upload and convert
+          items:
+            format: binary
+            type: string
+          type: array
+        request:
+          description: JSON string with FileRetainRequest model
+          title: Request
+          type: string
+      required:
+      - files
+      - request
+      title: Body_file_retain
     Budget:
       description: Budget levels for recall/reflect operations.
       enum:
@@ -3040,12 +3123,34 @@ components:
           description: Whether per-bank configuration API is enabled
           title: Bank Config Api
           type: boolean
+        file_upload_api:
+          description: Whether file upload/conversion API is enabled
+          title: File Upload Api
+          type: boolean
       required:
       - bank_config_api
+      - file_upload_api
       - mcp
       - observations
       - worker
       title: FeaturesInfo
+    FileRetainResponse:
+      description: Response model for file upload endpoint.
+      example:
+        operation_ids:
+        - 550e8400-e29b-41d4-a716-446655440000
+        - 550e8400-e29b-41d4-a716-446655440001
+        - 550e8400-e29b-41d4-a716-446655440002
+      properties:
+        operation_ids:
+          description: "Operation IDs for tracking file conversion operations. Use\
+            \ GET /v1/default/banks/{bank_id}/operations to list operations."
+          items:
+            type: string
+          type: array
+      required:
+      - operation_ids
+      title: FileRetainResponse
     GraphDataResponse:
       description: Response model for graph data endpoint.
       example:
@@ -4197,6 +4302,7 @@ components:
         api_version: 0.4.0
         features:
           bank_config_api: false
+          file_upload_api: true
           mcp: true
           observations: false
           worker: true

hindsight-clients/go/api_banks.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_directives.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_documents.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_entities.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_files.go

@@ -0,0 +1,209 @@
+/*
+Hindsight HTTP API
+
+HTTP API for Hindsight
+
+API version: 0.4.12
+*/
+
+// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
+
+package hindsight
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+)
+
+
+// FilesAPIService FilesAPI service
+type FilesAPIService service
+
+type ApiFileRetainRequest struct {
+	ctx context.Context
+	ApiService *FilesAPIService
+	bankId string
+	files []*os.File
+	request *string
+	authorization *string
+}
+
+// Files to upload and convert
+func (r ApiFileRetainRequest) Files(files []*os.File) ApiFileRetainRequest {
+	r.files = files
+	return r
+}
+
+// JSON string with FileRetainRequest model
+func (r ApiFileRetainRequest) Request(request string) ApiFileRetainRequest {
+	r.request = &request
+	return r
+}
+
+func (r ApiFileRetainRequest) Authorization(authorization string) ApiFileRetainRequest {
+	r.authorization = &authorization
+	return r
+}
+
+func (r ApiFileRetainRequest) Execute() (*FileRetainResponse, *http.Response, error) {
+	return r.ApiService.FileRetainExecute(r)
+}
+
+/*
+FileRetain Convert files to memories
+
+Upload files (PDF, DOCX, etc.), convert them to markdown, and retain as memories.
+
+This endpoint handles file upload, conversion, and memory creation in a single operation.
+
+**Features:**
+- Supports PDF, DOCX, PPTX, XLSX, images (with OCR), audio (with transcription)
+- Automatic file-to-markdown conversion using pluggable parsers
+- Files stored in object storage (PostgreSQL by default, S3 for production)
+- Each file becomes a separate document with optional metadata/tags
+- Always processes asynchronously — returns operation IDs immediately
+
+**The system automatically:**
+1. Stores uploaded files in object storage
+2. Converts files to markdown
+3. Creates document records with file metadata
+4. Extracts facts and creates memory units (same as regular retain)
+
+Use the operations endpoint to monitor progress.
+
+**Request format:** multipart/form-data with:
+- `files`: One or more files to upload
+- `request`: JSON string with FileRetainRequest model (files_metadata)
+
+**Note:** File parser is configured server-side via `HINDSIGHT_API_FILE_PARSER` (default: markitdown).
+
+ @param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
+ @param bankId
+ @return ApiFileRetainRequest
+*/
+func (a *FilesAPIService) FileRetain(ctx context.Context, bankId string) ApiFileRetainRequest {
+	return ApiFileRetainRequest{
+		ApiService: a,
+		ctx: ctx,
+		bankId: bankId,
+	}
+}
+
+// Execute executes the request
+//  @return FileRetainResponse
+func (a *FilesAPIService) FileRetainExecute(r ApiFileRetainRequest) (*FileRetainResponse, *http.Response, error) {
+	var (
+		localVarHTTPMethod   = http.MethodPost
+		localVarPostBody     interface{}
+		formFiles            []formFile
+		localVarReturnValue  *FileRetainResponse
+	)
+
+	localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "FilesAPIService.FileRetain")
+	if err != nil {
+		return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
+	}
+
+	localVarPath := localBasePath + "/v1/default/banks/{bank_id}/files/retain"
+	localVarPath = strings.Replace(localVarPath, "{"+"bank_id"+"}", url.PathEscape(parameterValueToString(r.bankId, "bankId")), -1)
+
+	localVarHeaderParams := make(map[string]string)
+	localVarQueryParams := url.Values{}
+	localVarFormParams := url.Values{}
+	if r.files == nil {
+		return localVarReturnValue, nil, reportError("files is required and must be specified")
+	}
+	if r.request == nil {
+		return localVarReturnValue, nil, reportError("request is required and must be specified")
+	}
+
+	// to determine the Content-Type header
+	localVarHTTPContentTypes := []string{"multipart/form-data"}
+
+	// set Content-Type header
+	localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
+	if localVarHTTPContentType != "" {
+		localVarHeaderParams["Content-Type"] = localVarHTTPContentType
+	}
+
+	// to determine the Accept header
+	localVarHTTPHeaderAccepts := []string{"application/json"}
+
+	// set Accept header
+	localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
+	if localVarHTTPHeaderAccept != "" {
+		localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
+	}
+	if r.authorization != nil {
+		parameterAddToHeaderOrQuery(localVarHeaderParams, "authorization", r.authorization, "simple", "")
+	}
+	var filesLocalVarFormFileName string
+	var filesLocalVarFileName     string
+	var filesLocalVarFileBytes    []byte
+
+	filesLocalVarFormFileName = "files"
+	filesLocalVarFile := r.files
+
+	if filesLocalVarFile != nil {
+		// loop through the array to prepare multiple files upload
+		for _, filesLocalVarFileValue := range filesLocalVarFile {
+			fbs, _ := io.ReadAll(filesLocalVarFileValue)
+
+			filesLocalVarFileBytes = fbs
+			filesLocalVarFileName = filesLocalVarFileValue.Name()
+			filesLocalVarFileValue.Close()
+			formFiles = append(formFiles, formFile{fileBytes: filesLocalVarFileBytes, fileName: filesLocalVarFileName, formFileName: filesLocalVarFormFileName})
+		}
+	}
+	parameterAddToHeaderOrQuery(localVarFormParams, "request", r.request, "", "")
+	req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
+	if err != nil {
+		return localVarReturnValue, nil, err
+	}
+
+	localVarHTTPResponse, err := a.client.callAPI(req)
+	if err != nil || localVarHTTPResponse == nil {
+		return localVarReturnValue, localVarHTTPResponse, err
+	}
+
+	localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
+	localVarHTTPResponse.Body.Close()
+	localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
+	if err != nil {
+		return localVarReturnValue, localVarHTTPResponse, err
+	}
+
+	if localVarHTTPResponse.StatusCode >= 300 {
+		newErr := &GenericOpenAPIError{
+			body:  localVarBody,
+			error: localVarHTTPResponse.Status,
+		}
+		if localVarHTTPResponse.StatusCode == 422 {
+			var v HTTPValidationError
+			err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+			if err != nil {
+				newErr.error = err.Error()
+				return localVarReturnValue, localVarHTTPResponse, newErr
+			}
+					newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
+					newErr.model = v
+		}
+		return localVarReturnValue, localVarHTTPResponse, newErr
+	}
+
+	err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+	if err != nil {
+		newErr := &GenericOpenAPIError{
+			body:  localVarBody,
+			error: err.Error(),
+		}
+		return localVarReturnValue, localVarHTTPResponse, newErr
+	}
+
+	return localVarReturnValue, localVarHTTPResponse, nil
+}

hindsight-clients/go/api_memory.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_mental_models.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_monitoring.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_operations.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/client.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -41,7 +41,7 @@ var (
 	queryDescape    = strings.NewReplacer( "%5B", "[", "%5D", "]" )
 )
 
-// APIClient manages communication with the Hindsight HTTP API API v0.4.11
+// APIClient manages communication with the Hindsight HTTP API API v0.4.12
 // In most cases there should be only one, shared, APIClient.
 type APIClient struct {
 	cfg    *Configuration
@@ -57,6 +57,8 @@ type APIClient struct {
 
 	EntitiesAPI *EntitiesAPIService
 
+	FilesAPI *FilesAPIService
+
 	MemoryAPI *MemoryAPIService
 
 	MentalModelsAPI *MentalModelsAPIService
@@ -86,6 +88,7 @@ func NewAPIClient(cfg *Configuration) *APIClient {
 	c.DirectivesAPI = (*DirectivesAPIService)(&c.common)
 	c.DocumentsAPI = (*DocumentsAPIService)(&c.common)
 	c.EntitiesAPI = (*EntitiesAPIService)(&c.common)
+	c.FilesAPI = (*FilesAPIService)(&c.common)
 	c.MemoryAPI = (*MemoryAPIService)(&c.common)
 	c.MentalModelsAPI = (*MentalModelsAPIService)(&c.common)
 	c.MonitoringAPI = (*MonitoringAPIService)(&c.common)

hindsight-clients/go/configuration.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/go.mod

@@ -1,4 +1,4 @@
-module github.com/vectorize-io/hindsight-client-go
+module github.com/vectorize-io/hindsight/hindsight-clients/go
 
 go 1.18
 

hindsight-clients/go/hindsight_client.go

@@ -0,0 +1,41 @@
+package hindsight
+
+import (
+	"net/http"
+	"time"
+)
+
+// NewAPIClientWithToken creates a new API client configured with a base URL and API token.
+// The token is sent as a Bearer token in the Authorization header for all requests.
+// Note: this uses http.DefaultClient which has no timeout. Use NewAPIClientWithTimeout
+// to set a request timeout.
+//
+// Example:
+//
+//	client := hindsight.NewAPIClientWithToken("https://api.example.com", "your-api-token")
+//	resp, _, err := client.MemoryAPI.RetainMemories(ctx, bankID).RetainRequest(req).Execute()
+func NewAPIClientWithToken(baseURL, token string) *APIClient {
+	cfg := NewConfiguration()
+	cfg.Servers = ServerConfigurations{
+		{URL: baseURL},
+	}
+	cfg.AddDefaultHeader("Authorization", "Bearer "+token)
+	return NewAPIClient(cfg)
+}
+
+// NewAPIClientWithTimeout creates a new API client configured with a base URL, API token,
+// and a request timeout. Use 0 for no timeout.
+//
+// Example:
+//
+//	client := hindsight.NewAPIClientWithTimeout("https://api.example.com", "your-api-token", 30*time.Second)
+//	resp, _, err := client.MemoryAPI.RetainMemories(ctx, bankID).RetainRequest(req).Execute()
+func NewAPIClientWithTimeout(baseURL, token string, timeout time.Duration) *APIClient {
+	cfg := NewConfiguration()
+	cfg.Servers = ServerConfigurations{
+		{URL: baseURL},
+	}
+	cfg.AddDefaultHeader("Authorization", "Bearer "+token)
+	cfg.HTTPClient = &http.Client{Timeout: timeout}
+	return NewAPIClient(cfg)
+}

hindsight-clients/go/model_add_background_request.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_async_operation_submit_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_background_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_config_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_config_update.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_list_item.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_list_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_profile_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_stats_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_budget.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_cancel_operation_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_child_operation_status.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_chunk_data.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_chunk_include_options.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_chunk_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_consolidation_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_create_bank_request.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_create_directive_request.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_create_mental_model_request.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_create_mental_model_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_delete_document_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_delete_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_directive_list_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_directive_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_disposition_traits.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_document_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_entity_detail_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_entity_include_options.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_entity_input.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_entity_list_item.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_entity_list_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_entity_observation_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_entity_state_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_features_info.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -29,6 +29,8 @@ type FeaturesInfo struct {
 	Worker bool `json:"worker"`
 	// Whether per-bank configuration API is enabled
 	BankConfigApi bool `json:"bank_config_api"`
+	// Whether file upload/conversion API is enabled
+	FileUploadApi bool `json:"file_upload_api"`
 }
 
 type _FeaturesInfo FeaturesInfo
@@ -37,12 +39,13 @@ type _FeaturesInfo FeaturesInfo
 // This constructor will assign default values to properties that have it defined,
 // and makes sure properties required by API are set, but the set of arguments
 // will change when the set of required properties is changed
-func NewFeaturesInfo(observations bool, mcp bool, worker bool, bankConfigApi bool) *FeaturesInfo {
+func NewFeaturesInfo(observations bool, mcp bool, worker bool, bankConfigApi bool, fileUploadApi bool) *FeaturesInfo {
 	this := FeaturesInfo{}
 	this.Observations = observations
 	this.Mcp = mcp
 	this.Worker = worker
 	this.BankConfigApi = bankConfigApi
+	this.FileUploadApi = fileUploadApi
 	return &this
 }
 
@@ -150,6 +153,30 @@ func (o *FeaturesInfo) SetBankConfigApi(v bool) {
 	o.BankConfigApi = v
 }
 
+// GetFileUploadApi returns the FileUploadApi field value
+func (o *FeaturesInfo) GetFileUploadApi() bool {
+	if o == nil {
+		var ret bool
+		return ret
+	}
+
+	return o.FileUploadApi
+}
+
+// GetFileUploadApiOk returns a tuple with the FileUploadApi field value
+// and a boolean to check if the value has been set.
+func (o *FeaturesInfo) GetFileUploadApiOk() (*bool, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return &o.FileUploadApi, true
+}
+
+// SetFileUploadApi sets field value
+func (o *FeaturesInfo) SetFileUploadApi(v bool) {
+	o.FileUploadApi = v
+}
+
 func (o FeaturesInfo) MarshalJSON() ([]byte, error) {
 	toSerialize,err := o.ToMap()
 	if err != nil {
@@ -164,6 +191,7 @@ func (o FeaturesInfo) ToMap() (map[string]interface{}, error) {
 	toSerialize["mcp"] = o.Mcp
 	toSerialize["worker"] = o.Worker
 	toSerialize["bank_config_api"] = o.BankConfigApi
+	toSerialize["file_upload_api"] = o.FileUploadApi
 	return toSerialize, nil
 }
 
@@ -176,6 +204,7 @@ func (o *FeaturesInfo) UnmarshalJSON(data []byte) (err error) {
 		"mcp",
 		"worker",
 		"bank_config_api",
+		"file_upload_api",
 	}
 
 	allProperties := make(map[string]interface{})

hindsight-clients/go/model_file_retain_response.go

@@ -0,0 +1,159 @@
+/*
+Hindsight HTTP API
+
+HTTP API for Hindsight
+
+API version: 0.4.12
+*/
+
+// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
+
+package hindsight
+
+import (
+	"encoding/json"
+	"bytes"
+	"fmt"
+)
+
+// checks if the FileRetainResponse type satisfies the MappedNullable interface at compile time
+var _ MappedNullable = &FileRetainResponse{}
+
+// FileRetainResponse Response model for file upload endpoint.
+type FileRetainResponse struct {
+	// Operation IDs for tracking file conversion operations. Use GET /v1/default/banks/{bank_id}/operations to list operations.
+	OperationIds []string `json:"operation_ids"`
+}
+
+type _FileRetainResponse FileRetainResponse
+
+// NewFileRetainResponse instantiates a new FileRetainResponse object
+// This constructor will assign default values to properties that have it defined,
+// and makes sure properties required by API are set, but the set of arguments
+// will change when the set of required properties is changed
+func NewFileRetainResponse(operationIds []string) *FileRetainResponse {
+	this := FileRetainResponse{}
+	this.OperationIds = operationIds
+	return &this
+}
+
+// NewFileRetainResponseWithDefaults instantiates a new FileRetainResponse object
+// This constructor will only assign default values to properties that have it defined,
+// but it doesn't guarantee that properties required by API are set
+func NewFileRetainResponseWithDefaults() *FileRetainResponse {
+	this := FileRetainResponse{}
+	return &this
+}
+
+// GetOperationIds returns the OperationIds field value
+func (o *FileRetainResponse) GetOperationIds() []string {
+	if o == nil {
+		var ret []string
+		return ret
+	}
+
+	return o.OperationIds
+}
+
+// GetOperationIdsOk returns a tuple with the OperationIds field value
+// and a boolean to check if the value has been set.
+func (o *FileRetainResponse) GetOperationIdsOk() ([]string, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.OperationIds, true
+}
+
+// SetOperationIds sets field value
+func (o *FileRetainResponse) SetOperationIds(v []string) {
+	o.OperationIds = v
+}
+
+func (o FileRetainResponse) MarshalJSON() ([]byte, error) {
+	toSerialize,err := o.ToMap()
+	if err != nil {
+		return []byte{}, err
+	}
+	return json.Marshal(toSerialize)
+}
+
+func (o FileRetainResponse) ToMap() (map[string]interface{}, error) {
+	toSerialize := map[string]interface{}{}
+	toSerialize["operation_ids"] = o.OperationIds
+	return toSerialize, nil
+}
+
+func (o *FileRetainResponse) UnmarshalJSON(data []byte) (err error) {
+	// This validates that all required properties are included in the JSON object
+	// by unmarshalling the object into a generic map with string keys and checking
+	// that every required field exists as a key in the generic map.
+	requiredProperties := []string{
+		"operation_ids",
+	}
+
+	allProperties := make(map[string]interface{})
+
+	err = json.Unmarshal(data, &allProperties)
+
+	if err != nil {
+		return err;
+	}
+
+	for _, requiredProperty := range(requiredProperties) {
+		if _, exists := allProperties[requiredProperty]; !exists {
+			return fmt.Errorf("no value given for required property %v", requiredProperty)
+		}
+	}
+
+	varFileRetainResponse := _FileRetainResponse{}
+
+	decoder := json.NewDecoder(bytes.NewReader(data))
+	decoder.DisallowUnknownFields()
+	err = decoder.Decode(&varFileRetainResponse)
+
+	if err != nil {
+		return err
+	}
+
+	*o = FileRetainResponse(varFileRetainResponse)
+
+	return err
+}
+
+type NullableFileRetainResponse struct {
+	value *FileRetainResponse
+	isSet bool
+}
+
+func (v NullableFileRetainResponse) Get() *FileRetainResponse {
+	return v.value
+}
+
+func (v *NullableFileRetainResponse) Set(val *FileRetainResponse) {
+	v.value = val
+	v.isSet = true
+}
+
+func (v NullableFileRetainResponse) IsSet() bool {
+	return v.isSet
+}
+
+func (v *NullableFileRetainResponse) Unset() {
+	v.value = nil
+	v.isSet = false
+}
+
+func NewNullableFileRetainResponse(val *FileRetainResponse) *NullableFileRetainResponse {
+	return &NullableFileRetainResponse{value: val, isSet: true}
+}
+
+func (v NullableFileRetainResponse) MarshalJSON() ([]byte, error) {
+	return json.Marshal(v.value)
+}
+
+func (v *NullableFileRetainResponse) UnmarshalJSON(src []byte) error {
+	v.isSet = true
+	return json.Unmarshal(src, &v.value)
+}
+
+

hindsight-clients/go/model_graph_data_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_http_validation_error.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_include_options.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_list_documents_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_list_memory_units_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_list_tags_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_memory_item.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_mental_model_list_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_mental_model_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_mental_model_trigger.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_operation_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_operation_status_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_operations_list_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_recall_request.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_recall_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.4.11
+API version: 0.4.12
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.