fix

* feat: accept pdf, images and office files

* refactor: rename FileConverter to FileParser, simplify file retain API

- Rename engine/converters/ → engine/parsers/, FileConverter → FileParser,
  ConverterRegistry → FileParserRegistry, MarkitdownConverter → MarkitdownParser
- Rename env var HINDSIGHT_API_FILE_CONVERTER → HINDSIGHT_API_FILE_PARSER
- Remove async/document_tags params from FileRetainRequest (always async now)
- Add retain_files() to Python Hindsight client and retainFiles() to TypeScript client
- Add sample.pdf to doc examples for working file upload demonstrations
- Update test_file_retain.py to use new parser names and always-async behavior
- Fix Go client missing os import in api_files.go
- Simplify postgresql.py storage to minimal schema

* fix: update rust CLI tests to use is_supported_file instead of is_text_file

* fix: patch Go api_files.go to add missing 'os' import after generation

* fix: insert 'os' import after 'net/url' in api_files.go patch for correct position

* chore: regenerate OpenAPI spec and clients (converter→parser description update)

docker/docker-compose/s3-file-storage/docker-compose.yaml

@@ -0,0 +1,83 @@
+# Docker Compose file for Hindsight with S3 file storage (SeaweedFS)
+#
+# SeaweedFS (Apache 2.0) provides an S3-compatible object storage backend
+# for storing uploaded files instead of PostgreSQL BYTEA storage.
+#
+# Make sure to set the required environment variables before running:
+# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
+# - Configure LLM provider variables as needed (see below in the hindsight service)
+#
+# Usage:
+# docker compose up -d
+#
+# Optional environment variables with defaults:
+# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
+# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
+# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
+# - HINDSIGHT_DB_VERSION: PostgreSQL version (default: 18)
+# - SEAWEEDFS_S3_ACCESS_KEY: S3 access key (default: hindsight_s3_key)
+# - SEAWEEDFS_S3_SECRET_KEY: S3 secret key (default: hindsight_s3_secret)
+
+services:
+  db:
+    image: pgvector/pgvector:pg${HINDSIGHT_DB_VERSION:-18}
+    container_name: hindsight-db
+    restart: always
+    environment:
+      POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
+      POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}
+      POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
+    volumes:
+      - pg_data:/var/lib/postgresql/${HINDSIGHT_DB_VERSION:-18}/docker
+    networks:
+      - hindsight-net
+
+  seaweedfs:
+    image: chrislusf/seaweedfs:latest
+    container_name: hindsight-seaweedfs
+    restart: always
+    # Single-node mode: master + volume + filer + S3 gateway all in one process
+    command: >
+      server
+      -s3
+      -s3.port=8333
+      -s3.config=/etc/seaweedfs/s3.json
+      -ip.bind=0.0.0.0
+    volumes:
+      - seaweedfs_data:/data
+      - ./s3.json:/etc/seaweedfs/s3.json:ro
+    # Expose S3 API port (uncomment to access from host)
+    # ports:
+    #   - "8333:8333"
+    networks:
+      - hindsight-net
+
+  hindsight:
+    image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
+    container_name: hindsight-app
+    ports:
+      - "8888:8888"
+      - "9999:9999"
+    environment:
+      - HINDSIGHT_API_LLM_API_KEY=${OPENAI_API_KEY?Please set the OPENAI_API_KEY env variable}
+      - HINDSIGHT_API_DATABASE_URL=postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:?Please set the HINDSIGHT_DB_PASSWORD env variable}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
+      # S3 file storage configuration (SeaweedFS)
+      - HINDSIGHT_API_FILE_STORAGE_TYPE=s3
+      - HINDSIGHT_API_FILE_STORAGE_S3_BUCKET=hindsight
+      - HINDSIGHT_API_FILE_STORAGE_S3_ENDPOINT=http://seaweedfs:8333
+      - HINDSIGHT_API_FILE_STORAGE_S3_REGION=us-east-1
+      - HINDSIGHT_API_FILE_STORAGE_S3_ACCESS_KEY_ID=${SEAWEEDFS_S3_ACCESS_KEY:-hindsight_s3_key}
+      - HINDSIGHT_API_FILE_STORAGE_S3_SECRET_ACCESS_KEY=${SEAWEEDFS_S3_SECRET_KEY:-hindsight_s3_secret}
+    depends_on:
+      - db
+      - seaweedfs
+    networks:
+      - hindsight-net
+
+networks:
+  hindsight-net:
+    driver: bridge
+
+volumes:
+  pg_data:
+  seaweedfs_data:

docker/docker-compose/s3-file-storage/s3.json

@@ -0,0 +1,19 @@
+{
+  "identities": [
+    {
+      "name": "hindsight",
+      "credentials": [
+        {
+          "accessKey": "hindsight_s3_key",
+          "secretKey": "hindsight_s3_secret"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "Write",
+        "List"
+      ]
+    }
+  ]
+}

hindsight-api/hindsight_api/alembic/versions/a1b2c3d4e5f6_add_file_storage_table.py

@@ -0,0 +1,70 @@
+"""Add file_storage table for BYTEA-based file storage
+
+Revision ID: a1b2c3d4e5f6
+Revises: y0t1u2v3w4x5
+Create Date: 2026-02-16
+
+Creates a dedicated table for storing uploaded files using BYTEA.
+This provides zero-config file storage that "just works" for development
+and small deployments. For production/scale, use S3-compatible storage.
+
+Files are stored in a separate table to avoid bloating the documents table.
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "a1b2c3d4e5f6"
+down_revision: str | Sequence[str] | None = "y0t1u2v3w4x5"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    """Create file_storage table for BYTEA storage."""
+    schema = _get_schema_prefix()
+
+    # Create file_storage table (minimal: just key + data)
+    op.execute(
+        f"""
+        CREATE TABLE {schema}file_storage (
+            storage_key TEXT PRIMARY KEY,
+            data BYTEA NOT NULL
+        )
+    """
+    )
+
+    # Add file tracking columns to documents table
+    op.execute(
+        f"""
+        ALTER TABLE {schema}documents
+        ADD COLUMN IF NOT EXISTS file_storage_key TEXT,
+        ADD COLUMN IF NOT EXISTS file_original_name TEXT,
+        ADD COLUMN IF NOT EXISTS file_content_type TEXT
+    """
+    )
+
+
+def downgrade() -> None:
+    """Remove file_storage table and related columns."""
+    schema = _get_schema_prefix()
+
+    # Drop columns from documents table
+    op.execute(
+        f"""
+        ALTER TABLE {schema}documents
+        DROP COLUMN IF EXISTS file_storage_key,
+        DROP COLUMN IF EXISTS file_original_name,
+        DROP COLUMN IF EXISTS file_content_type
+    """
+    )
+
+    # Drop file_storage table
+    op.execute(f"DROP TABLE IF EXISTS {schema}file_storage")

hindsight-api/hindsight_api/api/http.py

@@ -13,7 +13,7 @@ from contextlib import asynccontextmanager
 from datetime import datetime
 from typing import Any, Literal
 
-from fastapi import Depends, FastAPI, Header, HTTPException, Query
+from fastapi import Depends, FastAPI, File, Form, Header, HTTPException, Query, UploadFile
 
 from hindsight_api.extensions import AuthenticationError
 
@@ -430,6 +430,36 @@ class RetainRequest(BaseModel):
     )
 
 
+class FileRetainMetadata(BaseModel):
+    """Metadata for a single file in file retain request."""
+
+    document_id: str | None = Field(default=None, description="Document ID (auto-generated if not provided)")
+    context: str | None = Field(default=None, description="Context for the file")
+    metadata: dict[str, Any] | None = Field(default=None, description="Additional metadata")
+    tags: list[str] | None = Field(default=None, description="Tags for this file")
+    timestamp: str | None = Field(default=None, description="ISO timestamp")
+
+
+class FileRetainRequest(BaseModel):
+    """Request model for file retain endpoint."""
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "files_metadata": [
+                    {"document_id": "report_2024", "tags": ["quarterly"]},
+                    {"context": "meeting notes"},
+                ],
+            }
+        }
+    )
+
+    files_metadata: list[FileRetainMetadata] | None = Field(
+        default=None,
+        description="Metadata for each file (optional, must match number of files if provided)",
+    )
+
+
 class RetainResponse(BaseModel):
     """Response model for retain endpoint."""
 
@@ -454,7 +484,7 @@ class RetainResponse(BaseModel):
     )
     operation_id: str | None = Field(
         default=None,
-        description="Operation ID for tracking async operations. Use GET /v1/default/banks/{bank_id}/operations to list operations and find this ID. Only present when async=true.",
+        description="Operation ID for tracking async operations. Use GET /v1/default/banks/{bank_id}/operations to list operations. Only present when async=true.",
     )
     usage: TokenUsage | None = Field(
         default=None,
@@ -462,6 +492,26 @@ class RetainResponse(BaseModel):
     )
 
 
+class FileRetainResponse(BaseModel):
+    """Response model for file upload endpoint."""
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "operation_ids": [
+                    "550e8400-e29b-41d4-a716-446655440000",
+                    "550e8400-e29b-41d4-a716-446655440001",
+                    "550e8400-e29b-41d4-a716-446655440002",
+                ],
+            }
+        },
+    )
+
+    operation_ids: list[str] = Field(
+        description="Operation IDs for tracking file conversion operations. Use GET /v1/default/banks/{bank_id}/operations to list operations."
+    )
+
+
 class FactsIncludeOptions(BaseModel):
     """Options for including facts (based_on) in reflect results."""
 
@@ -1423,6 +1473,7 @@ class FeaturesInfo(BaseModel):
     mcp: bool = Field(description="Whether MCP (Model Context Protocol) server is enabled")
     worker: bool = Field(description="Whether the background worker is enabled")
     bank_config_api: bool = Field(description="Whether per-bank configuration API is enabled")
+    file_upload_api: bool = Field(description="Whether file upload/conversion API is enabled")
 
 
 class VersionResponse(BaseModel):
@@ -1437,6 +1488,7 @@ class VersionResponse(BaseModel):
                     "mcp": True,
                     "worker": True,
                     "bank_config_api": False,
+                    "file_upload_api": True,
                 },
             }
         }
@@ -1731,6 +1783,7 @@ def _register_routes(app: FastAPI):
                 mcp=config.mcp_enabled,
                 worker=config.worker_enabled,
                 bank_config_api=config.enable_bank_config_api,
+                file_upload_api=config.enable_file_upload_api,
             ),
         )
 
@@ -3632,6 +3685,147 @@ def _register_routes(app: FastAPI):
             logger.error(f"Error in /v1/default/banks/{bank_id}/memories (retain): {error_detail}")
             raise HTTPException(status_code=500, detail=str(e))
 
+    @app.post(
+        "/v1/default/banks/{bank_id}/files/retain",
+        response_model=FileRetainResponse,
+        summary="Convert files to memories",
+        description="Upload files (PDF, DOCX, etc.), convert them to markdown, and retain as memories.\n\n"
+        "This endpoint handles file upload, conversion, and memory creation in a single operation.\n\n"
+        "**Features:**\n"
+        "- Supports PDF, DOCX, PPTX, XLSX, images (with OCR), audio (with transcription)\n"
+        "- Automatic file-to-markdown conversion using pluggable parsers\n"
+        "- Files stored in object storage (PostgreSQL by default, S3 for production)\n"
+        "- Each file becomes a separate document with optional metadata/tags\n"
+        "- Always processes asynchronously — returns operation IDs immediately\n\n"
+        "**The system automatically:**\n"
+        "1. Stores uploaded files in object storage\n"
+        "2. Converts files to markdown\n"
+        "3. Creates document records with file metadata\n"
+        "4. Extracts facts and creates memory units (same as regular retain)\n\n"
+        "Use the operations endpoint to monitor progress.\n\n"
+        "**Request format:** multipart/form-data with:\n"
+        "- `files`: One or more files to upload\n"
+        "- `request`: JSON string with FileRetainRequest model (files_metadata)\n\n"
+        "**Note:** File parser is configured server-side via `HINDSIGHT_API_FILE_PARSER` (default: markitdown).",
+        operation_id="file_retain",
+        tags=["Files"],
+    )
+    async def api_file_retain(
+        bank_id: str,
+        files: list[UploadFile] = File(..., description="Files to upload and convert"),
+        request: str = Form(..., description="JSON string with FileRetainRequest model"),
+        request_context: RequestContext = Depends(get_request_context),
+    ):
+        """Upload and convert files to memories."""
+        from hindsight_api.config import get_config
+
+        config = get_config()
+
+        # Check if file upload API is enabled
+        if not config.enable_file_upload_api:
+            raise HTTPException(
+                status_code=404,
+                detail="File upload API is disabled. Set HINDSIGHT_API_ENABLE_FILE_UPLOAD_API=true to enable.",
+            )
+
+        try:
+            # Parse request JSON
+            try:
+                request_data = FileRetainRequest.model_validate_json(request)
+            except Exception as e:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Invalid request JSON: {str(e)}",
+                )
+
+            # Validate file count
+            if len(files) > config.file_conversion_max_batch_size:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Too many files. Maximum {config.file_conversion_max_batch_size} files per request.",
+                )
+
+            # Validate files_metadata count matches files count if provided
+            if request_data.files_metadata and len(request_data.files_metadata) != len(files):
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"files_metadata count ({len(request_data.files_metadata)}) must match files count ({len(files)})",
+                )
+
+            # Prepare file items and calculate total batch size
+            file_items = []
+            total_batch_size = 0
+
+            for i, file in enumerate(files):
+                # Read file content to check size
+                file_content = await file.read()
+                size = len(file_content)
+                total_batch_size += size
+
+                # Create a temporary file-like object from the bytes
+                import io
+
+                file_obj = io.BytesIO(file_content)
+
+                # Create a mock UploadFile with the necessary attributes
+                class FileWrapper:
+                    def __init__(self, content, filename, content_type):
+                        self._content = content
+                        self.filename = filename
+                        self.content_type = content_type
+                        self._buffer = io.BytesIO(content)
+
+                    async def read(self):
+                        return self._content
+
+                wrapped_file = FileWrapper(file_content, file.filename, file.content_type)
+
+                # Get per-file metadata
+                file_meta = request_data.files_metadata[i] if request_data.files_metadata else FileRetainMetadata()
+                doc_id = file_meta.document_id or f"file_{uuid.uuid4()}"
+
+                item = {
+                    "file": wrapped_file,
+                    "document_id": doc_id,
+                    "context": file_meta.context,
+                    "metadata": file_meta.metadata or {},
+                    "tags": file_meta.tags or [],
+                    "timestamp": file_meta.timestamp,
+                }
+                file_items.append(item)
+
+            # Check total batch size after processing all files
+            if total_batch_size > config.file_conversion_max_batch_size_bytes:
+                total_mb = total_batch_size / (1024 * 1024)
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Total batch size ({total_mb:.1f}MB) exceeds maximum of {config.file_conversion_max_batch_size_mb}MB",
+                )
+
+            result = await app.state.memory.submit_async_file_retain(
+                bank_id=bank_id,
+                file_items=file_items,
+                parser=config.file_parser,
+                document_tags=None,
+                request_context=request_context,
+            )
+            return FileRetainResponse.model_validate(
+                {
+                    "operation_ids": result["operation_ids"],
+                }
+            )
+
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in /v1/default/banks/{bank_id}/files/retain: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
     @app.delete(
         "/v1/default/banks/{bank_id}/memories",
         response_model=DeleteResponse,

hindsight-api/hindsight_api/config.py

@@ -259,6 +259,26 @@ ENV_RETAIN_BATCH_TOKENS = "HINDSIGHT_API_RETAIN_BATCH_TOKENS"
 ENV_RETAIN_BATCH_ENABLED = "HINDSIGHT_API_RETAIN_BATCH_ENABLED"
 ENV_RETAIN_BATCH_POLL_INTERVAL_SECONDS = "HINDSIGHT_API_RETAIN_BATCH_POLL_INTERVAL_SECONDS"
 
+# File storage configuration
+ENV_FILE_STORAGE_TYPE = "HINDSIGHT_API_FILE_STORAGE_TYPE"
+ENV_FILE_STORAGE_S3_BUCKET = "HINDSIGHT_API_FILE_STORAGE_S3_BUCKET"
+ENV_FILE_STORAGE_S3_REGION = "HINDSIGHT_API_FILE_STORAGE_S3_REGION"
+ENV_FILE_STORAGE_S3_ENDPOINT = "HINDSIGHT_API_FILE_STORAGE_S3_ENDPOINT"
+ENV_FILE_STORAGE_S3_ACCESS_KEY_ID = "HINDSIGHT_API_FILE_STORAGE_S3_ACCESS_KEY_ID"
+ENV_FILE_STORAGE_S3_SECRET_ACCESS_KEY = "HINDSIGHT_API_FILE_STORAGE_S3_SECRET_ACCESS_KEY"
+ENV_FILE_STORAGE_GCS_BUCKET = "HINDSIGHT_API_FILE_STORAGE_GCS_BUCKET"
+ENV_FILE_STORAGE_GCS_SERVICE_ACCOUNT_KEY = "HINDSIGHT_API_FILE_STORAGE_GCS_SERVICE_ACCOUNT_KEY"
+ENV_FILE_STORAGE_AZURE_CONTAINER = "HINDSIGHT_API_FILE_STORAGE_AZURE_CONTAINER"
+ENV_FILE_STORAGE_AZURE_ACCOUNT_NAME = "HINDSIGHT_API_FILE_STORAGE_AZURE_ACCOUNT_NAME"
+ENV_FILE_STORAGE_AZURE_ACCOUNT_KEY = "HINDSIGHT_API_FILE_STORAGE_AZURE_ACCOUNT_KEY"
+ENV_FILE_PARSER = "HINDSIGHT_API_FILE_PARSER"
+ENV_FILE_PARSER_IRIS_TOKEN = "HINDSIGHT_API_FILE_PARSER_IRIS_TOKEN"
+ENV_FILE_PARSER_IRIS_ORG_ID = "HINDSIGHT_API_FILE_PARSER_IRIS_ORG_ID"
+ENV_FILE_CONVERSION_MAX_BATCH_SIZE_MB = "HINDSIGHT_API_FILE_CONVERSION_MAX_BATCH_SIZE_MB"
+ENV_FILE_CONVERSION_MAX_BATCH_SIZE = "HINDSIGHT_API_FILE_CONVERSION_MAX_BATCH_SIZE"
+ENV_ENABLE_FILE_UPLOAD_API = "HINDSIGHT_API_ENABLE_FILE_UPLOAD_API"
+ENV_FILE_DELETE_AFTER_RETAIN = "HINDSIGHT_API_FILE_DELETE_AFTER_RETAIN"
+
 # Observations settings (consolidated knowledge from facts)
 ENV_ENABLE_OBSERVATIONS = "HINDSIGHT_API_ENABLE_OBSERVATIONS"
 ENV_CONSOLIDATION_BATCH_SIZE = "HINDSIGHT_API_CONSOLIDATION_BATCH_SIZE"
@@ -383,6 +403,14 @@ DEFAULT_RETAIN_BATCH_TOKENS = 10_000  # ~40KB of text  # Max chars per sub-batch
 DEFAULT_RETAIN_BATCH_ENABLED = False  # Use LLM Batch API for fact extraction (only when async=True)
 DEFAULT_RETAIN_BATCH_POLL_INTERVAL_SECONDS = 60  # Batch API polling interval in seconds
 
+# File storage defaults
+DEFAULT_FILE_STORAGE_TYPE = "native"  # PostgreSQL BYTEA storage
+DEFAULT_FILE_PARSER = "markitdown"  # File parser to use (markitdown is the only supported parser)
+DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE_MB = 100  # Max total batch size in MB (all files combined)
+DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE = 10  # Max files per batch upload
+DEFAULT_ENABLE_FILE_UPLOAD_API = True  # Enable file upload endpoint
+DEFAULT_FILE_DELETE_AFTER_RETAIN = True  # Delete file bytes after retain (saves storage)
+
 # Observations defaults (consolidated knowledge from facts)
 DEFAULT_ENABLE_OBSERVATIONS = True  # Observations enabled by default
 DEFAULT_CONSOLIDATION_BATCH_SIZE = 50  # Memories to load per batch (internal memory optimization)
@@ -607,6 +635,26 @@ class HindsightConfig:
     retain_batch_enabled: bool
     retain_batch_poll_interval_seconds: int
 
+    # File storage (static - server-level only)
+    file_storage_type: str  # "native" (PostgreSQL) or "s3" (S3-compatible)
+    file_storage_s3_bucket: str | None  # S3 bucket name (required for s3 storage)
+    file_storage_s3_region: str | None  # S3 region (optional, uses SDK default)
+    file_storage_s3_endpoint: str | None  # S3 endpoint URL (for MinIO, R2, etc.)
+    file_storage_s3_access_key_id: str | None  # S3 access key (optional, uses env/IAM)
+    file_storage_s3_secret_access_key: str | None  # S3 secret key (optional, uses env/IAM)
+    file_storage_gcs_bucket: str | None  # GCS bucket name (required for gcs storage)
+    file_storage_gcs_service_account_key: str | None  # GCS service account key JSON (optional, uses ADC)
+    file_storage_azure_container: str | None  # Azure container name (required for azure storage)
+    file_storage_azure_account_name: str | None  # Azure storage account name
+    file_storage_azure_account_key: str | None  # Azure storage account key
+    file_parser: str  # File parser to use (e.g., "markitdown", "iris")
+    file_parser_iris_token: str | None  # Vectorize API token for iris parser (VECTORIZE_TOKEN)
+    file_parser_iris_org_id: str | None  # Vectorize org ID for iris parser (VECTORIZE_ORG_ID)
+    file_conversion_max_batch_size_mb: int  # Max total batch size in MB (all files combined)
+    file_conversion_max_batch_size: int  # Max files per request
+    enable_file_upload_api: bool
+    file_delete_after_retain: bool
+
     # Observations settings (consolidated knowledge from facts)
     enable_observations: bool
     consolidation_batch_size: int
@@ -663,6 +711,13 @@ class HindsightConfig:
         "reranker_cohere_base_url",
         # Service Account Keys
         "llm_vertexai_service_account_key",
+        # File storage credentials
+        "file_storage_s3_access_key_id",
+        "file_storage_s3_secret_access_key",
+        "file_storage_gcs_service_account_key",
+        "file_storage_azure_account_key",
+        # File parser credentials
+        "file_parser_iris_token",
     }
 
     # CONFIGURABLE_FIELDS: Safe behavioral settings that can be customized per-tenant/bank
@@ -677,6 +732,11 @@ class HindsightConfig:
         "enable_observations",
     }
 
+    @property
+    def file_conversion_max_batch_size_bytes(self) -> int:
+        """Get maximum total batch size in bytes."""
+        return self.file_conversion_max_batch_size_mb * 1024 * 1024
+
     @classmethod
     def get_configurable_fields(cls) -> set[str]:
         """
@@ -963,6 +1023,33 @@ class HindsightConfig:
             retain_batch_poll_interval_seconds=int(
                 os.getenv(ENV_RETAIN_BATCH_POLL_INTERVAL_SECONDS, str(DEFAULT_RETAIN_BATCH_POLL_INTERVAL_SECONDS))
             ),
+            # File storage
+            file_storage_type=os.getenv(ENV_FILE_STORAGE_TYPE, DEFAULT_FILE_STORAGE_TYPE),
+            file_storage_s3_bucket=os.getenv(ENV_FILE_STORAGE_S3_BUCKET) or None,
+            file_storage_s3_region=os.getenv(ENV_FILE_STORAGE_S3_REGION) or None,
+            file_storage_s3_endpoint=os.getenv(ENV_FILE_STORAGE_S3_ENDPOINT) or None,
+            file_storage_s3_access_key_id=os.getenv(ENV_FILE_STORAGE_S3_ACCESS_KEY_ID) or None,
+            file_storage_s3_secret_access_key=os.getenv(ENV_FILE_STORAGE_S3_SECRET_ACCESS_KEY) or None,
+            file_storage_gcs_bucket=os.getenv(ENV_FILE_STORAGE_GCS_BUCKET) or None,
+            file_storage_gcs_service_account_key=os.getenv(ENV_FILE_STORAGE_GCS_SERVICE_ACCOUNT_KEY) or None,
+            file_storage_azure_container=os.getenv(ENV_FILE_STORAGE_AZURE_CONTAINER) or None,
+            file_storage_azure_account_name=os.getenv(ENV_FILE_STORAGE_AZURE_ACCOUNT_NAME) or None,
+            file_storage_azure_account_key=os.getenv(ENV_FILE_STORAGE_AZURE_ACCOUNT_KEY) or None,
+            file_parser=os.getenv(ENV_FILE_PARSER, DEFAULT_FILE_PARSER),
+            file_parser_iris_token=os.getenv(ENV_FILE_PARSER_IRIS_TOKEN) or None,
+            file_parser_iris_org_id=os.getenv(ENV_FILE_PARSER_IRIS_ORG_ID) or None,
+            file_conversion_max_batch_size_mb=int(
+                os.getenv(ENV_FILE_CONVERSION_MAX_BATCH_SIZE_MB, str(DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE_MB))
+            ),
+            file_conversion_max_batch_size=int(
+                os.getenv(ENV_FILE_CONVERSION_MAX_BATCH_SIZE, str(DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE))
+            ),
+            enable_file_upload_api=os.getenv(ENV_ENABLE_FILE_UPLOAD_API, str(DEFAULT_ENABLE_FILE_UPLOAD_API)).lower()
+            == "true",
+            file_delete_after_retain=os.getenv(
+                ENV_FILE_DELETE_AFTER_RETAIN, str(DEFAULT_FILE_DELETE_AFTER_RETAIN)
+            ).lower()
+            == "true",
             # Observations settings (consolidated knowledge from facts)
             enable_observations=os.getenv(ENV_ENABLE_OBSERVATIONS, str(DEFAULT_ENABLE_OBSERVATIONS)).lower() == "true",
             consolidation_batch_size=int(

hindsight-api/hindsight_api/engine/memory_engine.py

@@ -79,6 +79,7 @@ _PROTECTED_TABLES = frozenset(
         "documents",
         "chunks",
         "async_operations",
+        "file_storage",
     ]
 )
 
@@ -585,8 +586,165 @@ class MemoryEngine(MemoryEngineInterface):
             operation_id=operation_id,
         )
 
+        # If this retain was triggered by file conversion, update document with file metadata
+        file_metadata = task_dict.get("_file_metadata")
+        if file_metadata and len(contents) == 1:
+            doc_id = contents[0].get("document_id")
+            if doc_id:
+                pool = await self._get_pool()
+                async with acquire_with_retry(pool) as conn:
+                    await conn.execute(
+                        f"""
+                        UPDATE {fq_table("documents")}
+                        SET file_storage_key = $3,
+                            file_original_name = $4,
+                            file_content_type = $5,
+                            updated_at = NOW()
+                        WHERE id = $1 AND bank_id = $2
+                        """,
+                        doc_id,
+                        bank_id,
+                        file_metadata["file_storage_key"],
+                        file_metadata["file_original_name"],
+                        file_metadata["file_content_type"],
+                    )
+
         logger.info(f"[BATCH_RETAIN_TASK] Completed background batch retain for bank_id={bank_id}")
 
+    async def _handle_file_convert_retain(self, task_dict: dict[str, Any]):
+        """
+        Handler for file conversion tasks.
+
+        Converts a file to markdown, then submits a separate async retain operation
+        and marks this conversion as completed — all in a single transaction.
+        This avoids holding a worker slot during the expensive retain pipeline.
+
+        Args:
+            task_dict: Dict with 'bank_id', 'storage_key', 'parser', etc.
+
+        Raises:
+            ValueError: If required fields are missing
+            Exception: Any exception from conversion (includes filename in error)
+        """
+        bank_id = task_dict.get("bank_id")
+        storage_key = task_dict.get("storage_key")
+        document_id = task_dict.get("document_id")
+        operation_id = task_dict.get("operation_id")
+        filename = task_dict.get("original_filename", "unknown")
+
+        if not all([bank_id, storage_key, document_id]):
+            raise ValueError("bank_id, storage_key, and document_id are required for file_convert_retain task")
+
+        logger.info(f"[FILE_CONVERT_RETAIN] Starting for bank_id={bank_id}, document_id={document_id}, file={filename}")
+
+        try:
+            # Retrieve file from storage
+            file_data = await self._file_storage.retrieve(storage_key)
+
+            # Convert to markdown
+            parser = self._parser_registry.get_parser(
+                name=task_dict.get("parser"),
+                filename=filename,
+                content_type=task_dict.get("content_type"),
+            )
+            markdown_content = await parser.convert(file_data, filename)
+        except Exception as e:
+            # Re-raise with filename context for better error reporting
+            error_msg = f"Failed to parse file '{filename}': {str(e)}"
+            logger.error(f"[FILE_CONVERT_RETAIN] {error_msg}")
+            raise RuntimeError(error_msg) from e
+
+        logger.info(
+            f"[FILE_CONVERT_RETAIN] Converted file for bank_id={bank_id}, "
+            f"document_id={document_id}, {len(markdown_content)} chars. Submitting retain task."
+        )
+
+        # Build retain task payload
+        retain_contents = [
+            {
+                "content": markdown_content,
+                "document_id": document_id,
+                "context": task_dict.get("context"),
+                "metadata": task_dict.get("metadata", {}),
+                "tags": task_dict.get("tags", []),
+                "timestamp": task_dict.get("timestamp"),
+            }
+        ]
+        document_tags = task_dict.get("document_tags")
+
+        retain_task_payload: dict[str, Any] = {"contents": retain_contents}
+        if document_tags:
+            retain_task_payload["document_tags"] = document_tags
+
+        # Pass tenant/api_key context through to retain task
+        if task_dict.get("_tenant_id"):
+            retain_task_payload["_tenant_id"] = task_dict["_tenant_id"]
+        if task_dict.get("_api_key_id"):
+            retain_task_payload["_api_key_id"] = task_dict["_api_key_id"]
+
+        # File metadata to attach after retain creates the document
+        retain_task_payload["_file_metadata"] = {
+            "file_storage_key": storage_key,
+            "file_original_name": task_dict["original_filename"],
+            "file_content_type": task_dict["content_type"],
+        }
+
+        # In one transaction: create the retain async operation AND mark this conversion as completed
+        retain_operation_id = uuid.uuid4()
+        pool = await self._get_pool()
+        async with acquire_with_retry(pool) as conn:
+            async with conn.transaction():
+                # Create the retain operation record
+                await conn.execute(
+                    f"""
+                    INSERT INTO {fq_table("async_operations")}
+                    (operation_id, bank_id, operation_type, result_metadata, status)
+                    VALUES ($1, $2, $3, $4, $5)
+                    """,
+                    retain_operation_id,
+                    bank_id,
+                    "retain",
+                    json.dumps({}),
+                    "pending",
+                )
+
+                # Mark this file_convert_retain operation as completed
+                if operation_id:
+                    await conn.execute(
+                        f"""
+                        UPDATE {fq_table("async_operations")}
+                        SET status = 'completed', updated_at = NOW(), completed_at = NOW()
+                        WHERE operation_id = $1
+                        """,
+                        uuid.UUID(operation_id),
+                    )
+
+        # Submit the retain task to the task backend (outside the transaction)
+        full_retain_payload = {
+            "type": "batch_retain",
+            "operation_id": str(retain_operation_id),
+            "bank_id": bank_id,
+            **retain_task_payload,
+        }
+        await self._task_backend.submit_task(full_retain_payload)
+
+        logger.info(
+            f"[FILE_CONVERT_RETAIN] Completed conversion for bank_id={bank_id}, "
+            f"document_id={document_id}. Retain task submitted as operation {retain_operation_id}"
+        )
+
+        # Delete file bytes from storage if configured (saves storage costs)
+        from ..config import get_config
+
+        config = get_config()
+        if config.file_delete_after_retain:
+            try:
+                await self._file_storage.delete(storage_key)
+                logger.info(f"[FILE_CONVERT_RETAIN] Deleted file bytes for {storage_key} (conversion completed)")
+            except Exception as e:
+                # Non-fatal - log and continue
+                logger.warning(f"[FILE_CONVERT_RETAIN] Failed to delete file {storage_key}: {e}")
+
     async def _handle_consolidation(self, task_dict: dict[str, Any]):
         """
         Handler for consolidation tasks.
@@ -798,6 +956,8 @@ class MemoryEngine(MemoryEngineInterface):
         try:
             if task_type == "batch_retain":
                 await self._handle_batch_retain(task_dict)
+            elif task_type == "file_convert_retain":
+                await self._handle_file_convert_retain(task_dict)
             elif task_type == "consolidation":
                 await self._handle_consolidation(task_dict)
             elif task_type == "refresh_mental_model":
@@ -810,7 +970,8 @@ class MemoryEngine(MemoryEngineInterface):
                 return
 
             # Task succeeded - mark operation as completed
-            if operation_id:
+            # file_convert_retain marks itself as completed in a transaction, skip double-marking
+            if operation_id and task_type != "file_convert_retain":
                 await self._mark_operation_completed(operation_id)
 
         except Exception as e:
@@ -823,14 +984,19 @@ class MemoryEngine(MemoryEngineInterface):
             error_traceback = traceback.format_exc()
             traceback.print_exc()
 
-            if retry_count < max_retries:
+            # Don't retry file conversion - if conversion fails, it won't succeed on retry
+            # (missing OCR, corrupted file, unsupported format, etc.)
+            should_retry = retry_count < max_retries and task_type != "file_convert_retain"
+
+            if should_retry:
                 # Reschedule with incremented retry count
                 task_dict["retry_count"] = retry_count + 1
                 logger.info(f"Rescheduling task {task_type} (retry {retry_count + 1}/{max_retries})")
                 await self._task_backend.submit_task(task_dict)
             else:
-                # Max retries exceeded - mark operation as failed
-                logger.error(f"Max retries exceeded for task {task_type}, marking as failed")
+                # Max retries exceeded or non-retryable task - mark operation as failed
+                reason = "non-retryable task type" if task_type == "file_convert_retain" else "max retries exceeded"
+                logger.error(f"Not retrying task {task_type} ({reason}), marking as failed")
                 if operation_id:
                     await self._mark_operation_failed(operation_id, str(e), error_traceback)
 
@@ -1196,6 +1362,34 @@ class MemoryEngine(MemoryEngineInterface):
         self._config_resolver = ConfigResolver(pool=self._pool, tenant_extension=self._tenant_extension)
         logger.debug("Config resolver initialized for hierarchical configuration")
 
+        # Initialize file storage
+        from .storage import create_file_storage
+
+        config = get_config()
+        self._file_storage = create_file_storage(
+            storage_type=config.file_storage_type,
+            pool_getter=lambda: self._pool,
+            schema=get_current_schema() if get_current_schema() != config.database_schema else None,
+        )
+        logger.debug(f"File storage initialized ({config.file_storage_type})")
+
+        # Initialize parser registry
+        from .parsers import FileParserRegistry, IrisParser, MarkitdownParser
+
+        self._parser_registry = FileParserRegistry()
+        try:
+            self._parser_registry.register(MarkitdownParser())
+            logger.debug("Registered markitdown parser")
+        except ImportError:
+            logger.warning("markitdown not available - file parsing disabled")
+        iris_token = config.file_parser_iris_token
+        iris_org_id = config.file_parser_iris_org_id
+        if iris_token and iris_org_id:
+            self._parser_registry.register(IrisParser(token=iris_token, org_id=iris_org_id))
+            logger.debug("Registered iris parser")
+        else:
+            logger.debug("Iris parser not registered (VECTORIZE_TOKEN or VECTORIZE_ORG_ID not set)")
+
         # Set executor for task backend and initialize
         self._task_backend.set_executor(self.execute_task)
         await self._task_backend.initialize()
@@ -5938,13 +6132,14 @@ class MemoryEngine(MemoryEngineInterface):
         async with acquire_with_retry(pool) as conn:
             await conn.execute(
                 f"""
-                INSERT INTO {fq_table("async_operations")} (operation_id, bank_id, operation_type, result_metadata)
-                VALUES ($1, $2, $3, $4)
+                INSERT INTO {fq_table("async_operations")} (operation_id, bank_id, operation_type, result_metadata, status)
+                VALUES ($1, $2, $3, $4, $5)
                 """,
                 operation_id,
                 bank_id,
                 operation_type,
                 json.dumps(result_metadata or {}),
+                "pending",
             )
 
         # Build and submit task payload
@@ -6092,6 +6287,116 @@ class MemoryEngine(MemoryEngineInterface):
             "items_count": len(contents),
         }
 
+    async def submit_async_file_retain(
+        self,
+        bank_id: str,
+        file_items: list[dict[str, Any]],
+        parser: str,
+        document_tags: list[str] | None,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """
+        Submit batch file conversion + retain operation.
+
+        Each file is converted to markdown and then retained as a memory.
+        Files are stored in object storage and conversion happens asynchronously.
+
+        Args:
+            bank_id: Bank ID
+            file_items: List of file items, each containing:
+                - file: UploadFile object (FastAPI)
+                - document_id: Document ID
+                - context: Optional context
+                - metadata: Optional metadata dict
+                - tags: Optional tags list
+                - timestamp: Optional timestamp
+            parser: Parser name (e.g., "markitdown")
+            document_tags: Tags applied to all documents
+            request_context: Request context for authentication
+
+        Returns:
+            dict with operation_id and files_count
+        """
+        await self._authenticate_tenant(request_context)
+
+        config = get_config()
+
+        # Validate file count
+        if len(file_items) > config.file_conversion_max_batch_size:
+            raise ValueError(f"Too many files. Maximum {config.file_conversion_max_batch_size} files per request.")
+
+        # Read all files and validate total batch size
+        files_data = []
+        total_batch_size = 0
+
+        for item in file_items:
+            file = item["file"]
+            file_data = await file.read()
+            total_batch_size += len(file_data)
+            files_data.append((item, file, file_data))
+
+        # Validate total batch size
+        if total_batch_size > config.file_conversion_max_batch_size_bytes:
+            total_mb = total_batch_size / (1024 * 1024)
+            raise ValueError(
+                f"Total batch size ({total_mb:.1f}MB) exceeds maximum of {config.file_conversion_max_batch_size_mb}MB"
+            )
+
+        # Submit individual operation for each file
+        operation_ids = []
+        for item, file, file_data in files_data:
+            # Generate storage key
+            storage_key = f"banks/{bank_id}/files/{item['document_id']}/{file.filename}"
+
+            # Store file in object storage
+            await self._file_storage.store(
+                file_data=file_data,
+                key=storage_key,
+                metadata={
+                    "content_type": file.content_type or "application/octet-stream",
+                    "original_filename": file.filename,
+                    "bank_id": bank_id,
+                    "document_id": item["document_id"],
+                },
+            )
+
+            # Create individual operation and submit task
+            task_payload: dict[str, Any] = {
+                "document_id": item["document_id"],
+                "storage_key": storage_key,
+                "original_filename": file.filename,
+                "content_type": file.content_type or "application/octet-stream",
+                "parser": parser,
+                "context": item.get("context"),
+                "metadata": item.get("metadata", {}),
+                "tags": item.get("tags", []),
+                "document_tags": document_tags or [],
+                "timestamp": item.get("timestamp"),
+            }
+
+            # Pass tenant_id and api_key_id through task payload
+            if request_context.tenant_id:
+                task_payload["_tenant_id"] = request_context.tenant_id
+            if request_context.api_key_id:
+                task_payload["_api_key_id"] = request_context.api_key_id
+
+            result = await self._submit_async_operation(
+                bank_id=bank_id,
+                operation_type="file_convert_retain",
+                task_type="file_convert_retain",
+                task_payload=task_payload,
+                result_metadata={
+                    "original_filename": file.filename,
+                },
+                dedupe_by_bank=False,
+            )
+            operation_ids.append(result["operation_id"])
+
+        return {
+            "operation_ids": operation_ids,
+            "files_count": len(file_items),
+        }
+
     async def submit_async_consolidation(
         self,
         bank_id: str,

hindsight-api/hindsight_api/engine/parsers/__init__.py

@@ -0,0 +1,62 @@
+"""File parser implementations."""
+
+from .base import FileParser, UnsupportedFileTypeError
+from .iris import IrisParser
+from .markitdown import MarkitdownParser
+
+__all__ = ["FileParser", "UnsupportedFileTypeError", "IrisParser", "MarkitdownParser", "FileParserRegistry"]
+
+
+class FileParserRegistry:
+    """Registry for file parsers with auto-detection."""
+
+    def __init__(self):
+        """Initialize empty parser registry."""
+        self._parsers: dict[str, FileParser] = {}
+
+    def register(self, parser: FileParser):
+        """
+        Register a parser.
+
+        Args:
+            parser: FileParser instance
+        """
+        self._parsers[parser.name()] = parser
+
+    def get_parser(
+        self,
+        name: str | None,
+        filename: str,
+        content_type: str | None = None,
+    ) -> FileParser:
+        """
+        Get parser by name or auto-detect.
+
+        Args:
+            name: Parser name (e.g., "markitdown") or None for auto-detect
+            filename: File name for auto-detection
+            content_type: MIME type (optional)
+
+        Returns:
+            FileParser instance
+
+        Raises:
+            ValueError: If no suitable parser found
+        """
+        if name:
+            # Explicit parser requested — return it directly, let the parser
+            # raise UnsupportedFileTypeError from convert() if needed
+            if name not in self._parsers:
+                raise ValueError(f"Parser '{name}' not found. Available: {list(self._parsers.keys())}")
+            return self._parsers[name]
+
+        # Auto-detect parser
+        for parser in self._parsers.values():
+            if parser.supports(filename, content_type):
+                return parser
+
+        raise ValueError(f"No parser found for {filename}. Available parsers: {list(self._parsers.keys())}")
+
+    def list_parsers(self) -> list[str]:
+        """Get list of registered parser names."""
+        return list(self._parsers.keys())

hindsight-api/hindsight_api/engine/parsers/base.py

@@ -0,0 +1,58 @@
+"""Abstract base class for file parsers."""
+
+from abc import ABC, abstractmethod
+
+
+class UnsupportedFileTypeError(Exception):
+    """Raised by a parser when it does not support the given file type."""
+
+    pass
+
+
+class FileParser(ABC):
+    """Abstract base for file to markdown parsers."""
+
+    @abstractmethod
+    async def convert(self, file_data: bytes, filename: str) -> str:
+        """
+        Parse file to markdown.
+
+        Args:
+            file_data: Raw file bytes
+            filename: Original filename (used for format detection)
+
+        Returns:
+            Markdown content as string
+
+        Raises:
+            UnsupportedFileTypeError: If the file type is not supported by this parser
+            RuntimeError: If parsing fails for another reason
+        """
+        pass
+
+    def supports(self, filename: str, content_type: str | None = None) -> bool:
+        """
+        Check if parser supports this file type.
+
+        Override this for local/static extension-based filtering.
+        Parsers that delegate to a remote service should leave this as True
+        and raise UnsupportedFileTypeError from convert() instead.
+
+        Args:
+            filename: File name (used for extension check)
+            content_type: MIME type (optional)
+
+        Returns:
+            True if this parser can handle the file (default: True)
+        """
+        return True
+
+    @abstractmethod
+    def name(self) -> str:
+        """
+        Get parser name.
+
+        Returns:
+            Parser name (e.g., "markitdown")
+        """
+        pass

hindsight-api/hindsight_api/engine/parsers/iris.py

@@ -0,0 +1,137 @@
+"""Iris parser implementation using the Vectorize Iris HTTP API."""
+
+import asyncio
+import logging
+import mimetypes
+import time
+
+import httpx
+
+from .base import FileParser, UnsupportedFileTypeError
+
+logger = logging.getLogger(__name__)
+
+_IRIS_BASE_URL = "https://api.vectorize.io/v1"
+_DEFAULT_POLL_INTERVAL = 2.0  # seconds
+_DEFAULT_TIMEOUT = 300.0  # seconds
+
+
+class IrisParser(FileParser):
+    """
+    Iris file parser using the Vectorize Iris cloud extraction service.
+
+    Uploads files to the Vectorize Iris API, starts an extraction job,
+    and polls until the text is ready. The API determines which file types
+    are supported — UnsupportedFileTypeError is raised if the file is rejected.
+
+    Authentication:
+        Requires HINDSIGHT_API_FILE_PARSER_IRIS_TOKEN and
+        HINDSIGHT_API_FILE_PARSER_IRIS_ORG_ID environment variables,
+        or pass them explicitly via the constructor.
+    """
+
+    def __init__(
+        self,
+        token: str,
+        org_id: str,
+        poll_interval: float = _DEFAULT_POLL_INTERVAL,
+        timeout: float = _DEFAULT_TIMEOUT,
+    ):
+        """
+        Initialize iris parser.
+
+        Args:
+            token: Vectorize API token
+            org_id: Vectorize organization ID
+            poll_interval: Seconds between status poll requests (default: 2)
+            timeout: Maximum seconds to wait for extraction (default: 300)
+        """
+        self._token = token
+        self._org_id = org_id
+        self._poll_interval = poll_interval
+        self._timeout = timeout
+        self._auth_headers = {"Authorization": f"Bearer {token}"}
+
+    async def convert(self, file_data: bytes, filename: str) -> str:
+        """
+        Parse file to text using the Vectorize Iris API.
+
+        Raises:
+            UnsupportedFileTypeError: If the Iris API rejects the file type (4xx)
+            RuntimeError: If extraction fails for another reason
+        """
+        content_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
+
+        async with httpx.AsyncClient() as client:
+            # Step 1: Request a presigned upload URL
+            init_resp = await client.post(
+                f"{_IRIS_BASE_URL}/org/{self._org_id}/files",
+                headers=self._auth_headers,
+                json={"name": filename, "contentType": content_type},
+            )
+            _raise_for_status(init_resp, filename, "file upload init")
+            init_data = init_resp.json()
+            file_id: str = init_data["fileId"]
+            upload_url: str = init_data["uploadUrl"]
+
+            # Step 2: Upload the file bytes to the presigned URL (no auth header)
+            upload_resp = await client.put(
+                upload_url,
+                content=file_data,
+                headers={"Content-Type": content_type},
+            )
+            _raise_for_status(upload_resp, filename, "file upload")
+
+            # Step 3: Start extraction
+            extract_resp = await client.post(
+                f"{_IRIS_BASE_URL}/org/{self._org_id}/extraction",
+                headers=self._auth_headers,
+                json={"fileId": file_id},
+            )
+            _raise_for_status(extract_resp, filename, "start extraction")
+            extraction_id: str = extract_resp.json()["extractionId"]
+
+            # Step 4: Poll until ready or timeout
+            deadline = time.monotonic() + self._timeout
+            while True:
+                status_resp = await client.get(
+                    f"{_IRIS_BASE_URL}/org/{self._org_id}/extraction/{extraction_id}",
+                    headers=self._auth_headers,
+                )
+                _raise_for_status(status_resp, filename, "poll extraction status")
+                status_data = status_resp.json()
+
+                if status_data.get("ready"):
+                    data = status_data.get("data", {})
+                    if not data.get("success"):
+                        error = data.get("error", "unknown error")
+                        raise RuntimeError(f"Iris extraction failed for '{filename}': {error}")
+                    text = data.get("text")
+                    if not text:
+                        raise RuntimeError(f"No content extracted from '{filename}'")
+                    return text
+
+                if time.monotonic() >= deadline:
+                    raise RuntimeError(f"Iris extraction timed out after {self._timeout}s for '{filename}'")
+
+                await asyncio.sleep(self._poll_interval)
+
+    def name(self) -> str:
+        """Get parser name."""
+        return "iris"
+
+
+def _raise_for_status(response: httpx.Response, filename: str, step: str) -> None:
+    """
+    Raise an appropriate error including the response body on HTTP errors.
+
+    Raises UnsupportedFileTypeError for 4xx responses (file rejected by the API),
+    RuntimeError for other HTTP errors.
+    """
+    if not response.is_error:
+        return
+    body = response.text or "<empty>"
+    msg = f"Iris API error during {step} for '{filename}': {response.status_code} {response.reason_phrase} — {body}"
+    if response.is_client_error:
+        raise UnsupportedFileTypeError(msg)
+    raise RuntimeError(msg)

hindsight-api/hindsight_api/engine/parsers/markitdown.py

@@ -0,0 +1,109 @@
+"""Markitdown parser implementation."""
+
+import asyncio
+import logging
+import tempfile
+from pathlib import Path
+
+from .base import FileParser
+
+logger = logging.getLogger(__name__)
+
+
+class MarkitdownParser(FileParser):
+    """
+    Markitdown file parser.
+
+    Uses Microsoft's markitdown library to convert various file formats
+    to markdown including PDF, Office docs, images (via OCR), audio, HTML.
+
+    Supported formats:
+    - PDF (.pdf)
+    - Word (.docx, .doc)
+    - PowerPoint (.pptx, .ppt)
+    - Excel (.xlsx, .xls)
+    - Images (.jpg, .jpeg, .png) - with OCR
+    - HTML (.html, .htm)
+    - Text (.txt, .md)
+    - Audio (.mp3, .wav) - with transcription
+    """
+
+    def __init__(self):
+        """Initialize markitdown parser."""
+        # Lazy import to avoid requiring markitdown for all users
+        try:
+            from markitdown import MarkItDown
+
+            self._markitdown = MarkItDown()
+        except ImportError as e:
+            raise ImportError(
+                "markitdown package is required for file parsing. Install with: pip install markitdown"
+            ) from e
+
+    async def convert(self, file_data: bytes, filename: str) -> str:
+        """Parse file to markdown using markitdown."""
+        # markitdown is synchronous, so we run it in executor to avoid blocking
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, self._convert_sync, file_data, filename)
+
+    def _convert_sync(self, file_data: bytes, filename: str) -> str:
+        """Synchronous parsing (runs in thread pool)."""
+        # Write to temp file (markitdown requires file path)
+        with tempfile.NamedTemporaryFile(suffix=Path(filename).suffix, delete=False) as tmp:
+            tmp.write(file_data)
+            tmp_path = tmp.name
+
+        try:
+            # Parse using markitdown
+            result = self._markitdown.convert(tmp_path)
+
+            if not result or not result.text_content:
+                raise RuntimeError(f"No content extracted from '{filename}'")
+
+            return result.text_content
+
+        except Exception as e:
+            logger.error(f"Markitdown parsing failed for {filename}: {e}")
+            raise RuntimeError(f"Failed to parse '{filename}': {e}") from e
+
+        finally:
+            # Clean up temp file
+            try:
+                Path(tmp_path).unlink()
+            except Exception:
+                pass
+
+    def supports(self, filename: str, content_type: str | None = None) -> bool:
+        """Check if markitdown supports this file type."""
+        # Supported extensions (from markitdown docs)
+        supported_extensions = {
+            # Documents
+            ".pdf",
+            ".docx",
+            ".doc",
+            ".pptx",
+            ".ppt",
+            ".xlsx",
+            ".xls",
+            # Images (with OCR)
+            ".jpg",
+            ".jpeg",
+            ".png",
+            # Web
+            ".html",
+            ".htm",
+            # Text
+            ".txt",
+            ".md",
+            ".csv",
+            # Audio (with transcription)
+            ".mp3",
+            ".wav",
+        }
+
+        ext = Path(filename).suffix.lower()
+        return ext in supported_extensions
+
+    def name(self) -> str:
+        """Get parser name."""
+        return "markitdown"

hindsight-api/hindsight_api/engine/storage/__init__.py

@@ -0,0 +1,77 @@
+"""File storage backends for uploaded files."""
+
+from collections.abc import Callable
+
+from .base import FileStorage
+from .postgresql import PostgreSQLFileStorage
+
+__all__ = ["FileStorage", "PostgreSQLFileStorage", "create_file_storage"]
+
+
+def create_file_storage(
+    storage_type: str,
+    pool_getter: Callable | None = None,
+    schema: str | None = None,
+    **kwargs,
+) -> FileStorage:
+    """
+    Create file storage backend based on configuration.
+
+    Args:
+        storage_type: "native" (PostgreSQL BYTEA) or "s3" (S3-compatible object storage)
+        pool_getter: Database pool getter (required for native)
+        schema: Database schema (for native multi-tenant)
+        **kwargs: Additional args passed to storage backend
+
+    Returns:
+        FileStorage instance
+
+    Raises:
+        ValueError: If storage_type is unknown or required args are missing
+    """
+    if storage_type == "native":
+        if not pool_getter:
+            raise ValueError("pool_getter required for native (PostgreSQL) storage")
+        return PostgreSQLFileStorage(pool_getter=pool_getter, schema=schema)
+    elif storage_type == "s3":
+        from ...config import get_config
+        from .s3 import S3FileStorage
+
+        config = get_config()
+        bucket = config.file_storage_s3_bucket
+        if not bucket:
+            raise ValueError("HINDSIGHT_API_FILE_STORAGE_S3_BUCKET is required for S3 storage")
+        return S3FileStorage(
+            bucket=bucket,
+            region=config.file_storage_s3_region,
+            endpoint=config.file_storage_s3_endpoint,
+            access_key_id=config.file_storage_s3_access_key_id,
+            secret_access_key=config.file_storage_s3_secret_access_key,
+        )
+    elif storage_type == "gcs":
+        from ...config import get_config
+        from .gcs import GCSFileStorage
+
+        config = get_config()
+        bucket = config.file_storage_gcs_bucket
+        if not bucket:
+            raise ValueError("HINDSIGHT_API_FILE_STORAGE_GCS_BUCKET is required for GCS storage")
+        return GCSFileStorage(
+            bucket=bucket,
+            service_account_key=config.file_storage_gcs_service_account_key,
+        )
+    elif storage_type == "azure":
+        from ...config import get_config
+        from .azure import AzureFileStorage
+
+        config = get_config()
+        container = config.file_storage_azure_container
+        if not container:
+            raise ValueError("HINDSIGHT_API_FILE_STORAGE_AZURE_CONTAINER is required for Azure storage")
+        return AzureFileStorage(
+            container_name=container,
+            account_name=config.file_storage_azure_account_name,
+            account_key=config.file_storage_azure_account_key,
+        )
+    else:
+        raise ValueError(f"Unknown storage type: {storage_type}. Supported: 'native', 's3', 'gcs', 'azure'.")

hindsight-api/hindsight_api/engine/storage/azure.py

@@ -0,0 +1,62 @@
+"""Azure Blob Storage backend using obstore."""
+
+import logging
+from datetime import timedelta
+
+import obstore as obs
+from obstore.store import AzureStore
+
+from .base import FileStorage
+
+logger = logging.getLogger(__name__)
+
+
+class AzureFileStorage(FileStorage):
+    """
+    Azure Blob Storage backend.
+
+    Uses obstore (Rust-backed) for high-throughput async access to Azure Blob Storage.
+    Supports account key, SAS token, and default Azure credentials.
+    """
+
+    def __init__(
+        self,
+        container_name: str,
+        account_name: str | None = None,
+        account_key: str | None = None,
+    ):
+        kwargs: dict = {}
+        if account_name:
+            kwargs["account_name"] = account_name
+        if account_key:
+            kwargs["account_key"] = account_key
+
+        self._store = AzureStore(container_name, **kwargs)
+        logger.info(f"Initialized Azure file storage: container={container_name}, account={account_name}")
+
+    async def store(self, file_data: bytes, key: str, metadata: dict[str, str] | None = None) -> str:
+        await obs.put_async(self._store, key, file_data)
+        logger.debug(f"Stored file {key} ({len(file_data)} bytes) in Azure")
+        return key
+
+    async def retrieve(self, key: str) -> bytes:
+        try:
+            response = await obs.get_async(self._store, key)
+            return await response.bytes_async()
+        except Exception as e:
+            if "not found" in str(e).lower() or "BlobNotFound" in str(e):
+                raise FileNotFoundError(f"File not found: {key}") from e
+            raise
+
+    async def delete(self, key: str) -> None:
+        await obs.delete_async(self._store, key)
+
+    async def exists(self, key: str) -> bool:
+        try:
+            await obs.head_async(self._store, key)
+            return True
+        except Exception:
+            return False
+
+    async def get_download_url(self, key: str, expires_in: int = 3600) -> str:
+        return await obs.sign_async(self._store, "GET", key, timedelta(seconds=expires_in))

hindsight-api/hindsight_api/engine/storage/base.py

@@ -0,0 +1,83 @@
+"""Abstract base class for file storage backends."""
+
+from abc import ABC, abstractmethod
+
+
+class FileStorage(ABC):
+    """Abstract base for file storage backends."""
+
+    @abstractmethod
+    async def store(
+        self,
+        file_data: bytes,
+        key: str,
+        metadata: dict[str, str] | None = None,
+    ) -> str:
+        """
+        Store file and return storage key.
+
+        Args:
+            file_data: Raw file bytes
+            key: Storage key (e.g., "banks/{bank_id}/files/{file_id}.pdf")
+            metadata: Optional metadata to store with file
+
+        Returns:
+            Storage key that can be used to retrieve the file
+        """
+        pass
+
+    @abstractmethod
+    async def retrieve(self, key: str) -> bytes:
+        """
+        Retrieve file by storage key.
+
+        Args:
+            key: Storage key
+
+        Returns:
+            File data as bytes
+
+        Raises:
+            FileNotFoundError: If file does not exist
+        """
+        pass
+
+    @abstractmethod
+    async def delete(self, key: str) -> None:
+        """
+        Delete file by storage key.
+
+        Args:
+            key: Storage key
+        """
+        pass
+
+    @abstractmethod
+    async def exists(self, key: str) -> bool:
+        """
+        Check if file exists.
+
+        Args:
+            key: Storage key
+
+        Returns:
+            True if file exists, False otherwise
+        """
+        pass
+
+    @abstractmethod
+    async def get_download_url(self, key: str, expires_in: int = 3600) -> str:
+        """
+        Get a URL for downloading the file.
+
+        For PostgreSQL storage, this might be a relative API path.
+        For S3, this would be a pre-signed URL.
+
+        Args:
+            key: Storage key
+            expires_in: Expiration time in seconds (may be ignored for some backends)
+
+        Returns:
+            Download URL or path
+        """
+        pass

hindsight-api/hindsight_api/engine/storage/gcs.py

@@ -0,0 +1,59 @@
+"""Google Cloud Storage backend using obstore."""
+
+import logging
+from datetime import timedelta
+
+import obstore as obs
+from obstore.store import GCSStore
+
+from .base import FileStorage
+
+logger = logging.getLogger(__name__)
+
+
+class GCSFileStorage(FileStorage):
+    """
+    Google Cloud Storage backend.
+
+    Uses obstore (Rust-backed) for high-throughput async access to GCS.
+    Supports Application Default Credentials, service account keys, and explicit credentials.
+    """
+
+    def __init__(
+        self,
+        bucket: str,
+        service_account_key: str | None = None,
+    ):
+        kwargs: dict = {}
+        if service_account_key:
+            kwargs["service_account_key"] = service_account_key
+
+        self._store = GCSStore(bucket, **kwargs)
+        logger.info(f"Initialized GCS file storage: bucket={bucket}")
+
+    async def store(self, file_data: bytes, key: str, metadata: dict[str, str] | None = None) -> str:
+        await obs.put_async(self._store, key, file_data)
+        logger.debug(f"Stored file {key} ({len(file_data)} bytes) in GCS")
+        return key
+
+    async def retrieve(self, key: str) -> bytes:
+        try:
+            response = await obs.get_async(self._store, key)
+            return await response.bytes_async()
+        except Exception as e:
+            if "not found" in str(e).lower():
+                raise FileNotFoundError(f"File not found: {key}") from e
+            raise
+
+    async def delete(self, key: str) -> None:
+        await obs.delete_async(self._store, key)
+
+    async def exists(self, key: str) -> bool:
+        try:
+            await obs.head_async(self._store, key)
+            return True
+        except Exception:
+            return False
+
+    async def get_download_url(self, key: str, expires_in: int = 3600) -> str:
+        return await obs.sign_async(self._store, "GET", key, timedelta(seconds=expires_in))

hindsight-api/hindsight_api/engine/storage/postgresql.py

@@ -0,0 +1,139 @@
+"""PostgreSQL BYTEA-based file storage (default, zero-config)."""
+
+import logging
+from collections.abc import Callable
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import asyncpg
+
+from .base import FileStorage
+
+logger = logging.getLogger(__name__)
+
+
+def fq_table(table: str, schema: str | None = None) -> str:
+    """Get fully-qualified table name with optional schema prefix."""
+    if schema:
+        return f'"{schema}".{table}'
+    return table
+
+
+class PostgreSQLFileStorage(FileStorage):
+    """
+    PostgreSQL BYTEA-based file storage.
+
+    Stores files directly in PostgreSQL using BYTEA columns.
+    This is the default storage backend - zero configuration required!
+
+    Pros:
+    - Works out of the box (no external dependencies)
+    - Transactional consistency with database
+    - Simple backups (included in pg_dump)
+    - Good performance for <10MB files
+
+    Cons:
+    - Database bloat for large/many files
+    - Not ideal for distributed deployments
+    - Higher cost than object storage at scale
+
+    For production/scale, consider S3FileStorage instead.
+    """
+
+    def __init__(self, pool_getter: Callable[[], "asyncpg.Pool"], schema: str | None = None):
+        """
+        Initialize PostgreSQL file storage.
+
+        Args:
+            pool_getter: Function that returns asyncpg connection pool
+            schema: Database schema (for multi-tenant support)
+        """
+        self._pool_getter = pool_getter
+        self._schema = schema
+
+    async def store(
+        self,
+        file_data: bytes,
+        key: str,
+        metadata: dict[str, str] | None = None,
+    ) -> str:
+        """Store file in PostgreSQL."""
+        pool = self._pool_getter()
+
+        async with pool.acquire() as conn:
+            await conn.execute(
+                f"""
+                INSERT INTO {fq_table("file_storage", self._schema)}
+                (storage_key, data)
+                VALUES ($1, $2)
+                ON CONFLICT (storage_key) DO UPDATE SET
+                    data = EXCLUDED.data
+                """,
+                key,
+                file_data,
+            )
+
+        logger.debug(f"Stored file {key} ({len(file_data)} bytes) in PostgreSQL")
+        return key
+
+    async def retrieve(self, key: str) -> bytes:
+        """Retrieve file from PostgreSQL."""
+        pool = self._pool_getter()
+
+        async with pool.acquire() as conn:
+            row = await conn.fetchrow(
+                f"""
+                SELECT data FROM {fq_table("file_storage", self._schema)}
+                WHERE storage_key = $1
+                """,
+                key,
+            )
+
+            if not row:
+                raise FileNotFoundError(f"File not found: {key}")
+
+            return bytes(row["data"])
+
+    async def delete(self, key: str) -> None:
+        """Delete file from PostgreSQL."""
+        pool = self._pool_getter()
+
+        async with pool.acquire() as conn:
+            result = await conn.execute(
+                f"""
+                DELETE FROM {fq_table("file_storage", self._schema)}
+                WHERE storage_key = $1
+                """,
+                key,
+            )
+
+            # Check if anything was deleted
+            if result == "DELETE 0":
+                logger.warning(f"Attempted to delete non-existent file: {key}")
+
+    async def exists(self, key: str) -> bool:
+        """Check if file exists in PostgreSQL."""
+        pool = self._pool_getter()
+
+        async with pool.acquire() as conn:
+            row = await conn.fetchrow(
+                f"""
+                SELECT 1 FROM {fq_table("file_storage", self._schema)}
+                WHERE storage_key = $1
+                """,
+                key,
+            )
+
+            return row is not None
+
+    async def get_download_url(self, key: str, expires_in: int = 3600) -> str:
+        """
+        Get download URL for PostgreSQL-stored file.
+
+        Returns an API endpoint path (not a pre-signed URL since the file
+        is stored in the database). The expires_in parameter is ignored
+        for PostgreSQL storage.
+        """
+        # Return API path for download endpoint
+        # (expires_in ignored for database storage - auth handled at API level)
+        return f"/v1/default/files/download/{key}"

hindsight-api/hindsight_api/engine/storage/s3.py

@@ -0,0 +1,71 @@
+"""S3 object storage backend using obstore."""
+
+import logging
+from datetime import timedelta
+
+import obstore as obs
+from obstore.store import S3Store
+
+from .base import FileStorage
+
+logger = logging.getLogger(__name__)
+
+
+class S3FileStorage(FileStorage):
+    """
+    S3-compatible object storage backend.
+
+    Uses obstore (Rust-backed) for high-throughput async access to
+    Amazon S3, MinIO, Cloudflare R2, and other S3-compliant APIs.
+    """
+
+    def __init__(
+        self,
+        bucket: str,
+        region: str | None = None,
+        endpoint: str | None = None,
+        access_key_id: str | None = None,
+        secret_access_key: str | None = None,
+    ):
+        kwargs: dict = {}
+        if region:
+            kwargs["region"] = region
+        if endpoint:
+            kwargs["endpoint"] = endpoint
+            # Allow plain HTTP for local S3-compatible services (MinIO, LocalStack, etc.)
+            if endpoint.startswith("http://"):
+                kwargs["allow_http"] = True
+        if access_key_id:
+            kwargs["access_key_id"] = access_key_id
+        if secret_access_key:
+            kwargs["secret_access_key"] = secret_access_key
+
+        self._store = S3Store(bucket, **kwargs)
+        logger.info(f"Initialized S3 file storage: bucket={bucket}, region={region}, endpoint={endpoint}")
+
+    async def store(self, file_data: bytes, key: str, metadata: dict[str, str] | None = None) -> str:
+        await obs.put_async(self._store, key, file_data)
+        logger.debug(f"Stored file {key} ({len(file_data)} bytes) in S3")
+        return key
+
+    async def retrieve(self, key: str) -> bytes:
+        try:
+            response = await obs.get_async(self._store, key)
+            return await response.bytes_async()
+        except Exception as e:
+            if "not found" in str(e).lower() or "NoSuchKey" in str(e):
+                raise FileNotFoundError(f"File not found: {key}") from e
+            raise
+
+    async def delete(self, key: str) -> None:
+        await obs.delete_async(self._store, key)
+
+    async def exists(self, key: str) -> bool:
+        try:
+            await obs.head_async(self._store, key)
+            return True
+        except Exception:
+            return False
+
+    async def get_download_url(self, key: str, expires_in: int = 3600) -> str:
+        return await obs.sign_async(self._store, "GET", key, timedelta(seconds=expires_in))

hindsight-api/hindsight_api/main.py

@@ -250,6 +250,24 @@ def main():
             retain_batch_tokens=config.retain_batch_tokens,
             retain_batch_enabled=config.retain_batch_enabled,
             retain_batch_poll_interval_seconds=config.retain_batch_poll_interval_seconds,
+            file_storage_type=config.file_storage_type,
+            file_storage_s3_bucket=config.file_storage_s3_bucket,
+            file_storage_s3_region=config.file_storage_s3_region,
+            file_storage_s3_endpoint=config.file_storage_s3_endpoint,
+            file_storage_s3_access_key_id=config.file_storage_s3_access_key_id,
+            file_storage_s3_secret_access_key=config.file_storage_s3_secret_access_key,
+            file_storage_gcs_bucket=config.file_storage_gcs_bucket,
+            file_storage_gcs_service_account_key=config.file_storage_gcs_service_account_key,
+            file_storage_azure_container=config.file_storage_azure_container,
+            file_storage_azure_account_name=config.file_storage_azure_account_name,
+            file_storage_azure_account_key=config.file_storage_azure_account_key,
+            file_parser=config.file_parser,
+            file_parser_iris_token=config.file_parser_iris_token,
+            file_parser_iris_org_id=config.file_parser_iris_org_id,
+            file_conversion_max_batch_size_mb=config.file_conversion_max_batch_size_mb,
+            file_conversion_max_batch_size=config.file_conversion_max_batch_size,
+            enable_file_upload_api=config.enable_file_upload_api,
+            file_delete_after_retain=config.file_delete_after_retain,
             enable_observations=config.enable_observations,
             consolidation_batch_size=config.consolidation_batch_size,
             consolidation_max_tokens=config.consolidation_max_tokens,
@@ -351,6 +369,7 @@ def main():
         "proxy_headers": args.proxy_headers,
         "ws": "wsproto",  # Use wsproto instead of websockets to avoid deprecation warnings
         "loop": loop_impl,  # Explicitly set event loop implementation
+        "timeout_keep_alive": 30,  # Exceed aiohttp's 15s client timeout so the client always closes first
     }
 
     # Add optional parameters if provided

hindsight-api/hindsight_api/worker/poller.py

@@ -376,7 +376,12 @@ class WorkerPoller:
                         del self._in_flight_by_type[operation_type]
 
     async def _execute_task_inner(self, task: ClaimedTask):
-        """Inner task execution with error handling."""
+        """Inner task execution with error handling.
+
+        Note: The executor (MemoryEngine.execute_task) handles status marking internally
+        (marking operations as completed/failed and handling retries). This method should
+        NOT override those status updates.
+        """
         task_type = task.task_dict.get("type", "unknown")
         bank_id = task.task_dict.get("bank_id", "unknown")
 
@@ -386,12 +391,12 @@ class WorkerPoller:
             if task.schema:
                 task.task_dict["_schema"] = task.schema
             await self._executor(task.task_dict)
-            await self._mark_completed(task.operation_id, task.schema)
-            logger.debug(f"Task {task.operation_id} completed successfully")
+            logger.debug(f"Task {task.operation_id} execution finished")
         except Exception as e:
-            error_msg = f"{type(e).__name__}: {e}\n{traceback.format_exc()}"
-            logger.error(f"Task {task.operation_id} failed: {e}")
-            await self._retry_or_fail(task.operation_id, error_msg, task.schema)
+            # The executor should handle its own errors, but if an unexpected exception
+            # propagates (e.g., from schema setup), log it as a warning
+            logger.error(f"Task {task.operation_id} raised unexpected exception: {e}")
+            traceback.print_exc()
 
     async def recover_own_tasks(self) -> int:
         """

hindsight-api/pyproject.toml

@@ -43,6 +43,8 @@ dependencies = [
     "cohere>=5.0.0",
     "flashrank>=0.2.0",
     "litellm>=1.0.0",
+    "markitdown[pdf,docx,pptx,xlsx,xls]>=0.1.4",  # File to markdown conversion
+    "obstore>=0.4.0",  # S3/GCS/Azure object storage client (Rust-backed)
     # Local ML models for embeddings/reranking - can be excluded in Docker with INCLUDE_LOCAL_MODELS=false
     "sentence-transformers>=3.3.0",
     "transformers>=4.53.0", # Security fixes for ReDoS vulnerabilities
@@ -65,6 +67,7 @@ test = [
     "pytest-timeout>=2.4.0",
     "pytest-xdist>=3.0.0",
     "filelock>=3.20.1",  # TOCTOU race condition fix
+    "testcontainers>=4.0.0",
 ]
 
 [project.scripts]
@@ -114,6 +117,7 @@ dev = [
     "filelock>=3.20.1",  # TOCTOU race condition fix
     "ruff>=0.8.0",
     "ty>=0.0.1",
+    "testcontainers>=4.0.0",
 ]
 
 [tool.ruff]

hindsight-api/tests/test_file_retain.py

@@ -0,0 +1,553 @@
+"""
+End-to-end tests for file retain (upload, convert, retain) functionality.
+"""
+
+import io
+import json
+
+import pytest
+from httpx import ASGITransport, AsyncClient
+
+
+@pytest.fixture
+def sample_pdf_content():
+    """Create a simple PDF-like content for testing."""
+    # This is a minimal PDF that markitdown can parse
+    return b"""%PDF-1.4
+1 0 obj
+<<
+/Type /Catalog
+/Pages 2 0 R
+>>
+endobj
+2 0 obj
+<<
+/Type /Pages
+/Kids [3 0 R]
+/Count 1
+>>
+endobj
+3 0 obj
+<<
+/Type /Page
+/Parent 2 0 R
+/MediaBox [0 0 612 792]
+/Contents 4 0 R
+/Resources <<
+/Font <<
+/F1 <<
+/Type /Font
+/Subtype /Type1
+/BaseFont /Helvetica
+>>
+>>
+>>
+>>
+endobj
+4 0 obj
+<<
+/Length 44
+>>
+stream
+BT
+/F1 12 Tf
+100 700 Td
+(Test Document) Tj
+ET
+endstream
+endobj
+xref
+0 5
+0000000000 65535 f
+0000000009 00000 n
+0000000058 00000 n
+0000000115 00000 n
+0000000317 00000 n
+trailer
+<<
+/Size 5
+/Root 1 0 R
+>>
+startxref
+410
+%%EOF
+"""
+
+
+@pytest.fixture
+def sample_txt_content():
+    """Create simple text content."""
+    return b"This is a test document.\nIt contains some important information.\nAlice works at Google."
+
+
+@pytest.mark.asyncio
+async def test_file_retain_basic(memory_no_llm_verify, sample_txt_content):
+    """Test basic file upload and conversion."""
+    from hindsight_api.api.http import create_app
+
+    app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # Create a bank first
+        bank_response = await client.put("/v1/default/banks/test-file-bank", json={"name": "Test File Bank"})
+        assert bank_response.status_code in (200, 201)
+
+        # Upload file
+        request_data = {
+            "document_tags": ["test"],
+            "async": True,
+        }
+
+        files = {"files": ("test.txt", sample_txt_content, "text/plain")}
+        data = {"request": json.dumps(request_data)}
+
+        response = await client.post(
+            "/v1/default/banks/test-file-bank/files/retain",
+            files=files,
+            data=data,
+        )
+
+        assert response.status_code == 200
+        result = response.json()
+        assert "operation_ids" in result
+        assert len(result["operation_ids"]) == 1
+
+
+@pytest.mark.asyncio
+async def test_file_retain_with_metadata(memory_no_llm_verify, sample_txt_content):
+    """Test file upload with per-file metadata."""
+    from hindsight_api.api.http import create_app
+
+    app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # Create bank
+        bank_response = await client.put("/v1/default/banks/test-file-meta-bank", json={"name": "Test Meta Bank"})
+        assert bank_response.status_code in (200, 201)
+
+        # Upload file with metadata
+        request_data = {
+            "document_tags": ["work", "reports"],
+            "async": True,
+            "files_metadata": [
+                {
+                    "document_id": "test_doc_123",
+                    "context": "quarterly report",
+                    "metadata": {"author": "Alice", "year": "2024"},
+                    "tags": ["Q1"],
+                }
+            ],
+        }
+
+        files = {"files": ("report.txt", sample_txt_content, "text/plain")}
+        data = {"request": json.dumps(request_data)}
+
+        response = await client.post(
+            "/v1/default/banks/test-file-meta-bank/files/retain",
+            files=files,
+            data=data,
+        )
+
+        assert response.status_code == 200
+        result = response.json()
+        assert "operation_ids" in result
+        assert len(result["operation_ids"]) == 1
+
+
+@pytest.mark.asyncio
+async def test_file_retain_multiple_files(memory_no_llm_verify, sample_txt_content):
+    """Test uploading multiple files at once."""
+    from hindsight_api.api.http import create_app
+
+    app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # Create bank
+        bank_response = await client.put("/v1/default/banks/test-multi-file-bank", json={"name": "Test Multi Bank"})
+        assert bank_response.status_code in (200, 201)
+
+        # Upload multiple files
+        request_data = {
+            "async": True,
+            "files_metadata": [
+                {"document_id": "doc1", "tags": ["file1"]},
+                {"document_id": "doc2", "tags": ["file2"]},
+            ],
+        }
+
+        content1 = b"First document content"
+        content2 = b"Second document content"
+
+        files = [
+            ("files", ("file1.txt", content1, "text/plain")),
+            ("files", ("file2.txt", content2, "text/plain")),
+        ]
+        data = {"request": json.dumps(request_data)}
+
+        response = await client.post(
+            "/v1/default/banks/test-multi-file-bank/files/retain",
+            files=files,
+            data=data,
+        )
+
+        assert response.status_code == 200
+        result = response.json()
+        assert "operation_ids" in result
+        assert len(result["operation_ids"]) == 2
+
+
+@pytest.mark.asyncio
+async def test_file_retain_validation_errors(memory_no_llm_verify):
+    """Test validation errors."""
+    from hindsight_api.api.http import create_app
+
+    app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # Create bank
+        bank_response = await client.put("/v1/default/banks/test-validation-bank", json={"name": "Test Validation Bank"})
+        assert bank_response.status_code in (200, 201)
+
+        # Test: metadata count mismatch
+        request_data = {
+            "async": True,
+            "files_metadata": [
+                {"document_id": "doc1"},
+                {"document_id": "doc2"},  # 2 metadata entries
+            ],
+        }
+
+        files = {"files": ("file1.txt", b"content", "text/plain")}  # But only 1 file
+        data = {"request": json.dumps(request_data)}
+
+        response = await client.post(
+            "/v1/default/banks/test-validation-bank/files/retain",
+            files=files,
+            data=data,
+        )
+
+        assert response.status_code == 400
+        assert "files_metadata count" in response.json()["detail"]
+
+
+@pytest.mark.asyncio
+async def test_file_retain_no_files(memory_no_llm_verify):
+    """Test error when no files provided."""
+    from hindsight_api.api.http import create_app
+
+    app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # Create bank
+        bank_response = await client.put("/v1/default/banks/test-no-files-bank", json={"name": "Test No Files Bank"})
+        assert bank_response.status_code in (200, 201)
+
+        request_data = {
+            "async": True,
+        }
+
+        # No files provided
+        data = {"request": json.dumps(request_data)}
+
+        response = await client.post(
+            "/v1/default/banks/test-no-files-bank/files/retain",
+            data=data,
+        )
+
+        # FastAPI will return 422 for missing required field
+        assert response.status_code == 422
+
+
+@pytest.mark.asyncio
+async def test_file_retain_sync_not_supported(memory_no_llm_verify, sample_txt_content):
+    """Test that file retain is always async (sync is not supported)."""
+    from hindsight_api.api.http import create_app
+
+    app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # Create bank
+        bank_response = await client.put("/v1/default/banks/test-sync-bank", json={"name": "Test Sync Bank"})
+        assert bank_response.status_code in (200, 201)
+
+        # File retain is always async - just verify it succeeds and returns operation_ids
+        files = {"files": ("test.txt", sample_txt_content, "text/plain")}
+        data = {"request": json.dumps({})}
+
+        response = await client.post(
+            "/v1/default/banks/test-sync-bank/files/retain",
+            files=files,
+            data=data,
+        )
+
+        assert response.status_code == 200
+        result = response.json()
+        assert "operation_ids" in result
+
+
+@pytest.mark.asyncio
+async def test_file_storage_postgresql(memory_no_llm_verify, sample_txt_content):
+    """Test file storage in PostgreSQL."""
+    # Test that files are stored and retrieved correctly
+    storage = memory_no_llm_verify._file_storage
+
+    # Store a file
+    key = "test/file1.txt"
+    stored_key = await storage.store(
+        file_data=sample_txt_content,
+        key=key,
+        metadata={"content_type": "text/plain"},
+    )
+
+    assert stored_key == key
+
+    # Retrieve the file
+    retrieved = await storage.retrieve(key)
+    assert retrieved == sample_txt_content
+
+    # Check if file exists
+    exists = await storage.exists(key)
+    assert exists is True
+
+    # Delete the file
+    await storage.delete(key)
+
+    # Check file no longer exists
+    exists_after = await storage.exists(key)
+    assert exists_after is False
+
+
+@pytest.mark.asyncio
+async def test_markitdown_converter():
+    """Test markitdown parser."""
+    from hindsight_api.engine.parsers import MarkitdownParser
+
+    parser = MarkitdownParser()
+
+    # Test simple text file
+    text_content = b"This is a test document.\nWith multiple lines."
+    result = await parser.convert(text_content, "test.txt")
+
+    assert isinstance(result, str)
+    assert len(result) > 0
+    assert "test document" in result.lower() or "multiple lines" in result.lower()
+
+
+@pytest.mark.asyncio
+async def test_converter_registry():
+    """Test file parser registry."""
+    from hindsight_api.engine.parsers import FileParserRegistry, MarkitdownParser
+
+    registry = FileParserRegistry()
+    parser = MarkitdownParser()
+    registry.register(parser)
+
+    # Test get by name
+    retrieved = registry.get_parser("markitdown", "test.txt")
+    assert retrieved is parser
+
+    # Test auto-detection
+    auto = registry.get_parser(None, "test.pdf")
+    assert auto is parser
+
+    # Test unsupported format
+    with pytest.raises(ValueError, match="No parser found"):
+        registry.get_parser(None, "test.xyz")
+
+
+@pytest.mark.asyncio
+async def test_file_conversion_creates_separate_retain_operation(memory_no_llm_verify, sample_txt_content):
+    """Test that file conversion and retain are two separate async operations.
+
+    The file_convert_retain task should:
+    1. Convert the file to markdown
+    2. In a single transaction: create a separate 'retain' operation AND mark itself as 'completed'
+    3. Free the worker slot immediately after conversion
+
+    The retain then runs as its own task. This prevents deadlocks where file conversion
+    tasks hold worker slots while waiting for inline retain to finish.
+    """
+    from hindsight_api.models import RequestContext
+
+    bank_id = "test_file_two_phase_bank"
+
+    context = RequestContext(internal=True)
+    await memory_no_llm_verify.get_bank_profile(bank_id, request_context=context)
+
+    class MockFile:
+        def __init__(self, content, filename, content_type):
+            self.content = content
+            self.filename = filename
+            self.content_type = content_type
+
+        async def read(self):
+            return self.content
+
+    mock_file = MockFile(sample_txt_content, "test.txt", "text/plain")
+
+    file_items = [
+        {
+            "file": mock_file,
+            "document_id": "test_doc_two_phase",
+            "context": "test context",
+            "metadata": {"source": "test"},
+            "tags": ["test_tag"],
+            "timestamp": None,
+        }
+    ]
+
+    result = await memory_no_llm_verify.submit_async_file_retain(
+        bank_id=bank_id,
+        file_items=file_items,
+        parser="markitdown",
+        document_tags=["two_phase_test"],
+        request_context=context,
+    )
+
+    assert "operation_ids" in result
+    assert len(result["operation_ids"]) == 1
+    convert_operation_id = result["operation_ids"][0]
+
+    import asyncio
+
+    await asyncio.sleep(0.1)
+
+    pool = await memory_no_llm_verify._get_pool()
+    from hindsight_api.engine.memory_engine import get_current_schema
+
+    schema = get_current_schema()
+
+    async with pool.acquire() as conn:
+        # 1. The file_convert_retain operation must be completed
+        convert_op = await conn.fetchrow(
+            f"SELECT status, operation_type FROM {schema}.async_operations WHERE operation_id = $1",
+            convert_operation_id,
+        )
+        assert convert_op is not None
+        assert convert_op["operation_type"] == "file_convert_retain"
+        assert convert_op["status"] == "completed", (
+            f"file_convert_retain should be 'completed' after conversion, got '{convert_op['status']}'"
+        )
+
+        # 2. A separate retain operation must have been created
+        retain_op = await conn.fetchrow(
+            f"""
+            SELECT status, operation_type
+            FROM {schema}.async_operations
+            WHERE bank_id = $1 AND operation_type = 'retain' AND operation_id != $2
+            """,
+            bank_id,
+            convert_operation_id,
+        )
+        assert retain_op is not None, "A separate 'retain' operation should have been created by file conversion"
+        # With SyncTaskBackend the retain runs immediately, so it should be completed
+        assert retain_op["status"] == "completed"
+
+        # 3. The document should exist with file metadata and retained content
+        doc = await conn.fetchrow(
+            f"""
+            SELECT id, original_text, file_original_name, file_content_type
+            FROM {schema}.documents
+            WHERE id = $1 AND bank_id = $2
+            """,
+            "test_doc_two_phase",
+            bank_id,
+        )
+
+    assert doc is not None
+    assert doc["file_original_name"] == "test.txt"
+    assert doc["file_content_type"] == "text/plain"
+    assert doc["original_text"] is not None
+    assert len(doc["original_text"]) > 0
+
+
+@pytest.mark.asyncio
+async def test_file_conversion_failure_sets_status_to_failed(memory_no_llm_verify, sample_txt_content):
+    """Test that when file conversion fails, the operation status is set to 'failed' not 'completed'."""
+    from hindsight_api.engine.parsers.base import FileParser
+    from hindsight_api.models import RequestContext
+
+    bank_id = "test_file_failure_bank"
+
+    # Create a mock parser that always fails
+    class FailingParser(FileParser):
+        """Mock parser that raises an error."""
+
+        async def convert(self, file_data: bytes, filename: str) -> str:
+            # Simulate conversion failure
+            raise RuntimeError(f"Failed to convert '{filename}': Mock conversion error")
+
+        def supports(self, filename: str, content_type: str | None = None) -> bool:
+            return filename.endswith(".fail")
+
+        def name(self) -> str:
+            return "failing_converter"
+
+    # Register the failing parser
+    failing_converter = FailingParser()
+    memory_no_llm_verify._parser_registry.register(failing_converter)
+
+    # Create bank
+    context = RequestContext(internal=True)
+    await memory_no_llm_verify.get_bank_profile(bank_id, request_context=context)
+
+    # Create mock file
+    class MockFile:
+        def __init__(self, content, filename, content_type):
+            self.content = content
+            self.filename = filename
+            self.content_type = content_type
+
+        async def read(self):
+            return self.content
+
+    mock_file = MockFile(sample_txt_content, "test.fail", "application/octet-stream")
+
+    file_items = [
+        {
+            "file": mock_file,
+            "document_id": "test_doc_fail",
+            "context": None,
+            "metadata": {},
+            "tags": [],
+            "timestamp": None,
+        }
+    ]
+
+    # Submit async file retain with failing parser
+    result = await memory_no_llm_verify.submit_async_file_retain(
+        bank_id=bank_id,
+        file_items=file_items,
+        parser="failing_converter",
+        document_tags=None,
+        request_context=context,
+    )
+
+    assert "operation_ids" in result
+    assert len(result["operation_ids"]) == 1
+    operation_id = result["operation_ids"][0]
+
+    # Wait for async processing (with SyncTaskBackend, this is immediate)
+    import asyncio
+
+    await asyncio.sleep(0.2)
+
+    # Check operation status - should be 'failed' not 'completed'
+    pool = await memory_no_llm_verify._get_pool()
+    from hindsight_api.engine.memory_engine import get_current_schema
+
+    async with pool.acquire() as conn:
+        operation = await conn.fetchrow(
+            f"""
+            SELECT status, error_message
+            FROM {get_current_schema()}.async_operations
+            WHERE operation_id = $1
+            """,
+            operation_id,
+        )
+
+    assert operation is not None, f"Operation {operation_id} not found"
+    assert operation["status"] == "failed", f"Expected status 'failed' but got '{operation['status']}'"
+    assert operation["error_message"] is not None
+    assert "Mock conversion error" in operation["error_message"]
+    assert "test.fail" in operation["error_message"]

hindsight-api/tests/test_file_storage_s3.py

@@ -0,0 +1,252 @@
+"""
+Integration tests for S3FileStorage against a SeaweedFS Docker container.
+
+SeaweedFS (Apache 2.0) provides an S3-compatible API via `weed server -s3`.
+Requires Docker to be running. Tests are skipped automatically if Docker is unavailable.
+"""
+
+import json
+import logging
+import subprocess
+import tempfile
+import time
+import uuid
+
+import httpx
+import pytest
+from httpx import ASGITransport, AsyncClient
+
+logger = logging.getLogger(__name__)
+
+try:
+    from testcontainers.core.container import DockerContainer
+
+    _has_testcontainers = True
+except ImportError:
+    _has_testcontainers = False
+
+pytestmark = [
+    pytest.mark.skipif(not _has_testcontainers, reason="testcontainers not installed"),
+]
+
+SEAWEEDFS_S3_PORT = 8333
+TEST_BUCKET = "hindsight-test"
+ACCESS_KEY = "test_access_key"
+SECRET_KEY = "test_secret_key"
+
+# SeaweedFS S3 IAM config granting full access to our test credentials
+_S3_CONFIG = {
+    "identities": [
+        {
+            "name": "test-user",
+            "credentials": [{"accessKey": ACCESS_KEY, "secretKey": SECRET_KEY}],
+            "actions": ["Admin", "Read", "Write", "List"],
+        }
+    ]
+}
+
+
+def _docker_available() -> bool:
+    """Check if Docker daemon is running."""
+    try:
+        result = subprocess.run(
+            ["docker", "info"],
+            capture_output=True,
+            timeout=5,
+        )
+        return result.returncode == 0
+    except (FileNotFoundError, subprocess.TimeoutExpired):
+        return False
+
+
+def _wait_for_seaweedfs(endpoint: str, timeout: int = 30) -> None:
+    """Poll SeaweedFS S3 endpoint until ready."""
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        try:
+            resp = httpx.get(endpoint, timeout=2)
+            # 200 = no auth, 403 = auth enabled but gateway is up — either means ready
+            if resp.status_code in (200, 403):
+                logger.info("SeaweedFS S3 is ready at %s", endpoint)
+                return
+        except httpx.HTTPError:
+            pass
+        time.sleep(0.5)
+    raise TimeoutError(f"SeaweedFS did not become ready at {endpoint} within {timeout}s")
+
+
+@pytest.fixture(scope="module")
+def seaweedfs_container():
+    """Start a SeaweedFS container for the test module, shared across all tests.
+
+    Mounts an s3.json config file to set up S3 credentials for the test user.
+    """
+    if not _docker_available():
+        pytest.skip("Docker is not available")
+
+    # Write S3 IAM config to a temp file that persists for the module scope
+    s3_config_file = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False)
+    json.dump(_S3_CONFIG, s3_config_file)
+    s3_config_file.flush()
+
+    container = (
+        DockerContainer(image="chrislusf/seaweedfs:latest")
+        .with_exposed_ports(SEAWEEDFS_S3_PORT)
+        .with_volume_mapping(s3_config_file.name, "/etc/seaweedfs/s3.json", "ro")
+        .with_command(
+            f"server -s3 -s3.port={SEAWEEDFS_S3_PORT} -s3.config=/etc/seaweedfs/s3.json -ip.bind=0.0.0.0"
+        )
+    )
+
+    container.start()
+
+    try:
+        host = container.get_container_host_ip()
+        port = container.get_exposed_port(SEAWEEDFS_S3_PORT)
+        endpoint = f"http://{host}:{port}"
+
+        _wait_for_seaweedfs(endpoint)
+
+        # Create test bucket using obstore (proper SigV4 signing)
+        import obstore as obs
+        from obstore.store import S3Store
+
+        admin_store = S3Store(
+            TEST_BUCKET,
+            endpoint=endpoint,
+            region="us-east-1",
+            access_key_id=ACCESS_KEY,
+            secret_access_key=SECRET_KEY,
+            allow_http=True,
+        )
+        # SeaweedFS auto-creates buckets on first write
+        obs.put(admin_store, ".bucket-init", b"")
+        obs.delete(admin_store, ".bucket-init")
+        logger.info("Test bucket '%s' is ready", TEST_BUCKET)
+
+        yield {
+            "endpoint": endpoint,
+            "access_key": ACCESS_KEY,
+            "secret_key": SECRET_KEY,
+            "bucket": TEST_BUCKET,
+        }
+    finally:
+        container.stop()
+        import os
+
+        os.unlink(s3_config_file.name)
+
+
+@pytest.fixture
+def s3_storage(seaweedfs_container):
+    """Create an S3FileStorage instance pointing at the SeaweedFS container."""
+    from hindsight_api.engine.storage.s3 import S3FileStorage
+
+    return S3FileStorage(
+        bucket=seaweedfs_container["bucket"],
+        region="us-east-1",
+        endpoint=seaweedfs_container["endpoint"],
+        access_key_id=seaweedfs_container["access_key"],
+        secret_access_key=seaweedfs_container["secret_key"],
+    )
+
+
+@pytest.mark.asyncio
+async def test_s3_storage_store_and_retrieve(s3_storage):
+    """Store a file, retrieve it, verify bytes match."""
+    content = b"Hello, SeaweedFS! This is a test file."
+    key = f"test/{uuid.uuid4()}.txt"
+
+    stored_key = await s3_storage.store(
+        file_data=content,
+        key=key,
+        metadata={"content_type": "text/plain"},
+    )
+    assert stored_key == key
+
+    retrieved = await s3_storage.retrieve(key)
+    assert retrieved == content
+
+
+@pytest.mark.asyncio
+async def test_s3_storage_exists_and_delete(s3_storage):
+    """Store, check exists=True, delete, check exists=False."""
+    content = b"File to be deleted."
+    key = f"test/{uuid.uuid4()}.txt"
+
+    await s3_storage.store(file_data=content, key=key)
+
+    assert await s3_storage.exists(key) is True
+
+    await s3_storage.delete(key)
+
+    assert await s3_storage.exists(key) is False
+
+
+@pytest.mark.asyncio
+async def test_s3_storage_file_not_found(s3_storage):
+    """Retrieve a non-existent key, expect FileNotFoundError."""
+    with pytest.raises(FileNotFoundError):
+        await s3_storage.retrieve(f"nonexistent/{uuid.uuid4()}.txt")
+
+
+@pytest.mark.asyncio
+async def test_s3_storage_get_download_url(s3_storage):
+    """Store a file, get a presigned URL, verify it's a valid URL string."""
+    content = b"Presigned URL test content."
+    key = f"test/{uuid.uuid4()}.txt"
+
+    await s3_storage.store(file_data=content, key=key)
+
+    url = await s3_storage.get_download_url(key, expires_in=300)
+    assert isinstance(url, str)
+    assert url.startswith("http")
+    assert key in url
+
+
+@pytest.mark.asyncio
+async def test_s3_file_retain_api_end_to_end(seaweedfs_container, memory_no_llm_verify):
+    """Full HTTP API flow: upload file via /files/retain with S3 storage backend."""
+    from hindsight_api.api.http import create_app
+    from hindsight_api.engine.storage.s3 import S3FileStorage
+
+    # Swap the engine's file storage to use the SeaweedFS-backed S3 storage
+    original_storage = memory_no_llm_verify._file_storage
+    s3_storage = S3FileStorage(
+        bucket=seaweedfs_container["bucket"],
+        region="us-east-1",
+        endpoint=seaweedfs_container["endpoint"],
+        access_key_id=seaweedfs_container["access_key"],
+        secret_access_key=seaweedfs_container["secret_key"],
+    )
+    memory_no_llm_verify._file_storage = s3_storage
+
+    try:
+        app = create_app(memory_no_llm_verify, initialize_memory=False)
+
+        async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+            bank_id = f"test-s3-bank-{uuid.uuid4().hex[:8]}"
+            bank_response = await client.put(f"/v1/default/banks/{bank_id}", json={"name": "S3 Test Bank"})
+            assert bank_response.status_code in (200, 201)
+
+            txt_content = b"Alice works at Acme Corp. She joined in 2024."
+            request_data = {
+                "document_tags": ["s3-test"],
+                "async": True,
+            }
+
+            files = {"files": ("notes.txt", txt_content, "text/plain")}
+            data = {"request": json.dumps(request_data)}
+
+            response = await client.post(
+                f"/v1/default/banks/{bank_id}/files/retain",
+                files=files,
+                data=data,
+            )
+
+            assert response.status_code == 200
+            result = response.json()
+            assert "operation_ids" in result
+            assert len(result["operation_ids"]) == 1
+    finally:
+        memory_no_llm_verify._file_storage = original_storage

hindsight-api/tests/test_iris_parser.py

@@ -0,0 +1,72 @@
+"""
+Integration tests for the Iris file parser.
+
+Tests are skipped automatically if HINDSIGHT_API_FILE_PARSER_IRIS_TOKEN
+and HINDSIGHT_API_FILE_PARSER_IRIS_ORG_ID are not set in the environment.
+"""
+
+import os
+
+import pytest
+
+from hindsight_api.config import ENV_FILE_PARSER_IRIS_ORG_ID, ENV_FILE_PARSER_IRIS_TOKEN
+from hindsight_api.engine.parsers.iris import IrisParser
+
+_token = os.getenv(ENV_FILE_PARSER_IRIS_TOKEN)
+_org_id = os.getenv(ENV_FILE_PARSER_IRIS_ORG_ID)
+
+pytestmark = pytest.mark.skipif(
+    not (_token and _org_id),
+    reason="HINDSIGHT_API_FILE_PARSER_IRIS_TOKEN and HINDSIGHT_API_FILE_PARSER_IRIS_ORG_ID not set",
+)
+
+# Minimal valid PDF with the text "Hello from Hindsight"
+_SAMPLE_PDF = b"""%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+3 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792]
+   /Contents 4 0 R /Resources << /Font << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >> >>
+endobj
+4 0 obj
+<< /Length 44 >>
+stream
+BT /F1 12 Tf 100 700 Td (Hello from Hindsight) Tj ET
+endstream
+endobj
+xref
+0 5
+0000000000 65535 f
+0000000009 00000 n
+0000000058 00000 n
+0000000115 00000 n
+0000000274 00000 n
+trailer << /Size 5 /Root 1 0 R >>
+startxref
+369
+%%EOF"""
+
+
+@pytest.fixture
+def iris_parser() -> IrisParser:
+    return IrisParser(token=_token, org_id=_org_id)
+
+
+@pytest.mark.asyncio
+async def test_iris_parser_converts_pdf(iris_parser: IrisParser):
+    """IrisParser should extract text from a valid PDF."""
+    result = await iris_parser.convert(_SAMPLE_PDF, "sample.pdf")
+    assert isinstance(result, str)
+    assert len(result) > 0
+
+
+@pytest.mark.asyncio
+async def test_iris_parser_name(iris_parser: IrisParser):
+    """IrisParser.name() should return 'iris'."""
+    assert iris_parser.name() == "iris"
+
+

hindsight-api/tests/test_main_module.py

@@ -352,6 +352,47 @@ class TestMainModuleExtensionLoading:
             "main.py should use import string when workers > 1"
         assert uvicorn_calls[0]["workers"] == 2
 
+    def test_main_sets_keepalive_timeout(self, monkeypatch):
+        """
+        Verify that uvicorn is configured with timeout_keep_alive > aiohttp's
+        default client keepalive timeout (15s), so the server never closes
+        connections before the client does.
+        """
+        monkeypatch.setenv("HINDSIGHT_API_WORKERS", "1")
+        monkeypatch.delenv("HINDSIGHT_API_TENANT_EXTENSION", raising=False)
+
+        uvicorn_calls = []
+
+        def capture_uvicorn_run(**kwargs):
+            uvicorn_calls.append(kwargs)
+
+        with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
+             patch("hindsight_api.main.create_app") as mock_create_app, \
+             patch("hindsight_api.main._get_raw_config") as mock_get_config, \
+             patch("hindsight_api.main.print_banner"), \
+             patch("uvicorn.run", side_effect=capture_uvicorn_run):
+
+            mock_config = MagicMock()
+            mock_config.host = "0.0.0.0"
+            mock_config.port = 8888
+            mock_config.log_level = "info"
+            mock_config.mcp_enabled = False
+            mock_config.run_migrations_on_startup = False
+            mock_config.database_url = "postgresql://test:test@localhost/test"
+            mock_get_config.return_value = mock_config
+            mock_engine.return_value = MagicMock()
+            mock_create_app.return_value = MagicMock()
+
+            with patch.object(sys, 'argv', ['hindsight-api']):
+                from hindsight_api.main import main
+                main()
+
+        assert len(uvicorn_calls) == 1
+        assert "timeout_keep_alive" in uvicorn_calls[0], \
+            "uvicorn config must set timeout_keep_alive"
+        assert uvicorn_calls[0]["timeout_keep_alive"] > 15, \
+            "timeout_keep_alive must exceed aiohttp's 15s client default"
+
 
 # Mock extensions for testing
 from hindsight_api.extensions import (

hindsight-api/tests/test_worker.py

@@ -206,8 +206,12 @@ class TestWorkerPoller:
         assert len(claimed) == 3
 
     @pytest.mark.asyncio
-    async def test_execute_task_marks_completed(self, pool, clean_operations):
-        """Test that successful task execution marks task as completed."""
+    async def test_execute_task_executor_marks_completed(self, pool, clean_operations):
+        """Test that executor's status marking is preserved by the poller.
+
+        The executor (MemoryEngine.execute_task) handles marking operations as completed/failed.
+        The poller should NOT override those status updates.
+        """
         from hindsight_api.worker import WorkerPoller
         from hindsight_api.worker.poller import ClaimedTask
 
@@ -228,7 +232,16 @@ class TestWorkerPoller:
         executed = []
 
         async def mock_executor(task_dict):
+            """Executor that marks its own status as completed (like MemoryEngine.execute_task)."""
             executed.append(task_dict)
+            await pool.execute(
+                """
+                UPDATE async_operations
+                SET status = 'completed', completed_at = now(), updated_at = now()
+                WHERE operation_id = $1
+                """,
+                op_id,
+            )
 
         poller = WorkerPoller(
             pool=pool,
@@ -246,7 +259,7 @@ class TestWorkerPoller:
         assert completed, "Task did not complete within timeout"
         assert len(executed) == 1
 
-        # Verify task is marked as completed
+        # Verify task is marked as completed (by executor, not overridden by poller)
         row = await pool.fetchrow(
             "SELECT status, completed_at FROM async_operations WHERE operation_id = $1",
             op_id,
@@ -255,19 +268,24 @@ class TestWorkerPoller:
         assert row["completed_at"] is not None
 
     @pytest.mark.asyncio
-    async def test_execute_task_retries_on_failure(self, pool, clean_operations):
-        """Test that failed task execution triggers retry mechanism."""
+    async def test_executor_exception_does_not_crash_poller(self, pool, clean_operations):
+        """Test that unexpected exceptions from executor are caught and don't crash the poller.
+
+        If the executor raises an unexpected exception (which MemoryEngine.execute_task should NOT do,
+        but could happen from schema setup or other infrastructure issues), the poller should catch it
+        gracefully. Status remains 'processing' since neither executor nor poller handled it.
+        """
         from hindsight_api.worker import WorkerPoller
         from hindsight_api.worker.poller import ClaimedTask
 
-        # Create a pending task with retry_count=0
+        # Create a pending task
         bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
         op_id = uuid.uuid4()
         payload = json.dumps({"type": "test_task", "operation_id": str(op_id), "bank_id": bank_id})
         await pool.execute(
             """
-            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id, retry_count)
-            VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'test-worker-1', 0)
+            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id)
+            VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'test-worker-1')
             """,
             op_id,
             bank_id,
@@ -275,16 +293,15 @@ class TestWorkerPoller:
         )
 
         async def failing_executor(task_dict):
-            raise ValueError("Simulated failure")
+            raise ValueError("Unexpected infrastructure failure")
 
         poller = WorkerPoller(
             pool=pool,
             worker_id="test-worker-1",
             executor=failing_executor,
-            max_retries=3,
         )
 
-        # Execute (should fail and retry) - fire-and-forget
+        # Execute - should catch exception without crashing
         task_dict = json.loads(payload)
         claimed_task = ClaimedTask(operation_id=str(op_id), task_dict=task_dict, schema=None)
         await poller.execute_task(claimed_task)
@@ -293,61 +310,84 @@ class TestWorkerPoller:
         completed = await poller.wait_for_active_tasks(timeout=5.0)
         assert completed, "Task did not complete within timeout"
 
-        # Verify task is back to pending with incremented retry_count
+        # Status stays 'processing' since the poller no longer manages status
         row = await pool.fetchrow(
-            "SELECT status, retry_count, worker_id FROM async_operations WHERE operation_id = $1",
+            "SELECT status FROM async_operations WHERE operation_id = $1",
             op_id,
         )
-        assert row["status"] == "pending"
-        assert row["retry_count"] == 1
-        assert row["worker_id"] is None  # Worker ID cleared for retry
+        assert row["status"] == "processing"
 
     @pytest.mark.asyncio
-    async def test_execute_task_fails_after_max_retries(self, pool, clean_operations):
-        """Test that task is marked failed after exceeding max retries."""
+    async def test_executor_failed_status_not_overridden(self, pool, clean_operations):
+        """REGRESSION TEST: Verify poller does NOT overwrite executor's 'failed' status to 'completed'.
+
+        This test catches the bug where the poller always called _mark_completed() after executor
+        returned, overwriting the 'failed' status that the executor had already set.
+
+        Scenario:
+        1. Executor catches an internal error and marks the operation as 'failed' in the DB
+        2. Executor returns normally (does NOT re-raise) - this is how MemoryEngine.execute_task works
+        3. The poller must NOT overwrite the 'failed' status to 'completed'
+
+        With the old buggy code, this test would FAIL (status would be 'completed').
+        """
         from hindsight_api.worker import WorkerPoller
         from hindsight_api.worker.poller import ClaimedTask
 
-        # Create a task that has already used all retries
         bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
         op_id = uuid.uuid4()
         payload = json.dumps({"type": "test_task", "operation_id": str(op_id), "bank_id": bank_id})
         await pool.execute(
             """
-            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id, retry_count)
-            VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'test-worker-1', 3)
+            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload, worker_id)
+            VALUES ($1, $2, 'test', 'processing', $3::jsonb, 'test-worker-1')
             """,
             op_id,
             bank_id,
             payload,
         )
 
-        async def failing_executor(task_dict):
-            raise ValueError("Simulated failure")
+        async def executor_that_marks_failed(task_dict):
+            """Simulates MemoryEngine.execute_task behavior on internal error.
+
+            The executor catches the error, marks the operation as 'failed',
+            and returns normally (does NOT re-raise the exception).
+            """
+            # Simulate internal failure handling (like MemoryEngine._mark_operation_failed)
+            await pool.execute(
+                """
+                UPDATE async_operations
+                SET status = 'failed', error_message = $2, completed_at = now(), updated_at = now()
+                WHERE operation_id = $1
+                """,
+                op_id,
+                "Simulated conversion error: file format not supported",
+            )
+            # Returns normally - this is the key: executor does NOT re-raise
 
         poller = WorkerPoller(
             pool=pool,
             worker_id="test-worker-1",
-            executor=failing_executor,
-            max_retries=3,
+            executor=executor_that_marks_failed,
         )
 
-        # Execute (should fail permanently) - fire-and-forget
         task_dict = json.loads(payload)
         claimed_task = ClaimedTask(operation_id=str(op_id), task_dict=task_dict, schema=None)
         await poller.execute_task(claimed_task)
 
-        # Wait for background task to complete
         completed = await poller.wait_for_active_tasks(timeout=5.0)
         assert completed, "Task did not complete within timeout"
 
-        # Verify task is marked as failed
+        # THE KEY ASSERTION: Status must be 'failed', NOT 'completed'
         row = await pool.fetchrow(
             "SELECT status, error_message FROM async_operations WHERE operation_id = $1",
             op_id,
         )
-        assert row["status"] == "failed"
-        assert "Max retries" in row["error_message"]
+        assert row["status"] == "failed", (
+            f"REGRESSION: Poller overwrote executor's 'failed' status to '{row['status']}'. "
+            "The poller must not override status set by the executor."
+        )
+        assert "Simulated conversion error" in row["error_message"]
 
     @pytest.mark.asyncio
     async def test_claim_batch_skips_consolidation_when_same_bank_processing(self, pool, clean_operations):

hindsight-cli/Cargo.toml

@@ -20,8 +20,8 @@ clap = { version = "4.5", features = ["derive", "env"] }
 # Async runtime
 tokio = { version = "1", features = ["full"] }
 
-# HTTP client (for timeout configuration)
-reqwest = "0.12"
+# HTTP client (for timeout configuration and multipart file uploads)
+reqwest = { version = "0.12", features = ["multipart"] }
 
 # Serialization (for config and output formatting)
 serde = { version = "1.0", features = ["derive"] }

hindsight-cli/src/api.rs

@@ -58,9 +58,16 @@ pub struct MemoryPutResult {
     pub operation_id: Option<String>,
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct FileRetainResult {
+    pub operation_ids: Vec<String>,
+}
+
 #[derive(Clone)]
 pub struct ApiClient {
     client: AsyncClient,
+    http_client: reqwest::Client,
+    base_url: String,
     runtime: std::sync::Arc<tokio::runtime::Runtime>,
 }
 
@@ -84,8 +91,8 @@ impl ApiClient {
 
         let http_client = client_builder.build()?;
 
-        let client = AsyncClient::new_with_client(&base_url, http_client);
-        Ok(ApiClient { client, runtime })
+        let client = AsyncClient::new_with_client(&base_url, http_client.clone());
+        Ok(ApiClient { client, http_client, base_url, runtime })
     }
 
     pub fn list_agents(&self, _verbose: bool) -> Result<Vec<types::BankListItem>> {
@@ -168,6 +175,67 @@ impl ApiClient {
         })
     }
 
+    /// Upload files to the file retain endpoint (multipart/form-data).
+    /// Returns a list of operation IDs for tracking. Always async server-side.
+    pub fn file_retain(
+        &self,
+        bank_id: &str,
+        files: Vec<(String, Vec<u8>)>,
+        context: Option<String>,
+        verbose: bool,
+    ) -> Result<FileRetainResult> {
+        self.runtime.block_on(async {
+            let url = format!("{}/v1/default/banks/{}/files/retain", self.base_url, bank_id);
+
+            let files_metadata: Vec<serde_json::Value> = files
+                .iter()
+                .map(|(name, _)| {
+                    let mut meta = serde_json::json!({});
+                    if let Some(ctx) = &context {
+                        meta["context"] = serde_json::Value::String(ctx.clone());
+                    }
+                    // Use filename stem as document_id for deduplication
+                    if let Some(stem) = std::path::Path::new(name)
+                        .file_stem()
+                        .and_then(|s| s.to_str())
+                    {
+                        meta["document_id"] = serde_json::Value::String(stem.to_string());
+                    }
+                    meta
+                })
+                .collect();
+
+            let request_json = serde_json::json!({
+                "files_metadata": files_metadata,
+            });
+
+            let mut form = reqwest::multipart::Form::new()
+                .text("request", request_json.to_string());
+
+            for (filename, content) in files {
+                let part = reqwest::multipart::Part::bytes(content)
+                    .file_name(filename)
+                    .mime_str("application/octet-stream")?;
+                form = form.part("files", part);
+            }
+
+            if verbose {
+                eprintln!("POST {}", url);
+            }
+
+            let response = self.http_client.post(&url).multipart(form).send().await?;
+
+            if !response.status().is_success() {
+                let status = response.status();
+                let text = response.text().await.unwrap_or_default();
+                anyhow::bail!("File retain failed ({}): {}", status, text);
+            }
+
+            let result: FileRetainResult = response.json().await?;
+            Ok(result)
+        })
+    }
+
     /// Poll an operation until it completes or fails.
     /// Returns Ok(true) if completed successfully, Ok(false) if failed, Err if polling error.
     pub fn poll_operation(&self, agent_id: &str, operation_id: &str, verbose: bool) -> Result<(bool, Option<String>)> {

hindsight-cli/src/commands/memory.rs

@@ -220,14 +220,23 @@ pub fn get(
     }
 }
 
-// Helper function to check if a file has a text-based extension
-fn is_text_file(path: &std::path::Path) -> bool {
-    const TEXT_EXTENSIONS: &[&str] = &[
-        "txt", "md", "json", "yaml", "yml", "toml", "xml", "csv", "log", "rst", "adoc",
+// Helper function to check if a file is supported by the file converter (markitdown)
+fn is_supported_file(path: &std::path::Path) -> bool {
+    const SUPPORTED_EXTENSIONS: &[&str] = &[
+        // Documents
+        "pdf", "docx", "doc", "pptx", "ppt", "xlsx", "xls",
+        // Images (OCR)
+        "jpg", "jpeg", "png", "gif", "bmp", "webp", "tiff",
+        // Web / markup
+        "html", "htm",
+        // Text / data
+        "txt", "md", "csv", "json", "yaml", "yml", "toml", "xml", "rst", "adoc", "log",
+        // Audio (transcription)
+        "mp3", "wav", "ogg", "flac",
     ];
     path.extension()
         .and_then(|ext| ext.to_str())
-        .map(|ext| TEXT_EXTENSIONS.contains(&ext.to_lowercase().as_str()))
+        .map(|ext| SUPPORTED_EXTENSIONS.contains(&ext.to_lowercase().as_str()))
         .unwrap_or(false)
 }
 
@@ -427,10 +436,10 @@ pub fn retain_files(
         anyhow::bail!("Path does not exist: {}", path.display());
     }
 
-    let mut files = Vec::new();
+    let mut file_paths = Vec::new();
 
     if path.is_file() {
-        files.push(path);
+        file_paths.push(path);
     } else if path.is_dir() {
         if recursive {
             for entry in WalkDir::new(&path)
@@ -438,133 +447,110 @@ pub fn retain_files(
                 .filter_map(|e| e.ok())
                 .filter(|e| e.file_type().is_file())
             {
-                let path = entry.path();
-                if is_text_file(&path) {
-                    files.push(path.to_path_buf());
+                let file_path = entry.path();
+                if is_supported_file(file_path) {
+                    file_paths.push(file_path.to_path_buf());
                 }
             }
         } else {
             for entry in fs::read_dir(&path)? {
                 let entry = entry?;
-                let path = entry.path();
-                if path.is_file() && is_text_file(&path) {
-                    files.push(path);
+                let file_path = entry.path();
+                if file_path.is_file() && is_supported_file(&file_path) {
+                    file_paths.push(file_path);
                 }
             }
         }
     }
 
-    if files.is_empty() {
-        ui::print_warning("No text files found (supported: txt, md, json, yaml, yml, toml, xml, csv, log, rst, adoc)");
+    if file_paths.is_empty() {
+        ui::print_warning("No supported files found. Supported formats: pdf, docx, pptx, xlsx, jpg, png, html, txt, md, csv, mp3, wav, and more.");
         return Ok(());
     }
 
-    ui::print_info(&format!("Found {} files to import", files.len()));
+    ui::print_info(&format!("Found {} file(s) to import", file_paths.len()));
 
-    let pb = ui::create_progress_bar(files.len() as u64, "Processing files");
+    // Batch files (max 10 per request)
+    const BATCH_SIZE: usize = 10;
+    let batches: Vec<&[PathBuf]> = file_paths.chunks(BATCH_SIZE).collect();
+    let mut all_operation_ids: Vec<String> = Vec::new();
 
-    let mut items = Vec::new();
+    let pb = ui::create_progress_bar(file_paths.len() as u64, "Uploading files");
 
-    for file_path in &files {
-        let content = fs::read_to_string(file_path)
-            .with_context(|| format!("Failed to read file: {}", file_path.display()))?;
+    for batch in &batches {
+        let mut file_data: Vec<(String, Vec<u8>)> = Vec::new();
+        for file_path in *batch {
+            let filename = file_path
+                .file_name()
+                .and_then(|n| n.to_str())
+                .map(|s| s.to_string())
+                .unwrap_or_else(|| "file".to_string());
+            let content = fs::read(file_path)
+                .with_context(|| format!("Failed to read file: {}", file_path.display()))?;
+            file_data.push((filename, content));
+            pb.inc(1);
+        }
 
-        let doc_id = file_path
-            .file_stem()
-            .and_then(|s| s.to_str())
-            .map(|s| s.to_string())
-            .unwrap_or_else(config::generate_doc_id);
-
-        items.push(MemoryItem {
-            content,
-            context: context.clone(),
-            metadata: None,
-            timestamp: None,
-            document_id: Some(doc_id),
-            entities: None,
-            tags: None,
-        });
-
-        pb.inc(1);
+        let result = client.file_retain(agent_id, file_data, context.clone(), verbose)?;
+        all_operation_ids.extend(result.operation_ids);
     }
 
-    pb.finish_with_message("Files processed");
+    pb.finish_with_message("Files uploaded");
 
-    // Always use async mode for the API call
-    let request = RetainRequest {
-        items,
-        async_: true,
-        document_tags: None,
-    };
-
-    let spinner = if output_format == OutputFormat::Pretty {
-        Some(ui::create_spinner("Submitting retain request..."))
+    if r#async {
+        if output_format == OutputFormat::Pretty {
+            ui::print_success("Files queued for processing");
+            println!("  Files: {}", file_paths.len());
+            for op_id in &all_operation_ids {
+                println!("  Operation ID: {}", op_id);
+            }
+        } else {
+            let result = serde_json::json!({ "operation_ids": all_operation_ids });
+            output::print_output(&result, output_format)?;
+        }
     } else {
-        None
-    };
+        // Poll all operations until they complete
+        let poll_spinner = if output_format == OutputFormat::Pretty {
+            Some(ui::create_spinner("Processing files..."))
+        } else {
+            None
+        };
 
-    let response = client.retain(agent_id, &request, true, verbose);
+        let mut failed = Vec::new();
+        for op_id in &all_operation_ids {
+            let (success, error_msg) = client.poll_operation(agent_id, op_id, verbose)?;
+            if !success {
+                failed.push(error_msg.unwrap_or_else(|| "Unknown error".to_string()));
+            }
+        }
 
-    if let Some(mut sp) = spinner {
-        sp.finish();
-    }
+        if let Some(mut sp) = poll_spinner {
+            sp.finish();
+        }
 
-    match response {
-        Ok(result) => {
-            if r#async {
-                // User requested async mode - return immediately
-                if output_format == OutputFormat::Pretty {
-                    ui::print_success("Files queued for processing");
-                    println!("  Items: {}", result.items_count);
-                    if let Some(op_id) = &result.operation_id {
-                        println!("  Operation ID: {}", op_id);
-                    }
-                } else {
-                    output::print_output(&result, output_format)?;
-                }
+        if failed.is_empty() {
+            if output_format == OutputFormat::Pretty {
+                ui::print_success("Files retained successfully");
+                println!("  Files processed: {}", file_paths.len());
             } else {
-                // Poll until completion
-                if let Some(operation_id) = &result.operation_id {
-                    let poll_spinner = if output_format == OutputFormat::Pretty {
-                        Some(ui::create_spinner("Processing memories..."))
-                    } else {
-                        None
-                    };
-
-                    let (success, error_msg) = client.poll_operation(agent_id, operation_id, verbose)?;
-
-                    if let Some(mut sp) = poll_spinner {
-                        sp.finish();
-                    }
-
-                    if success {
-                        if output_format == OutputFormat::Pretty {
-                            ui::print_success("Files retained successfully");
-                            println!("  Items processed: {}", result.items_count);
-                        } else {
-                            output::print_output(&result, output_format)?;
-                        }
-                    } else {
-                        let msg = error_msg.unwrap_or_else(|| "Unknown error".to_string());
-                        if output_format == OutputFormat::Pretty {
-                            ui::print_error(&format!("Retain operation failed: {}", msg));
-                        }
-                        anyhow::bail!("Retain operation failed: {}", msg);
-                    }
-                } else {
-                    // No operation ID returned, shouldn't happen with async=true
-                    if output_format == OutputFormat::Pretty {
-                        ui::print_success("Files retained successfully");
-                        println!("  Items processed: {}", result.items_count);
-                    } else {
-                        output::print_output(&result, output_format)?;
-                    }
+                let result = serde_json::json!({
+                    "success": true,
+                    "files_count": file_paths.len(),
+                    "operation_ids": all_operation_ids,
+                });
+                output::print_output(&result, output_format)?;
+            }
+        } else {
+            for msg in &failed {
+                if output_format == OutputFormat::Pretty {
+                    ui::print_error(&format!("Retain operation failed: {}", msg));
                 }
             }
-            Ok(())
+            anyhow::bail!("{} operation(s) failed", failed.len());
         }
-        Err(e) => Err(e)
     }
+
+    Ok(())
 }
 
 pub fn delete(
@@ -679,55 +665,71 @@ mod tests {
     use std::path::Path;
 
     #[test]
-    fn test_is_text_file_supported_extensions() {
+    fn test_is_supported_file_text_extensions() {
         let supported = [
             "file.txt", "file.md", "file.json", "file.yaml", "file.yml",
             "file.toml", "file.xml", "file.csv", "file.log", "file.rst", "file.adoc",
         ];
         for filename in supported {
             assert!(
-                is_text_file(Path::new(filename)),
-                "{} should be recognized as a text file",
+                is_supported_file(Path::new(filename)),
+                "{} should be recognized as a supported file",
                 filename
             );
         }
     }
 
     #[test]
-    fn test_is_text_file_case_insensitive() {
-        assert!(is_text_file(Path::new("file.JSON")));
-        assert!(is_text_file(Path::new("file.TXT")));
-        assert!(is_text_file(Path::new("file.Md")));
-        assert!(is_text_file(Path::new("file.YAML")));
+    fn test_is_supported_file_binary_extensions() {
+        let supported = [
+            "file.pdf", "file.docx", "file.pptx", "file.xlsx",
+            "file.png", "file.jpg", "file.jpeg", "file.gif",
+            "file.mp3", "file.wav",
+        ];
+        for filename in supported {
+            assert!(
+                is_supported_file(Path::new(filename)),
+                "{} should be recognized as a supported file",
+                filename
+            );
+        }
     }
 
     #[test]
-    fn test_is_text_file_unsupported_extensions() {
+    fn test_is_supported_file_case_insensitive() {
+        assert!(is_supported_file(Path::new("file.JSON")));
+        assert!(is_supported_file(Path::new("file.TXT")));
+        assert!(is_supported_file(Path::new("file.Md")));
+        assert!(is_supported_file(Path::new("file.YAML")));
+        assert!(is_supported_file(Path::new("file.PDF")));
+    }
+
+    #[test]
+    fn test_is_supported_file_unsupported_extensions() {
         let unsupported = [
-            "file.pdf", "file.doc", "file.docx", "file.png", "file.jpg",
             "file.exe", "file.bin", "file.zip", "file.tar", "file.gz",
         ];
         for filename in unsupported {
             assert!(
-                !is_text_file(Path::new(filename)),
-                "{} should NOT be recognized as a text file",
+                !is_supported_file(Path::new(filename)),
+                "{} should NOT be recognized as a supported file",
                 filename
             );
         }
     }
 
     #[test]
-    fn test_is_text_file_no_extension() {
-        assert!(!is_text_file(Path::new("README")));
-        assert!(!is_text_file(Path::new("Makefile")));
-        assert!(!is_text_file(Path::new(".gitignore")));
+    fn test_is_supported_file_no_extension() {
+        assert!(!is_supported_file(Path::new("README")));
+        assert!(!is_supported_file(Path::new("Makefile")));
+        assert!(!is_supported_file(Path::new(".gitignore")));
     }
 
     #[test]
-    fn test_is_text_file_with_path() {
-        assert!(is_text_file(Path::new("/some/path/to/file.json")));
-        assert!(is_text_file(Path::new("../relative/path/file.md")));
-        assert!(!is_text_file(Path::new("/path/to/image.png")));
+    fn test_is_supported_file_with_path() {
+        assert!(is_supported_file(Path::new("/some/path/to/file.json")));
+        assert!(is_supported_file(Path::new("../relative/path/file.md")));
+        assert!(is_supported_file(Path::new("/path/to/image.png")));
     }
 
     #[test]

hindsight-clients/go/api/openapi.yaml

@@ -2124,6 +2124,73 @@ paths:
       summary: Retain memories
       tags:
       - Memory
+  /v1/default/banks/{bank_id}/files/retain:
+    post:
+      description: |-
+        Upload files (PDF, DOCX, etc.), convert them to markdown, and retain as memories.
+
+        This endpoint handles file upload, conversion, and memory creation in a single operation.
+
+        **Features:**
+        - Supports PDF, DOCX, PPTX, XLSX, images (with OCR), audio (with transcription)
+        - Automatic file-to-markdown conversion using pluggable parsers
+        - Files stored in object storage (PostgreSQL by default, S3 for production)
+        - Each file becomes a separate document with optional metadata/tags
+        - Always processes asynchronously — returns operation IDs immediately
+
+        **The system automatically:**
+        1. Stores uploaded files in object storage
+        2. Converts files to markdown
+        3. Creates document records with file metadata
+        4. Extracts facts and creates memory units (same as regular retain)
+
+        Use the operations endpoint to monitor progress.
+
+        **Request format:** multipart/form-data with:
+        - `files`: One or more files to upload
+        - `request`: JSON string with FileRetainRequest model (files_metadata)
+
+        **Note:** File parser is configured server-side via `HINDSIGHT_API_FILE_PARSER` (default: markitdown).
+      operationId: file_retain
+      parameters:
+      - explode: false
+        in: path
+        name: bank_id
+        required: true
+        schema:
+          title: Bank Id
+          type: string
+        style: simple
+      - explode: false
+        in: header
+        name: authorization
+        required: false
+        schema:
+          nullable: true
+          type: string
+        style: simple
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/Body_file_retain'
+        required: true
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/FileRetainResponse'
+          description: Successful Response
+        "422":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      summary: Convert files to memories
+      tags:
+      - Files
 components:
   schemas:
     AddBackgroundRequest:
@@ -2394,6 +2461,22 @@ components:
       - total_links
       - total_nodes
       title: BankStatsResponse
+    Body_file_retain:
+      properties:
+        files:
+          description: Files to upload and convert
+          items:
+            format: binary
+            type: string
+          type: array
+        request:
+          description: JSON string with FileRetainRequest model
+          title: Request
+          type: string
+      required:
+      - files
+      - request
+      title: Body_file_retain
     Budget:
       description: Budget levels for recall/reflect operations.
       enum:
@@ -3040,12 +3123,34 @@ components:
           description: Whether per-bank configuration API is enabled
           title: Bank Config Api
           type: boolean
+        file_upload_api:
+          description: Whether file upload/conversion API is enabled
+          title: File Upload Api
+          type: boolean
       required:
       - bank_config_api
+      - file_upload_api
       - mcp
       - observations
       - worker
       title: FeaturesInfo
+    FileRetainResponse:
+      description: Response model for file upload endpoint.
+      example:
+        operation_ids:
+        - 550e8400-e29b-41d4-a716-446655440000
+        - 550e8400-e29b-41d4-a716-446655440001
+        - 550e8400-e29b-41d4-a716-446655440002
+      properties:
+        operation_ids:
+          description: "Operation IDs for tracking file conversion operations. Use\
+            \ GET /v1/default/banks/{bank_id}/operations to list operations."
+          items:
+            type: string
+          type: array
+      required:
+      - operation_ids
+      title: FileRetainResponse
     GraphDataResponse:
       description: Response model for graph data endpoint.
       example:
@@ -4197,6 +4302,7 @@ components:
         api_version: 0.4.0
         features:
           bank_config_api: false
+          file_upload_api: true
           mcp: true
           observations: false
           worker: true

hindsight-clients/go/api_files.go

@@ -0,0 +1,209 @@
+/*
+Hindsight HTTP API
+
+HTTP API for Hindsight
+
+API version: 0.4.11
+*/
+
+// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
+
+package hindsight
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+)
+
+
+// FilesAPIService FilesAPI service
+type FilesAPIService service
+
+type ApiFileRetainRequest struct {
+	ctx context.Context
+	ApiService *FilesAPIService
+	bankId string
+	files []*os.File
+	request *string
+	authorization *string
+}
+
+// Files to upload and convert
+func (r ApiFileRetainRequest) Files(files []*os.File) ApiFileRetainRequest {
+	r.files = files
+	return r
+}
+
+// JSON string with FileRetainRequest model
+func (r ApiFileRetainRequest) Request(request string) ApiFileRetainRequest {
+	r.request = &request
+	return r
+}
+
+func (r ApiFileRetainRequest) Authorization(authorization string) ApiFileRetainRequest {
+	r.authorization = &authorization
+	return r
+}
+
+func (r ApiFileRetainRequest) Execute() (*FileRetainResponse, *http.Response, error) {
+	return r.ApiService.FileRetainExecute(r)
+}
+
+/*
+FileRetain Convert files to memories
+
+Upload files (PDF, DOCX, etc.), convert them to markdown, and retain as memories.
+
+This endpoint handles file upload, conversion, and memory creation in a single operation.
+
+**Features:**
+- Supports PDF, DOCX, PPTX, XLSX, images (with OCR), audio (with transcription)
+- Automatic file-to-markdown conversion using pluggable parsers
+- Files stored in object storage (PostgreSQL by default, S3 for production)
+- Each file becomes a separate document with optional metadata/tags
+- Always processes asynchronously — returns operation IDs immediately
+
+**The system automatically:**
+1. Stores uploaded files in object storage
+2. Converts files to markdown
+3. Creates document records with file metadata
+4. Extracts facts and creates memory units (same as regular retain)
+
+Use the operations endpoint to monitor progress.
+
+**Request format:** multipart/form-data with:
+- `files`: One or more files to upload
+- `request`: JSON string with FileRetainRequest model (files_metadata)
+
+**Note:** File parser is configured server-side via `HINDSIGHT_API_FILE_PARSER` (default: markitdown).
+
+ @param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
+ @param bankId
+ @return ApiFileRetainRequest
+*/
+func (a *FilesAPIService) FileRetain(ctx context.Context, bankId string) ApiFileRetainRequest {
+	return ApiFileRetainRequest{
+		ApiService: a,
+		ctx: ctx,
+		bankId: bankId,
+	}
+}
+
+// Execute executes the request
+//  @return FileRetainResponse
+func (a *FilesAPIService) FileRetainExecute(r ApiFileRetainRequest) (*FileRetainResponse, *http.Response, error) {
+	var (
+		localVarHTTPMethod   = http.MethodPost
+		localVarPostBody     interface{}
+		formFiles            []formFile
+		localVarReturnValue  *FileRetainResponse
+	)
+
+	localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "FilesAPIService.FileRetain")
+	if err != nil {
+		return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
+	}
+
+	localVarPath := localBasePath + "/v1/default/banks/{bank_id}/files/retain"
+	localVarPath = strings.Replace(localVarPath, "{"+"bank_id"+"}", url.PathEscape(parameterValueToString(r.bankId, "bankId")), -1)
+
+	localVarHeaderParams := make(map[string]string)
+	localVarQueryParams := url.Values{}
+	localVarFormParams := url.Values{}
+	if r.files == nil {
+		return localVarReturnValue, nil, reportError("files is required and must be specified")
+	}
+	if r.request == nil {
+		return localVarReturnValue, nil, reportError("request is required and must be specified")
+	}
+
+	// to determine the Content-Type header
+	localVarHTTPContentTypes := []string{"multipart/form-data"}
+
+	// set Content-Type header
+	localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
+	if localVarHTTPContentType != "" {
+		localVarHeaderParams["Content-Type"] = localVarHTTPContentType
+	}
+
+	// to determine the Accept header
+	localVarHTTPHeaderAccepts := []string{"application/json"}
+
+	// set Accept header
+	localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
+	if localVarHTTPHeaderAccept != "" {
+		localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
+	}
+	if r.authorization != nil {
+		parameterAddToHeaderOrQuery(localVarHeaderParams, "authorization", r.authorization, "simple", "")
+	}
+	var filesLocalVarFormFileName string
+	var filesLocalVarFileName     string
+	var filesLocalVarFileBytes    []byte
+
+	filesLocalVarFormFileName = "files"
+	filesLocalVarFile := r.files
+
+	if filesLocalVarFile != nil {
+		// loop through the array to prepare multiple files upload
+		for _, filesLocalVarFileValue := range filesLocalVarFile {
+			fbs, _ := io.ReadAll(filesLocalVarFileValue)
+
+			filesLocalVarFileBytes = fbs
+			filesLocalVarFileName = filesLocalVarFileValue.Name()
+			filesLocalVarFileValue.Close()
+			formFiles = append(formFiles, formFile{fileBytes: filesLocalVarFileBytes, fileName: filesLocalVarFileName, formFileName: filesLocalVarFormFileName})
+		}
+	}
+	parameterAddToHeaderOrQuery(localVarFormParams, "request", r.request, "", "")
+	req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
+	if err != nil {
+		return localVarReturnValue, nil, err
+	}
+
+	localVarHTTPResponse, err := a.client.callAPI(req)
+	if err != nil || localVarHTTPResponse == nil {
+		return localVarReturnValue, localVarHTTPResponse, err
+	}
+
+	localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
+	localVarHTTPResponse.Body.Close()
+	localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
+	if err != nil {
+		return localVarReturnValue, localVarHTTPResponse, err
+	}
+
+	if localVarHTTPResponse.StatusCode >= 300 {
+		newErr := &GenericOpenAPIError{
+			body:  localVarBody,
+			error: localVarHTTPResponse.Status,
+		}
+		if localVarHTTPResponse.StatusCode == 422 {
+			var v HTTPValidationError
+			err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+			if err != nil {
+				newErr.error = err.Error()
+				return localVarReturnValue, localVarHTTPResponse, newErr
+			}
+					newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
+					newErr.model = v
+		}
+		return localVarReturnValue, localVarHTTPResponse, newErr
+	}
+
+	err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+	if err != nil {
+		newErr := &GenericOpenAPIError{
+			body:  localVarBody,
+			error: err.Error(),
+		}
+		return localVarReturnValue, localVarHTTPResponse, newErr
+	}
+
+	return localVarReturnValue, localVarHTTPResponse, nil
+}

hindsight-clients/go/client.go

@@ -57,6 +57,8 @@ type APIClient struct {
 
 	EntitiesAPI *EntitiesAPIService
 
+	FilesAPI *FilesAPIService
+
 	MemoryAPI *MemoryAPIService
 
 	MentalModelsAPI *MentalModelsAPIService
@@ -86,6 +88,7 @@ func NewAPIClient(cfg *Configuration) *APIClient {
 	c.DirectivesAPI = (*DirectivesAPIService)(&c.common)
 	c.DocumentsAPI = (*DocumentsAPIService)(&c.common)
 	c.EntitiesAPI = (*EntitiesAPIService)(&c.common)
+	c.FilesAPI = (*FilesAPIService)(&c.common)
 	c.MemoryAPI = (*MemoryAPIService)(&c.common)
 	c.MentalModelsAPI = (*MentalModelsAPIService)(&c.common)
 	c.MonitoringAPI = (*MonitoringAPIService)(&c.common)

hindsight-clients/go/model_features_info.go

@@ -29,6 +29,8 @@ type FeaturesInfo struct {
 	Worker bool `json:"worker"`
 	// Whether per-bank configuration API is enabled
 	BankConfigApi bool `json:"bank_config_api"`
+	// Whether file upload/conversion API is enabled
+	FileUploadApi bool `json:"file_upload_api"`
 }
 
 type _FeaturesInfo FeaturesInfo
@@ -37,12 +39,13 @@ type _FeaturesInfo FeaturesInfo
 // This constructor will assign default values to properties that have it defined,
 // and makes sure properties required by API are set, but the set of arguments
 // will change when the set of required properties is changed
-func NewFeaturesInfo(observations bool, mcp bool, worker bool, bankConfigApi bool) *FeaturesInfo {
+func NewFeaturesInfo(observations bool, mcp bool, worker bool, bankConfigApi bool, fileUploadApi bool) *FeaturesInfo {
 	this := FeaturesInfo{}
 	this.Observations = observations
 	this.Mcp = mcp
 	this.Worker = worker
 	this.BankConfigApi = bankConfigApi
+	this.FileUploadApi = fileUploadApi
 	return &this
 }
 
@@ -150,6 +153,30 @@ func (o *FeaturesInfo) SetBankConfigApi(v bool) {
 	o.BankConfigApi = v
 }
 
+// GetFileUploadApi returns the FileUploadApi field value
+func (o *FeaturesInfo) GetFileUploadApi() bool {
+	if o == nil {
+		var ret bool
+		return ret
+	}
+
+	return o.FileUploadApi
+}
+
+// GetFileUploadApiOk returns a tuple with the FileUploadApi field value
+// and a boolean to check if the value has been set.
+func (o *FeaturesInfo) GetFileUploadApiOk() (*bool, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return &o.FileUploadApi, true
+}
+
+// SetFileUploadApi sets field value
+func (o *FeaturesInfo) SetFileUploadApi(v bool) {
+	o.FileUploadApi = v
+}
+
 func (o FeaturesInfo) MarshalJSON() ([]byte, error) {
 	toSerialize,err := o.ToMap()
 	if err != nil {
@@ -164,6 +191,7 @@ func (o FeaturesInfo) ToMap() (map[string]interface{}, error) {
 	toSerialize["mcp"] = o.Mcp
 	toSerialize["worker"] = o.Worker
 	toSerialize["bank_config_api"] = o.BankConfigApi
+	toSerialize["file_upload_api"] = o.FileUploadApi
 	return toSerialize, nil
 }
 
@@ -176,6 +204,7 @@ func (o *FeaturesInfo) UnmarshalJSON(data []byte) (err error) {
 		"mcp",
 		"worker",
 		"bank_config_api",
+		"file_upload_api",
 	}
 
 	allProperties := make(map[string]interface{})

hindsight-clients/go/model_file_retain_response.go

@@ -0,0 +1,159 @@
+/*
+Hindsight HTTP API
+
+HTTP API for Hindsight
+
+API version: 0.4.11
+*/
+
+// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
+
+package hindsight
+
+import (
+	"encoding/json"
+	"bytes"
+	"fmt"
+)
+
+// checks if the FileRetainResponse type satisfies the MappedNullable interface at compile time
+var _ MappedNullable = &FileRetainResponse{}
+
+// FileRetainResponse Response model for file upload endpoint.
+type FileRetainResponse struct {
+	// Operation IDs for tracking file conversion operations. Use GET /v1/default/banks/{bank_id}/operations to list operations.
+	OperationIds []string `json:"operation_ids"`
+}
+
+type _FileRetainResponse FileRetainResponse
+
+// NewFileRetainResponse instantiates a new FileRetainResponse object
+// This constructor will assign default values to properties that have it defined,
+// and makes sure properties required by API are set, but the set of arguments
+// will change when the set of required properties is changed
+func NewFileRetainResponse(operationIds []string) *FileRetainResponse {
+	this := FileRetainResponse{}
+	this.OperationIds = operationIds
+	return &this
+}
+
+// NewFileRetainResponseWithDefaults instantiates a new FileRetainResponse object
+// This constructor will only assign default values to properties that have it defined,
+// but it doesn't guarantee that properties required by API are set
+func NewFileRetainResponseWithDefaults() *FileRetainResponse {
+	this := FileRetainResponse{}
+	return &this
+}
+
+// GetOperationIds returns the OperationIds field value
+func (o *FileRetainResponse) GetOperationIds() []string {
+	if o == nil {
+		var ret []string
+		return ret
+	}
+
+	return o.OperationIds
+}
+
+// GetOperationIdsOk returns a tuple with the OperationIds field value
+// and a boolean to check if the value has been set.
+func (o *FileRetainResponse) GetOperationIdsOk() ([]string, bool) {
+	if o == nil {
+		return nil, false
+	}
+	return o.OperationIds, true
+}
+
+// SetOperationIds sets field value
+func (o *FileRetainResponse) SetOperationIds(v []string) {
+	o.OperationIds = v
+}
+
+func (o FileRetainResponse) MarshalJSON() ([]byte, error) {
+	toSerialize,err := o.ToMap()
+	if err != nil {
+		return []byte{}, err
+	}
+	return json.Marshal(toSerialize)
+}
+
+func (o FileRetainResponse) ToMap() (map[string]interface{}, error) {
+	toSerialize := map[string]interface{}{}
+	toSerialize["operation_ids"] = o.OperationIds
+	return toSerialize, nil
+}
+
+func (o *FileRetainResponse) UnmarshalJSON(data []byte) (err error) {
+	// This validates that all required properties are included in the JSON object
+	// by unmarshalling the object into a generic map with string keys and checking
+	// that every required field exists as a key in the generic map.
+	requiredProperties := []string{
+		"operation_ids",
+	}
+
+	allProperties := make(map[string]interface{})
+
+	err = json.Unmarshal(data, &allProperties)
+
+	if err != nil {
+		return err;
+	}
+
+	for _, requiredProperty := range(requiredProperties) {
+		if _, exists := allProperties[requiredProperty]; !exists {
+			return fmt.Errorf("no value given for required property %v", requiredProperty)
+		}
+	}
+
+	varFileRetainResponse := _FileRetainResponse{}
+
+	decoder := json.NewDecoder(bytes.NewReader(data))
+	decoder.DisallowUnknownFields()
+	err = decoder.Decode(&varFileRetainResponse)
+
+	if err != nil {
+		return err
+	}
+
+	*o = FileRetainResponse(varFileRetainResponse)
+
+	return err
+}
+
+type NullableFileRetainResponse struct {
+	value *FileRetainResponse
+	isSet bool
+}
+
+func (v NullableFileRetainResponse) Get() *FileRetainResponse {
+	return v.value
+}
+
+func (v *NullableFileRetainResponse) Set(val *FileRetainResponse) {
+	v.value = val
+	v.isSet = true
+}
+
+func (v NullableFileRetainResponse) IsSet() bool {
+	return v.isSet
+}
+
+func (v *NullableFileRetainResponse) Unset() {
+	v.value = nil
+	v.isSet = false
+}
+
+func NewNullableFileRetainResponse(val *FileRetainResponse) *NullableFileRetainResponse {
+	return &NullableFileRetainResponse{value: val, isSet: true}
+}
+
+func (v NullableFileRetainResponse) MarshalJSON() ([]byte, error) {
+	return json.Marshal(v.value)
+}
+
+func (v *NullableFileRetainResponse) UnmarshalJSON(src []byte) error {
+	v.isSet = true
+	return json.Unmarshal(src, &v.value)
+}
+
+

hindsight-clients/python/.openapi-generator/FILES

@@ -4,6 +4,7 @@ hindsight_client_api/api/banks_api.py
 hindsight_client_api/api/directives_api.py
 hindsight_client_api/api/documents_api.py
 hindsight_client_api/api/entities_api.py
+hindsight_client_api/api/files_api.py
 hindsight_client_api/api/memory_api.py
 hindsight_client_api/api/mental_models_api.py
 hindsight_client_api/api/monitoring_api.py
@@ -47,6 +48,7 @@ hindsight_client_api/models/entity_list_response.py
 hindsight_client_api/models/entity_observation_response.py
 hindsight_client_api/models/entity_state_response.py
 hindsight_client_api/models/features_info.py
+hindsight_client_api/models/file_retain_response.py
 hindsight_client_api/models/graph_data_response.py
 hindsight_client_api/models/http_validation_error.py
 hindsight_client_api/models/include_options.py

hindsight-clients/python/hindsight_client/hindsight_client.py

@@ -6,11 +6,13 @@ easy-to-use interface on top of the auto-generated OpenAPI client.
 """
 
 import asyncio
+import json
 from datetime import datetime
+from pathlib import Path
 from typing import Any, Literal
 
 import hindsight_client_api
-from hindsight_client_api.api import banks_api, directives_api, memory_api, mental_models_api
+from hindsight_client_api.api import banks_api, directives_api, files_api, memory_api, mental_models_api
 from hindsight_client_api.models import (
     memory_item,
     recall_request,
@@ -18,6 +20,7 @@ from hindsight_client_api.models import (
     retain_request,
 )
 from hindsight_client_api.models.bank_profile_response import BankProfileResponse
+from hindsight_client_api.models.file_retain_response import FileRetainResponse
 from hindsight_client_api.models.list_memory_units_response import ListMemoryUnitsResponse
 from hindsight_client_api.models.recall_response import RecallResponse
 from hindsight_client_api.models.recall_result import RecallResult
@@ -80,6 +83,7 @@ class Hindsight:
         self._banks_api = banks_api.BanksApi(self._api_client)
         self._mental_models_api = mental_models_api.MentalModelsApi(self._api_client)
         self._directives_api = directives_api.DirectivesApi(self._api_client)
+        self._files_api = files_api.FilesApi(self._api_client)
 
     def __enter__(self):
         """Context manager entry."""
@@ -199,6 +203,41 @@ class Hindsight:
 
         return _run_async(self._memory_api.retain_memories(bank_id, request_obj))
 
+    def retain_files(
+        self,
+        bank_id: str,
+        files: list[str | Path],
+        context: str | None = None,
+        files_metadata: list[dict[str, Any]] | None = None,
+    ) -> FileRetainResponse:
+        """
+        Upload files and retain their contents as memories.
+
+        Files are automatically converted to text (PDF, DOCX, images via OCR, audio via
+        transcription, and more) and ingested as memories. Processing is always asynchronous
+        — use the returned operation IDs to track progress.
+
+        Args:
+            bank_id: The memory bank ID
+            files: List of file paths to upload
+            context: Optional context description applied to all files
+            files_metadata: Optional per-file metadata list. If provided, must match the
+                length of `files`. Each entry can have: context, document_id, tags, metadata.
+
+        Returns:
+            FileRetainResponse with operation_ids for tracking progress
+        """
+        file_data = []
+        for file_path in files:
+            path = Path(file_path)
+            file_data.append((path.name, path.read_bytes()))
+
+        meta = files_metadata or [{"context": context} if context else {} for _ in files]
+
+        request_body = json.dumps({"files_metadata": meta})
+
+        return _run_async(self._files_api.file_retain(bank_id=bank_id, files=file_data, request=request_body))
+
     def recall(
         self,
         bank_id: str,
@@ -373,6 +412,55 @@ class Hindsight:
 
     # Async methods (native async, no _run_async wrapper)
 
+    async def acreate_bank(
+        self,
+        bank_id: str,
+        name: str | None = None,
+        mission: str | None = None,
+        disposition: dict[str, float] | None = None,
+    ) -> BankProfileResponse:
+        """Create or update a memory bank (async).
+
+        Args:
+            bank_id: Unique identifier for the bank
+            name: Human-readable display name
+            mission: Instructions guiding what Hindsight should learn and remember (for mental models)
+            disposition: Optional disposition traits (skepticism, literalism, empathy)
+        """
+        from hindsight_client_api.models import create_bank_request, disposition_traits
+
+        disposition_obj = None
+        if disposition:
+            disposition_obj = disposition_traits.DispositionTraits(**disposition)
+
+        request_obj = create_bank_request.CreateBankRequest(
+            name=name,
+            mission=mission,
+            disposition=disposition_obj,
+        )
+
+        return await self._banks_api.create_or_update_bank(bank_id, request_obj)
+
+    async def aset_mission(
+        self,
+        bank_id: str,
+        mission: str,
+    ) -> BankProfileResponse:
+        """
+        Set or update the mission for a memory bank (async).
+
+        Args:
+            bank_id: The memory bank ID
+            mission: The mission text describing the agent's purpose
+
+        Returns:
+            BankProfileResponse with updated bank profile
+        """
+        from hindsight_client_api.models import create_bank_request
+
+        request_obj = create_bank_request.CreateBankRequest(mission=mission)
+        return await self._banks_api.create_or_update_bank(bank_id, request_obj)
+
     async def aretain_batch(
         self,
         bank_id: str,
@@ -471,9 +559,15 @@ class Hindsight:
         types: list[str] | None = None,
         max_tokens: int = 4096,
         budget: str = "mid",
+        trace: bool = False,
+        query_timestamp: str | None = None,
+        include_entities: bool = False,
+        max_entity_tokens: int = 500,
+        include_chunks: bool = False,
+        max_chunk_tokens: int = 8192,
         tags: list[str] | None = None,
         tags_match: Literal["any", "all", "any_strict", "all_strict"] = "any",
-    ) -> list[RecallResult]:
+    ) -> RecallResponse:
         """
         Recall memories using semantic similarity (async).
 
@@ -483,25 +577,41 @@ class Hindsight:
             types: Optional list of fact types to filter (world, experience, opinion, observation)
             max_tokens: Maximum tokens in results (default: 4096)
             budget: Budget level for recall - "low", "mid", or "high" (default: "mid")
+            trace: Enable trace output (default: False)
+            query_timestamp: Optional ISO format date string (e.g., '2023-05-30T23:40:00')
+            include_entities: Include entity observations in results (default: False)
+            max_entity_tokens: Maximum tokens for entity observations (default: 500)
+            include_chunks: Include raw text chunks in results (default: False)
+            max_chunk_tokens: Maximum tokens for chunks (default: 8192)
             tags: Optional list of tags to filter memories by
             tags_match: How to match tags - "any" (OR, includes untagged), "all" (AND, includes untagged),
                 "any_strict" (OR, excludes untagged), "all_strict" (AND, excludes untagged). Default: "any"
 
         Returns:
-            List of RecallResult objects
+            RecallResponse with results, optional entities, optional chunks, and optional trace
         """
+        from hindsight_client_api.models import chunk_include_options, entity_include_options, include_options
+
+        include_opts = include_options.IncludeOptions(
+            entities=entity_include_options.EntityIncludeOptions(max_tokens=max_entity_tokens)
+            if include_entities
+            else None,
+            chunks=chunk_include_options.ChunkIncludeOptions(max_tokens=max_chunk_tokens) if include_chunks else None,
+        )
+
         request_obj = recall_request.RecallRequest(
             query=query,
             types=types,
             budget=budget,
             max_tokens=max_tokens,
-            trace=False,
+            trace=trace,
+            query_timestamp=query_timestamp,
+            include=include_opts,
             tags=tags,
             tags_match=tags_match,
         )
 
-        response = await self._memory_api.recall_memories(bank_id, request_obj)
-        return response.results if hasattr(response, "results") else []
+        return await self._memory_api.recall_memories(bank_id, request_obj)
 
     async def areflect(
         self,
@@ -509,6 +619,8 @@ class Hindsight:
         query: str,
         budget: str = "low",
         context: str | None = None,
+        max_tokens: int | None = None,
+        response_schema: dict[str, Any] | None = None,
         tags: list[str] | None = None,
         tags_match: Literal["any", "all", "any_strict", "all_strict"] = "any",
     ) -> ReflectResponse:
@@ -520,17 +632,24 @@ class Hindsight:
             query: The question or prompt
             budget: Budget level for reflection - "low", "mid", or "high" (default: "low")
             context: Optional additional context
+            max_tokens: Maximum tokens for the response (server default: 4096)
+            response_schema: Optional JSON Schema for structured output. When provided,
+                the response will include a 'structured_output' field with the LLM
+                response parsed according to this schema.
             tags: Optional list of tags to filter memories by
             tags_match: How to match tags - "any" (OR, includes untagged), "all" (AND, includes untagged),
                 "any_strict" (OR, excludes untagged), "all_strict" (AND, excludes untagged). Default: "any"
 
         Returns:
-            ReflectResponse with answer text and optionally facts used
+            ReflectResponse with answer text, optionally facts used, and optionally
+            structured_output if response_schema was provided
         """
         request_obj = reflect_request.ReflectRequest(
             query=query,
             budget=budget,
             context=context,
+            max_tokens=max_tokens,
+            response_schema=response_schema,
             tags=tags,
             tags_match=tags_match,
         )
@@ -786,3 +905,12 @@ class Hindsight:
             bank_id: The memory bank ID
         """
         return _run_async(self._banks_api.delete_bank(bank_id))
+
+    async def adelete_bank(self, bank_id: str):
+        """
+        Delete a memory bank (async).
+
+        Args:
+            bank_id: The memory bank ID
+        """
+        return await self._banks_api.delete_bank(bank_id)

hindsight-clients/python/hindsight_client_api/__init__.py

@@ -21,6 +21,7 @@ from hindsight_client_api.api.banks_api import BanksApi
 from hindsight_client_api.api.directives_api import DirectivesApi
 from hindsight_client_api.api.documents_api import DocumentsApi
 from hindsight_client_api.api.entities_api import EntitiesApi
+from hindsight_client_api.api.files_api import FilesApi
 from hindsight_client_api.api.memory_api import MemoryApi
 from hindsight_client_api.api.mental_models_api import MentalModelsApi
 from hindsight_client_api.api.monitoring_api import MonitoringApi
@@ -72,6 +73,7 @@ from hindsight_client_api.models.entity_list_response import EntityListResponse
 from hindsight_client_api.models.entity_observation_response import EntityObservationResponse
 from hindsight_client_api.models.entity_state_response import EntityStateResponse
 from hindsight_client_api.models.features_info import FeaturesInfo
+from hindsight_client_api.models.file_retain_response import FileRetainResponse
 from hindsight_client_api.models.graph_data_response import GraphDataResponse
 from hindsight_client_api.models.http_validation_error import HTTPValidationError
 from hindsight_client_api.models.include_options import IncludeOptions

hindsight-clients/python/hindsight_client_api/api/__init__.py

@@ -5,6 +5,7 @@ from hindsight_client_api.api.banks_api import BanksApi
 from hindsight_client_api.api.directives_api import DirectivesApi
 from hindsight_client_api.api.documents_api import DocumentsApi
 from hindsight_client_api.api.entities_api import EntitiesApi
+from hindsight_client_api.api.files_api import FilesApi
 from hindsight_client_api.api.memory_api import MemoryApi
 from hindsight_client_api.api.mental_models_api import MentalModelsApi
 from hindsight_client_api.api.monitoring_api import MonitoringApi

hindsight-clients/python/hindsight_client_api/api/files_api.py

@@ -0,0 +1,361 @@
+# coding: utf-8
+
+"""
+    Hindsight HTTP API
+
+    HTTP API for Hindsight
+
+    The version of the OpenAPI document: 0.4.11
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+import warnings
+from pydantic import validate_call, Field, StrictFloat, StrictStr, StrictInt
+from typing import Any, Dict, List, Optional, Tuple, Union
+from typing_extensions import Annotated
+
+from pydantic import Field, StrictBytes, StrictStr
+from typing import List, Optional, Tuple, Union
+from typing_extensions import Annotated
+from hindsight_client_api.models.file_retain_response import FileRetainResponse
+
+from hindsight_client_api.api_client import ApiClient, RequestSerialized
+from hindsight_client_api.api_response import ApiResponse
+from hindsight_client_api.rest import RESTResponseType
+
+
+class FilesApi:
+    """NOTE: This class is auto generated by OpenAPI Generator
+    Ref: https://openapi-generator.tech
+
+    Do not edit the class manually.
+    """
+
+    def __init__(self, api_client=None) -> None:
+        if api_client is None:
+            api_client = ApiClient.get_default()
+        self.api_client = api_client
+
+
+    @validate_call
+    async def file_retain(
+        self,
+        bank_id: StrictStr,
+        files: Annotated[List[Union[StrictBytes, StrictStr, Tuple[StrictStr, StrictBytes]]], Field(description="Files to upload and convert")],
+        request: Annotated[StrictStr, Field(description="JSON string with FileRetainRequest model")],
+        authorization: Optional[StrictStr] = None,
+        _request_timeout: Union[
+            None,
+            Annotated[StrictFloat, Field(gt=0)],
+            Tuple[
+                Annotated[StrictFloat, Field(gt=0)],
+                Annotated[StrictFloat, Field(gt=0)]
+            ]
+        ] = None,
+        _request_auth: Optional[Dict[StrictStr, Any]] = None,
+        _content_type: Optional[StrictStr] = None,
+        _headers: Optional[Dict[StrictStr, Any]] = None,
+        _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
+    ) -> FileRetainResponse:
+        """Convert files to memories
+
+        Upload files (PDF, DOCX, etc.), convert them to markdown, and retain as memories.  This endpoint handles file upload, conversion, and memory creation in a single operation.  **Features:** - Supports PDF, DOCX, PPTX, XLSX, images (with OCR), audio (with transcription) - Automatic file-to-markdown conversion using pluggable parsers - Files stored in object storage (PostgreSQL by default, S3 for production) - Each file becomes a separate document with optional metadata/tags - Always processes asynchronously — returns operation IDs immediately  **The system automatically:** 1. Stores uploaded files in object storage 2. Converts files to markdown 3. Creates document records with file metadata 4. Extracts facts and creates memory units (same as regular retain)  Use the operations endpoint to monitor progress.  **Request format:** multipart/form-data with: - `files`: One or more files to upload - `request`: JSON string with FileRetainRequest model (files_metadata)  **Note:** File parser is configured server-side via `HINDSIGHT_API_FILE_PARSER` (default: markitdown).
+
+        :param bank_id: (required)
+        :type bank_id: str
+        :param files: Files to upload and convert (required)
+        :type files: List[bytearray]
+        :param request: JSON string with FileRetainRequest model (required)
+        :type request: str
+        :param authorization:
+        :type authorization: str
+        :param _request_timeout: timeout setting for this request. If one
+                                 number provided, it will be total request
+                                 timeout. It can also be a pair (tuple) of
+                                 (connection, read) timeouts.
+        :type _request_timeout: int, tuple(int, int), optional
+        :param _request_auth: set to override the auth_settings for an a single
+                              request; this effectively ignores the
+                              authentication in the spec for a single request.
+        :type _request_auth: dict, optional
+        :param _content_type: force content-type for the request.
+        :type _content_type: str, Optional
+        :param _headers: set to override the headers for a single
+                         request; this effectively ignores the headers
+                         in the spec for a single request.
+        :type _headers: dict, optional
+        :param _host_index: set to override the host_index for a single
+                            request; this effectively ignores the host_index
+                            in the spec for a single request.
+        :type _host_index: int, optional
+        :return: Returns the result object.
+        """ # noqa: E501
+
+        _param = self._file_retain_serialize(
+            bank_id=bank_id,
+            files=files,
+            request=request,
+            authorization=authorization,
+            _request_auth=_request_auth,
+            _content_type=_content_type,
+            _headers=_headers,
+            _host_index=_host_index
+        )
+
+        _response_types_map: Dict[str, Optional[str]] = {
+            '200': "FileRetainResponse",
+            '422': "HTTPValidationError",
+        }
+        response_data = await self.api_client.call_api(
+            *_param,
+            _request_timeout=_request_timeout
+        )
+        await response_data.read()
+        return self.api_client.response_deserialize(
+            response_data=response_data,
+            response_types_map=_response_types_map,
+        ).data
+
+
+    @validate_call
+    async def file_retain_with_http_info(
+        self,
+        bank_id: StrictStr,
+        files: Annotated[List[Union[StrictBytes, StrictStr, Tuple[StrictStr, StrictBytes]]], Field(description="Files to upload and convert")],
+        request: Annotated[StrictStr, Field(description="JSON string with FileRetainRequest model")],
+        authorization: Optional[StrictStr] = None,
+        _request_timeout: Union[
+            None,
+            Annotated[StrictFloat, Field(gt=0)],
+            Tuple[
+                Annotated[StrictFloat, Field(gt=0)],
+                Annotated[StrictFloat, Field(gt=0)]
+            ]
+        ] = None,
+        _request_auth: Optional[Dict[StrictStr, Any]] = None,
+        _content_type: Optional[StrictStr] = None,
+        _headers: Optional[Dict[StrictStr, Any]] = None,
+        _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
+    ) -> ApiResponse[FileRetainResponse]:
+        """Convert files to memories
+
+        Upload files (PDF, DOCX, etc.), convert them to markdown, and retain as memories.  This endpoint handles file upload, conversion, and memory creation in a single operation.  **Features:** - Supports PDF, DOCX, PPTX, XLSX, images (with OCR), audio (with transcription) - Automatic file-to-markdown conversion using pluggable parsers - Files stored in object storage (PostgreSQL by default, S3 for production) - Each file becomes a separate document with optional metadata/tags - Always processes asynchronously — returns operation IDs immediately  **The system automatically:** 1. Stores uploaded files in object storage 2. Converts files to markdown 3. Creates document records with file metadata 4. Extracts facts and creates memory units (same as regular retain)  Use the operations endpoint to monitor progress.  **Request format:** multipart/form-data with: - `files`: One or more files to upload - `request`: JSON string with FileRetainRequest model (files_metadata)  **Note:** File parser is configured server-side via `HINDSIGHT_API_FILE_PARSER` (default: markitdown).
+
+        :param bank_id: (required)
+        :type bank_id: str
+        :param files: Files to upload and convert (required)
+        :type files: List[bytearray]
+        :param request: JSON string with FileRetainRequest model (required)
+        :type request: str
+        :param authorization:
+        :type authorization: str
+        :param _request_timeout: timeout setting for this request. If one
+                                 number provided, it will be total request
+                                 timeout. It can also be a pair (tuple) of
+                                 (connection, read) timeouts.
+        :type _request_timeout: int, tuple(int, int), optional
+        :param _request_auth: set to override the auth_settings for an a single
+                              request; this effectively ignores the
+                              authentication in the spec for a single request.
+        :type _request_auth: dict, optional
+        :param _content_type: force content-type for the request.
+        :type _content_type: str, Optional
+        :param _headers: set to override the headers for a single
+                         request; this effectively ignores the headers
+                         in the spec for a single request.
+        :type _headers: dict, optional
+        :param _host_index: set to override the host_index for a single
+                            request; this effectively ignores the host_index
+                            in the spec for a single request.
+        :type _host_index: int, optional
+        :return: Returns the result object.
+        """ # noqa: E501
+
+        _param = self._file_retain_serialize(
+            bank_id=bank_id,
+            files=files,
+            request=request,
+            authorization=authorization,
+            _request_auth=_request_auth,
+            _content_type=_content_type,
+            _headers=_headers,
+            _host_index=_host_index
+        )
+
+        _response_types_map: Dict[str, Optional[str]] = {
+            '200': "FileRetainResponse",
+            '422': "HTTPValidationError",
+        }
+        response_data = await self.api_client.call_api(
+            *_param,
+            _request_timeout=_request_timeout
+        )
+        await response_data.read()
+        return self.api_client.response_deserialize(
+            response_data=response_data,
+            response_types_map=_response_types_map,
+        )
+
+
+    @validate_call
+    async def file_retain_without_preload_content(
+        self,
+        bank_id: StrictStr,
+        files: Annotated[List[Union[StrictBytes, StrictStr, Tuple[StrictStr, StrictBytes]]], Field(description="Files to upload and convert")],
+        request: Annotated[StrictStr, Field(description="JSON string with FileRetainRequest model")],
+        authorization: Optional[StrictStr] = None,
+        _request_timeout: Union[
+            None,
+            Annotated[StrictFloat, Field(gt=0)],
+            Tuple[
+                Annotated[StrictFloat, Field(gt=0)],
+                Annotated[StrictFloat, Field(gt=0)]
+            ]
+        ] = None,
+        _request_auth: Optional[Dict[StrictStr, Any]] = None,
+        _content_type: Optional[StrictStr] = None,
+        _headers: Optional[Dict[StrictStr, Any]] = None,
+        _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
+    ) -> RESTResponseType:
+        """Convert files to memories
+
+        Upload files (PDF, DOCX, etc.), convert them to markdown, and retain as memories.  This endpoint handles file upload, conversion, and memory creation in a single operation.  **Features:** - Supports PDF, DOCX, PPTX, XLSX, images (with OCR), audio (with transcription) - Automatic file-to-markdown conversion using pluggable parsers - Files stored in object storage (PostgreSQL by default, S3 for production) - Each file becomes a separate document with optional metadata/tags - Always processes asynchronously — returns operation IDs immediately  **The system automatically:** 1. Stores uploaded files in object storage 2. Converts files to markdown 3. Creates document records with file metadata 4. Extracts facts and creates memory units (same as regular retain)  Use the operations endpoint to monitor progress.  **Request format:** multipart/form-data with: - `files`: One or more files to upload - `request`: JSON string with FileRetainRequest model (files_metadata)  **Note:** File parser is configured server-side via `HINDSIGHT_API_FILE_PARSER` (default: markitdown).
+
+        :param bank_id: (required)
+        :type bank_id: str
+        :param files: Files to upload and convert (required)
+        :type files: List[bytearray]
+        :param request: JSON string with FileRetainRequest model (required)
+        :type request: str
+        :param authorization:
+        :type authorization: str
+        :param _request_timeout: timeout setting for this request. If one
+                                 number provided, it will be total request
+                                 timeout. It can also be a pair (tuple) of
+                                 (connection, read) timeouts.
+        :type _request_timeout: int, tuple(int, int), optional
+        :param _request_auth: set to override the auth_settings for an a single
+                              request; this effectively ignores the
+                              authentication in the spec for a single request.
+        :type _request_auth: dict, optional
+        :param _content_type: force content-type for the request.
+        :type _content_type: str, Optional
+        :param _headers: set to override the headers for a single
+                         request; this effectively ignores the headers
+                         in the spec for a single request.
+        :type _headers: dict, optional
+        :param _host_index: set to override the host_index for a single
+                            request; this effectively ignores the host_index
+                            in the spec for a single request.
+        :type _host_index: int, optional
+        :return: Returns the result object.
+        """ # noqa: E501
+
+        _param = self._file_retain_serialize(
+            bank_id=bank_id,
+            files=files,
+            request=request,
+            authorization=authorization,
+            _request_auth=_request_auth,
+            _content_type=_content_type,
+            _headers=_headers,
+            _host_index=_host_index
+        )
+
+        _response_types_map: Dict[str, Optional[str]] = {
+            '200': "FileRetainResponse",
+            '422': "HTTPValidationError",
+        }
+        response_data = await self.api_client.call_api(
+            *_param,
+            _request_timeout=_request_timeout
+        )
+        return response_data.response
+
+
+    def _file_retain_serialize(
+        self,
+        bank_id,
+        files,
+        request,
+        authorization,
+        _request_auth,
+        _content_type,
+        _headers,
+        _host_index,
+    ) -> RequestSerialized:
+
+        _host = None
+
+        _collection_formats: Dict[str, str] = {
+            'files': 'csv',
+        }
+
+        _path_params: Dict[str, str] = {}
+        _query_params: List[Tuple[str, str]] = []
+        _header_params: Dict[str, Optional[str]] = _headers or {}
+        _form_params: List[Tuple[str, str]] = []
+        _files: Dict[
+            str, Union[str, bytes, List[str], List[bytes], List[Tuple[str, bytes]]]
+        ] = {}
+        _body_params: Optional[bytes] = None
+
+        # process the path parameters
+        if bank_id is not None:
+            _path_params['bank_id'] = bank_id
+        # process the query parameters
+        # process the header parameters
+        if authorization is not None:
+            _header_params['authorization'] = authorization
+        # process the form parameters
+        if files is not None:
+            _files['files'] = files
+        if request is not None:
+            _form_params.append(('request', request))
+        # process the body parameter
+
+
+        # set the HTTP header `Accept`
+        if 'Accept' not in _header_params:
+            _header_params['Accept'] = self.api_client.select_header_accept(
+                [
+                    'application/json'
+                ]
+            )
+
+        # set the HTTP header `Content-Type`
+        if _content_type:
+            _header_params['Content-Type'] = _content_type
+        else:
+            _default_content_type = (
+                self.api_client.select_header_content_type(
+                    [
+                        'multipart/form-data'
+                    ]
+                )
+            )
+            if _default_content_type is not None:
+                _header_params['Content-Type'] = _default_content_type
+
+        # authentication setting
+        _auth_settings: List[str] = [
+        ]
+
+        return self.api_client.param_serialize(
+            method='POST',
+            resource_path='/v1/default/banks/{bank_id}/files/retain',
+            path_params=_path_params,
+            query_params=_query_params,
+            header_params=_header_params,
+            body=_body_params,
+            post_params=_form_params,
+            files=_files,
+            auth_settings=_auth_settings,
+            collection_formats=_collection_formats,
+            _host=_host,
+            _request_auth=_request_auth
+        )
+
+

hindsight-clients/python/hindsight_client_api/models/__init__.py

@@ -48,6 +48,7 @@ from hindsight_client_api.models.entity_list_response import EntityListResponse
 from hindsight_client_api.models.entity_observation_response import EntityObservationResponse
 from hindsight_client_api.models.entity_state_response import EntityStateResponse
 from hindsight_client_api.models.features_info import FeaturesInfo
+from hindsight_client_api.models.file_retain_response import FileRetainResponse
 from hindsight_client_api.models.graph_data_response import GraphDataResponse
 from hindsight_client_api.models.http_validation_error import HTTPValidationError
 from hindsight_client_api.models.include_options import IncludeOptions

hindsight-clients/python/hindsight_client_api/models/features_info.py

@@ -30,7 +30,8 @@ class FeaturesInfo(BaseModel):
     mcp: StrictBool = Field(description="Whether MCP (Model Context Protocol) server is enabled")
     worker: StrictBool = Field(description="Whether the background worker is enabled")
     bank_config_api: StrictBool = Field(description="Whether per-bank configuration API is enabled")
-    __properties: ClassVar[List[str]] = ["observations", "mcp", "worker", "bank_config_api"]
+    file_upload_api: StrictBool = Field(description="Whether file upload/conversion API is enabled")
+    __properties: ClassVar[List[str]] = ["observations", "mcp", "worker", "bank_config_api", "file_upload_api"]
 
     model_config = ConfigDict(
         populate_by_name=True,
@@ -86,7 +87,8 @@ class FeaturesInfo(BaseModel):
             "observations": obj.get("observations"),
             "mcp": obj.get("mcp"),
             "worker": obj.get("worker"),
-            "bank_config_api": obj.get("bank_config_api")
+            "bank_config_api": obj.get("bank_config_api"),
+            "file_upload_api": obj.get("file_upload_api")
         })
         return _obj
 

hindsight-clients/python/hindsight_client_api/models/file_retain_response.py

@@ -0,0 +1,87 @@
+# coding: utf-8
+
+"""
+    Hindsight HTTP API
+
+    HTTP API for Hindsight
+
+    The version of the OpenAPI document: 0.4.11
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+from pydantic import BaseModel, ConfigDict, Field, StrictStr
+from typing import Any, ClassVar, Dict, List
+from typing import Optional, Set
+from typing_extensions import Self
+
+class FileRetainResponse(BaseModel):
+    """
+    Response model for file upload endpoint.
+    """ # noqa: E501
+    operation_ids: List[StrictStr] = Field(description="Operation IDs for tracking file conversion operations. Use GET /v1/default/banks/{bank_id}/operations to list operations.")
+    __properties: ClassVar[List[str]] = ["operation_ids"]
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        validate_assignment=True,
+        protected_namespaces=(),
+    )
+
+
+    def to_str(self) -> str:
+        """Returns the string representation of the model using alias"""
+        return pprint.pformat(self.model_dump(by_alias=True))
+
+    def to_json(self) -> str:
+        """Returns the JSON representation of the model using alias"""
+        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str) -> Optional[Self]:
+        """Create an instance of FileRetainResponse from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return the dictionary representation of the model using alias.
+
+        This has the following differences from calling pydantic's
+        `self.model_dump(by_alias=True)`:
+
+        * `None` is only added to the output dict for nullable fields that
+          were set at model initialization. Other fields with value `None`
+          are ignored.
+        """
+        excluded_fields: Set[str] = set([
+        ])
+
+        _dict = self.model_dump(
+            by_alias=True,
+            exclude=excluded_fields,
+            exclude_none=True,
+        )
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
+        """Create an instance of FileRetainResponse from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return cls.model_validate(obj)
+
+        _obj = cls.model_validate({
+            "operation_ids": obj.get("operation_ids")
+        })
+        return _obj
+
+

hindsight-clients/python/tests/test_main_operations.py

@@ -522,11 +522,11 @@ class TestTags:
         client.retain_batch(
             bank_id=bank_id,
             items=[
-                {"content": "Project X meeting notes from Monday", "tags": ["project_x", "meetings"]},
-                {"content": "Project X design document", "tags": ["project_x", "docs"]},
-                {"content": "Project Y sprint planning", "tags": ["project_y", "meetings"]},
-                {"content": "General company announcement", "tags": ["company"]},
-                {"content": "Untagged memory about random things"},  # no tags
+                {"content": "Alice presented the Q3 roadmap at the Monday standup", "tags": ["project_x", "meetings"]},
+                {"content": "Bob wrote the architecture document for the new auth system", "tags": ["project_x", "docs"]},
+                {"content": "Charlie led the sprint planning session for the mobile app", "tags": ["project_y", "meetings"]},
+                {"content": "Diana announced the company picnic for next Friday", "tags": ["company"]},
+                {"content": "Eve mentioned she likes pineapple on pizza"},  # no tags
             ],
             retain_async=False,
         )
@@ -535,68 +535,73 @@ class TestTags:
         """Test recall with tags using 'any' match (includes untagged)."""
         response = client.recall(
             bank_id=bank_id,
-            query="What are the documents?",
+            query="What has everyone been working on?",
             tags=["project_x"],
             tags_match="any",
+            max_tokens=16000,
         )
 
         assert response is not None
         assert response.results is not None
-        # Should include project_x tagged items and potentially untagged items
-        result_texts = [r.text.lower() for r in response.results]
-        assert any("project x" in text for text in result_texts)
+        assert len(response.results) > 0
+        # 'any' mode: results should include items matching the tag or untagged items
+        result_tags = [set(r.tags) if r.tags else set() for r in response.results]
+        assert any("project_x" in tags for tags in result_tags)
 
     def test_recall_with_tags_any_strict(self, client, bank_id):
         """Test recall with tags using 'any_strict' match (excludes untagged)."""
         response = client.recall(
             bank_id=bank_id,
-            query="meetings",
+            query="What has everyone been working on?",
             tags=["project_x"],
             tags_match="any_strict",
+            max_tokens=16000,
         )
 
         assert response is not None
         assert response.results is not None
-        # All results should have project_x tag - no untagged items
-        result_texts = [r.text.lower() for r in response.results]
-        # Should find project_x items only
-        for text in result_texts:
-            assert "project x" in text or "untagged" not in text
+        assert len(response.results) > 0
+        # any_strict: every result must have the project_x tag
+        for r in response.results:
+            assert r.tags is not None
+            assert "project_x" in r.tags
 
     def test_recall_with_tags_all_strict(self, client, bank_id):
         """Test recall with tags using 'all_strict' match (AND matching)."""
         response = client.recall(
             bank_id=bank_id,
-            query="meeting notes",
+            query="What has everyone been working on?",
             tags=["project_x", "meetings"],
             tags_match="all_strict",
+            max_tokens=16000,
         )
 
         assert response is not None
         assert response.results is not None
-        # Should only return items tagged with BOTH project_x AND meetings
-        if len(response.results) > 0:
-            result_texts = [r.text.lower() for r in response.results]
-            # The "Project X meeting notes" should be found
-            assert any("project x" in text and "meeting" in text for text in result_texts)
+        assert len(response.results) > 0
+        # all_strict: every result must have BOTH tags
+        for r in response.results:
+            assert r.tags is not None
+            assert "project_x" in r.tags
+            assert "meetings" in r.tags
 
     def test_recall_with_multiple_tags_any(self, client, bank_id):
-        """Test recall with multiple tags using 'any' match (OR)."""
+        """Test recall with multiple tags using 'any_strict' match (OR)."""
         response = client.recall(
             bank_id=bank_id,
-            query="What's happening?",
+            query="What has everyone been working on?",
             tags=["project_x", "project_y"],
             tags_match="any_strict",
+            max_tokens=16000,
         )
 
         assert response is not None
         assert response.results is not None
-        # Should include items from both project_x and project_y
-        result_texts = [r.text.lower() for r in response.results]
-        has_project_x = any("project x" in text for text in result_texts)
-        has_project_y = any("project y" in text for text in result_texts)
-        # At least one of them should be present
-        assert has_project_x or has_project_y
+        assert len(response.results) > 0
+        # any_strict with multiple tags: every result must have at least one of the tags
+        for r in response.results:
+            assert r.tags is not None
+            assert "project_x" in r.tags or "project_y" in r.tags
 
     def test_reflect_with_tags(self, client, bank_id):
         """Test reflect with tags filtering."""
@@ -615,7 +620,7 @@ class TestTags:
         """Test storing a memory with tags."""
         response = client.retain(
             bank_id=bank_id,
-            content="New feature implementation for project Z",
+            content="Frank deployed the billing microservice to production on Tuesday",
             tags=["project_z", "features"],
         )
 
@@ -625,13 +630,17 @@ class TestTags:
         # Verify we can recall it with the tag
         recall_response = client.recall(
             bank_id=bank_id,
-            query="project Z features",
+            query="What did Frank deploy?",
             tags=["project_z"],
             tags_match="any_strict",
+            max_tokens=16000,
         )
         assert recall_response is not None
-        result_texts = [r.text.lower() for r in recall_response.results]
-        assert any("project z" in text for text in result_texts)
+        assert len(recall_response.results) > 0
+        # any_strict: every result must have the project_z tag
+        for r in recall_response.results:
+            assert r.tags is not None
+            assert "project_z" in r.tags
 
     def test_retain_batch_with_document_tags(self, client, bank_id):
         """Test batch retain with document-level tags."""
@@ -702,3 +711,173 @@ class TestMission:
         assert response is not None
         assert response.bank_id == bank_id
         assert response.mission == "Be a helpful PM tracking sprint progress and team capacity"
+
+
+class TestAsyncRecall:
+    """Tests for async recall with full feature parity."""
+
+    @pytest.fixture(autouse=True)
+    async def setup_memories(self, client, bank_id):
+        """Setup: Store some test memories before search tests."""
+        await client.aretain_batch(
+            bank_id=bank_id,
+            items=[
+                {"content": "Alice loves programming in Python"},
+                {"content": "Bob enjoys hiking and outdoor adventures"},
+                {"content": "Charlie is interested in quantum physics"},
+            ],
+        )
+
+    @pytest.mark.asyncio
+    async def test_arecall_returns_recall_response(self, client, bank_id):
+        """Test that arecall returns a RecallResponse (not a list)."""
+        from hindsight_client_api.models.recall_response import RecallResponse
+
+        response = await client.arecall(
+            bank_id=bank_id,
+            query="What does Alice like?",
+        )
+
+        assert isinstance(response, RecallResponse)
+        assert response.results is not None
+        assert len(response.results) > 0
+
+    @pytest.mark.asyncio
+    async def test_arecall_with_include_chunks(self, client, bank_id):
+        """Test arecall with include_chunks returns raw text chunks."""
+        response = await client.arecall(
+            bank_id=bank_id,
+            query="programming",
+            include_chunks=True,
+        )
+
+        assert response is not None
+        assert response.results is not None
+        # chunks should be present (may be empty dict if not yet consolidated)
+        assert response.chunks is not None
+
+    @pytest.mark.asyncio
+    async def test_arecall_with_include_entities(self, client, bank_id):
+        """Test arecall with include_entities."""
+        response = await client.arecall(
+            bank_id=bank_id,
+            query="Alice",
+            include_entities=True,
+            max_entity_tokens=500,
+        )
+
+        assert response is not None
+        assert response.results is not None
+
+    @pytest.mark.asyncio
+    async def test_arecall_with_trace(self, client, bank_id):
+        """Test arecall with trace enabled."""
+        response = await client.arecall(
+            bank_id=bank_id,
+            query="outdoor activities",
+            trace=True,
+        )
+
+        assert response is not None
+        assert response.results is not None
+
+    @pytest.mark.asyncio
+    async def test_arecall_full_featured(self, client, bank_id):
+        """Test arecall with all parameters."""
+        response = await client.arecall(
+            bank_id=bank_id,
+            query="What are people's interests?",
+            types=["world"],
+            max_tokens=2048,
+            budget="high",
+            trace=True,
+            include_chunks=True,
+            max_chunk_tokens=4096,
+            include_entities=True,
+            max_entity_tokens=500,
+        )
+
+        assert response is not None
+        assert response.results is not None
+        assert response.chunks is not None
+
+
+class TestAsyncReflect:
+    """Tests for async reflect with full feature parity."""
+
+    @pytest.fixture(autouse=True)
+    async def setup_memories(self, client, bank_id):
+        """Setup: Store some test memories and bank background."""
+        await client.acreate_bank(
+            bank_id=bank_id,
+            mission="I am a helpful AI assistant interested in technology and science.",
+        )
+
+        await client.aretain_batch(
+            bank_id=bank_id,
+            items=[
+                {"content": "The Python programming language is great for data science"},
+                {"content": "Machine learning models can recognize patterns in data"},
+            ],
+        )
+
+    @pytest.mark.asyncio
+    async def test_areflect_with_max_tokens(self, client, bank_id):
+        """Test areflect with max_tokens parameter."""
+        response = await client.areflect(
+            bank_id=bank_id,
+            query="What do you think about Python?",
+            max_tokens=500,
+        )
+
+        assert response is not None
+        assert response.text is not None
+        assert len(response.text) > 0
+
+    @pytest.mark.asyncio
+    async def test_areflect_with_structured_output(self, client, bank_id):
+        """Test areflect with response_schema for structured output."""
+        from pydantic import BaseModel
+
+        class RecommendationResponse(BaseModel):
+            recommendation: str
+            reasons: list[str]
+            confidence: str | None = None
+
+        response = await client.areflect(
+            bank_id=bank_id,
+            query="What programming language should I learn for data science?",
+            response_schema=RecommendationResponse.model_json_schema(),
+            max_tokens=10000,
+        )
+
+        assert response is not None
+        assert response.structured_output is not None
+        result = RecommendationResponse.model_validate(response.structured_output)
+        assert result.recommendation
+        assert isinstance(result.reasons, list)
+
+
+class TestAsyncDeleteBank:
+    """Tests for async bank deletion."""
+
+    @pytest.mark.asyncio
+    async def test_adelete_bank(self, client):
+        """Test deleting a bank using the async method."""
+        bank_id = f"test_bank_adelete_{uuid.uuid4().hex[:12]}"
+
+        # Create bank with some data using async API
+        await client.acreate_bank(
+            bank_id=bank_id,
+            mission="This bank will be deleted via async",
+        )
+        await client.aretain(
+            bank_id=bank_id,
+            content="Some memory to store",
+        )
+
+        # Delete using async method
+        response = await client.adelete_bank(bank_id=bank_id)
+
+        assert response is not None
+        assert response.success is True

hindsight-clients/rust/build.rs

@@ -13,6 +13,36 @@ fn convert_31_to_30(spec: &mut serde_json::Value) {
     convert_anyof_to_nullable(spec);
 }
 
+/// Remove paths with multipart/form-data content type (not supported by progenitor)
+fn filter_multipart_endpoints(spec: &mut serde_json::Value) {
+    if let Some(paths) = spec.get_mut("paths").and_then(|v| v.as_object_mut()) {
+        let mut paths_to_remove = Vec::new();
+
+        for (path_name, path_item) in paths.iter() {
+            if let Some(operations) = path_item.as_object() {
+                for (_method, operation) in operations.iter() {
+                    if let Some(request_body) = operation.get("requestBody") {
+                        if let Some(content) = request_body.get("content") {
+                            if let Some(content_obj) = content.as_object() {
+                                if content_obj.contains_key("multipart/form-data") {
+                                    eprintln!("Filtering out endpoint with multipart/form-data: {}", path_name);
+                                    paths_to_remove.push(path_name.clone());
+                                    break;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        // Remove the paths
+        for path in paths_to_remove {
+            paths.remove(&path);
+        }
+    }
+}
+
 fn convert_anyof_to_nullable(value: &mut serde_json::Value) {
     match value {
         serde_json::Value::Object(obj) => {
@@ -103,6 +133,9 @@ fn main() {
         }
     }
 
+    // Filter out multipart/form-data endpoints (progenitor doesn't support them)
+    filter_multipart_endpoints(&mut spec_json);
+
     // Now parse as OpenAPI struct
     let spec: openapiv3::OpenAPI = serde_json::from_value(spec_json)
         .expect("Failed to parse converted OpenAPI spec");

hindsight-clients/typescript/generated/sdk.gen.ts

@@ -1,6 +1,11 @@
 // This file is auto-generated by @hey-api/openapi-ts
 
-import type { Client, Options as Options2, TDataShape } from "./client";
+import {
+  type Client,
+  formDataBodySerializer,
+  type Options as Options2,
+  type TDataShape,
+} from "./client";
 import { client } from "./client.gen";
 import type {
   AddBankBackgroundData,
@@ -36,6 +41,9 @@ import type {
   DeleteMentalModelData,
   DeleteMentalModelErrors,
   DeleteMentalModelResponses,
+  FileRetainData,
+  FileRetainErrors,
+  FileRetainResponses,
   GetAgentStatsData,
   GetAgentStatsErrors,
   GetAgentStatsResponses,
@@ -937,3 +945,48 @@ export const retainMemories = <ThrowOnError extends boolean = false>(
       ...options.headers,
     },
   });
+
+/**
+ * Convert files to memories
+ *
+ * Upload files (PDF, DOCX, etc.), convert them to markdown, and retain as memories.
+ *
+ * This endpoint handles file upload, conversion, and memory creation in a single operation.
+ *
+ * **Features:**
+ * - Supports PDF, DOCX, PPTX, XLSX, images (with OCR), audio (with transcription)
+ * - Automatic file-to-markdown conversion using pluggable parsers
+ * - Files stored in object storage (PostgreSQL by default, S3 for production)
+ * - Each file becomes a separate document with optional metadata/tags
+ * - Always processes asynchronously — returns operation IDs immediately
+ *
+ * **The system automatically:**
+ * 1. Stores uploaded files in object storage
+ * 2. Converts files to markdown
+ * 3. Creates document records with file metadata
+ * 4. Extracts facts and creates memory units (same as regular retain)
+ *
+ * Use the operations endpoint to monitor progress.
+ *
+ * **Request format:** multipart/form-data with:
+ * - `files`: One or more files to upload
+ * - `request`: JSON string with FileRetainRequest model (files_metadata)
+ *
+ * **Note:** File parser is configured server-side via `HINDSIGHT_API_FILE_PARSER` (default: markitdown).
+ */
+export const fileRetain = <ThrowOnError extends boolean = false>(
+  options: Options<FileRetainData, ThrowOnError>,
+) =>
+  (options.client ?? client).post<
+    FileRetainResponses,
+    FileRetainErrors,
+    ThrowOnError
+  >({
+    ...formDataBodySerializer,
+    url: "/v1/default/banks/{bank_id}/files/retain",
+    ...options,
+    headers: {
+      "Content-Type": null,
+      ...options.headers,
+    },
+  });

hindsight-clients/typescript/generated/types.gen.ts

@@ -251,6 +251,24 @@ export type BankStatsResponse = {
   total_observations?: number;
 };
 
+/**
+ * Body_file_retain
+ */
+export type BodyFileRetain = {
+  /**
+   * Files
+   *
+   * Files to upload and convert
+   */
+  files: Array<Blob | File>;
+  /**
+   * Request
+   *
+   * JSON string with FileRetainRequest model
+   */
+  request: string;
+};
+
 /**
  * Budget
  *
@@ -896,6 +914,26 @@ export type FeaturesInfo = {
    * Whether per-bank configuration API is enabled
    */
   bank_config_api: boolean;
+  /**
+   * File Upload Api
+   *
+   * Whether file upload/conversion API is enabled
+   */
+  file_upload_api: boolean;
+};
+
+/**
+ * FileRetainResponse
+ *
+ * Response model for file upload endpoint.
+ */
+export type FileRetainResponse = {
+  /**
+   * Operation Ids
+   *
+   * Operation IDs for tracking file conversion operations. Use GET /v1/default/banks/{bank_id}/operations to list operations.
+   */
+  operation_ids: Array<string>;
 };
 
 /**
@@ -1760,7 +1798,7 @@ export type RetainResponse = {
   /**
    * Operation Id
    *
-   * Operation ID for tracking async operations. Use GET /v1/default/banks/{bank_id}/operations to list operations and find this ID. Only present when async=true.
+   * Operation ID for tracking async operations. Use GET /v1/default/banks/{bank_id}/operations to list operations. Only present when async=true.
    */
   operation_id?: string | null;
   /**
@@ -3753,3 +3791,40 @@ export type RetainMemoriesResponses = {
 
 export type RetainMemoriesResponse =
   RetainMemoriesResponses[keyof RetainMemoriesResponses];
+
+export type FileRetainData = {
+  body: BodyFileRetain;
+  headers?: {
+    /**
+     * Authorization
+     */
+    authorization?: string | null;
+  };
+  path: {
+    /**
+     * Bank Id
+     */
+    bank_id: string;
+  };
+  query?: never;
+  url: "/v1/default/banks/{bank_id}/files/retain";
+};
+
+export type FileRetainErrors = {
+  /**
+   * Validation Error
+   */
+  422: HttpValidationError;
+};
+
+export type FileRetainError = FileRetainErrors[keyof FileRetainErrors];
+
+export type FileRetainResponses = {
+  /**
+   * Successful Response
+   */
+  200: FileRetainResponse;
+};
+
+export type FileRetainResponse2 =
+  FileRetainResponses[keyof FileRetainResponses];

hindsight-clients/typescript/src/index.ts

@@ -36,6 +36,7 @@ import type {
     RecallResult,
     ReflectRequest,
     ReflectResponse,
+    FileRetainResponse,
     ListMemoryUnitsResponse,
     BankProfileResponse,
     CreateBankRequest,
@@ -211,6 +212,40 @@ export class HindsightClient {
         return this.validateResponse(response, 'retainBatch');
     }
 
+    /**
+     * Upload files and retain their contents as memories.
+     *
+     * Files are automatically converted to text (PDF, DOCX, images via OCR, audio via
+     * transcription, and more) and ingested as memories. Processing is always asynchronous —
+     * use the returned operation IDs to track progress via the operations endpoint.
+     *
+     * @param bankId - The memory bank ID
+     * @param files - Array of File or Blob objects to upload
+     * @param options - Optional settings: context, documentTags, filesMetadata
+     */
+    async retainFiles(
+        bankId: string,
+        files: Array<File | Blob>,
+        options?: {
+            context?: string;
+            filesMetadata?: Array<{ context?: string; document_id?: string; tags?: string[]; metadata?: Record<string, string> }>;
+        }
+    ): Promise<FileRetainResponse> {
+        const meta = options?.filesMetadata ?? files.map(() => options?.context ? { context: options.context } : {});
+
+        const requestBody = JSON.stringify({
+            files_metadata: meta,
+        });
+
+        const response = await sdk.fileRetain({
+            client: this.client,
+            path: { bank_id: bankId },
+            body: { files, request: requestBody },
+        });
+
+        return this.validateResponse(response, 'retainFiles');
+    }
+
     /**
      * Recall memories with a natural language query.
      */
@@ -580,6 +615,7 @@ export type {
     RecallResult,
     ReflectRequest,
     ReflectResponse,
+    FileRetainResponse,
     ListMemoryUnitsResponse,
     BankProfileResponse,
     CreateBankRequest,

hindsight-control-plane/src/app/api/files/retain/route.ts

@@ -0,0 +1,52 @@
+import { NextRequest, NextResponse } from "next/server";
+import { DATAPLANE_URL } from "@/lib/hindsight-client";
+
+export async function POST(request: NextRequest) {
+  try {
+    // Clone the form data to read bank_id without consuming it
+    const formData = await request.formData();
+
+    // Extract bank_id from request JSON
+    const requestJson = formData.get("request");
+    if (!requestJson || typeof requestJson !== "string") {
+      return NextResponse.json({ error: "Missing request data" }, { status: 400 });
+    }
+
+    const requestData = JSON.parse(requestJson);
+    const bankId = requestData.bank_id;
+
+    if (!bankId) {
+      return NextResponse.json({ error: "Missing bank_id" }, { status: 400 });
+    }
+
+    // Use the shared dataplane URL configuration
+    const url = `${DATAPLANE_URL}/v1/default/banks/${bankId}/files/retain`;
+
+    // Forward the form data to the dataplane
+    const response = await fetch(url, {
+      method: "POST",
+      body: formData,
+      // Don't set Content-Type - let fetch handle multipart boundary
+    });
+
+    if (!response.ok) {
+      const errorText = await response.text();
+      let errorData;
+      try {
+        errorData = JSON.parse(errorText);
+      } catch {
+        errorData = { error: errorText };
+      }
+      return NextResponse.json(errorData, { status: response.status });
+    }
+
+    const data = await response.json();
+    return NextResponse.json(data, { status: 200 });
+  } catch (error) {
+    console.error("Error uploading files:", error);
+    return NextResponse.json(
+      { error: error instanceof Error ? error.message : "Failed to upload files" },
+      { status: 500 }
+    );
+  }
+}

hindsight-control-plane/src/components/bank-selector.tsx

@@ -34,6 +34,7 @@ import {
   Tag,
   Upload,
   X,
+  Lock,
 } from "lucide-react";
 import { useTheme } from "@/lib/theme-context";
 import Image from "next/image";
@@ -42,22 +43,6 @@ import { Checkbox } from "@/components/ui/checkbox";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { cn } from "@/lib/utils";
 
-// Supported text file extensions (matching CLI)
-const TEXT_EXTENSIONS = [
-  "txt",
-  "md",
-  "json",
-  "yaml",
-  "yml",
-  "toml",
-  "xml",
-  "csv",
-  "log",
-  "rst",
-  "adoc",
-];
-const ACCEPT_FILES = TEXT_EXTENSIONS.map((ext) => `.${ext}`).join(",");
-
 function BankSelectorInner() {
   const router = useRouter();
   const searchParams = useSearchParams();
@@ -85,6 +70,21 @@ function BankSelectorInner() {
   const [uploadProgress, setUploadProgress] = React.useState<string>("");
   const fileInputRef = React.useRef<HTMLInputElement>(null);
 
+  // Feature flags
+  const [fileUploadEnabled, setFileUploadEnabled] = React.useState<boolean | null>(null);
+
+  // Load feature flags
+  React.useEffect(() => {
+    client
+      .getVersion()
+      .then((version) => {
+        setFileUploadEnabled(version.features.file_upload_api);
+      })
+      .catch(() => {
+        setFileUploadEnabled(false);
+      });
+  }, []);
+
   const sortedBanks = React.useMemo(() => {
     return [...banks].sort((a, b) => a.localeCompare(b));
   }, [banks]);
@@ -112,12 +112,7 @@ function BankSelectorInner() {
 
   const handleFileSelect = (e: React.ChangeEvent<HTMLInputElement>) => {
     const files = Array.from(e.target.files || []);
-    // Filter to only supported extensions
-    const validFiles = files.filter((file) => {
-      const ext = file.name.split(".").pop()?.toLowerCase();
-      return ext && TEXT_EXTENSIONS.includes(ext);
-    });
-    setSelectedFiles((prev) => [...prev, ...validFiles]);
+    setSelectedFiles((prev) => [...prev, ...files]);
     // Reset input to allow selecting same file again
     if (fileInputRef.current) {
       fileInputRef.current.value = "";
@@ -128,15 +123,6 @@ function BankSelectorInner() {
     setSelectedFiles((prev) => prev.filter((_, i) => i !== index));
   };
 
-  const readFileContent = (file: File): Promise<string> => {
-    return new Promise((resolve, reject) => {
-      const reader = new FileReader();
-      reader.onload = (e) => resolve(e.target?.result as string);
-      reader.onerror = (e) => reject(e);
-      reader.readAsText(file);
-    });
-  };
-
   const handleUploadFiles = async () => {
     if (!currentBank || selectedFiles.length === 0) return;
 
@@ -149,33 +135,15 @@ function BankSelectorInner() {
         .map((t) => t.trim())
         .filter((t) => t.length > 0);
 
-      // Read all files and create items
-      const items: any[] = [];
-      for (let i = 0; i < selectedFiles.length; i++) {
-        const file = selectedFiles[i];
-        setUploadProgress(`Reading file ${i + 1}/${selectedFiles.length}: ${file.name}`);
-        const content = await readFileContent(file);
-        const item: any = {
-          content,
-          context: `File: ${file.name}`,
-        };
-        if (parsedTags.length > 0) item.tags = parsedTags;
-        items.push(item);
-      }
+      setUploadProgress(`Uploading ${selectedFiles.length} file(s)...`);
 
-      setUploadProgress(`Uploading ${items.length} file(s)...`);
-
-      const params: any = {
+      // Use the new file upload API (always async, converter configured server-side)
+      await client.uploadFiles({
         bank_id: currentBank,
-        items,
-      };
-      if (parsedTags.length > 0) params.document_tags = parsedTags;
-
-      if (docAsync) {
-        await client.retain({ ...params, async: true });
-      } else {
-        await client.retain(params);
-      }
+        files: selectedFiles,
+        document_tags: parsedTags.length > 0 ? parsedTags : undefined,
+        async: true,
+      });
 
       // Reset form and close dialog
       setDocDialogOpen(false);
@@ -422,8 +390,16 @@ function BankSelectorInner() {
                   <FileText className="h-4 w-4" />
                   Text
                 </TabsTrigger>
-                <TabsTrigger value="upload" className="flex items-center gap-2">
-                  <Upload className="h-4 w-4" />
+                <TabsTrigger
+                  value="upload"
+                  className="flex items-center gap-2"
+                  disabled={fileUploadEnabled === false}
+                >
+                  {fileUploadEnabled === false ? (
+                    <Lock className="h-4 w-4" />
+                  ) : (
+                    <Upload className="h-4 w-4" />
+                  )}
                   Upload Files
                 </TabsTrigger>
               </TabsList>
@@ -505,91 +481,96 @@ function BankSelectorInner() {
               </TabsContent>
 
               <TabsContent value="upload" className="space-y-4 mt-4">
-                <div>
-                  <input
-                    ref={fileInputRef}
-                    type="file"
-                    multiple
-                    accept={ACCEPT_FILES}
-                    onChange={handleFileSelect}
-                    className="hidden"
-                    id="file-upload"
-                  />
-                  <label
-                    htmlFor="file-upload"
-                    className="flex flex-col items-center justify-center w-full h-32 border-2 border-dashed border-muted-foreground/25 rounded-lg cursor-pointer hover:border-primary/50 hover:bg-accent/50 transition-colors"
-                  >
-                    <Upload className="h-8 w-8 text-muted-foreground mb-2" />
-                    <span className="text-sm text-muted-foreground">
-                      Click to select files or drag and drop
-                    </span>
-                    <span className="text-xs text-muted-foreground mt-1">
-                      Supported: {TEXT_EXTENSIONS.join(", ")}
-                    </span>
-                  </label>
-                </div>
-
-                {selectedFiles.length > 0 && (
-                  <div className="space-y-2">
-                    <label className="font-bold block text-sm text-foreground">
-                      Selected Files ({selectedFiles.length})
-                    </label>
-                    <div className="max-h-[150px] overflow-y-auto space-y-1">
-                      {selectedFiles.map((file, index) => (
-                        <div
-                          key={`${file.name}-${index}`}
-                          className="flex items-center justify-between p-2 bg-muted rounded-md"
-                        >
-                          <div className="flex items-center gap-2 min-w-0">
-                            <FileText className="h-4 w-4 flex-shrink-0 text-muted-foreground" />
-                            <span className="text-sm truncate">{file.name}</span>
-                            <span className="text-xs text-muted-foreground">
-                              ({(file.size / 1024).toFixed(1)} KB)
-                            </span>
-                          </div>
-                          <Button
-                            variant="ghost"
-                            size="sm"
-                            className="h-6 w-6 p-0"
-                            onClick={() => removeFile(index)}
-                          >
-                            <X className="h-4 w-4" />
-                          </Button>
-                        </div>
-                      ))}
+                {fileUploadEnabled === false ? (
+                  <div className="flex flex-col items-center justify-center py-8 text-center space-y-3">
+                    <Lock className="h-12 w-12 text-muted-foreground/50" />
+                    <div>
+                      <p className="font-semibold text-foreground">File Upload API Disabled</p>
+                      <p className="text-sm text-muted-foreground mt-1">
+                        File upload is not enabled on this server.
+                      </p>
+                      <p className="text-xs text-muted-foreground mt-2">
+                        To enable, set{" "}
+                        <code className="bg-muted px-1 py-0.5 rounded">
+                          HINDSIGHT_API_ENABLE_FILE_UPLOAD_API=true
+                        </code>
+                      </p>
                     </div>
                   </div>
-                )}
+                ) : (
+                  <>
+                    <div>
+                      <input
+                        ref={fileInputRef}
+                        type="file"
+                        multiple
+                        onChange={handleFileSelect}
+                        className="hidden"
+                        id="file-upload"
+                      />
+                      <label
+                        htmlFor="file-upload"
+                        className="flex flex-col items-center justify-center w-full h-32 border-2 border-dashed border-muted-foreground/25 rounded-lg cursor-pointer hover:border-primary/50 hover:bg-accent/50 transition-colors"
+                      >
+                        <Upload className="h-8 w-8 text-muted-foreground mb-2" />
+                        <span className="text-sm text-muted-foreground">
+                          Click to select files or drag and drop
+                        </span>
+                      </label>
+                    </div>
 
-                <div>
-                  <label className="font-bold block mb-1 text-sm text-foreground flex items-center gap-2">
-                    <Tag className="h-4 w-4" />
-                    Tags (applied to all files)
-                  </label>
-                  <Input
-                    type="text"
-                    value={docTags}
-                    onChange={(e) => setDocTags(e.target.value)}
-                    placeholder="user_alice, session_123, project_x"
-                  />
-                  <p className="text-xs text-muted-foreground mt-1">
-                    Comma-separated tags for filtering during recall/reflect
-                  </p>
-                </div>
+                    {selectedFiles.length > 0 && (
+                      <div className="space-y-2">
+                        <label className="font-bold block text-sm text-foreground">
+                          Selected Files ({selectedFiles.length})
+                        </label>
+                        <div className="max-h-[150px] overflow-y-auto space-y-1">
+                          {selectedFiles.map((file, index) => (
+                            <div
+                              key={`${file.name}-${index}`}
+                              className="flex items-center justify-between p-2 bg-muted rounded-md"
+                            >
+                              <div className="flex items-center gap-2 min-w-0">
+                                <FileText className="h-4 w-4 flex-shrink-0 text-muted-foreground" />
+                                <span className="text-sm truncate">{file.name}</span>
+                                <span className="text-xs text-muted-foreground">
+                                  ({(file.size / 1024).toFixed(1)} KB)
+                                </span>
+                              </div>
+                              <Button
+                                variant="ghost"
+                                size="sm"
+                                className="h-6 w-6 p-0"
+                                onClick={() => removeFile(index)}
+                              >
+                                <X className="h-4 w-4" />
+                              </Button>
+                            </div>
+                          ))}
+                        </div>
+                      </div>
+                    )}
 
-                <div className="flex items-center gap-2">
-                  <Checkbox
-                    id="async-upload"
-                    checked={docAsync}
-                    onCheckedChange={(checked) => setDocAsync(checked as boolean)}
-                  />
-                  <label htmlFor="async-upload" className="text-sm cursor-pointer text-foreground">
-                    Process in background (async)
-                  </label>
-                </div>
+                    <div>
+                      <label className="font-bold block mb-1 text-sm text-foreground flex items-center gap-2">
+                        <Tag className="h-4 w-4" />
+                        Tags (applied to all files)
+                      </label>
+                      <Input
+                        type="text"
+                        value={docTags}
+                        onChange={(e) => setDocTags(e.target.value)}
+                        placeholder="user_alice, session_123, project_x"
+                      />
+                      <p className="text-xs text-muted-foreground mt-1">
+                        Comma-separated tags for filtering during recall/reflect
+                      </p>
+                    </div>
 
-                {uploadProgress && (
-                  <p className="text-sm text-muted-foreground">{uploadProgress}</p>
+                    {uploadProgress && (
+                      <p className="text-sm text-muted-foreground">{uploadProgress}</p>
+                    )}
+                  </>
                 )}
               </TabsContent>
             </Tabs>

hindsight-control-plane/src/lib/api.ts

@@ -745,10 +745,69 @@ export class ControlPlaneClient {
         mcp: boolean;
         worker: boolean;
         bank_config_api: boolean;
+        file_upload_api: boolean;
       };
     }>("/api/version");
   }
 
+  /**
+   * Upload files for retain (uses file conversion API)
+   * Requires file_upload_api feature flag to be enabled
+   * Converter is configured server-side via HINDSIGHT_API_FILE_CONVERTER
+   */
+  async uploadFiles(params: {
+    bank_id: string;
+    files: File[];
+    document_tags?: string[];
+    async?: boolean;
+    files_metadata?: Array<{
+      document_id?: string;
+      context?: string;
+      metadata?: Record<string, any>;
+      tags?: string[];
+      timestamp?: string;
+    }>;
+  }) {
+    const formData = new FormData();
+
+    // Add files
+    params.files.forEach((file) => {
+      formData.append("files", file);
+    });
+
+    // Add request JSON (including bank_id)
+    const requestData: any = {
+      bank_id: params.bank_id,
+      async: params.async ?? true,
+    };
+    if (params.document_tags) requestData.document_tags = params.document_tags;
+    if (params.files_metadata) requestData.files_metadata = params.files_metadata;
+
+    formData.append("request", JSON.stringify(requestData));
+
+    // Use fetch directly for multipart/form-data
+    const response = await fetch(`/api/files/retain`, {
+      method: "POST",
+      body: formData,
+      // Don't set Content-Type - browser will set it with boundary
+    });
+
+    if (!response.ok) {
+      let errorMessage = `HTTP ${response.status}`;
+      try {
+        const errorData = await response.json();
+        errorMessage = errorData.error || errorMessage;
+      } catch {
+        // Ignore parse errors
+      }
+      const error = new Error(errorMessage);
+      (error as any).status = response.status;
+      throw error;
+    }
+
+    return response.json();
+  }
+
   /**
    * Get bank configuration (resolved with hierarchy)
    */

hindsight-control-plane/src/lib/hindsight-client.ts

@@ -14,6 +14,14 @@ import {
 export const DATAPLANE_URL = process.env.HINDSIGHT_CP_DATAPLANE_API_URL || "http://localhost:8888";
 const DATAPLANE_API_KEY = process.env.HINDSIGHT_CP_DATAPLANE_API_KEY || "";
 
+// Log dataplane configuration at startup
+console.log(`[Control Plane] Connecting to dataplane at: ${DATAPLANE_URL}`);
+if (DATAPLANE_API_KEY) {
+  console.log("[Control Plane] Using API key authentication");
+} else {
+  console.log("[Control Plane] No API key configured (public access)");
+}
+
 /**
  * Auth headers for direct fetch calls to the dataplane API.
  */

hindsight-docs/docs/developer/api/retain.mdx

@@ -102,20 +102,47 @@ The `document_id` groups related memories for later management.
 
 ## Store from Files
 
+Upload files directly — Hindsight automatically converts them to text and extracts memories. File processing always runs asynchronously and returns operation IDs for tracking.
+
+**Supported formats:** PDF, DOCX, DOC, PPTX, PPT, XLSX, XLS, images (JPG, PNG, GIF, etc. — OCR), audio (MP3, WAV, FLAC, etc. — transcription), HTML, and plain text formats (TXT, MD, CSV, JSON, YAML, etc.)
+
 <Tabs>
 <TabItem value="cli" label="CLI">
-
-```bash
-# Single file
-hindsight memory retain-files my-bank document.txt
-
-# Directory (recursive by default)
-hindsight memory retain-files my-bank ./documents/
-```
-
+<CodeSnippet code={retainSh} section="retain-files" language="bash" />
+</TabItem>
+<TabItem value="curl" label="HTTP">
+<CodeSnippet code={retainSh} section="retain-files-curl" language="bash" />
+</TabItem>
+<TabItem value="python" label="Python">
+<CodeSnippet code={retainPy} section="retain-files" language="python" />
+</TabItem>
+<TabItem value="node" label="Node.js">
+<CodeSnippet code={retainMjs} section="retain-files" language="javascript" />
 </TabItem>
 </Tabs>
 
+### File Retain Response
+
+The file retain endpoint always returns asynchronously:
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `operation_ids` | string[] | One operation ID per uploaded file. Use `GET /v1/default/banks/{bank_id}/operations` to track progress. |
+
+### Batch File Uploads
+
+Upload up to 10 files per request (max 100 MB total). Each file becomes a separate document with optional per-file metadata:
+
+<Tabs>
+<TabItem value="python" label="Python">
+<CodeSnippet code={retainPy} section="retain-files-batch" language="python" />
+</TabItem>
+</Tabs>
+
+:::info File Storage
+Uploaded files are stored server-side (PostgreSQL by default, or S3/GCS/Azure for production). Configure storage via `HINDSIGHT_API_FILE_STORAGE_TYPE`. See [Configuration](../configuration#file-processing) for details.
+:::
+
 
 ## Async Ingestion
 

hindsight-docs/docs/developer/configuration.md

@@ -629,6 +629,136 @@ CONSOLIDATE related technical discussions into ONE fact when possible.
 Ask yourself: 'Would this technical context be useful in 6 months?' If no, skip it."
 ```
 
+### File Processing
+
+Configuration for the file upload and conversion pipeline (used by `POST /v1/default/banks/{bank_id}/files/retain`).
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `HINDSIGHT_API_ENABLE_FILE_UPLOAD_API` | Enable the file upload API endpoint | `true` |
+| `HINDSIGHT_API_FILE_PARSER` | File parser to use (`markitdown`, `iris`) | `markitdown` |
+| `HINDSIGHT_API_FILE_CONVERSION_MAX_BATCH_SIZE` | Max files per upload request | `10` |
+| `HINDSIGHT_API_FILE_CONVERSION_MAX_BATCH_SIZE_MB` | Max total upload size per request (MB) | `100` |
+| `HINDSIGHT_API_FILE_DELETE_AFTER_RETAIN` | Delete stored files after memory extraction completes | `true` |
+
+#### Parser: markitdown (default)
+
+Local file-to-markdown conversion using [Microsoft's markitdown](https://github.com/microsoft/markitdown). No external service required.
+
+**Supported formats:** PDF, DOCX, DOC, PPTX, PPT, XLSX, XLS, images (JPG, PNG — OCR), audio (MP3, WAV — transcription), HTML, TXT, MD, CSV.
+
+#### Parser: iris
+
+Cloud-based extraction via [Vectorize Iris](https://docs.vectorize.io/build-deploy/extract-information/understanding-iris/). Higher quality extraction for complex documents, powered by a remote AI service.
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `HINDSIGHT_API_FILE_PARSER_IRIS_TOKEN` | Vectorize API token | — |
+| `HINDSIGHT_API_FILE_PARSER_IRIS_ORG_ID` | Vectorize organization ID | — |
+
+**Supported formats:** PDF, DOCX, DOC, PPTX, PPT, XLSX, XLS, images (JPG, JPEG, PNG, GIF, BMP, TIFF, WEBP), HTML, TXT, MD, CSV.
+
+```bash
+# Use iris parser (requires Vectorize account)
+export HINDSIGHT_API_FILE_PARSER=iris
+export HINDSIGHT_API_FILE_PARSER_IRIS_TOKEN=your-vectorize-token
+export HINDSIGHT_API_FILE_PARSER_IRIS_ORG_ID=your-org-id
+```
+
+```bash
+# Increase batch limits for large file imports
+export HINDSIGHT_API_FILE_CONVERSION_MAX_BATCH_SIZE=20
+export HINDSIGHT_API_FILE_CONVERSION_MAX_BATCH_SIZE_MB=500
+
+# Keep files after processing (useful for debugging or re-processing)
+export HINDSIGHT_API_FILE_DELETE_AFTER_RETAIN=false
+```
+
+### File Storage
+
+Files uploaded via the file retain API are stored in an object storage backend before conversion. Choose the backend that fits your infrastructure.
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `HINDSIGHT_API_FILE_STORAGE_TYPE` | Storage backend: `native`, `s3`, `gcs`, or `azure` | `native` |
+
+#### Native (PostgreSQL)
+
+Files are stored as `BYTEA` in the `file_storage` table. No additional infrastructure required. Suitable for development and small deployments.
+
+```bash
+# Native storage is the default — no additional configuration needed
+export HINDSIGHT_API_FILE_STORAGE_TYPE=native
+```
+
+#### S3 / S3-Compatible
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `HINDSIGHT_API_FILE_STORAGE_S3_BUCKET` | S3 bucket name | - |
+| `HINDSIGHT_API_FILE_STORAGE_S3_REGION` | AWS region | - |
+| `HINDSIGHT_API_FILE_STORAGE_S3_ENDPOINT` | Custom endpoint URL (for S3-compatible stores like MinIO, Cloudflare R2) | AWS default |
+| `HINDSIGHT_API_FILE_STORAGE_S3_ACCESS_KEY_ID` | AWS access key ID | - |
+| `HINDSIGHT_API_FILE_STORAGE_S3_SECRET_ACCESS_KEY` | AWS secret access key | - |
+
+```bash
+# AWS S3
+export HINDSIGHT_API_FILE_STORAGE_TYPE=s3
+export HINDSIGHT_API_FILE_STORAGE_S3_BUCKET=my-hindsight-files
+export HINDSIGHT_API_FILE_STORAGE_S3_REGION=us-east-1
+export HINDSIGHT_API_FILE_STORAGE_S3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
+export HINDSIGHT_API_FILE_STORAGE_S3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+
+# S3-compatible (MinIO, Cloudflare R2, etc.)
+export HINDSIGHT_API_FILE_STORAGE_TYPE=s3
+export HINDSIGHT_API_FILE_STORAGE_S3_BUCKET=my-bucket
+export HINDSIGHT_API_FILE_STORAGE_S3_ENDPOINT=https://your-minio.example.com
+export HINDSIGHT_API_FILE_STORAGE_S3_ACCESS_KEY_ID=minioadmin
+export HINDSIGHT_API_FILE_STORAGE_S3_SECRET_ACCESS_KEY=minioadmin
+```
+
+#### Google Cloud Storage
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `HINDSIGHT_API_FILE_STORAGE_GCS_BUCKET` | GCS bucket name | - |
+| `HINDSIGHT_API_FILE_STORAGE_GCS_SERVICE_ACCOUNT_KEY` | Path to service account JSON key file | ADC if not set |
+
+```bash
+export HINDSIGHT_API_FILE_STORAGE_TYPE=gcs
+export HINDSIGHT_API_FILE_STORAGE_GCS_BUCKET=my-hindsight-files
+# Optional: use service account key file (otherwise falls back to ADC)
+export HINDSIGHT_API_FILE_STORAGE_GCS_SERVICE_ACCOUNT_KEY=/path/to/key.json
+```
+
+#### Azure Blob Storage
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `HINDSIGHT_API_FILE_STORAGE_AZURE_CONTAINER` | Azure container name | - |
+| `HINDSIGHT_API_FILE_STORAGE_AZURE_ACCOUNT_NAME` | Azure storage account name | - |
+| `HINDSIGHT_API_FILE_STORAGE_AZURE_ACCOUNT_KEY` | Azure storage account key | - |
+
+```bash
+export HINDSIGHT_API_FILE_STORAGE_TYPE=azure
+export HINDSIGHT_API_FILE_STORAGE_AZURE_CONTAINER=hindsight-files
+export HINDSIGHT_API_FILE_STORAGE_AZURE_ACCOUNT_NAME=mystorageaccount
+export HINDSIGHT_API_FILE_STORAGE_AZURE_ACCOUNT_KEY=base64encodedkey==
+```
+
+#### Storage Backend Comparison
+
+| Backend | Best For | Notes |
+|---------|----------|-------|
+| `native` | Development, small deployments | No extra infrastructure, stored in PostgreSQL |
+| `s3` | Production, AWS deployments | Works with any S3-compatible store |
+| `gcs` | Production, GCP deployments | Supports ADC for keyless auth |
+| `azure` | Production, Azure deployments | Uses account key auth |
+
+:::tip Production Recommendation
+For production deployments, use `s3`, `gcs`, or `azure` to avoid storing large binary files in your PostgreSQL database. Set `HINDSIGHT_API_FILE_DELETE_AFTER_RETAIN=true` (the default) to delete files after memory extraction, which minimizes storage costs.
+:::
+
 ### Observations (Experimental)
 
 Observations are consolidated knowledge synthesized from facts.
@@ -989,6 +1119,13 @@ HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx
 # HINDSIGHT_API_TENANT_EXTENSION=hindsight_api.extensions.builtin.tenant:ApiKeyTenantExtension
 # HINDSIGHT_API_TENANT_API_KEY=your-secret-api-key
 
+# File storage (optional, defaults to PostgreSQL native storage)
+# HINDSIGHT_API_FILE_STORAGE_TYPE=s3
+# HINDSIGHT_API_FILE_STORAGE_S3_BUCKET=my-hindsight-files
+# HINDSIGHT_API_FILE_STORAGE_S3_REGION=us-east-1
+# HINDSIGHT_API_FILE_STORAGE_S3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
+# HINDSIGHT_API_FILE_STORAGE_S3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+
 # Control Plane
 HINDSIGHT_CP_DATAPLANE_API_URL=http://localhost:8888
 ```

hindsight-docs/docusaurus.config.ts

@@ -201,23 +201,10 @@ const config: Config = {
           className: 'navbar-item-sdks',
         },
         {
-          to: '/api-reference',
+          to: '/faq',
           position: 'left',
-          label: 'API Reference',
-          className: 'navbar-item-api',
-        },
-        {
-          type: 'doc',
-          docId: 'cookbook/index',
-          position: 'left',
-          label: 'Cookbook',
-          className: 'navbar-item-cookbook',
-        },
-        {
-          to: '/blog',
-          position: 'left',
-          label: 'Blog',
-          className: 'navbar-item-blog',
+          label: 'FAQ',
+          className: 'navbar-item-faq',
         },
         {
           to: '/changelog',
@@ -225,6 +212,27 @@ const config: Config = {
           label: 'Changelog',
           className: 'navbar-item-changelog',
         },
+        {
+          type: 'dropdown',
+          label: 'Resources',
+          position: 'left',
+          className: 'navbar-item-resources',
+          items: [
+            {
+              type: 'doc',
+              docId: 'cookbook/index',
+              label: 'Cookbook',
+            },
+            {
+              to: '/blog',
+              label: 'Blog',
+            },
+            {
+              to: '/api-reference',
+              label: 'API Reference',
+            },
+          ],
+        },
         {
           href: 'https://ui.hindsight.vectorize.io/signup',
           position: 'right',

hindsight-docs/examples/api/retain.mjs

@@ -49,6 +49,22 @@ await client.retainBatch('my-bank', [
 // [/docs:retain-async]
 
 
+// [docs:retain-files]
+// Upload files and retain their contents as memories.
+// Supports: PDF, DOCX, PPTX, XLSX, images (OCR), audio (transcription), and text formats.
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const pdfBytes = readFileSync(join(__dirname, 'sample.pdf'));
+const result = await client.retainFiles('my-bank', [
+    new File([pdfBytes], 'sample.pdf'),
+], { context: 'quarterly report' });
+console.log(result.operation_ids);  // Track processing via the operations endpoint
+// [/docs:retain-files]
+
+
 // =============================================================================
 // Cleanup (not shown in docs)
 // =============================================================================

hindsight-docs/examples/api/retain.py

@@ -5,8 +5,10 @@ Run: python examples/api/retain.py
 """
 import os
 import requests
+from pathlib import Path
 
 HINDSIGHT_URL = os.getenv("HINDSIGHT_API_URL", "http://localhost:8888")
+EXAMPLES_DIR = Path(__file__).parent
 
 # =============================================================================
 # Setup (not shown in docs)
@@ -98,6 +100,37 @@ client.retain_batch(
 # [/docs:retain-with-document-tags]
 
 
+# [docs:retain-files]
+# Upload files and retain their contents as memories.
+# Supports: PDF, DOCX, PPTX, XLSX, images (OCR), audio (transcription), and text formats.
+# Pass file paths — the client reads and uploads them automatically.
+result = client.retain_files(
+    bank_id="my-bank",
+    files=[EXAMPLES_DIR / "sample.pdf"],
+    context="quarterly report",
+)
+print(result.operation_ids)  # Track processing via the operations endpoint
+# [/docs:retain-files]
+
+
+# [docs:retain-files-batch]
+# Upload multiple files with per-file metadata (up to 10 files per batch).
+# Each file gets its own context, document_id, and tags.
+result = client.retain_files(
+    bank_id="my-bank",
+    files=[
+        EXAMPLES_DIR / "sample.pdf",
+        EXAMPLES_DIR / "sample.pdf",  # Replace with a second file path
+    ],
+    files_metadata=[
+        {"context": "quarterly report", "document_id": "q1-report", "tags": ["project:alpha"]},
+        {"context": "meeting notes", "document_id": "q1-notes", "tags": ["project:alpha"]},
+    ],
+)
+print(result.operation_ids)  # One operation ID per file
+# [/docs:retain-files-batch]
+
+
 # [docs:retain-list-tags]
 # List all tags in a bank
 response = requests.get(f"{HINDSIGHT_URL}/v1/default/banks/my-bank/tags")

hindsight-docs/examples/api/retain.sh

@@ -26,6 +26,29 @@ hindsight memory retain my-bank "Meeting notes" --async
 # [/docs:retain-async]
 
 
+# [docs:retain-files]
+# Upload a single file (PDF, DOCX, PPTX, XLSX, images, audio, and more)
+hindsight memory retain-files my-bank report.pdf
+
+# Upload a directory of files
+hindsight memory retain-files my-bank ./documents/
+
+# Upload and wait for processing to complete (polls until done)
+hindsight memory retain-files my-bank report.pdf
+
+# Queue files for background processing (returns immediately)
+hindsight memory retain-files my-bank ./documents/ --async
+# [/docs:retain-files]
+
+
+# [docs:retain-files-curl]
+# Via HTTP API (multipart/form-data)
+curl -X POST "${HINDSIGHT_URL}/v1/default/banks/my-bank/files/retain" \
+    -F "[email protected];type=application/octet-stream" \
+    -F "request={\"files_metadata\": [{\"context\": \"quarterly report\"}]}"
+# [/docs:retain-files-curl]
+
+
 # =============================================================================
 # Cleanup (not shown in docs)
 # =============================================================================

hindsight-docs/src/pages/faq.md

@@ -0,0 +1,151 @@
+---
+title: Frequently Asked Questions
+description: Common questions and answers about Hindsight
+hide_table_of_contents: true
+---
+
+# Frequently Asked Questions
+
+### What is Hindsight and how does it differ from RAG?
+
+Hindsight is an agent memory system that provides long-term memory for AI agents using biomimetic data structures. Unlike traditional RAG (Retrieval-Augmented Generation), Hindsight:
+
+- **Stores structured facts** instead of raw document chunks
+- **Builds mental models** that consolidate knowledge over time
+- **Uses graph-based relationships** between entities and concepts
+- **Supports temporal reasoning** with time-aware retrieval
+- **Enables disposition-aware reflection** for nuanced reasoning
+
+For a detailed comparison, see [RAG vs Memory](/developer/rag-vs-hindsight).
+
+---
+
+### Why use Hindsight instead of other solutions?
+
+Hindsight is purpose-built for agent memory with unique advantages:
+
+- **State-of-the-art accuracy**: Ranked #1 LongMemEval benchmarks for agent memory (see [details](https://benchmarks.hindsight.vectorize.io/))
+- **Built on proven technology**: PostgreSQL - battle-tested, reliable, and widely understood
+- **Cloud-native architecture**: Designed for modern cloud deployments with horizontal scalability
+- **Flexible deployment**: Self-host or use Hindsight Cloud - works with any LLM provider
+- **True long-term memory**: Builds mental models that consolidate knowledge over time, not just retrieval
+- **Graph-based reasoning**: Understands relationships between entities and concepts for richer context
+- **Production-ready**: Scales to millions of memories with 50-500ms recall latency
+- **Developer-friendly**: Simple APIs (retain, recall, reflect), SDKs for Python/TypeScript/Go/Rust, integrations with LiteLLM/Vercel AI SDK
+
+Unlike vector databases (just search) or RAG systems (document retrieval), Hindsight provides **living memory** that evolves with your users.
+
+---
+
+### Which LLM providers are supported?
+
+Hindsight supports:
+- **OpenAI**
+- **Anthropic**
+- **Google Gemini**
+- **Groq**
+- **Ollama** (local models)
+- **LM Studio** (local models)
+- **Any OpenAI-compatible provider** (Together AI, Fireworks, DeepInfra, etc.)
+- **Any Anthropic-compatible provider**
+
+**Using local models with Ollama:**
+```bash
+HINDSIGHT_API_LLM_PROVIDER=ollama
+HINDSIGHT_API_LLM_MODEL=llama3.1
+HINDSIGHT_API_LLM_BASE_URL=http://localhost:11434
+```
+
+**Using local models with LM Studio:**
+```bash
+HINDSIGHT_API_LLM_PROVIDER=lmstudio
+HINDSIGHT_API_LLM_MODEL=your-model-name
+HINDSIGHT_API_LLM_BASE_URL=http://localhost:1234/v1
+```
+
+Configure your provider using the `HINDSIGHT_API_LLM_PROVIDER` environment variable. See [Configuration](/developer/configuration) and [Models](/developer/models) for details.
+
+---
+
+### Do I need to host my own infrastructure?
+
+No! You have two options:
+
+1. **Hindsight Cloud** - Fully managed service at [ui.hindsight.vectorize.io](https://ui.hindsight.vectorize.io)
+2. **Self-hosted** - Deploy on your own infrastructure using Docker or direct installation
+
+See [Installation](/developer/installation) for self-hosting instructions.
+
+---
+
+### What are the minimum system requirements for self-hosting?
+
+For running the Hindsight API server locally:
+- Python 3.11+
+- 4GB RAM minimum (8GB recommended for production)
+- LLM API key (OpenAI, Anthropic, etc.) or local LLM setup
+
+See [Installation](/developer/installation) for setup instructions.
+
+---
+
+### How do I isolate user data?
+
+A **memory bank** is an isolated memory store (like a "brain") that contains its own memories, entities, relationships, and optional disposition traits (skepticism, literalism, empathy). Banks are completely isolated from each other with no data leakage.
+
+There are two approaches for multi-user applications:
+
+**1. Per-user memory banks** (recommended for most use cases)
+- Create one bank per user (e.g., `bank_id="user-123"`)
+- Easiest setup and strongest data isolation
+- Perfect for per-user queries and personalization
+- Each bank can have unique disposition traits and background context
+- **Limitation**: Cannot perform cross-user analysis (e.g., "What is the most mentioned topic across all users?")
+
+**2. Single bank with tags** (for applications needing aggregated insights)
+- Use one bank for the entire application
+- Tag memories with user identifiers during retain (e.g., `tags={"user_id": "user-123"}`)
+- Filter by tags during recall/reflect for per-user queries
+- **Advantage**: Enables both per-user AND cross-user queries (e.g., analyze specific users or aggregate across all users)
+
+Choose per-user banks for simplicity and privacy, or single bank with tags if you need holistic reasoning across users. See [Memory Banks](/developer/api/memory-banks) for management details.
+
+--- 
+### What's the difference between retain, recall, and reflect?
+
+Hindsight has three core operations:
+
+- **Retain**: Store data (facts, entities, relationships)
+- **Recall**: Search and retrieve raw memory data based on a query
+- **Reflect**: Use an AI agent to answer a query using retrieved memories
+
+See [Operations](/developer/api/operations) for API details.
+
+---
+
+### When should I use mental models?
+
+**Mental models** are consolidated knowledge patterns synthesized from individual facts over time. Use them when you need:
+
+- Higher-level understanding beyond raw facts (e.g., "User prefers functional programming patterns")
+- Long-term behavioral patterns (e.g., "Customer is price-sensitive but values quality")
+- Context for AI agent reasoning during **reflect** operations
+
+Mental models are automatically built during retain and used by reflect to provide richer, more contextual responses. See [Mental Models](/developer/api/mental-models).
+
+---
+
+
+### What's the typical latency for recall operations?
+
+Typical latencies:
+- **Without reranking**: 50-100ms
+- **With reranking**: 200-500ms (depends on reranker model and installation)
+
+See [Performance](/developer/performance) for tuning options.
+
+
+
+## Still have questions?
+
+Join our [Slack community](https://join.slack.com/t/hindsight-space/shared_invite/zt-3nhbm4w29-LeSJ5Ixi6j8PdiYOCPlOgg) or report issues on [GitHub](https://github.com/vectorize-io/hindsight/issues).

hindsight-docs/static/openapi.json

@@ -3134,6 +3134,75 @@
           }
         }
       }
+    },
+    "/v1/default/banks/{bank_id}/files/retain": {
+      "post": {
+        "tags": [
+          "Files"
+        ],
+        "summary": "Convert files to memories",
+        "description": "Upload files (PDF, DOCX, etc.), convert them to markdown, and retain as memories.\n\nThis endpoint handles file upload, conversion, and memory creation in a single operation.\n\n**Features:**\n- Supports PDF, DOCX, PPTX, XLSX, images (with OCR), audio (with transcription)\n- Automatic file-to-markdown conversion using pluggable parsers\n- Files stored in object storage (PostgreSQL by default, S3 for production)\n- Each file becomes a separate document with optional metadata/tags\n- Always processes asynchronously \u2014 returns operation IDs immediately\n\n**The system automatically:**\n1. Stores uploaded files in object storage\n2. Converts files to markdown\n3. Creates document records with file metadata\n4. Extracts facts and creates memory units (same as regular retain)\n\nUse the operations endpoint to monitor progress.\n\n**Request format:** multipart/form-data with:\n- `files`: One or more files to upload\n- `request`: JSON string with FileRetainRequest model (files_metadata)\n\n**Note:** File parser is configured server-side via `HINDSIGHT_API_FILE_PARSER` (default: markitdown).",
+        "operationId": "file_retain",
+        "parameters": [
+          {
+            "name": "bank_id",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string",
+              "title": "Bank Id"
+            }
+          },
+          {
+            "name": "authorization",
+            "in": "header",
+            "required": false,
+            "schema": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Authorization"
+            }
+          }
+        ],
+        "requestBody": {
+          "required": true,
+          "content": {
+            "multipart/form-data": {
+              "schema": {
+                "$ref": "#/components/schemas/Body_file_retain"
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "Successful Response",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/FileRetainResponse"
+                }
+              }
+            }
+          },
+          "422": {
+            "description": "Validation Error",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/HTTPValidationError"
+                }
+              }
+            }
+          }
+        }
+      }
     }
   },
   "components": {
@@ -3566,6 +3635,30 @@
           "total_observations": 45
         }
       },
+      "Body_file_retain": {
+        "properties": {
+          "files": {
+            "items": {
+              "type": "string",
+              "format": "binary"
+            },
+            "type": "array",
+            "title": "Files",
+            "description": "Files to upload and convert"
+          },
+          "request": {
+            "type": "string",
+            "title": "Request",
+            "description": "JSON string with FileRetainRequest model"
+          }
+        },
+        "type": "object",
+        "required": [
+          "files",
+          "request"
+        ],
+        "title": "Body_file_retain"
+      },
       "Budget": {
         "type": "string",
         "enum": [
@@ -4547,6 +4640,11 @@
             "type": "boolean",
             "title": "Bank Config Api",
             "description": "Whether per-bank configuration API is enabled"
+          },
+          "file_upload_api": {
+            "type": "boolean",
+            "title": "File Upload Api",
+            "description": "Whether file upload/conversion API is enabled"
           }
         },
         "type": "object",
@@ -4554,11 +4652,37 @@
           "observations",
           "mcp",
           "worker",
-          "bank_config_api"
+          "bank_config_api",
+          "file_upload_api"
         ],
         "title": "FeaturesInfo",
         "description": "Feature flags indicating which capabilities are enabled."
       },
+      "FileRetainResponse": {
+        "properties": {
+          "operation_ids": {
+            "items": {
+              "type": "string"
+            },
+            "type": "array",
+            "title": "Operation Ids",
+            "description": "Operation IDs for tracking file conversion operations. Use GET /v1/default/banks/{bank_id}/operations to list operations."
+          }
+        },
+        "type": "object",
+        "required": [
+          "operation_ids"
+        ],
+        "title": "FileRetainResponse",
+        "description": "Response model for file upload endpoint.",
+        "example": {
+          "operation_ids": [
+            "550e8400-e29b-41d4-a716-446655440000",
+            "550e8400-e29b-41d4-a716-446655440001",
+            "550e8400-e29b-41d4-a716-446655440002"
+          ]
+        }
+      },
       "GraphDataResponse": {
         "properties": {
           "nodes": {
@@ -6272,7 +6396,7 @@
               }
             ],
             "title": "Operation Id",
-            "description": "Operation ID for tracking async operations. Use GET /v1/default/banks/{bank_id}/operations to list operations and find this ID. Only present when async=true."
+            "description": "Operation ID for tracking async operations. Use GET /v1/default/banks/{bank_id}/operations to list operations. Only present when async=true."
           },
           "usage": {
             "anyOf": [
@@ -6593,6 +6717,7 @@
           "api_version": "0.4.0",
           "features": {
             "bank_config_api": false,
+            "file_upload_api": true,
             "mcp": true,
             "observations": false,
             "worker": true

hindsight-integrations/openclaw/src/client.ts

@@ -1,5 +1,9 @@
 import { exec } from 'child_process';
 import { promisify } from 'util';
+import { writeFile, mkdir, rm } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { randomBytes } from 'crypto';
 import type {
   RetainRequest,
   RetainResponse,
@@ -9,6 +13,15 @@ import type {
 
 const execAsync = promisify(exec);
 
+/**
+ * Sanitize a string for use as a cross-platform filename.
+ * Replaces characters illegal on Windows or Unix with underscores.
+ */
+function sanitizeFilename(name: string): string {
+  // Replace characters illegal on Windows (\/:*?"<>|) and control chars
+  return name.replace(/[\\/:*?"<>|\x00-\x1f]/g, '_').slice(0, 200) || 'content';
+}
+
 /**
  * Escape a string for use as a single-quoted shell argument.
  *
@@ -91,17 +104,25 @@ export class HindsightClient {
   }
 
   async retain(request: RetainRequest): Promise<RetainResponse> {
-    const content = escapeShellArg(request.content);
-    const docId = escapeShellArg(request.document_id || 'conversation');
+    const docId = request.document_id || 'conversation';
 
-    const embedCmd = this.getEmbedCommandPrefix();
-    const cmd = `${embedCmd} --profile openclaw memory retain ${this.bankId} '${content}' --doc-id '${docId}' --async`;
+    // Write content to a temp file to avoid E2BIG (ARG_MAX) errors when passing
+    // large conversations as shell arguments via execAsync.
+    const tempDir = join(tmpdir(), `hindsight_${randomBytes(8).toString('hex')}`);
+    const safeFilename = sanitizeFilename(docId);
+    const tempFile = join(tempDir, `${safeFilename}.txt`);
 
     try {
+      await mkdir(tempDir, { recursive: true });
+      await writeFile(tempFile, request.content, 'utf8');
+
+      const escapedTempFile = escapeShellArg(tempFile);
+      const embedCmd = this.getEmbedCommandPrefix();
+      const cmd = `${embedCmd} --profile openclaw memory retain-files ${this.bankId} '${escapedTempFile}' --async`;
+
       const { stdout } = await execAsync(cmd);
       console.log(`[Hindsight] Retained (async): ${stdout.trim()}`);
 
-      // Return a simple response
       return {
         message: 'Memory queued for background processing',
         document_id: docId,
@@ -109,6 +130,8 @@ export class HindsightClient {
       };
     } catch (error) {
       throw new Error(`Failed to retain memory: ${error}`);
+    } finally {
+      await rm(tempDir, { recursive: true, force: true }).catch(() => {});
     }
   }
 

scripts/generate-clients.sh

@@ -396,6 +396,13 @@ else
     [ -f "$TEMP_DIR/trace_test.go" ] && mv "$TEMP_DIR/trace_test.go" .
     rm -rf "$TEMP_DIR"
 
+    # Fix known generator issue: api_files.go uses os.File but generator omits "os" import
+    if [ -f "api_files.go" ] && grep -q 'os\.File' api_files.go && ! grep -q '"os"' api_files.go; then
+        echo "Patching api_files.go: adding missing 'os' import..."
+        sed -i.bak 's|"net/url"|"net/url"\n\t"os"|' api_files.go
+        rm -f api_files.go.bak
+    fi
+
     # Initialize module and build
     echo "Building Go client..."
     go mod tidy

uv.lock

@@ -2,8 +2,14 @@ version = 1
 revision = 1
 requires-python = ">=3.11"
 resolution-markers = [
-    "python_full_version >= '3.12'",
-    "python_full_version < '3.12'",
+    "python_full_version >= '3.14' and sys_platform == 'win32'",
+    "python_full_version == '3.13.*' and sys_platform == 'win32'",
+    "python_full_version >= '3.14' and sys_platform != 'win32'",
+    "python_full_version == '3.13.*' and sys_platform != 'win32'",
+    "python_full_version == '3.12.*' and sys_platform == 'win32'",
+    "python_full_version == '3.12.*' and sys_platform != 'win32'",
+    "python_full_version < '3.12' and sys_platform == 'win32'",
+    "python_full_version < '3.12' and sys_platform != 'win32'",
 ]
 
 [manifest]
@@ -627,6 +633,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228 },
 ]
 
+[[package]]
+name = "cobble"
+version = "0.1.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/54/7a/a507c709be2c96e1bb6102eb7b7f4026c5e5e223ef7d745a17d239e9d844/cobble-0.1.4.tar.gz", hash = "sha256:de38be1539992c8a06e569630717c485a5f91be2192c461ea2b220607dfa78aa", size = 3805 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d5/e1/3714a2f371985215c219c2a70953d38e3eed81ef165aed061d21de0e998b/cobble-0.1.4-py3-none-any.whl", hash = "sha256:36c91b1655e599fd428e2b95fdd5f0da1ca2e9f1abb0bc871dec21a0e78a2b44", size = 3984 },
+]
+
 [[package]]
 name = "cohere"
 version = "5.20.1"
@@ -759,6 +774,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/87/22/f020c047ae1346613db9322638186468238bcfa8849b4668a22b97faad65/dateparser-1.2.2-py3-none-any.whl", hash = "sha256:5a5d7211a09013499867547023a2a0c91d5a27d15dd4dbcea676ea9fe66f2482", size = 315453 },
 ]
 
+[[package]]
+name = "defusedxml"
+version = "0.7.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604 },
+]
+
 [[package]]
 name = "diskcache"
 version = "5.6.3"
@@ -786,6 +810,20 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094 },
 ]
 
+[[package]]
+name = "docker"
+version = "7.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pywin32", marker = "sys_platform == 'win32'" },
+    { name = "requests" },
+    { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774 },
+]
+
 [[package]]
 name = "docstring-parser"
 version = "0.17.0"
@@ -817,6 +855,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604 },
 ]
 
+[[package]]
+name = "et-xmlfile"
+version = "2.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059 },
+]
+
 [[package]]
 name = "exceptiongroup"
 version = "1.3.0"
@@ -1423,6 +1470,8 @@ dependencies = [
     { name = "langchain-core" },
     { name = "langchain-text-splitters" },
     { name = "litellm" },
+    { name = "markitdown", extra = ["docx", "pdf", "pptx", "xls", "xlsx"] },
+    { name = "obstore" },
     { name = "openai" },
     { name = "opentelemetry-api" },
     { name = "opentelemetry-exporter-otlp-proto-http" },
@@ -1458,6 +1507,7 @@ test = [
     { name = "pytest-asyncio" },
     { name = "pytest-timeout" },
     { name = "pytest-xdist" },
+    { name = "testcontainers" },
 ]
 
 [package.dev-dependencies]
@@ -1469,6 +1519,7 @@ dev = [
     { name = "pytest-xdist" },
     { name = "python-dotenv" },
     { name = "ruff" },
+    { name = "testcontainers" },
     { name = "ty" },
 ]
 
@@ -1494,6 +1545,8 @@ requires-dist = [
     { name = "langchain-core", specifier = ">=1.2.5" },
     { name = "langchain-text-splitters", specifier = ">=0.3.0" },
     { name = "litellm", specifier = ">=1.0.0" },
+    { name = "markitdown", extras = ["pdf", "docx", "pptx", "xlsx", "xls"], specifier = ">=0.1.4" },
+    { name = "obstore", specifier = ">=0.4.0" },
     { name = "openai", specifier = ">=1.0.0" },
     { name = "opentelemetry-api", specifier = ">=1.20.0" },
     { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.20.0" },
@@ -1516,6 +1569,7 @@ requires-dist = [
     { name = "rich", specifier = ">=13.0.0" },
     { name = "sentence-transformers", specifier = ">=3.3.0" },
     { name = "sqlalchemy", specifier = ">=2.0.44" },
+    { name = "testcontainers", marker = "extra == 'test'", specifier = ">=4.0.0" },
     { name = "tiktoken", specifier = ">=0.12.0" },
     { name = "torch", specifier = ">=2.6.0" },
     { name = "transformers", specifier = ">=4.53.0" },
@@ -1536,6 +1590,7 @@ dev = [
     { name = "pytest-xdist", specifier = ">=3.8.0" },
     { name = "python-dotenv", specifier = ">=1.2.1" },
     { name = "ruff", specifier = ">=0.8.0" },
+    { name = "testcontainers", specifier = ">=4.0.0" },
     { name = "ty", specifier = ">=0.0.1" },
 ]
 
@@ -2160,6 +2215,126 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7d/5e/db903ce9cf82c48d6b91bf6d63ae4c8d0d17958939a4e04ba6b9f38b8643/lupa-2.6-cp314-cp314t-win_amd64.whl", hash = "sha256:fc1498d1a4fc028bc521c26d0fad4ca00ed63b952e32fb95949bda76a04bad52", size = 1913818 },
 ]
 
+[[package]]
+name = "lxml"
+version = "6.0.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/aa/88/262177de60548e5a2bfc46ad28232c9e9cbde697bd94132aeb80364675cb/lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62", size = 4073426 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/77/d5/becbe1e2569b474a23f0c672ead8a29ac50b2dc1d5b9de184831bda8d14c/lxml-6.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:13e35cbc684aadf05d8711a5d1b5857c92e5e580efa9a0d2be197199c8def607", size = 8634365 },
+    { url = "https://files.pythonhosted.org/packages/28/66/1ced58f12e804644426b85d0bb8a4478ca77bc1761455da310505f1a3526/lxml-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b1675e096e17c6fe9c0e8c81434f5736c0739ff9ac6123c87c2d452f48fc938", size = 4650793 },
+    { url = "https://files.pythonhosted.org/packages/11/84/549098ffea39dfd167e3f174b4ce983d0eed61f9d8d25b7bf2a57c3247fc/lxml-6.0.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac6e5811ae2870953390452e3476694196f98d447573234592d30488147404d", size = 4944362 },
+    { url = "https://files.pythonhosted.org/packages/ac/bd/f207f16abf9749d2037453d56b643a7471d8fde855a231a12d1e095c4f01/lxml-6.0.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5aa0fc67ae19d7a64c3fe725dc9a1bb11f80e01f78289d05c6f62545affec438", size = 5083152 },
+    { url = "https://files.pythonhosted.org/packages/15/ae/bd813e87d8941d52ad5b65071b1affb48da01c4ed3c9c99e40abb266fbff/lxml-6.0.2-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de496365750cc472b4e7902a485d3f152ecf57bd3ba03ddd5578ed8ceb4c5964", size = 5023539 },
+    { url = "https://files.pythonhosted.org/packages/02/cd/9bfef16bd1d874fbe0cb51afb00329540f30a3283beb9f0780adbb7eec03/lxml-6.0.2-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:200069a593c5e40b8f6fc0d84d86d970ba43138c3e68619ffa234bc9bb806a4d", size = 5344853 },
+    { url = "https://files.pythonhosted.org/packages/b8/89/ea8f91594bc5dbb879734d35a6f2b0ad50605d7fb419de2b63d4211765cc/lxml-6.0.2-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d2de809c2ee3b888b59f995625385f74629707c9355e0ff856445cdcae682b7", size = 5225133 },
+    { url = "https://files.pythonhosted.org/packages/b9/37/9c735274f5dbec726b2db99b98a43950395ba3d4a1043083dba2ad814170/lxml-6.0.2-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:b2c3da8d93cf5db60e8858c17684c47d01fee6405e554fb55018dd85fc23b178", size = 4677944 },
+    { url = "https://files.pythonhosted.org/packages/20/28/7dfe1ba3475d8bfca3878365075abe002e05d40dfaaeb7ec01b4c587d533/lxml-6.0.2-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:442de7530296ef5e188373a1ea5789a46ce90c4847e597856570439621d9c553", size = 5284535 },
+    { url = "https://files.pythonhosted.org/packages/e7/cf/5f14bc0de763498fc29510e3532bf2b4b3a1c1d5d0dff2e900c16ba021ef/lxml-6.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2593c77efde7bfea7f6389f1ab249b15ed4aa5bc5cb5131faa3b843c429fbedb", size = 5067343 },
+    { url = "https://files.pythonhosted.org/packages/1c/b0/bb8275ab5472f32b28cfbbcc6db7c9d092482d3439ca279d8d6fa02f7025/lxml-6.0.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3e3cb08855967a20f553ff32d147e14329b3ae70ced6edc2f282b94afbc74b2a", size = 4725419 },
+    { url = "https://files.pythonhosted.org/packages/25/4c/7c222753bc72edca3b99dbadba1b064209bc8ed4ad448af990e60dcce462/lxml-6.0.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2ed6c667fcbb8c19c6791bbf40b7268ef8ddf5a96940ba9404b9f9a304832f6c", size = 5275008 },
+    { url = "https://files.pythonhosted.org/packages/6c/8c/478a0dc6b6ed661451379447cdbec77c05741a75736d97e5b2b729687828/lxml-6.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b8f18914faec94132e5b91e69d76a5c1d7b0c73e2489ea8929c4aaa10b76bbf7", size = 5248906 },
+    { url = "https://files.pythonhosted.org/packages/2d/d9/5be3a6ab2784cdf9accb0703b65e1b64fcdd9311c9f007630c7db0cfcce1/lxml-6.0.2-cp311-cp311-win32.whl", hash = "sha256:6605c604e6daa9e0d7f0a2137bdc47a2e93b59c60a65466353e37f8272f47c46", size = 3610357 },
+    { url = "https://files.pythonhosted.org/packages/e2/7d/ca6fb13349b473d5732fb0ee3eec8f6c80fc0688e76b7d79c1008481bf1f/lxml-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e5867f2651016a3afd8dd2c8238baa66f1e2802f44bc17e236f547ace6647078", size = 4036583 },
+    { url = "https://files.pythonhosted.org/packages/ab/a2/51363b5ecd3eab46563645f3a2c3836a2fc67d01a1b87c5017040f39f567/lxml-6.0.2-cp311-cp311-win_arm64.whl", hash = "sha256:4197fb2534ee05fd3e7afaab5d8bfd6c2e186f65ea7f9cd6a82809c887bd1285", size = 3680591 },
+    { url = "https://files.pythonhosted.org/packages/f3/c8/8ff2bc6b920c84355146cd1ab7d181bc543b89241cfb1ebee824a7c81457/lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456", size = 8661887 },
+    { url = "https://files.pythonhosted.org/packages/37/6f/9aae1008083bb501ef63284220ce81638332f9ccbfa53765b2b7502203cf/lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924", size = 4667818 },
+    { url = "https://files.pythonhosted.org/packages/f1/ca/31fb37f99f37f1536c133476674c10b577e409c0a624384147653e38baf2/lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f", size = 4950807 },
+    { url = "https://files.pythonhosted.org/packages/da/87/f6cb9442e4bada8aab5ae7e1046264f62fdbeaa6e3f6211b93f4c0dd97f1/lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534", size = 5109179 },
+    { url = "https://files.pythonhosted.org/packages/c8/20/a7760713e65888db79bbae4f6146a6ae5c04e4a204a3c48896c408cd6ed2/lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564", size = 5023044 },
+    { url = "https://files.pythonhosted.org/packages/a2/b0/7e64e0460fcb36471899f75831509098f3fd7cd02a3833ac517433cb4f8f/lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f", size = 5359685 },
+    { url = "https://files.pythonhosted.org/packages/b9/e1/e5df362e9ca4e2f48ed6411bd4b3a0ae737cc842e96877f5bf9428055ab4/lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0", size = 5654127 },
+    { url = "https://files.pythonhosted.org/packages/c6/d1/232b3309a02d60f11e71857778bfcd4acbdb86c07db8260caf7d008b08f8/lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192", size = 5253958 },
+    { url = "https://files.pythonhosted.org/packages/35/35/d955a070994725c4f7d80583a96cab9c107c57a125b20bb5f708fe941011/lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0", size = 4711541 },
+    { url = "https://files.pythonhosted.org/packages/1e/be/667d17363b38a78c4bd63cfd4b4632029fd68d2c2dc81f25ce9eb5224dd5/lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092", size = 5267426 },
+    { url = "https://files.pythonhosted.org/packages/ea/47/62c70aa4a1c26569bc958c9ca86af2bb4e1f614e8c04fb2989833874f7ae/lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f", size = 5064917 },
+    { url = "https://files.pythonhosted.org/packages/bd/55/6ceddaca353ebd0f1908ef712c597f8570cc9c58130dbb89903198e441fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8", size = 4788795 },
+    { url = "https://files.pythonhosted.org/packages/cf/e8/fd63e15da5e3fd4c2146f8bbb3c14e94ab850589beab88e547b2dbce22e1/lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f", size = 5676759 },
+    { url = "https://files.pythonhosted.org/packages/76/47/b3ec58dc5c374697f5ba37412cd2728f427d056315d124dd4b61da381877/lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6", size = 5255666 },
+    { url = "https://files.pythonhosted.org/packages/19/93/03ba725df4c3d72afd9596eef4a37a837ce8e4806010569bedfcd2cb68fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322", size = 5277989 },
+    { url = "https://files.pythonhosted.org/packages/c6/80/c06de80bfce881d0ad738576f243911fccf992687ae09fd80b734712b39c/lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849", size = 3611456 },
+    { url = "https://files.pythonhosted.org/packages/f7/d7/0cdfb6c3e30893463fb3d1e52bc5f5f99684a03c29a0b6b605cfae879cd5/lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f", size = 4011793 },
+    { url = "https://files.pythonhosted.org/packages/ea/7b/93c73c67db235931527301ed3785f849c78991e2e34f3fd9a6663ffda4c5/lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6", size = 3672836 },
+    { url = "https://files.pythonhosted.org/packages/53/fd/4e8f0540608977aea078bf6d79f128e0e2c2bba8af1acf775c30baa70460/lxml-6.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9b33d21594afab46f37ae58dfadd06636f154923c4e8a4d754b0127554eb2e77", size = 8648494 },
+    { url = "https://files.pythonhosted.org/packages/5d/f4/2a94a3d3dfd6c6b433501b8d470a1960a20ecce93245cf2db1706adf6c19/lxml-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c8963287d7a4c5c9a432ff487c52e9c5618667179c18a204bdedb27310f022f", size = 4661146 },
+    { url = "https://files.pythonhosted.org/packages/25/2e/4efa677fa6b322013035d38016f6ae859d06cac67437ca7dc708a6af7028/lxml-6.0.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1941354d92699fb5ffe6ed7b32f9649e43c2feb4b97205f75866f7d21aa91452", size = 4946932 },
+    { url = "https://files.pythonhosted.org/packages/ce/0f/526e78a6d38d109fdbaa5049c62e1d32fdd70c75fb61c4eadf3045d3d124/lxml-6.0.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb2f6ca0ae2d983ded09357b84af659c954722bbf04dea98030064996d156048", size = 5100060 },
+    { url = "https://files.pythonhosted.org/packages/81/76/99de58d81fa702cc0ea7edae4f4640416c2062813a00ff24bd70ac1d9c9b/lxml-6.0.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb2a12d704f180a902d7fa778c6d71f36ceb7b0d317f34cdc76a5d05aa1dd1df", size = 5019000 },
+    { url = "https://files.pythonhosted.org/packages/b5/35/9e57d25482bc9a9882cb0037fdb9cc18f4b79d85df94fa9d2a89562f1d25/lxml-6.0.2-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6ec0e3f745021bfed19c456647f0298d60a24c9ff86d9d051f52b509663feeb1", size = 5348496 },
+    { url = "https://files.pythonhosted.org/packages/a6/8e/cb99bd0b83ccc3e8f0f528e9aa1f7a9965dfec08c617070c5db8d63a87ce/lxml-6.0.2-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:846ae9a12d54e368933b9759052d6206a9e8b250291109c48e350c1f1f49d916", size = 5643779 },
+    { url = "https://files.pythonhosted.org/packages/d0/34/9e591954939276bb679b73773836c6684c22e56d05980e31d52a9a8deb18/lxml-6.0.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef9266d2aa545d7374938fb5c484531ef5a2ec7f2d573e62f8ce722c735685fd", size = 5244072 },
+    { url = "https://files.pythonhosted.org/packages/8d/27/b29ff065f9aaca443ee377aff699714fcbffb371b4fce5ac4ca759e436d5/lxml-6.0.2-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:4077b7c79f31755df33b795dc12119cb557a0106bfdab0d2c2d97bd3cf3dffa6", size = 4718675 },
+    { url = "https://files.pythonhosted.org/packages/2b/9f/f756f9c2cd27caa1a6ef8c32ae47aadea697f5c2c6d07b0dae133c244fbe/lxml-6.0.2-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a7c5d5e5f1081955358533be077166ee97ed2571d6a66bdba6ec2f609a715d1a", size = 5255171 },
+    { url = "https://files.pythonhosted.org/packages/61/46/bb85ea42d2cb1bd8395484fd72f38e3389611aa496ac7772da9205bbda0e/lxml-6.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f8d0cbd0674ee89863a523e6994ac25fd5be9c8486acfc3e5ccea679bad2679", size = 5057175 },
+    { url = "https://files.pythonhosted.org/packages/95/0c/443fc476dcc8e41577f0af70458c50fe299a97bb6b7505bb1ae09aa7f9ac/lxml-6.0.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2cbcbf6d6e924c28f04a43f3b6f6e272312a090f269eff68a2982e13e5d57659", size = 4785688 },
+    { url = "https://files.pythonhosted.org/packages/48/78/6ef0b359d45bb9697bc5a626e1992fa5d27aa3f8004b137b2314793b50a0/lxml-6.0.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dfb874cfa53340009af6bdd7e54ebc0d21012a60a4e65d927c2e477112e63484", size = 5660655 },
+    { url = "https://files.pythonhosted.org/packages/ff/ea/e1d33808f386bc1339d08c0dcada6e4712d4ed8e93fcad5f057070b7988a/lxml-6.0.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb8dae0b6b8b7f9e96c26fdd8121522ce5de9bb5538010870bd538683d30e9a2", size = 5247695 },
+    { url = "https://files.pythonhosted.org/packages/4f/47/eba75dfd8183673725255247a603b4ad606f4ae657b60c6c145b381697da/lxml-6.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:358d9adae670b63e95bc59747c72f4dc97c9ec58881d4627fe0120da0f90d314", size = 5269841 },
+    { url = "https://files.pythonhosted.org/packages/76/04/5c5e2b8577bc936e219becb2e98cdb1aca14a4921a12995b9d0c523502ae/lxml-6.0.2-cp313-cp313-win32.whl", hash = "sha256:e8cd2415f372e7e5a789d743d133ae474290a90b9023197fd78f32e2dc6873e2", size = 3610700 },
+    { url = "https://files.pythonhosted.org/packages/fe/0a/4643ccc6bb8b143e9f9640aa54e38255f9d3b45feb2cbe7ae2ca47e8782e/lxml-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:b30d46379644fbfc3ab81f8f82ae4de55179414651f110a1514f0b1f8f6cb2d7", size = 4010347 },
+    { url = "https://files.pythonhosted.org/packages/31/ef/dcf1d29c3f530577f61e5fe2f1bd72929acf779953668a8a47a479ae6f26/lxml-6.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:13dcecc9946dca97b11b7c40d29fba63b55ab4170d3c0cf8c0c164343b9bfdcf", size = 3671248 },
+    { url = "https://files.pythonhosted.org/packages/03/15/d4a377b385ab693ce97b472fe0c77c2b16ec79590e688b3ccc71fba19884/lxml-6.0.2-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:b0c732aa23de8f8aec23f4b580d1e52905ef468afb4abeafd3fec77042abb6fe", size = 8659801 },
+    { url = "https://files.pythonhosted.org/packages/c8/e8/c128e37589463668794d503afaeb003987373c5f94d667124ffd8078bbd9/lxml-6.0.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4468e3b83e10e0317a89a33d28f7aeba1caa4d1a6fd457d115dd4ffe90c5931d", size = 4659403 },
+    { url = "https://files.pythonhosted.org/packages/00/ce/74903904339decdf7da7847bb5741fc98a5451b42fc419a86c0c13d26fe2/lxml-6.0.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:abd44571493973bad4598a3be7e1d807ed45aa2adaf7ab92ab7c62609569b17d", size = 4966974 },
+    { url = "https://files.pythonhosted.org/packages/1f/d3/131dec79ce61c5567fecf82515bd9bc36395df42501b50f7f7f3bd065df0/lxml-6.0.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:370cd78d5855cfbffd57c422851f7d3864e6ae72d0da615fca4dad8c45d375a5", size = 5102953 },
+    { url = "https://files.pythonhosted.org/packages/3a/ea/a43ba9bb750d4ffdd885f2cd333572f5bb900cd2408b67fdda07e85978a0/lxml-6.0.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:901e3b4219fa04ef766885fb40fa516a71662a4c61b80c94d25336b4934b71c0", size = 5055054 },
+    { url = "https://files.pythonhosted.org/packages/60/23/6885b451636ae286c34628f70a7ed1fcc759f8d9ad382d132e1c8d3d9bfd/lxml-6.0.2-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:a4bf42d2e4cf52c28cc1812d62426b9503cdb0c87a6de81442626aa7d69707ba", size = 5352421 },
+    { url = "https://files.pythonhosted.org/packages/48/5b/fc2ddfc94ddbe3eebb8e9af6e3fd65e2feba4967f6a4e9683875c394c2d8/lxml-6.0.2-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2c7fdaa4d7c3d886a42534adec7cfac73860b89b4e5298752f60aa5984641a0", size = 5673684 },
+    { url = "https://files.pythonhosted.org/packages/29/9c/47293c58cc91769130fbf85531280e8cc7868f7fbb6d92f4670071b9cb3e/lxml-6.0.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98a5e1660dc7de2200b00d53fa00bcd3c35a3608c305d45a7bbcaf29fa16e83d", size = 5252463 },
+    { url = "https://files.pythonhosted.org/packages/9b/da/ba6eceb830c762b48e711ded880d7e3e89fc6c7323e587c36540b6b23c6b/lxml-6.0.2-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:dc051506c30b609238d79eda75ee9cab3e520570ec8219844a72a46020901e37", size = 4698437 },
+    { url = "https://files.pythonhosted.org/packages/a5/24/7be3f82cb7990b89118d944b619e53c656c97dc89c28cfb143fdb7cd6f4d/lxml-6.0.2-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8799481bbdd212470d17513a54d568f44416db01250f49449647b5ab5b5dccb9", size = 5269890 },
+    { url = "https://files.pythonhosted.org/packages/1b/bd/dcfb9ea1e16c665efd7538fc5d5c34071276ce9220e234217682e7d2c4a5/lxml-6.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9261bb77c2dab42f3ecd9103951aeca2c40277701eb7e912c545c1b16e0e4917", size = 5097185 },
+    { url = "https://files.pythonhosted.org/packages/21/04/a60b0ff9314736316f28316b694bccbbabe100f8483ad83852d77fc7468e/lxml-6.0.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:65ac4a01aba353cfa6d5725b95d7aed6356ddc0a3cd734de00124d285b04b64f", size = 4745895 },
+    { url = "https://files.pythonhosted.org/packages/d6/bd/7d54bd1846e5a310d9c715921c5faa71cf5c0853372adf78aee70c8d7aa2/lxml-6.0.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:b22a07cbb82fea98f8a2fd814f3d1811ff9ed76d0fc6abc84eb21527596e7cc8", size = 5695246 },
+    { url = "https://files.pythonhosted.org/packages/fd/32/5643d6ab947bc371da21323acb2a6e603cedbe71cb4c99c8254289ab6f4e/lxml-6.0.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d759cdd7f3e055d6bc8d9bec3ad905227b2e4c785dc16c372eb5b5e83123f48a", size = 5260797 },
+    { url = "https://files.pythonhosted.org/packages/33/da/34c1ec4cff1eea7d0b4cd44af8411806ed943141804ac9c5d565302afb78/lxml-6.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:945da35a48d193d27c188037a05fec5492937f66fb1958c24fc761fb9d40d43c", size = 5277404 },
+    { url = "https://files.pythonhosted.org/packages/82/57/4eca3e31e54dc89e2c3507e1cd411074a17565fa5ffc437c4ae0a00d439e/lxml-6.0.2-cp314-cp314-win32.whl", hash = "sha256:be3aaa60da67e6153eb15715cc2e19091af5dc75faef8b8a585aea372507384b", size = 3670072 },
+    { url = "https://files.pythonhosted.org/packages/e3/e0/c96cf13eccd20c9421ba910304dae0f619724dcf1702864fd59dd386404d/lxml-6.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:fa25afbadead523f7001caf0c2382afd272c315a033a7b06336da2637d92d6ed", size = 4080617 },
+    { url = "https://files.pythonhosted.org/packages/d5/5d/b3f03e22b3d38d6f188ef044900a9b29b2fe0aebb94625ce9fe244011d34/lxml-6.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:063eccf89df5b24e361b123e257e437f9e9878f425ee9aae3144c77faf6da6d8", size = 3754930 },
+    { url = "https://files.pythonhosted.org/packages/5e/5c/42c2c4c03554580708fc738d13414801f340c04c3eff90d8d2d227145275/lxml-6.0.2-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:6162a86d86893d63084faaf4ff937b3daea233e3682fb4474db07395794fa80d", size = 8910380 },
+    { url = "https://files.pythonhosted.org/packages/bf/4f/12df843e3e10d18d468a7557058f8d3733e8b6e12401f30b1ef29360740f/lxml-6.0.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:414aaa94e974e23a3e92e7ca5b97d10c0cf37b6481f50911032c69eeb3991bba", size = 4775632 },
+    { url = "https://files.pythonhosted.org/packages/e4/0c/9dc31e6c2d0d418483cbcb469d1f5a582a1cd00a1f4081953d44051f3c50/lxml-6.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48461bd21625458dd01e14e2c38dd0aea69addc3c4f960c30d9f59d7f93be601", size = 4975171 },
+    { url = "https://files.pythonhosted.org/packages/e7/2b/9b870c6ca24c841bdd887504808f0417aa9d8d564114689266f19ddf29c8/lxml-6.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:25fcc59afc57d527cfc78a58f40ab4c9b8fd096a9a3f964d2781ffb6eb33f4ed", size = 5110109 },
+    { url = "https://files.pythonhosted.org/packages/bf/0c/4f5f2a4dd319a178912751564471355d9019e220c20d7db3fb8307ed8582/lxml-6.0.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5179c60288204e6ddde3f774a93350177e08876eaf3ab78aa3a3649d43eb7d37", size = 5041061 },
+    { url = "https://files.pythonhosted.org/packages/12/64/554eed290365267671fe001a20d72d14f468ae4e6acef1e179b039436967/lxml-6.0.2-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:967aab75434de148ec80597b75062d8123cadf2943fb4281f385141e18b21338", size = 5306233 },
+    { url = "https://files.pythonhosted.org/packages/7a/31/1d748aa275e71802ad9722df32a7a35034246b42c0ecdd8235412c3396ef/lxml-6.0.2-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d100fcc8930d697c6561156c6810ab4a508fb264c8b6779e6e61e2ed5e7558f9", size = 5604739 },
+    { url = "https://files.pythonhosted.org/packages/8f/41/2c11916bcac09ed561adccacceaedd2bf0e0b25b297ea92aab99fd03d0fa/lxml-6.0.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ca59e7e13e5981175b8b3e4ab84d7da57993eeff53c07764dcebda0d0e64ecd", size = 5225119 },
+    { url = "https://files.pythonhosted.org/packages/99/05/4e5c2873d8f17aa018e6afde417c80cc5d0c33be4854cce3ef5670c49367/lxml-6.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:957448ac63a42e2e49531b9d6c0fa449a1970dbc32467aaad46f11545be9af1d", size = 4633665 },
+    { url = "https://files.pythonhosted.org/packages/0f/c9/dcc2da1bebd6275cdc723b515f93edf548b82f36a5458cca3578bc899332/lxml-6.0.2-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b7fc49c37f1786284b12af63152fe1d0990722497e2d5817acfe7a877522f9a9", size = 5234997 },
+    { url = "https://files.pythonhosted.org/packages/9c/e2/5172e4e7468afca64a37b81dba152fc5d90e30f9c83c7c3213d6a02a5ce4/lxml-6.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e19e0643cc936a22e837f79d01a550678da8377d7d801a14487c10c34ee49c7e", size = 5090957 },
+    { url = "https://files.pythonhosted.org/packages/a5/b3/15461fd3e5cd4ddcb7938b87fc20b14ab113b92312fc97afe65cd7c85de1/lxml-6.0.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1db01e5cf14345628e0cbe71067204db658e2fb8e51e7f33631f5f4735fefd8d", size = 4764372 },
+    { url = "https://files.pythonhosted.org/packages/05/33/f310b987c8bf9e61c4dd8e8035c416bd3230098f5e3cfa69fc4232de7059/lxml-6.0.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:875c6b5ab39ad5291588aed6925fac99d0097af0dd62f33c7b43736043d4a2ec", size = 5634653 },
+    { url = "https://files.pythonhosted.org/packages/70/ff/51c80e75e0bc9382158133bdcf4e339b5886c6ee2418b5199b3f1a61ed6d/lxml-6.0.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cdcbed9ad19da81c480dfd6dd161886db6096083c9938ead313d94b30aadf272", size = 5233795 },
+    { url = "https://files.pythonhosted.org/packages/56/4d/4856e897df0d588789dd844dbed9d91782c4ef0b327f96ce53c807e13128/lxml-6.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:80dadc234ebc532e09be1975ff538d154a7fa61ea5031c03d25178855544728f", size = 5257023 },
+    { url = "https://files.pythonhosted.org/packages/0f/85/86766dfebfa87bea0ab78e9ff7a4b4b45225df4b4d3b8cc3c03c5cd68464/lxml-6.0.2-cp314-cp314t-win32.whl", hash = "sha256:da08e7bb297b04e893d91087df19638dc7a6bb858a954b0cc2b9f5053c922312", size = 3911420 },
+    { url = "https://files.pythonhosted.org/packages/fe/1a/b248b355834c8e32614650b8008c69ffeb0ceb149c793961dd8c0b991bb3/lxml-6.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:252a22982dca42f6155125ac76d3432e548a7625d56f5a273ee78a5057216eca", size = 4406837 },
+    { url = "https://files.pythonhosted.org/packages/92/aa/df863bcc39c5e0946263454aba394de8a9084dbaff8ad143846b0d844739/lxml-6.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:bb4c1847b303835d89d785a18801a883436cdfd5dc3d62947f9c49e24f0f5a2c", size = 3822205 },
+    { url = "https://files.pythonhosted.org/packages/0b/11/29d08bc103a62c0eba8016e7ed5aeebbf1e4312e83b0b1648dd203b0e87d/lxml-6.0.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1c06035eafa8404b5cf475bb37a9f6088b0aca288d4ccc9d69389750d5543700", size = 3949829 },
+    { url = "https://files.pythonhosted.org/packages/12/b3/52ab9a3b31e5ab8238da241baa19eec44d2ab426532441ee607165aebb52/lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c7d13103045de1bdd6fe5d61802565f1a3537d70cd3abf596aa0af62761921ee", size = 4226277 },
+    { url = "https://files.pythonhosted.org/packages/a0/33/1eaf780c1baad88224611df13b1c2a9dfa460b526cacfe769103ff50d845/lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a3c150a95fbe5ac91de323aa756219ef9cf7fde5a3f00e2281e30f33fa5fa4f", size = 4330433 },
+    { url = "https://files.pythonhosted.org/packages/7a/c1/27428a2ff348e994ab4f8777d3a0ad510b6b92d37718e5887d2da99952a2/lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60fa43be34f78bebb27812ed90f1925ec99560b0fa1decdb7d12b84d857d31e9", size = 4272119 },
+    { url = "https://files.pythonhosted.org/packages/f0/d0/3020fa12bcec4ab62f97aab026d57c2f0cfd480a558758d9ca233bb6a79d/lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21c73b476d3cfe836be731225ec3421fa2f048d84f6df6a8e70433dff1376d5a", size = 4417314 },
+    { url = "https://files.pythonhosted.org/packages/6c/77/d7f491cbc05303ac6801651aabeb262d43f319288c1ea96c66b1d2692ff3/lxml-6.0.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:27220da5be049e936c3aca06f174e8827ca6445a4353a1995584311487fc4e3e", size = 3518768 },
+]
+
+[[package]]
+name = "magika"
+version = "0.6.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+    { name = "numpy" },
+    { name = "onnxruntime" },
+    { name = "python-dotenv" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a3/f3/3d1dcdd7b9c41d589f5cff252d32ed91cdf86ba84391cfc81d9d8773571d/magika-0.6.3.tar.gz", hash = "sha256:7cc52aa7359af861957043e2bf7265ed4741067251c104532765cd668c0c0cb1", size = 3042784 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a2/e4/35c323beb3280482c94299d61626116856ac2d4ec16ecef50afc4fdd4291/magika-0.6.3-py3-none-any.whl", hash = "sha256:eda443d08006ee495e02083b32e51b98cb3696ab595a7d13900d8e2ef506ec9d", size = 2969474 },
+    { url = "https://files.pythonhosted.org/packages/25/8f/132b0d7cd51c02c39fd52658a5896276c30c8cc2fd453270b19db8c40f7e/magika-0.6.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:86901e64b05dde5faff408c9b8245495b2e1fd4c226e3393d3d2a3fee65c504b", size = 13358841 },
+    { url = "https://files.pythonhosted.org/packages/c4/03/5ed859be502903a68b7b393b17ae0283bf34195cfcca79ce2dc25b9290e7/magika-0.6.3-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:3d9661eedbdf445ac9567e97e7ceefb93545d77a6a32858139ea966b5806fb64", size = 15367335 },
+    { url = "https://files.pythonhosted.org/packages/7b/9e/f8ee7d644affa3b80efdd623a3d75865c8f058f3950cb87fb0c48e3559bc/magika-0.6.3-py3-none-win_amd64.whl", hash = "sha256:e57f75674447b20cab4db928ae58ab264d7d8582b55183a0b876711c2b2787f3", size = 12692831 },
+]
+
 [[package]]
 name = "mako"
 version = "1.3.10"
@@ -2172,6 +2347,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509 },
 ]
 
+[[package]]
+name = "mammoth"
+version = "1.11.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cobble" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ed/3c/a58418d2af00f2da60d4a51e18cd0311307b72d48d2fffec36a97b4a5e44/mammoth-1.11.0.tar.gz", hash = "sha256:a0f59e442f34d5b6447f4b0999306cbf3e67aaabfa8cb516f878fb1456744637", size = 53142 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ca/54/2e39566a131b13f6d8d193f974cb6a34e81bb7cc2fa6f7e03de067b36588/mammoth-1.11.0-py2.py3-none-any.whl", hash = "sha256:c077ab0d450bd7c0c6ecd529a23bf7e0fa8190c929e28998308ff4eada3f063b", size = 54752 },
+]
+
 [[package]]
 name = "markdown-it-py"
 version = "4.0.0"
@@ -2184,6 +2371,58 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321 },
 ]
 
+[[package]]
+name = "markdownify"
+version = "1.2.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "beautifulsoup4" },
+    { name = "six" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3f/bc/c8c8eea5335341306b0fa7e1cb33c5e1c8d24ef70ddd684da65f41c49c92/markdownify-1.2.2.tar.gz", hash = "sha256:b274f1b5943180b031b699b199cbaeb1e2ac938b75851849a31fd0c3d6603d09", size = 18816 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/43/ce/f1e3e9d959db134cedf06825fae8d5b294bd368aacdd0831a3975b7c4d55/markdownify-1.2.2-py3-none-any.whl", hash = "sha256:3f02d3cc52714084d6e589f70397b6fc9f2f3a8531481bf35e8cc39f975e186a", size = 15724 },
+]
+
+[[package]]
+name = "markitdown"
+version = "0.1.5b1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "beautifulsoup4" },
+    { name = "charset-normalizer" },
+    { name = "defusedxml" },
+    { name = "magika" },
+    { name = "markdownify" },
+    { name = "onnxruntime", marker = "sys_platform == 'win32'" },
+    { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/23/ae/c1f5e3fbd63883f27aec6bb6f8af2852253928bc673617bd02f2740a3057/markitdown-0.1.5b1.tar.gz", hash = "sha256:d5bffdb9d7aff9cac423a6d80d9e3a970937cec61338167bd855555f8a1c591c", size = 44408 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3c/10/e3a4e65265c239b15b34199ca340b56b1163ffbc8d0a658a734f7fa2f8b3/markitdown-0.1.5b1-py3-none-any.whl", hash = "sha256:31b667ce9858bc7ff50b7c7aec5fab2c3103d3ca2cb69203b3edabdda5d3a568", size = 62710 },
+]
+
+[package.optional-dependencies]
+docx = [
+    { name = "lxml" },
+    { name = "mammoth" },
+]
+pdf = [
+    { name = "pdfminer-six" },
+    { name = "pdfplumber" },
+]
+pptx = [
+    { name = "python-pptx" },
+]
+xls = [
+    { name = "pandas" },
+    { name = "xlrd" },
+]
+xlsx = [
+    { name = "openpyxl" },
+    { name = "pandas" },
+]
+
 [[package]]
 name = "markupsafe"
 version = "3.0.3"
@@ -2563,7 +2802,7 @@ name = "nvidia-cudnn-cu12"
 version = "9.10.2.21"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cublas-cu12" },
+    { name = "nvidia-cublas-cu12", marker = "sys_platform != 'win32'" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467 },
@@ -2574,7 +2813,7 @@ name = "nvidia-cufft-cu12"
 version = "11.3.3.83"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-nvjitlink-cu12" },
+    { name = "nvidia-nvjitlink-cu12", marker = "sys_platform != 'win32'" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695 },
@@ -2601,9 +2840,9 @@ name = "nvidia-cusolver-cu12"
 version = "11.7.3.90"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cublas-cu12" },
-    { name = "nvidia-cusparse-cu12" },
-    { name = "nvidia-nvjitlink-cu12" },
+    { name = "nvidia-cublas-cu12", marker = "sys_platform != 'win32'" },
+    { name = "nvidia-cusparse-cu12", marker = "sys_platform != 'win32'" },
+    { name = "nvidia-nvjitlink-cu12", marker = "sys_platform != 'win32'" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905 },
@@ -2614,7 +2853,7 @@ name = "nvidia-cusparse-cu12"
 version = "12.5.8.93"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-nvjitlink-cu12" },
+    { name = "nvidia-nvjitlink-cu12", marker = "sys_platform != 'win32'" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466 },
@@ -2669,9 +2908,71 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065 },
 ]
 
+[[package]]
+name = "obstore"
+version = "0.8.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a3/8c/9ec984edd0f3b72226adfaa19b1c61b15823b35b52f311ca4af36d009d15/obstore-0.8.2.tar.gz", hash = "sha256:a467bc4e97169e2ba749981b4fd0936015428d9b8f3fb83a5528536b1b6f377f", size = 168852 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2f/c4/018f90701f1e5ea3fbd57f61463f42e1ef5218e548d3adcf12b6be021c34/obstore-0.8.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:2edaa97687c191c5324bb939d72f6fe86a7aa8191c410f1648c14e8296d05c1c", size = 3622568 },
+    { url = "https://files.pythonhosted.org/packages/a8/62/72dd1e7d52fc554bb1fdb1a9499bda219cf3facea5865a1d97fdc00b3a1b/obstore-0.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c4fb7ef8108f08d14edc8bec9e9a6a2e5c4d14eddb8819f5d0da498aff6e8888", size = 3356109 },
+    { url = "https://files.pythonhosted.org/packages/e0/ae/089fe5b9207091252fe5ce352551214f04560f85eb8f2cc4f716a6a1a57e/obstore-0.8.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fda8f658c0edf799ab1e264f9b12c7c184cd09a5272dc645d42e987810ff2772", size = 3454588 },
+    { url = "https://files.pythonhosted.org/packages/ea/10/1865ae2d1ba45e8ae85fb0c1aada2dc9533baf60c4dfe74dab905348d74a/obstore-0.8.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87fe2bc15ce4051ecb56abd484feca323c2416628beb62c1c7b6712114564d6e", size = 3688627 },
+    { url = "https://files.pythonhosted.org/packages/a6/09/5d7ba6d0aeac563ea5f5586401c677bace4f782af83522b1fdf15430e152/obstore-0.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2482aa2562ab6a4ca40250b26bea33f8375b59898a9b5615fd412cab81098123", size = 3959896 },
+    { url = "https://files.pythonhosted.org/packages/16/15/2b3eda59914761a9ff4d840e2daec5697fd29b293bd18d3dc11c593aed06/obstore-0.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4153b928f5d2e9c6cb645e83668a53e0b42253d1e8bcb4e16571fc0a1434599a", size = 3933162 },
+    { url = "https://files.pythonhosted.org/packages/14/7a/5fc63b41526587067537fb1498c59a210884664c65ccf0d1f8f823b0875a/obstore-0.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dbfa9c38620cc191be98c8b5558c62071e495dc6b1cc724f38293ee439aa9f92", size = 3769605 },
+    { url = "https://files.pythonhosted.org/packages/77/4e/2208ab6e1fc021bf8b7e117249a10ab75d0ed24e0f2de1a8d7cd67d885b5/obstore-0.8.2-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:0822836eae8d52499f10daef17f26855b4c123119c6eb984aa4f2d525ec2678d", size = 3534396 },
+    { url = "https://files.pythonhosted.org/packages/1d/8f/a0e2882edd6bd285c82b8a5851c4ecf386c93fe75b6e340d5d9d30e809fc/obstore-0.8.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ef6435dfd586d83b4f778e7927a5d5b0d8b771e9ba914bc809a13d7805410e6", size = 3697777 },
+    { url = "https://files.pythonhosted.org/packages/94/78/ebf0c33bed5c9a8eed3b00eefafbcc0a687eeb1e05451c76fcf199d29ff8/obstore-0.8.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:0f2cba91f4271ca95a932a51aa8dda1537160342b33f7836c75e1eb9d40621a2", size = 3681546 },
+    { url = "https://files.pythonhosted.org/packages/af/21/9bf4fb9e53fd5f01af580b6538de2eae857e31d24b0ebfc4d916c306a1e4/obstore-0.8.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:23c876d603af0627627808d19a58d43eb5d8bfd02eecd29460bc9a58030fed55", size = 3765336 },
+    { url = "https://files.pythonhosted.org/packages/dd/3c/7f6895c23719482d231b2d6ed328e3223fdf99785f6850fba8d2fc5a86ee/obstore-0.8.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ff3c4b5d07629b70b9dee494cd6b94fff8465c3864752181a1cb81a77190fe42", size = 3941142 },
+    { url = "https://files.pythonhosted.org/packages/93/a4/56ccdb756161595680a28f4b0def2c04f7048ffacf128029be8394367b26/obstore-0.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:aadb2cb72de7227d07f4570f82729625ffc77522fadca5cf13c3a37fbe8c8de9", size = 3970172 },
+    { url = "https://files.pythonhosted.org/packages/2b/dc/60fefbb5736e69eab56657bca04ca64dc07fdeccb3814164a31b62ad066b/obstore-0.8.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bb70ce297a47392b1d9a3e310f18d59cd5ebbb9453428210fef02ed60e4d75d1", size = 3612955 },
+    { url = "https://files.pythonhosted.org/packages/d2/8b/844e8f382e5a12b8a3796a05d76a03e12c7aedc13d6900419e39207d7868/obstore-0.8.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1619bf618428abf1f607e0b219b2e230a966dcf697b717deccfa0983dd91f646", size = 3346564 },
+    { url = "https://files.pythonhosted.org/packages/89/73/8537f99e09a38a54a6a15ede907aa25d4da089f767a808f0b2edd9c03cec/obstore-0.8.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a4605c3ed7c9515aeb4c619b5f7f2c9986ed4a79fe6045e536b5e59b804b1476", size = 3460809 },
+    { url = "https://files.pythonhosted.org/packages/b4/99/7714dec721e43f521d6325a82303a002cddad089437640f92542b84e9cc8/obstore-0.8.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce42670417876dd8668cbb8659e860e9725e5f26bbc86449fd259970e2dd9d18", size = 3692081 },
+    { url = "https://files.pythonhosted.org/packages/ec/bd/4ac4175fe95a24c220a96021c25c432bcc0c0212f618be0737184eebbaad/obstore-0.8.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4a3e893b2a06585f651c541c1972fe1e3bf999ae2a5fda052ee55eb7e6516f5", size = 3957466 },
+    { url = "https://files.pythonhosted.org/packages/4e/04/caa288fb735484fc5cb019bdf3d896eaccfae0ac4622e520d05692c46790/obstore-0.8.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08462b32f95a9948ed56ed63e88406e2e5a4cae1fde198f9682e0fb8487100ed", size = 3951293 },
+    { url = "https://files.pythonhosted.org/packages/44/2f/d380239da2d6a1fda82e17df5dae600a404e8a93a065784518ff8325d5f6/obstore-0.8.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a0bf7763292a8fc47d01cd66e6f19002c5c6ad4b3ed4e6b2729f5e190fa8a0d", size = 3766199 },
+    { url = "https://files.pythonhosted.org/packages/28/41/d391be069d3da82969b54266948b2582aeca5dd735abeda4d63dba36e07b/obstore-0.8.2-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:bcd47f8126cb192cbe86942b8f73b1c45a651ce7e14c9a82c5641dfbf8be7603", size = 3529678 },
+    { url = "https://files.pythonhosted.org/packages/b9/4c/4862fdd1a3abde459ee8eea699b1797df638a460af235b18ca82c8fffb72/obstore-0.8.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:57eda9fd8c757c3b4fe36cf3918d7e589cc1286591295cc10b34122fa36dd3fd", size = 3698079 },
+    { url = "https://files.pythonhosted.org/packages/68/ca/014e747bc53b570059c27e3565b2316fbe5c107d4134551f4cd3e24aa667/obstore-0.8.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:ea44442aad8992166baa69f5069750979e4c5d9ffce772e61565945eea5774b9", size = 3687154 },
+    { url = "https://files.pythonhosted.org/packages/6f/89/6db5f8edd93028e5b8bfbeee15e6bd3e56f72106107d31cb208b57659de4/obstore-0.8.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:41496a3ab8527402db4142aaaf0d42df9d7d354b13ba10d9c33e0e48dd49dd96", size = 3773444 },
+    { url = "https://files.pythonhosted.org/packages/26/e5/c9e2cc540689c873beb61246e1615d6e38301e6a34dec424f5a5c63c1afd/obstore-0.8.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:43da209803f052df96c7c3cbec512d310982efd2407e4a435632841a51143170", size = 3939315 },
+    { url = "https://files.pythonhosted.org/packages/4d/c9/bb53280ca50103c1ffda373cdc9b0f835431060039c2897cbc87ddd92e42/obstore-0.8.2-cp312-cp312-win_amd64.whl", hash = "sha256:1836f5dcd49f9f2950c75889ab5c51fb290d3ea93cdc39a514541e0be3af016e", size = 3978234 },
+    { url = "https://files.pythonhosted.org/packages/f0/5d/8c3316cc958d386d5e6ab03e9db9ddc27f8e2141cee4a6777ae5b92f3aac/obstore-0.8.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:212f033e53fe6e53d64957923c5c88949a400e9027f7038c705ec2e9038be563", size = 3612027 },
+    { url = "https://files.pythonhosted.org/packages/ea/4d/699359774ce6330130536d008bfc32827fab0c25a00238d015a5974a3d1d/obstore-0.8.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bee21fa4ba148d08fa90e47a96df11161661ed31e09c056a373cb2154b0f2852", size = 3344686 },
+    { url = "https://files.pythonhosted.org/packages/82/37/55437341f10512906e02fd9fa69a8a95ad3f2f6a916d3233fda01763d110/obstore-0.8.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4c66594b59832ff1ced4c72575d9beb8b5f9b4e404ac1150a42bfb226617fd50", size = 3459860 },
+    { url = "https://files.pythonhosted.org/packages/7a/51/4245a616c94ee4851965e33f7a563ab4090cc81f52cc73227ff9ceca2e46/obstore-0.8.2-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:089f33af5c2fe132d00214a0c1f40601b28f23a38e24ef9f79fb0576f2730b74", size = 3691648 },
+    { url = "https://files.pythonhosted.org/packages/4e/f1/4e2fb24171e3ca3641a4653f006be826e7e17634b11688a5190553b00b83/obstore-0.8.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d87f658dfd340d5d9ea2d86a7c90d44da77a0db9e00c034367dca335735110cf", size = 3956867 },
+    { url = "https://files.pythonhosted.org/packages/42/f5/b703115361c798c9c1744e1e700d5908d904a8c2e2bd38bec759c9ffb469/obstore-0.8.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6e2e4fa92828c4fbc2d487f3da2d3588701a1b67d9f6ca3c97cc2afc912e9c63", size = 3950599 },
+    { url = "https://files.pythonhosted.org/packages/53/20/08c6dc0f20c1394e2324b9344838e4e7af770cdcb52c30757a475f50daeb/obstore-0.8.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab440e89c5c37a8ec230857dd65147d4b923e0cada33297135d05e0f937d696a", size = 3765865 },
+    { url = "https://files.pythonhosted.org/packages/77/20/77907765e29b2eba6bd8821872284d91170d7084f670855b2dfcb249ea14/obstore-0.8.2-cp313-cp313-manylinux_2_24_aarch64.whl", hash = "sha256:b9beed107c5c9cd995d4a73263861fcfbc414d58773ed65c14f80eb18258a932", size = 3529807 },
+    { url = "https://files.pythonhosted.org/packages/a5/f5/f629d39cc30d050f52b1bf927e4d65c1cc7d7ffbb8a635cd546b5c5219a0/obstore-0.8.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b75b4e7746292c785e31edcd5aadc8b758238372a19d4c5e394db5c305d7d175", size = 3693629 },
+    { url = "https://files.pythonhosted.org/packages/30/ff/106763fd10f2a1cb47f2ef1162293c78ad52f4e73223d8d43fc6b755445d/obstore-0.8.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:f33e6c366869d05ab0b7f12efe63269e631c5450d95d6b4ba4c5faf63f69de70", size = 3686176 },
+    { url = "https://files.pythonhosted.org/packages/ce/0c/d2ccb6f32feeca906d5a7c4255340df5262af8838441ca06c9e4e37b67d5/obstore-0.8.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:12c885a9ce5ceb09d13cc186586c0c10b62597eff21b985f6ce8ff9dab963ad3", size = 3773081 },
+    { url = "https://files.pythonhosted.org/packages/fa/79/40d1cc504cefc89c9b3dd8874287f3fddc7d963a8748d6dffc5880222013/obstore-0.8.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4accc883b93349a81c9931e15dd318cc703b02bbef2805d964724c73d006d00e", size = 3938589 },
+    { url = "https://files.pythonhosted.org/packages/14/dd/916c6777222db3271e9fb3cf9a97ed92b3a9b3e465bdeec96de9ab809d53/obstore-0.8.2-cp313-cp313-win_amd64.whl", hash = "sha256:ec850adf9980e5788a826ccfd5819989724e2a2f712bfa3258e85966c8d9981e", size = 3977768 },
+    { url = "https://files.pythonhosted.org/packages/f1/61/66f8dc98bbf5613bbfe5bf21747b4c8091442977f4bd897945895ab7325c/obstore-0.8.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:1431e40e9bb4773a261e51b192ea6489d0799b9d4d7dbdf175cdf813eb8c0503", size = 3623364 },
+    { url = "https://files.pythonhosted.org/packages/1a/66/6d527b3027e42f625c8fc816ac7d19b0d6228f95bfe7666e4d6b081d2348/obstore-0.8.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ddb39d4da303f50b959da000aa42734f6da7ac0cc0be2d5a7838b62c97055bb9", size = 3347764 },
+    { url = "https://files.pythonhosted.org/packages/0d/79/c00103302b620192ea447a948921ad3fed031ce3d19e989f038e1183f607/obstore-0.8.2-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e01f4e13783db453e17e005a4a3ceff09c41c262e44649ba169d253098c775e8", size = 3460981 },
+    { url = "https://files.pythonhosted.org/packages/3d/d9/bfe4ed4b1aebc45b56644dd5b943cf8e1673505cccb352e66878a457e807/obstore-0.8.2-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df0fc2d0bc17caff9b538564ddc26d7616f7e8b7c65b1a3c90b5048a8ad2e797", size = 3692711 },
+    { url = "https://files.pythonhosted.org/packages/13/47/cd6c2cbb18e1f40c77e7957a4a03d2d83f1859a2e876a408f1ece81cad4c/obstore-0.8.2-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e439d06c99a140348f046c9f598ee349cc2dcd9105c15540a4b231f9cc48bbae", size = 3958362 },
+    { url = "https://files.pythonhosted.org/packages/3d/ea/5ee82bf23abd71c7d6a3f2d008197ae8f8f569d41314c26a8f75318245be/obstore-0.8.2-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0e37d9046669fcc59522d0faf1d105fcbfd09c84cccaaa1e809227d8e030f32c", size = 3957082 },
+    { url = "https://files.pythonhosted.org/packages/cb/ee/46650405e50fdaa8d95f30375491f9c91fac9517980e8a28a4a6af66927f/obstore-0.8.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2646fdcc4bbe92dc2bb5bcdff15574da1211f5806c002b66d514cee2a23c7cb8", size = 3775539 },
+    { url = "https://files.pythonhosted.org/packages/35/d6/348a7ebebe2ca3d94dfc75344ea19675ae45472823e372c1852844078307/obstore-0.8.2-cp314-cp314-manylinux_2_24_aarch64.whl", hash = "sha256:e31a7d37675056d93dfc244605089dee67f5bba30f37c88436623c8c5ad9ba9d", size = 3535048 },
+    { url = "https://files.pythonhosted.org/packages/41/07/b7a16cc0da91a4b902d47880ad24016abfe7880c63f7cdafda45d89a2f91/obstore-0.8.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:656313dd8170dde0f0cd471433283337a63912e8e790a121f7cc7639c83e3816", size = 3699035 },
+    { url = "https://files.pythonhosted.org/packages/7f/74/3269a3a58347e0b019742d888612c4b765293c9c75efa44e144b1e884c0d/obstore-0.8.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:329038c9645d6d1741e77fe1a53e28a14b1a5c1461cfe4086082ad39ebabf981", size = 3687307 },
+    { url = "https://files.pythonhosted.org/packages/01/f9/4fd4819ad6a49d2f462a45be453561f4caebded0dc40112deeffc34b89b1/obstore-0.8.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:1e4df99b369790c97c752d126b286dc86484ea49bff5782843a265221406566f", size = 3776076 },
+    { url = "https://files.pythonhosted.org/packages/14/dd/7c4f958fa0b9fc4778fb3d232e38b37db8c6b260f641022fbba48b049d7e/obstore-0.8.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9e1c65c65e20cc990414a8a9af88209b1bbc0dd9521b5f6b0293c60e19439bb7", size = 3947445 },
+]
+
 [[package]]
 name = "onnxruntime"
-version = "1.23.2"
+version = "1.20.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "coloredlogs" },
@@ -2682,23 +2983,22 @@ dependencies = [
     { name = "sympy" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/44/be/467b00f09061572f022ffd17e49e49e5a7a789056bad95b54dfd3bee73ff/onnxruntime-1.23.2-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:6f91d2c9b0965e86827a5ba01531d5b669770b01775b23199565d6c1f136616c", size = 17196113 },
-    { url = "https://files.pythonhosted.org/packages/9f/a8/3c23a8f75f93122d2b3410bfb74d06d0f8da4ac663185f91866b03f7da1b/onnxruntime-1.23.2-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:87d8b6eaf0fbeb6835a60a4265fde7a3b60157cf1b2764773ac47237b4d48612", size = 19153857 },
-    { url = "https://files.pythonhosted.org/packages/3f/d8/506eed9af03d86f8db4880a4c47cd0dffee973ef7e4f4cff9f1d4bcf7d22/onnxruntime-1.23.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbfd2fca76c855317568c1b36a885ddea2272c13cb0e395002c402f2360429a6", size = 15220095 },
-    { url = "https://files.pythonhosted.org/packages/e9/80/113381ba832d5e777accedc6cb41d10f9eca82321ae31ebb6bcede530cea/onnxruntime-1.23.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da44b99206e77734c5819aa2142c69e64f3b46edc3bd314f6a45a932defc0b3e", size = 17372080 },
-    { url = "https://files.pythonhosted.org/packages/3a/db/1b4a62e23183a0c3fe441782462c0ede9a2a65c6bbffb9582fab7c7a0d38/onnxruntime-1.23.2-cp311-cp311-win_amd64.whl", hash = "sha256:902c756d8b633ce0dedd889b7c08459433fbcf35e9c38d1c03ddc020f0648c6e", size = 13468349 },
-    { url = "https://files.pythonhosted.org/packages/1b/9e/f748cd64161213adeef83d0cb16cb8ace1e62fa501033acdd9f9341fff57/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:b8f029a6b98d3cf5be564d52802bb50a8489ab73409fa9db0bf583eabb7c2321", size = 17195929 },
-    { url = "https://files.pythonhosted.org/packages/91/9d/a81aafd899b900101988ead7fb14974c8a58695338ab6a0f3d6b0100f30b/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:218295a8acae83905f6f1aed8cacb8e3eb3bd7513a13fe4ba3b2664a19fc4a6b", size = 19157705 },
-    { url = "https://files.pythonhosted.org/packages/3c/35/4e40f2fba272a6698d62be2cd21ddc3675edfc1a4b9ddefcc4648f115315/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76ff670550dc23e58ea9bc53b5149b99a44e63b34b524f7b8547469aaa0dcb8c", size = 15226915 },
-    { url = "https://files.pythonhosted.org/packages/ef/88/9cc25d2bafe6bc0d4d3c1db3ade98196d5b355c0b273e6a5dc09c5d5d0d5/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f9b4ae77f8e3c9bee50c27bc1beede83f786fe1d52e99ac85aa8d65a01e9b77", size = 17382649 },
-    { url = "https://files.pythonhosted.org/packages/c0/b4/569d298f9fc4d286c11c45e85d9ffa9e877af12ace98af8cab52396e8f46/onnxruntime-1.23.2-cp312-cp312-win_amd64.whl", hash = "sha256:25de5214923ce941a3523739d34a520aac30f21e631de53bba9174dc9c004435", size = 13470528 },
-    { url = "https://files.pythonhosted.org/packages/3d/41/fba0cabccecefe4a1b5fc8020c44febb334637f133acefc7ec492029dd2c/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:2ff531ad8496281b4297f32b83b01cdd719617e2351ffe0dba5684fb283afa1f", size = 17196337 },
-    { url = "https://files.pythonhosted.org/packages/fe/f9/2d49ca491c6a986acce9f1d1d5fc2099108958cc1710c28e89a032c9cfe9/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:162f4ca894ec3de1a6fd53589e511e06ecdc3ff646849b62a9da7489dee9ce95", size = 19157691 },
-    { url = "https://files.pythonhosted.org/packages/1c/a1/428ee29c6eaf09a6f6be56f836213f104618fb35ac6cc586ff0f477263eb/onnxruntime-1.23.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45d127d6e1e9b99d1ebeae9bcd8f98617a812f53f46699eafeb976275744826b", size = 15226898 },
-    { url = "https://files.pythonhosted.org/packages/f2/2b/b57c8a2466a3126dbe0a792f56ad7290949b02f47b86216cd47d857e4b77/onnxruntime-1.23.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8bace4e0d46480fbeeb7bbe1ffe1f080e6663a42d1086ff95c1551f2d39e7872", size = 17382518 },
-    { url = "https://files.pythonhosted.org/packages/4a/93/aba75358133b3a941d736816dd392f687e7eab77215a6e429879080b76b6/onnxruntime-1.23.2-cp313-cp313-win_amd64.whl", hash = "sha256:1f9cc0a55349c584f083c1c076e611a7c35d5b867d5d6e6d6c823bf821978088", size = 13470276 },
-    { url = "https://files.pythonhosted.org/packages/7c/3d/6830fa61c69ca8e905f237001dbfc01689a4e4ab06147020a4518318881f/onnxruntime-1.23.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9d2385e774f46ac38f02b3a91a91e30263d41b2f1f4f26ae34805b2a9ddef466", size = 15229610 },
-    { url = "https://files.pythonhosted.org/packages/b6/ca/862b1e7a639460f0ca25fd5b6135fb42cf9deea86d398a92e44dfda2279d/onnxruntime-1.23.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2b9233c4947907fd1818d0e581c049c41ccc39b2856cc942ff6d26317cee145", size = 17394184 },
+    { url = "https://files.pythonhosted.org/packages/95/8d/2634e2959b34aa8a0037989f4229e9abcfa484e9c228f99633b3241768a6/onnxruntime-1.20.1-cp311-cp311-macosx_13_0_universal2.whl", hash = "sha256:06bfbf02ca9ab5f28946e0f912a562a5f005301d0c419283dc57b3ed7969bb7b", size = 30998725 },
+    { url = "https://files.pythonhosted.org/packages/a5/da/c44bf9bd66cd6d9018a921f053f28d819445c4d84b4dd4777271b0fe52a2/onnxruntime-1.20.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6243e34d74423bdd1edf0ae9596dd61023b260f546ee17d701723915f06a9f7", size = 11955227 },
+    { url = "https://files.pythonhosted.org/packages/11/ac/4120dfb74c8e45cce1c664fc7f7ce010edd587ba67ac41489f7432eb9381/onnxruntime-1.20.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5eec64c0269dcdb8d9a9a53dc4d64f87b9e0c19801d9321246a53b7eb5a7d1bc", size = 13331703 },
+    { url = "https://files.pythonhosted.org/packages/12/f1/cefacac137f7bb7bfba57c50c478150fcd3c54aca72762ac2c05ce0532c1/onnxruntime-1.20.1-cp311-cp311-win32.whl", hash = "sha256:a19bc6e8c70e2485a1725b3d517a2319603acc14c1f1a017dda0afe6d4665b41", size = 9813977 },
+    { url = "https://files.pythonhosted.org/packages/2c/2d/2d4d202c0bcfb3a4cc2b171abb9328672d7f91d7af9ea52572722c6d8d96/onnxruntime-1.20.1-cp311-cp311-win_amd64.whl", hash = "sha256:8508887eb1c5f9537a4071768723ec7c30c28eb2518a00d0adcd32c89dea3221", size = 11329895 },
+    { url = "https://files.pythonhosted.org/packages/e5/39/9335e0874f68f7d27103cbffc0e235e32e26759202df6085716375c078bb/onnxruntime-1.20.1-cp312-cp312-macosx_13_0_universal2.whl", hash = "sha256:22b0655e2bf4f2161d52706e31f517a0e54939dc393e92577df51808a7edc8c9", size = 31007580 },
+    { url = "https://files.pythonhosted.org/packages/c5/9d/a42a84e10f1744dd27c6f2f9280cc3fb98f869dd19b7cd042e391ee2ab61/onnxruntime-1.20.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1f56e898815963d6dc4ee1c35fc6c36506466eff6d16f3cb9848cea4e8c8172", size = 11952833 },
+    { url = "https://files.pythonhosted.org/packages/47/42/2f71f5680834688a9c81becbe5c5bb996fd33eaed5c66ae0606c3b1d6a02/onnxruntime-1.20.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bb71a814f66517a65628c9e4a2bb530a6edd2cd5d87ffa0af0f6f773a027d99e", size = 13333903 },
+    { url = "https://files.pythonhosted.org/packages/c8/f1/aabfdf91d013320aa2fc46cf43c88ca0182860ff15df872b4552254a9680/onnxruntime-1.20.1-cp312-cp312-win32.whl", hash = "sha256:bd386cc9ee5f686ee8a75ba74037750aca55183085bf1941da8efcfe12d5b120", size = 9814562 },
+    { url = "https://files.pythonhosted.org/packages/dd/80/76979e0b744307d488c79e41051117634b956612cc731f1028eb17ee7294/onnxruntime-1.20.1-cp312-cp312-win_amd64.whl", hash = "sha256:19c2d843eb074f385e8bbb753a40df780511061a63f9def1b216bf53860223fb", size = 11331482 },
+    { url = "https://files.pythonhosted.org/packages/f7/71/c5d980ac4189589267a06f758bd6c5667d07e55656bed6c6c0580733ad07/onnxruntime-1.20.1-cp313-cp313-macosx_13_0_universal2.whl", hash = "sha256:cc01437a32d0042b606f462245c8bbae269e5442797f6213e36ce61d5abdd8cc", size = 31007574 },
+    { url = "https://files.pythonhosted.org/packages/81/0d/13bbd9489be2a6944f4a940084bfe388f1100472f38c07080a46fbd4ab96/onnxruntime-1.20.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fb44b08e017a648924dbe91b82d89b0c105b1adcfe31e90d1dc06b8677ad37be", size = 11951459 },
+    { url = "https://files.pythonhosted.org/packages/c0/ea/4454ae122874fd52bbb8a961262de81c5f932edeb1b72217f594c700d6ef/onnxruntime-1.20.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bda6aebdf7917c1d811f21d41633df00c58aff2bef2f598f69289c1f1dabc4b3", size = 13331620 },
+    { url = "https://files.pythonhosted.org/packages/d8/e0/50db43188ca1c945decaa8fc2a024c33446d31afed40149897d4f9de505f/onnxruntime-1.20.1-cp313-cp313-win_amd64.whl", hash = "sha256:d30367df7e70f1d9fc5a6a68106f5961686d39b54d3221f760085524e8d38e16", size = 11331758 },
+    { url = "https://files.pythonhosted.org/packages/d8/55/3821c5fd60b52a6c82a00bba18531793c93c4addfe64fbf061e235c5617a/onnxruntime-1.20.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c9158465745423b2b5d97ed25aa7740c7d38d2993ee2e5c3bfacb0c4145c49d8", size = 11950342 },
+    { url = "https://files.pythonhosted.org/packages/14/56/fd990ca222cef4f9f4a9400567b9a15b220dee2eafffb16b2adbc55c8281/onnxruntime-1.20.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0df6f2df83d61f46e842dbcde610ede27218947c33e994545a22333491e72a3b", size = 13337040 },
 ]
 
 [[package]]
@@ -2732,6 +3032,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/12/cf/03675d8bd8ecbf4445504d8071adab19f5f993676795708e36402ab38263/openapi_pydantic-0.5.1-py3-none-any.whl", hash = "sha256:a3a09ef4586f5bd760a8df7f43028b60cafb6d9f61de2acba9574766255ab146", size = 96381 },
 ]
 
+[[package]]
+name = "openpyxl"
+version = "3.1.5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "et-xmlfile" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910 },
+]
+
 [[package]]
 name = "opentelemetry-api"
 version = "1.39.1"
@@ -3033,6 +3345,33 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9a/70/875f4a23bfc4731703a5835487d0d2fb999031bd415e7d17c0ae615c18b7/pathvalidate-3.3.1-py3-none-any.whl", hash = "sha256:5263baab691f8e1af96092fa5137ee17df5bdfbd6cff1fcac4d6ef4bc2e1735f", size = 24305 },
 ]
 
+[[package]]
+name = "pdfminer-six"
+version = "20251230"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "charset-normalizer" },
+    { name = "cryptography" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/46/9a/d79d8fa6d47a0338846bb558b39b9963b8eb2dfedec61867c138c1b17eeb/pdfminer_six-20251230.tar.gz", hash = "sha256:e8f68a14c57e00c2d7276d26519ea64be1b48f91db1cdc776faa80528ca06c1e", size = 8511285 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/65/d7/b288ea32deb752a09aab73c75e1e7572ab2a2b56c3124a5d1eb24c62ceb3/pdfminer_six-20251230-py3-none-any.whl", hash = "sha256:9ff2e3466a7dfc6de6fd779478850b6b7c2d9e9405aa2a5869376a822771f485", size = 6591909 },
+]
+
+[[package]]
+name = "pdfplumber"
+version = "0.11.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pdfminer-six" },
+    { name = "pillow" },
+    { name = "pypdfium2" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/38/37/9ca3519e92a8434eb93be570b131476cc0a4e840bb39c62ddb7813a39d53/pdfplumber-0.11.9.tar.gz", hash = "sha256:481224b678b2bbdbf376e2c39bf914144eef7c3d301b4a28eebf0f7f6109d6dc", size = 102768 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8b/c8/cdbc975f5b634e249cfa6597e37c50f3078412474f21c015e508bfbfe3c3/pdfplumber-0.11.9-py3-none-any.whl", hash = "sha256:33ec5580959ba524e9100138746e090879504c42955df1b8a997604dd326c443", size = 60045 },
+]
+
 [[package]]
 name = "pg0-embedded"
 version = "0.11.0"
@@ -3634,6 +3973,35 @@ crypto = [
     { name = "cryptography" },
 ]
 
+[[package]]
+name = "pypdfium2"
+version = "5.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/99/23/b3979a1d4f536fabce02e3d9f332e8aeeed064d9df9391f2a77160f4ab36/pypdfium2-5.4.0.tar.gz", hash = "sha256:7219e55048fb3999fc8adcaea467088507207df4676ff9e521a3ae15a67d99c4", size = 269136 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/4b/c0/3d707bff5e973272b5412556d19e8c6889ce859a235465f0049cc8d35bc3/pypdfium2-5.4.0-py3-none-android_23_arm64_v8a.whl", hash = "sha256:8bc51a12a8c8eabbdbd7499d3e5ec47bcf56ba18e07b52bdd07d321cc1252c90", size = 2759769 },
+    { url = "https://files.pythonhosted.org/packages/1b/6b/306cafcb0b18d5fab41687d9ed76eabea86a9ff78bc568bee1bfa34e526d/pypdfium2-5.4.0-py3-none-android_23_armeabi_v7a.whl", hash = "sha256:a414ef5b685824cc6c7acbe19b7dbc735de2023cf473321a8ebfe8d7f5d8a41f", size = 2301913 },
+    { url = "https://files.pythonhosted.org/packages/7a/37/3d737c7eb84fb22939ab0a643aa0183dbc0745c309e962b4d61eeff8211b/pypdfium2-5.4.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:0e83657db8da5971434ff5683bf3faa007ee1f3a56b61f245b8aa5b60442c23a", size = 2814181 },
+    { url = "https://files.pythonhosted.org/packages/96/d7/0895737ec3d95ad607ade42e98fa8868b91e35b1170ec39b8c1b5fdb124c/pypdfium2-5.4.0-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:e42b1d14db642e96bb3a57167f620b4247e9c843d22b9fb569b16a7c35a18f47", size = 2943476 },
+    { url = "https://files.pythonhosted.org/packages/9a/53/f8ab449997d3efa52737b8e6c494f1c3f09dc0642161fadc934f16a57cf0/pypdfium2-5.4.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0698c9a002f839127e74ec0185147e08b64e47a1e6caeaee95df434c05b26e8c", size = 2976675 },
+    { url = "https://files.pythonhosted.org/packages/c6/28/b8a4d4c1557019101bb722c88ba532ec9c14640117ab1c272c80774d83d7/pypdfium2-5.4.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:22e9d4c73fc48b18b022977ea6fe78df43adf95440e1135020ed35fea9595017", size = 2762396 },
+    { url = "https://files.pythonhosted.org/packages/0b/4a/6c765f6e0b69d792e2d4c7ef2359301896c82df265d60f9a56e87618ec50/pypdfium2-5.4.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f0619f8a8ae3eb71b2cdc1fbd2a8f5d43f0fc6bee66d1b3aac2c9c23e44a3bf", size = 3068559 },
+    { url = "https://files.pythonhosted.org/packages/1c/17/4464e4ab6dd98ac3783c10eb799d8da49cb551a769c987eb9c6ba72a5ccf/pypdfium2-5.4.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:50124415d815c41de8ce7e21cee5450f74f6f1240a140573bb71ccac804d5e5f", size = 3419384 },
+    { url = "https://files.pythonhosted.org/packages/92/08/fa315a2ab353b41501b7088be72dc6cf8ad2bd4f1ebdfdb90c41b7f29155/pypdfium2-5.4.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce482d76e5447e745d761307401eaa366616ca44032b86cf7fbe6be918ade64e", size = 2998123 },
+    { url = "https://files.pythonhosted.org/packages/02/7a/a171d313d54a028d9437dea2c5d07fc9e1592f4daf5c39cbf514fca75242/pypdfium2-5.4.0-py3-none-manylinux_2_27_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16b9c6b07f3dbe7eda209bf7aaf131ca9614e1dae527e9764180dd58bcbaf411", size = 3673594 },
+    { url = "https://files.pythonhosted.org/packages/0b/c0/60416f011f7e5a4ca29f40ae94907f34975239f3c6dd7fcb51f99e110f3b/pypdfium2-5.4.0-py3-none-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3b08d48b7cca3b51aefaad7855bc0e9e251432a6eef1356d532ff438be84855e", size = 2965025 },
+    { url = "https://files.pythonhosted.org/packages/75/e2/8e36144b5e933c707b6aeab7dc6638eee8208697925b48b5b78ef68fb52a/pypdfium2-5.4.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0a1526e2a2bde7f2f13bec0f471d9fd475f7bbac2c0c860d48c35af8394d5931", size = 4130551 },
+    { url = "https://files.pythonhosted.org/packages/a0/64/8cda96259a8fdecd457f5d14a9d650315d7bdf496f96055d1d55900b3881/pypdfium2-5.4.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:40cea0bceb1e60a71b3855e2b04d175d2199b7da06212bb80f0c78067d065810", size = 3746587 },
+    { url = "https://files.pythonhosted.org/packages/33/6b/7764491269f188a922bd6b254359d718899fc3092c90f0f68c2f6e451921/pypdfium2-5.4.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:7a116f8fbeae7aa3a18ff2d1fa331ac647831cc16b589d4fbbbb66d64ecc8793", size = 4336703 },
+    { url = "https://files.pythonhosted.org/packages/87/b0/2484bd3c20ead51ecea2082deaf94a3e91bad709fa14f049ca7fb598dc9a/pypdfium2-5.4.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:55c7fc894718db5fa2981d46dee45fe3a4fcd60d26f5095ad8f7779600fa8b6f", size = 4375051 },
+    { url = "https://files.pythonhosted.org/packages/c0/ac/5f0536be885c3cadc09422de0324a193a21c165488a574029d9d2db92ecb/pypdfium2-5.4.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:dfc1c0c7e6e7ba258ebb338aaf664eb933bff1854cda76e4ee530886ea39b31a", size = 3928935 },
+    { url = "https://files.pythonhosted.org/packages/13/b9/693b665df0939555491bece0777cafda1270e208734e925006de313abb5b/pypdfium2-5.4.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:4c0a48ede7180f804c029c509c2b6ea0c66813a3fde9eb9afc390183f947164d", size = 4997642 },
+    { url = "https://files.pythonhosted.org/packages/fb/ea/ba585acdfbefe309ee2fe5ebfeb097e36abe1d33c2a5108828c493c070bb/pypdfium2-5.4.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:dea22d15c44a275702fd95ad664ba6eaa3c493d53d58b4d69272a04bdfb0df70", size = 4179914 },
+    { url = "https://files.pythonhosted.org/packages/97/47/238383e89081a0ed1ca2bf4ef44f7e512fa0c72ffc51adc7df83bfcfd9b9/pypdfium2-5.4.0-py3-none-win32.whl", hash = "sha256:35c643827ed0f4dae9cedf3caf836f94cba5b31bd2c115b80a7c85f004636de9", size = 2995844 },
+    { url = "https://files.pythonhosted.org/packages/08/37/f1338a0600c6c6e31759f8f80d7ab20aa0bc43b11594da67091300e051d4/pypdfium2-5.4.0-py3-none-win_amd64.whl", hash = "sha256:f9d9ce3c6901294d6984004d4a797dea110f8248b1bde33a823d25b45d3c2685", size = 3104198 },
+    { url = "https://files.pythonhosted.org/packages/65/17/18ad82f070da18ab970928f730fbd44d9b05aafcb52a2ebb6470eaae53f9/pypdfium2-5.4.0-py3-none-win_arm64.whl", hash = "sha256:2b78ea216fb92e7709b61c46241ebf2cc0c60cf18ad2fb4633af665d7b4e21e6", size = 2938727 },
+]
+
 [[package]]
 name = "pyperclip"
 version = "1.11.0"
@@ -3766,6 +4134,21 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546 },
 ]
 
+[[package]]
+name = "python-pptx"
+version = "1.0.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "lxml" },
+    { name = "pillow" },
+    { name = "typing-extensions" },
+    { name = "xlsxwriter" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/52/a9/0c0db8d37b2b8a645666f7fd8accea4c6224e013c42b1d5c17c93590cd06/python_pptx-1.0.2.tar.gz", hash = "sha256:479a8af0eaf0f0d76b6f00b0887732874ad2e3188230315290cd1f9dd9cc7095", size = 10109297 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788 },
+]
+
 [[package]]
 name = "pytz"
 version = "2025.2"
@@ -4421,8 +4804,8 @@ name = "secretstorage"
 version = "3.4.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "cryptography" },
-    { name = "jeepney" },
+    { name = "cryptography", marker = "sys_platform != 'win32'" },
+    { name = "jeepney", marker = "sys_platform != 'win32'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/32/8a/ed6747b1cc723c81f526d4c12c1b1d43d07190e1e8258dbf934392fc850e/secretstorage-3.4.1.tar.gz", hash = "sha256:a799acf5be9fb93db609ebaa4ab6e8f1f3ed5ae640e0fa732bfea59e9c3b50e8", size = 19871 }
 wheels = [
@@ -4635,6 +5018,22 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248 },
 ]
 
+[[package]]
+name = "testcontainers"
+version = "4.14.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "docker" },
+    { name = "python-dotenv" },
+    { name = "typing-extensions" },
+    { name = "urllib3" },
+    { name = "wrapt" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8b/02/ef62dec9e4f804189c44df23f0b86897c738d38e9c48282fcd410308632f/testcontainers-4.14.1.tar.gz", hash = "sha256:316f1bb178d829c003acd650233e3ff3c59a833a08d8661c074f58a4fbd42a64", size = 80148 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c8/31/5e7b23f9e43ff7fd46d243808d70c5e8daf3bc08ecf5a7fb84d5e38f7603/testcontainers-4.14.1-py3-none-any.whl", hash = "sha256:03dfef4797b31c82e7b762a454b6afec61a2a512ad54af47ab41e4fa5415f891", size = 125640 },
+]
+
 [[package]]
 name = "threadpoolctl"
 version = "3.6.0"
@@ -5266,6 +5665,24 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405 },
 ]
 
+[[package]]
+name = "xlrd"
+version = "2.0.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/07/5a/377161c2d3538d1990d7af382c79f3b2372e880b65de21b01b1a2b78691e/xlrd-2.0.2.tar.gz", hash = "sha256:08b5e25de58f21ce71dc7db3b3b8106c1fa776f3024c54e45b45b374e89234c9", size = 100167 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/1a/62/c8d562e7766786ba6587d09c5a8ba9f718ed3fa8af7f4553e8f91c36f302/xlrd-2.0.2-py2.py3-none-any.whl", hash = "sha256:ea762c3d29f4cca48d82df517b6d89fbce4db3107f9d78713e48cd321d5c9aa9", size = 96555 },
+]
+
+[[package]]
+name = "xlsxwriter"
+version = "3.2.9"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/46/2c/c06ef49dc36e7954e55b802a8b231770d286a9758b3d936bd1e04ce5ba88/xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c", size = 215940 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3a/0c/3662f4a66880196a590b202f0db82d919dd2f89e99a27fadef91c4a33d41/xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3", size = 175315 },
+]
+
 [[package]]
 name = "yarl"
 version = "1.22.0"