feat(cli): cover every OpenAPI endpoint and request-body param

Wires the Rust CLI up to every endpoint exposed by the Hindsight OpenAPI
spec and adds CI enforcement so new endpoints or new request-body fields
cannot slip in without matching CLI coverage.

Endpoints
- New `hindsight webhook {list,create,update,delete,deliveries}` and
  `hindsight audit {list,stats}` subcommands.
- `hindsight bank` gains `set-disposition`, `consolidation-recover`,
  `export-template`, `import-template`, `template-schema`.
- `hindsight memory` gains `history` and per-memory `clear-observations`.
- `hindsight document update`, `hindsight operation retry` added.
- Brings CLI coverage from 46/62 to 62/62 operations.

Request-body parameters
- Expose missing flags that the CLI was silently hardcoding: directive
  `--priority`; mental-model `--tags` / `--max-tokens` /
  `--trigger-refresh-after-consolidation`; recall `--query-timestamp`;
  reflect `--fact-types` / `--exclude-mental-models` /
  `--exclude-mental-model-ids`; retain `--document-tags`.

CI enforcement
- New `cli-coverage-check` entry point in `hindsight-dev` parses
  openapi.json and verifies that (a) every operationId is called from
  hindsight-cli/src/ (the progenitor client method names match the
  operationId), and (b) every request-body property is present in
  main.rs as a clap field or `long = "..."` attribute.
- Intentional non-exposures live in `hindsight-cli/.openapi-coverage.toml`
  under `[skip]` / `[fields.<op>]` with a reason each (38 documented
  field skips for flattened structs, nested structs, or fields surfaced
  via a different subcommand).
- New `check-cli-coverage` job in .github/workflows/test.yml, triggered
  on cli/core/dev/ci path changes, runs the script on every PR.
- smoke-test.sh exercises the new webhook / audit / bank-template /
  set-disposition / consolidation-recover commands.

.github/workflows/release-integration.yml

@@ -82,15 +82,6 @@ jobs:
         node-version: '22'
         registry-url: 'https://registry.npmjs.org'
 
-    # Guard: fail fast if the integration's lockfile resolves any dep from a
-    # monorepo workspace (link=true) or a relative file path. The release
-    # runner has no pre-built workspace `dist/` so `npm run build` would
-    # later fail at tsc with "Cannot find module". See:
-    # https://github.com/vectorize-io/hindsight/issues/… (0.6.0 openclaw retry)
-    - name: Check integration lockfile
-      if: steps.type.outputs.type == 'typescript'
-      run: ./scripts/check-integration-lockfiles.sh
-
     - name: Install dependencies
       if: steps.type.outputs.type == 'typescript'
       working-directory: ./hindsight-integrations/${{ steps.info.outputs.integration }}

.github/workflows/release.yml

@@ -540,7 +540,7 @@ jobs:
         ls -la release-assets/
 
     - name: Create GitHub Release
-      uses: softprops/action-gh-release@v3
+      uses: softprops/action-gh-release@v2
       with:
         files: release-assets/*
         generate_release_notes: true

.github/workflows/test.yml

@@ -47,8 +47,6 @@ jobs:
       integrations-llamaindex: ${{ steps.filter.outputs.integrations-llamaindex }}
       integrations-paperclip: ${{ steps.filter.outputs.integrations-paperclip }}
       integrations-opencode: ${{ steps.filter.outputs.integrations-opencode }}
-      integrations-cloudflare-oauth-proxy: ${{ steps.filter.outputs.integrations-cloudflare-oauth-proxy }}
-      integrations-lockfiles: ${{ steps.filter.outputs.integrations-lockfiles }}
       dev: ${{ steps.filter.outputs.dev }}
       ci: ${{ steps.filter.outputs.ci }}
       # Secrets are available for internal PRs, pull_request_review, and workflow_dispatch.
@@ -127,40 +125,11 @@ jobs:
             - 'hindsight-integrations/paperclip/**'
           integrations-opencode:
             - 'hindsight-integrations/opencode/**'
-          integrations-cloudflare-oauth-proxy:
-            - 'hindsight-integrations/cloudflare-oauth-proxy/**'
-          integrations-lockfiles:
-            - 'hindsight-integrations/*/package-lock.json'
-            - 'hindsight-integrations/*/package.json'
-            - 'scripts/check-integration-lockfiles.sh'
           dev:
             - 'hindsight-dev/**'
           ci:
             - '.github/**'
 
-  # Fail fast if any hindsight-integrations/*/package-lock.json was regenerated
-  # from the monorepo root and ended up symlinked at a workspace path instead
-  # of the npm registry. That bit us on the 0.6.0 openclaw release — tsc in
-  # the release workflow couldn't find `@vectorize-io/hindsight-client`
-  # because its `resolved` url pointed at a workspace dir whose `dist/` was
-  # gitignored and unbuilt. Catching this at PR time means the release CI
-  # never hits that class of failure.
-  check-integration-lockfiles:
-    needs: [detect-changes]
-    if: >-
-      github.event_name != 'pull_request_review' &&
-      (github.event_name == 'workflow_dispatch' ||
-      needs.detect-changes.outputs.integrations-lockfiles == 'true' ||
-      needs.detect-changes.outputs.ci == 'true')
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        ref: ${{ github.event.pull_request.head.sha || '' }}
-
-    - name: Check integration lockfiles resolve from the npm registry
-      run: ./scripts/check-integration-lockfiles.sh
-
   build-api-python-versions:
     needs: [detect-changes]
     if: >-
@@ -302,61 +271,6 @@ jobs:
       working-directory: ./hindsight-integrations/openclaw
       run: npm test
 
-  smoke-openclaw-install:
-    needs: [detect-changes, build-openclaw-integration]
-    if: >-
-      github.event_name != 'pull_request_review' &&
-      (github.event_name == 'workflow_dispatch' ||
-      needs.detect-changes.outputs.integrations-openclaw == 'true' ||
-      needs.detect-changes.outputs.clients-ts == 'true' ||
-      needs.detect-changes.outputs.all-npm == 'true' ||
-      needs.detect-changes.outputs.ci == 'true')
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        ref: ${{ github.event.pull_request.head.sha || '' }}
-
-    - name: Set up Node.js
-      uses: actions/setup-node@v6
-      with:
-        node-version: '22'
-        cache: 'npm'
-        cache-dependency-path: package-lock.json
-
-    # Install the openclaw CLI globally. The smoke test exercises the real
-    # `openclaw plugins install` / `openclaw config set` / `openclaw plugins
-    # doctor` commands — not the in-repo integration tests — so a real CLI
-    # must be on PATH.
-    - name: Install openclaw CLI
-      run: npm install -g openclaw
-
-    - name: Verify openclaw CLI
-      run: openclaw --version
-
-    # openclaw depends on the workspace packages via published version
-    # ranges (^0.1.0 / ^0.5.0), not file: paths, so the smoke test's
-    # `openclaw plugins install <tarball>` resolves them straight from the
-    # npm registry. These builds are just for `npm pack` / local unit
-    # tests, not for resolving the plugin's runtime deps.
-    - name: Install root workspace dependencies
-      run: npm ci
-
-    - name: Build hindsight-client (openclaw dep)
-      run: npm run build --workspace=hindsight-clients/typescript
-
-    - name: Build hindsight-all-npm (openclaw dep)
-      run: npm run build --workspace=hindsight-all-npm
-
-    - name: Install openclaw dependencies
-      working-directory: ./hindsight-integrations/openclaw
-      run: npm ci
-
-    - name: Run openclaw install smoke test
-      working-directory: ./hindsight-integrations/openclaw
-      run: ./scripts/smoke-test.sh
-
   test-claude-code-integration:
     needs: [detect-changes]
     if: >-
@@ -503,37 +417,6 @@ jobs:
       working-directory: ./hindsight-integrations/opencode
       run: npm run build
 
-  test-cloudflare-oauth-proxy-integration:
-    needs: [detect-changes]
-    if: >-
-      github.event_name != 'pull_request_review' &&
-      (github.event_name == 'workflow_dispatch' ||
-      needs.detect-changes.outputs.integrations-cloudflare-oauth-proxy == 'true' ||
-      needs.detect-changes.outputs.ci == 'true')
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        ref: ${{ github.event.pull_request.head.sha || '' }}
-
-    - name: Set up Node.js
-      uses: actions/setup-node@v6
-      with:
-        node-version: '22'
-
-    - name: Install dependencies
-      working-directory: ./hindsight-integrations/cloudflare-oauth-proxy
-      run: npm ci
-
-    - name: Typecheck
-      working-directory: ./hindsight-integrations/cloudflare-oauth-proxy
-      run: npm run typecheck
-
-    - name: Run tests
-      working-directory: ./hindsight-integrations/cloudflare-oauth-proxy
-      run: npm test
-
   build-chat-integration:
     needs: [detect-changes]
     if: >-
@@ -2668,17 +2551,14 @@ jobs:
     if: github.event_name == 'pull_request_review' && github.event.review.state == 'approved' && always()
     needs:
       - detect-changes
-      - check-integration-lockfiles
       - build-api-python-versions
       - build-typescript-client
       - build-openclaw-integration
-      - smoke-openclaw-install
       - test-claude-code-integration
       - test-codex-integration
       - build-ai-sdk-integration
       - test-ai-sdk-integration-deno
       - test-opencode-integration
-      - test-cloudflare-oauth-proxy-integration
       - build-chat-integration
       - test-paperclip-integration
       - build-control-plane

CLAUDE.md

@@ -222,10 +222,6 @@ Every new integration in `hindsight-integrations/` must satisfy all of the follo
 
 If any of these are missing, the integration is incomplete and must not be pushed or merged.
 
-### Changelogs
-
-Never add "Unreleased" entries to changelogs (e.g. `hindsight-docs/src/pages/changelog/**`). Changelog entries are written by the release script (`./scripts/release-integration.sh`) when a version is actually cut. If a bug fix or feature needs documenting before release, describe it in the PR/commit — the release tooling will surface it in the published changelog section.
-
 ### Adding New API Configuration Flags
 
 Configuration follows a hierarchical system: **Global (env vars) → Tenant (via extension) → Bank (database)**.

helm/hindsight/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: hindsight
 description: Hindsight helm chart
 type: application
-version: 0.5.1
-appVersion: "0.5.1"
+version: 0.5.0
+appVersion: "0.5.0"
 keywords:
   - ai
   - memory

hindsight-all-npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vectorize-io/hindsight-all",
-  "version": "0.5.1",
+  "version": "0.5.0",
   "description": "Node.js programmatic lifecycle manager for Hindsight — embeds a local hindsight daemon in a Node application. Pair with @vectorize-io/hindsight-client for memory operations.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

hindsight-all-slim/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hindsight-all-slim"
-version = "0.5.1"
+version = "0.5.0"
 description = "Hindsight: Agent Memory That Works Like Human Memory - Slim All-in-One Bundle"
 readme = "README.md"
 requires-python = ">=3.11"

hindsight-all/hindsight/embedded.py

@@ -190,32 +190,12 @@ class HindsightEmbedded:
         if self._closed:
             return
 
-        acquired = self._lock.acquire(timeout=5.0)
-        if not acquired:
-            # Lock is held by another thread (e.g. _ensure_started).
-            # Mark closed to prevent new operations but skip shared-state
-            # teardown — the daemon's idle timeout handles the rest.
-            logger.warning(
-                "Cleanup lock acquisition timed out for profile '%s'; "
-                "marking closed, daemon will idle-stop on its own",
-                self.profile,
-            )
-            self._closed = True
-            return
-
-        try:
+        with self._lock:
             if self._closed:
                 return
 
             if self._client is not None:
-                try:
-                    self._client.close()
-                except Exception:
-                    logger.debug(
-                        "Error closing client for profile '%s'",
-                        self.profile,
-                        exc_info=True,
-                    )
+                self._client.close()
                 self._client = None
 
             # Stop UI if it was started
@@ -229,8 +209,6 @@ class HindsightEmbedded:
                 self._manager.stop(self.profile)
 
             self._closed = True
-        finally:
-            self._lock.release()
 
     def close(self, stop_daemon: bool = False):
         """

hindsight-all/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-all"
-version = "0.5.1"
+version = "0.5.0"
 description = "Hindsight: Agent Memory That Works Like Human Memory - All-in-One Bundle"
 readme = "README.md"
 requires-python = ">=3.11"

hindsight-all/tests/test_cleanup_timeout.py

@@ -1,56 +0,0 @@
-"""
-Unit test for _cleanup lock timeout behavior.
-
-Verifies that _cleanup completes even when the lock is held by another thread,
-instead of hanging indefinitely (fixes #952).
-"""
-
-import threading
-import time
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-
-def test_cleanup_completes_when_lock_held():
-    """
-    _cleanup should complete (best-effort) even when self._lock is held
-    by another thread, e.g. during a long _ensure_started call.
-    """
-    with patch.dict("sys.modules", {
-        "hindsight_client": MagicMock(),
-        "hindsight_embed": MagicMock(),
-        "hindsight.api_namespaces": MagicMock(),
-    }):
-        from hindsight.embedded import HindsightEmbedded
-
-        client = HindsightEmbedded.__new__(HindsightEmbedded)
-        client.profile = "test"
-        client._lock = threading.Lock()
-        client._closed = False
-        client._client = None
-        client._started = False
-        client._ui = False
-
-        # Simulate another thread holding the lock
-        client._lock.acquire()
-
-        cleanup_done = threading.Event()
-
-        def run_cleanup():
-            client._cleanup()
-            cleanup_done.set()
-
-        t = threading.Thread(target=run_cleanup)
-        t.start()
-
-        # Cleanup should complete within the timeout (5s) + margin
-        assert cleanup_done.wait(timeout=8.0), (
-            "_cleanup hung instead of timing out on lock acquisition"
-        )
-
-        # Release the lock from the simulating thread
-        client._lock.release()
-        t.join(timeout=1.0)
-
-        assert client._closed, "Client should be marked as closed after cleanup"

hindsight-api-slim/hindsight_api/__init__.py

@@ -46,4 +46,4 @@ __all__ = [
     "RemoteTEICrossEncoder",
     "LLMConfig",
 ]
-__version__ = "0.5.1"
+__version__ = "0.5.0"

hindsight-api-slim/hindsight_api/api/http.py

@@ -1526,27 +1526,6 @@ class MentalModelTrigger(BaseModel):
             "Supports nested and/or/not expressions for complex tag-based scoping."
         ),
     )
-    include_chunks: bool | None = Field(
-        default=None,
-        description=(
-            "Override whether the internal recall used during refresh returns raw chunk text. "
-            "None means use the bank/global config default (recall_include_chunks)."
-        ),
-    )
-    recall_max_tokens: int | None = Field(
-        default=None,
-        description=(
-            "Override the token budget for facts returned by the internal recall during refresh. "
-            "None means use the bank/global config default (recall_max_tokens)."
-        ),
-    )
-    recall_chunks_max_tokens: int | None = Field(
-        default=None,
-        description=(
-            "Override the token budget for raw chunks returned by the internal recall during refresh. "
-            "None means use the bank/global config default (recall_chunks_max_tokens)."
-        ),
-    )
 
     @field_validator("fact_types")
     @classmethod
@@ -1694,36 +1673,6 @@ class BankTemplateConfig(BaseModel):
     entities_allow_free_form: bool | None = Field(
         default=None, description="Allow entities outside the label vocabulary"
     )
-    retain_default_strategy: str | None = Field(
-        default=None, description="Name of the default retain strategy (key into retain_strategies map)"
-    )
-    retain_strategies: dict | None = Field(
-        default=None, description="Map of retain strategy name to per-strategy config dict"
-    )
-    retain_chunk_batch_size: int | None = Field(
-        default=None, description="Max chunks per streaming batch (0 disables batching)"
-    )
-    mcp_enabled_tools: list[str] | None = Field(
-        default=None, description="MCP tool allowlist for this bank (None = all tools)"
-    )
-    consolidation_llm_batch_size: int | None = Field(
-        default=None, description="LLM batch size for observation consolidation"
-    )
-    consolidation_source_facts_max_tokens: int | None = Field(
-        default=None, description="Max tokens of source facts per consolidation batch"
-    )
-    consolidation_source_facts_max_tokens_per_observation: int | None = Field(
-        default=None, description="Max tokens of source facts per observation"
-    )
-    max_observations_per_scope: int | None = Field(
-        default=None, description="Max observations to retain per consolidation scope"
-    )
-    reflect_source_facts_max_tokens: int | None = Field(
-        default=None, description="Max tokens of source facts per reflect call"
-    )
-    llm_gemini_safety_settings: list | None = Field(
-        default=None, description="Per-bank Gemini/VertexAI safety filter settings"
-    )
 
     def get_config_updates(self) -> dict[str, Any]:
         """Return only the fields that were explicitly set (non-None)."""
@@ -1851,150 +1800,6 @@ class BankTemplateImportResponse(BaseModel):
     dry_run: bool = Field(default=False, description="True if this was a validation-only run")
 
 
-def validate_bank_template(manifest: "BankTemplateManifest") -> list[str]:
-    """Validate a parsed manifest beyond Pydantic's structural checks.
-
-    Returns a list of human-readable error strings (e.g. invalid
-    extraction mode values, conflicting settings).
-    """
-    errors: list[str] = []
-    if manifest.bank:
-        bank = manifest.bank
-        if bank.retain_extraction_mode is not None:
-            valid_modes = ("concise", "verbose", "custom", "chunks")
-            if bank.retain_extraction_mode not in valid_modes:
-                errors.append(
-                    f"bank.retain_extraction_mode: must be one of {valid_modes}, got '{bank.retain_extraction_mode}'"
-                )
-        if bank.retain_custom_instructions and bank.retain_extraction_mode != "custom":
-            errors.append("bank.retain_custom_instructions: requires retain_extraction_mode='custom'")
-    if manifest.mental_models:
-        for i, mm in enumerate(manifest.mental_models):
-            if not mm.name.strip():
-                errors.append(f"mental_models[{i}].name: must not be empty")
-            if not mm.source_query.strip():
-                errors.append(f"mental_models[{i}].source_query: must not be empty")
-    if manifest.directives:
-        for i, d in enumerate(manifest.directives):
-            if not d.name.strip():
-                errors.append(f"directives[{i}].name: must not be empty")
-            if not d.content.strip():
-                errors.append(f"directives[{i}].content: must not be empty")
-    return errors
-
-
-async def apply_bank_template_manifest(
-    memory,
-    bank_id: str,
-    manifest: "BankTemplateManifest",
-    request_context: "RequestContext",
-) -> "BankTemplateImportResponse":
-    """Apply a validated BankTemplateManifest to an existing bank.
-
-    Shared by the /import endpoint and the default-template-on-create hook
-    driven by HINDSIGHT_API_DEFAULT_BANK_TEMPLATE. The bank MUST already
-    exist; caller is responsible for validation (Pydantic + validate_bank_template).
-    """
-    config_applied = False
-    if manifest.bank:
-        config_updates = manifest.bank.get_config_updates()
-        if config_updates:
-            await memory._config_resolver.update_bank_config(bank_id, config_updates, request_context)
-            config_applied = True
-
-    created_ids: list[str] = []
-    updated_ids: list[str] = []
-    operation_ids: list[str] = []
-
-    if manifest.mental_models:
-        # Fetch existing mental models to decide create vs update
-        existing = await memory.list_mental_models(bank_id=bank_id, request_context=request_context)
-        existing_by_id = {m["id"]: m for m in existing}
-
-        for mm in manifest.mental_models:
-            if mm.id in existing_by_id:
-                await memory.update_mental_model(
-                    bank_id=bank_id,
-                    mental_model_id=mm.id,
-                    name=mm.name,
-                    source_query=mm.source_query,
-                    max_tokens=mm.max_tokens,
-                    tags=mm.tags if mm.tags else None,
-                    trigger=mm.trigger.model_dump() if mm.trigger else None,
-                    request_context=request_context,
-                )
-                result = await memory.submit_async_refresh_mental_model(
-                    bank_id=bank_id,
-                    mental_model_id=mm.id,
-                    request_context=request_context,
-                )
-                operation_ids.append(result["operation_id"])
-                updated_ids.append(mm.id)
-            else:
-                mental_model = await memory.create_mental_model(
-                    bank_id=bank_id,
-                    name=mm.name,
-                    source_query=mm.source_query,
-                    content="Generating content...",
-                    mental_model_id=mm.id,
-                    tags=mm.tags if mm.tags else None,
-                    max_tokens=mm.max_tokens,
-                    trigger=mm.trigger.model_dump() if mm.trigger else None,
-                    request_context=request_context,
-                )
-                result = await memory.submit_async_refresh_mental_model(
-                    bank_id=bank_id,
-                    mental_model_id=mental_model["id"],
-                    request_context=request_context,
-                )
-                operation_ids.append(result["operation_id"])
-                created_ids.append(mm.id)
-
-    directives_created: list[str] = []
-    directives_updated: list[str] = []
-
-    if manifest.directives:
-        existing_directives = await memory.list_directives(
-            bank_id=bank_id, active_only=False, request_context=request_context
-        )
-        existing_by_name = {d["name"]: d for d in existing_directives}
-
-        for directive in manifest.directives:
-            if directive.name in existing_by_name:
-                await memory.update_directive(
-                    bank_id=bank_id,
-                    directive_id=existing_by_name[directive.name]["id"],
-                    content=directive.content,
-                    priority=directive.priority,
-                    is_active=directive.is_active,
-                    tags=directive.tags if directive.tags else None,
-                    request_context=request_context,
-                )
-                directives_updated.append(directive.name)
-            else:
-                await memory.create_directive(
-                    bank_id=bank_id,
-                    name=directive.name,
-                    content=directive.content,
-                    priority=directive.priority,
-                    is_active=directive.is_active,
-                    tags=directive.tags if directive.tags else None,
-                    request_context=request_context,
-                )
-                directives_created.append(directive.name)
-
-    return BankTemplateImportResponse(
-        bank_id=bank_id,
-        config_applied=config_applied,
-        mental_models_created=created_ids,
-        mental_models_updated=updated_ids,
-        directives_created=directives_created,
-        directives_updated=directives_updated,
-        operation_ids=operation_ids,
-        dry_run=False,
-    )
-
-
 class OperationResponse(BaseModel):
     """Response model for a single async operation."""
 
@@ -2135,10 +1940,6 @@ class OperationStatusResponse(BaseModel):
     child_operations: list[ChildOperationStatus] | None = Field(
         default=None, description="Child operations for batch operations (if applicable)"
     )
-    task_payload: dict[str, Any] | None = Field(
-        default=None,
-        description="Raw task payload (params the operation was submitted with). Only populated when include_payload=true.",
-    )
 
 
 class AsyncOperationSubmitResponse(BaseModel):
@@ -4223,13 +4024,7 @@ def _register_routes(app: FastAPI):
         tags=["Operations"],
     )
     async def api_get_operation_status(
-        bank_id: str,
-        operation_id: str,
-        include_payload: bool = Query(
-            default=False,
-            description="Include the raw task payload (submission params) in the response. May be large.",
-        ),
-        request_context: RequestContext = Depends(get_request_context),
+        bank_id: str, operation_id: str, request_context: RequestContext = Depends(get_request_context)
     ):
         """Get the status of an async operation."""
         try:
@@ -4239,9 +4034,7 @@ def _register_routes(app: FastAPI):
             except ValueError:
                 raise HTTPException(status_code=400, detail=f"Invalid operation_id format: {operation_id}")
 
-            result = await app.state.memory.get_operation_status(
-                bank_id, operation_id, request_context=request_context, include_payload=include_payload
-            )
+            result = await app.state.memory.get_operation_status(bank_id, operation_id, request_context=request_context)
             return OperationStatusResponse(**result)
         except OperationValidationError as e:
             raise HTTPException(status_code=e.status_code, detail=e.reason)
@@ -4584,6 +4377,38 @@ def _register_routes(app: FastAPI):
     # Bank Template Import / Export
     # =====================================================================
 
+    def _validate_template(manifest: BankTemplateManifest) -> list[str]:
+        """Validate a parsed manifest beyond Pydantic's structural checks.
+
+        Returns a list of human-readable error strings (e.g. invalid
+        extraction mode values, conflicting settings).
+        """
+        errors: list[str] = []
+        if manifest.bank:
+            bank = manifest.bank
+            if bank.retain_extraction_mode is not None:
+                valid_modes = ("concise", "verbose", "custom", "chunks")
+                if bank.retain_extraction_mode not in valid_modes:
+                    errors.append(
+                        f"bank.retain_extraction_mode: must be one of {valid_modes}, "
+                        f"got '{bank.retain_extraction_mode}'"
+                    )
+            if bank.retain_custom_instructions and bank.retain_extraction_mode != "custom":
+                errors.append("bank.retain_custom_instructions: requires retain_extraction_mode='custom'")
+        if manifest.mental_models:
+            for i, mm in enumerate(manifest.mental_models):
+                if not mm.name.strip():
+                    errors.append(f"mental_models[{i}].name: must not be empty")
+                if not mm.source_query.strip():
+                    errors.append(f"mental_models[{i}].source_query: must not be empty")
+        if manifest.directives:
+            for i, d in enumerate(manifest.directives):
+                if not d.name.strip():
+                    errors.append(f"directives[{i}].name: must not be empty")
+                if not d.content.strip():
+                    errors.append(f"directives[{i}].content: must not be empty")
+        return errors
+
     @app.post(
         "/v1/default/banks/{bank_id}/import",
         response_model=BankTemplateImportResponse,
@@ -4619,7 +4444,7 @@ def _register_routes(app: FastAPI):
                 )
 
             # Semantic validation beyond Pydantic structural checks
-            validation_errors = validate_bank_template(body)
+            validation_errors = _validate_template(body)
             if validation_errors:
                 raise HTTPException(
                     status_code=400,
@@ -4637,11 +4462,107 @@ def _register_routes(app: FastAPI):
             # Ensure bank exists (auto-creates with defaults if needed)
             await app.state.memory.get_bank_profile(bank_id, request_context=request_context)
 
-            return await apply_bank_template_manifest(
-                memory=app.state.memory,
+            config_applied = False
+            if body.bank:
+                config_updates = body.bank.get_config_updates()
+                if config_updates:
+                    await app.state.memory._config_resolver.update_bank_config(bank_id, config_updates, request_context)
+                    config_applied = True
+
+            created_ids: list[str] = []
+            updated_ids: list[str] = []
+            operation_ids: list[str] = []
+
+            if body.mental_models:
+                # Fetch existing mental models to decide create vs update
+                existing = await app.state.memory.list_mental_models(bank_id=bank_id, request_context=request_context)
+                existing_by_id = {m["id"]: m for m in existing}
+
+                for mm in body.mental_models:
+                    if mm.id in existing_by_id:
+                        # Update existing mental model metadata
+                        await app.state.memory.update_mental_model(
+                            bank_id=bank_id,
+                            mental_model_id=mm.id,
+                            name=mm.name,
+                            source_query=mm.source_query,
+                            max_tokens=mm.max_tokens,
+                            tags=mm.tags if mm.tags else None,
+                            trigger=mm.trigger.model_dump() if mm.trigger else None,
+                            request_context=request_context,
+                        )
+                        # Schedule a refresh to regenerate content with updated query
+                        result = await app.state.memory.submit_async_refresh_mental_model(
+                            bank_id=bank_id,
+                            mental_model_id=mm.id,
+                            request_context=request_context,
+                        )
+                        operation_ids.append(result["operation_id"])
+                        updated_ids.append(mm.id)
+                    else:
+                        # Create new mental model
+                        mental_model = await app.state.memory.create_mental_model(
+                            bank_id=bank_id,
+                            name=mm.name,
+                            source_query=mm.source_query,
+                            content="Generating content...",
+                            mental_model_id=mm.id,
+                            tags=mm.tags if mm.tags else None,
+                            max_tokens=mm.max_tokens,
+                            trigger=mm.trigger.model_dump() if mm.trigger else None,
+                            request_context=request_context,
+                        )
+                        result = await app.state.memory.submit_async_refresh_mental_model(
+                            bank_id=bank_id,
+                            mental_model_id=mental_model["id"],
+                            request_context=request_context,
+                        )
+                        operation_ids.append(result["operation_id"])
+                        created_ids.append(mm.id)
+
+            directives_created: list[str] = []
+            directives_updated: list[str] = []
+
+            if body.directives:
+                # Fetch existing directives to decide create vs update (matched by name)
+                existing_directives = await app.state.memory.list_directives(
+                    bank_id=bank_id, active_only=False, request_context=request_context
+                )
+                existing_by_name = {d["name"]: d for d in existing_directives}
+
+                for directive in body.directives:
+                    if directive.name in existing_by_name:
+                        await app.state.memory.update_directive(
+                            bank_id=bank_id,
+                            directive_id=existing_by_name[directive.name]["id"],
+                            content=directive.content,
+                            priority=directive.priority,
+                            is_active=directive.is_active,
+                            tags=directive.tags if directive.tags else None,
+                            request_context=request_context,
+                        )
+                        directives_updated.append(directive.name)
+                    else:
+                        await app.state.memory.create_directive(
+                            bank_id=bank_id,
+                            name=directive.name,
+                            content=directive.content,
+                            priority=directive.priority,
+                            is_active=directive.is_active,
+                            tags=directive.tags if directive.tags else None,
+                            request_context=request_context,
+                        )
+                        directives_created.append(directive.name)
+
+            return BankTemplateImportResponse(
                 bank_id=bank_id,
-                manifest=body,
-                request_context=request_context,
+                config_applied=config_applied,
+                mental_models_created=created_ids,
+                mental_models_updated=updated_ids,
+                directives_created=directives_created,
+                directives_updated=directives_updated,
+                operation_ids=operation_ids,
+                dry_run=False,
             )
         except ValueError as e:
             raise HTTPException(status_code=400, detail=str(e))
@@ -5043,9 +4964,7 @@ def _register_routes(app: FastAPI):
             from hindsight_api.engine.retain import bank_utils
 
             # Ensure the bank row exists before inserting into webhooks (FK constraint).
-            _, created = await bank_utils.get_or_create_bank_profile(pool, bank_id)
-            if created:
-                await app.state.memory._apply_default_bank_template(bank_id, request_context)
+            await bank_utils.get_bank_profile(pool, bank_id)
 
             webhook_id = uuid.uuid4()
             now = datetime.now(timezone.utc).isoformat()

hindsight-api-slim/hindsight_api/config.py

@@ -247,11 +247,6 @@ ENV_RERANKER_ZEROENTROPY_API_KEY = "HINDSIGHT_API_RERANKER_ZEROENTROPY_API_KEY"
 ENV_RERANKER_ZEROENTROPY_MODEL = "HINDSIGHT_API_RERANKER_ZEROENTROPY_MODEL"
 ENV_RERANKER_ZEROENTROPY_BASE_URL = "HINDSIGHT_API_RERANKER_ZEROENTROPY_BASE_URL"
 
-# SiliconFlow configuration (reranker only; Cohere-compatible /rerank endpoint)
-ENV_RERANKER_SILICONFLOW_API_KEY = "HINDSIGHT_API_RERANKER_SILICONFLOW_API_KEY"
-ENV_RERANKER_SILICONFLOW_MODEL = "HINDSIGHT_API_RERANKER_SILICONFLOW_MODEL"
-ENV_RERANKER_SILICONFLOW_BASE_URL = "HINDSIGHT_API_RERANKER_SILICONFLOW_BASE_URL"
-
 # Google Discovery Engine reranker configuration
 ENV_RERANKER_GOOGLE_MODEL = "HINDSIGHT_API_RERANKER_GOOGLE_MODEL"
 ENV_RERANKER_GOOGLE_PROJECT_ID = "HINDSIGHT_API_RERANKER_GOOGLE_PROJECT_ID"
@@ -270,7 +265,6 @@ ENV_MCP_ENABLED = "HINDSIGHT_API_MCP_ENABLED"
 ENV_MCP_ENABLED_TOOLS = "HINDSIGHT_API_MCP_ENABLED_TOOLS"
 ENV_MCP_STATELESS = "HINDSIGHT_API_MCP_STATELESS"
 ENV_ENABLE_BANK_CONFIG_API = "HINDSIGHT_API_ENABLE_BANK_CONFIG_API"
-ENV_DEFAULT_BANK_TEMPLATE = "HINDSIGHT_API_DEFAULT_BANK_TEMPLATE"
 ENV_GRAPH_RETRIEVER = "HINDSIGHT_API_GRAPH_RETRIEVER"
 ENV_RECALL_MAX_CONCURRENT = "HINDSIGHT_API_RECALL_MAX_CONCURRENT"
 ENV_RECALL_CONNECTION_BUDGET = "HINDSIGHT_API_RECALL_CONNECTION_BUDGET"
@@ -387,9 +381,6 @@ ENV_REFLECT_MAX_CONTEXT_TOKENS = "HINDSIGHT_API_REFLECT_MAX_CONTEXT_TOKENS"
 ENV_REFLECT_WALL_TIMEOUT = "HINDSIGHT_API_REFLECT_WALL_TIMEOUT"
 ENV_REFLECT_MISSION = "HINDSIGHT_API_REFLECT_MISSION"
 ENV_REFLECT_SOURCE_FACTS_MAX_TOKENS = "HINDSIGHT_API_REFLECT_SOURCE_FACTS_MAX_TOKENS"
-ENV_RECALL_INCLUDE_CHUNKS = "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS"
-ENV_RECALL_MAX_TOKENS = "HINDSIGHT_API_RECALL_MAX_TOKENS"
-ENV_RECALL_CHUNKS_MAX_TOKENS = "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS"
 
 # Audit log settings
 ENV_AUDIT_LOG_ENABLED = "HINDSIGHT_API_AUDIT_LOG_ENABLED"
@@ -482,9 +473,6 @@ DEFAULT_RERANKER_OPENROUTER_MODEL = "cohere/rerank-v3.5"
 
 DEFAULT_RERANKER_ZEROENTROPY_MODEL = "zerank-2"
 
-DEFAULT_RERANKER_SILICONFLOW_MODEL = "BAAI/bge-reranker-v2-m3"
-DEFAULT_RERANKER_SILICONFLOW_BASE_URL = "https://api.siliconflow.cn/v1"
-
 DEFAULT_RERANKER_GOOGLE_MODEL = "semantic-ranker-default-004"
 
 # Vector extension (pgvector, vchord, or pgvectorscale)
@@ -514,7 +502,6 @@ DEFAULT_MCP_ENABLED = True
 DEFAULT_MCP_ENABLED_TOOLS: list[str] | None = None  # None = all tools enabled
 DEFAULT_MCP_STATELESS = False  # False = stateful (supports SSE/GET); True = stateless (POST-only)
 DEFAULT_ENABLE_BANK_CONFIG_API = True
-DEFAULT_DEFAULT_BANK_TEMPLATE: dict | None = None  # BankTemplateManifest dict applied to newly-created banks
 DEFAULT_GRAPH_RETRIEVER = "link_expansion"
 DEFAULT_RECALL_MAX_CONCURRENT = 32  # Max concurrent recall operations per worker
 DEFAULT_RECALL_CONNECTION_BUDGET = 4  # Max concurrent DB connections per recall operation
@@ -590,9 +577,6 @@ DEFAULT_REFLECT_MAX_ITERATIONS = 10  # Max tool call iterations before forcing r
 DEFAULT_REFLECT_MAX_CONTEXT_TOKENS = 100_000  # Max accumulated context tokens before forcing final prompt
 DEFAULT_REFLECT_WALL_TIMEOUT = 300  # Wall-clock timeout in seconds for the entire reflect operation (5 minutes)
 DEFAULT_REFLECT_SOURCE_FACTS_MAX_TOKENS = -1  # Token budget for source facts in search_observations (-1 = disabled)
-DEFAULT_RECALL_INCLUDE_CHUNKS = True  # Whether internal recall (e.g. mental model refresh) returns raw chunks
-DEFAULT_RECALL_MAX_TOKENS = 2048  # Token budget for facts returned by internal recall
-DEFAULT_RECALL_CHUNKS_MAX_TOKENS = 1000  # Token budget for raw chunks returned by internal recall
 
 # Disposition defaults (None = not set, fall back to bank DB value or 3)
 DEFAULT_DISPOSITION_SKEPTICISM = None
@@ -692,26 +676,6 @@ def _get_default_model_for_provider(provider: str) -> str:
     return PROVIDER_DEFAULT_MODELS.get(provider.lower(), DEFAULT_LLM_MODEL)
 
 
-def _parse_default_bank_template(raw: str | None) -> dict | None:
-    """
-    Parse HINDSIGHT_API_DEFAULT_BANK_TEMPLATE as JSON.
-
-    The env var holds a BankTemplateManifest (JSON object) applied verbatim to
-    every newly-created bank. Full Pydantic validation is deferred to bank
-    creation time (to avoid pulling API models into config.py), but we fail
-    fast here if the value is not valid JSON or not a JSON object.
-    """
-    if raw is None or raw.strip() == "":
-        return DEFAULT_DEFAULT_BANK_TEMPLATE
-    try:
-        parsed = json.loads(raw)
-    except json.JSONDecodeError as e:
-        raise ValueError(f"Invalid {ENV_DEFAULT_BANK_TEMPLATE}: expected a JSON object, got invalid JSON: {e}") from e
-    if not isinstance(parsed, dict):
-        raise ValueError(f"Invalid {ENV_DEFAULT_BANK_TEMPLATE}: expected a JSON object, got {type(parsed).__name__}")
-    return parsed
-
-
 @dataclass
 class HindsightConfig:
     """Configuration container for Hindsight API."""
@@ -842,9 +806,6 @@ class HindsightConfig:
     reranker_zeroentropy_api_key: str | None
     reranker_zeroentropy_model: str
     reranker_zeroentropy_base_url: str | None
-    reranker_siliconflow_api_key: str | None
-    reranker_siliconflow_model: str
-    reranker_siliconflow_base_url: str
     reranker_google_model: str
     reranker_google_project_id: str | None
     reranker_google_service_account_key: str | None
@@ -859,9 +820,6 @@ class HindsightConfig:
     mcp_enabled_tools: list[str] | None  # None = all tools; explicit list = allowlist
     mcp_stateless: bool  # True = stateless HTTP (POST-only); False = stateful (supports GET/SSE)
     enable_bank_config_api: bool
-    # Default bank template (static, server-level only). When set, the manifest is applied
-    # to every newly-created bank, overriding the env/config defaults for any fields it sets.
-    default_bank_template: dict | None
 
     # Recall
     graph_retriever: str
@@ -931,11 +889,6 @@ class HindsightConfig:
     reflect_mission: str | None
     reflect_source_facts_max_tokens: int
 
-    # Recall settings (used by internal recall, e.g. during mental model refresh)
-    recall_include_chunks: bool
-    recall_max_tokens: int
-    recall_chunks_max_tokens: int
-
     # Disposition settings (hierarchical - can be overridden per bank; None = fall back to DB)
     disposition_skepticism: int | None
     disposition_literalism: int | None
@@ -1006,7 +959,6 @@ class HindsightConfig:
         "reranker_tei_base_url",
         "reranker_cohere_base_url",
         "reranker_zeroentropy_base_url",
-        "reranker_siliconflow_base_url",
         # Service Account Keys
         "llm_vertexai_service_account_key",
         "embeddings_vertexai_service_account_key",
@@ -1049,10 +1001,6 @@ class HindsightConfig:
         # Reflect settings
         "reflect_mission",
         "reflect_source_facts_max_tokens",
-        # Recall settings (used by internal recall, e.g. mental model refresh)
-        "recall_include_chunks",
-        "recall_max_tokens",
-        "recall_chunks_max_tokens",
         # Disposition settings
         "disposition_skepticism",
         "disposition_literalism",
@@ -1379,12 +1327,6 @@ class HindsightConfig:
             reranker_zeroentropy_api_key=os.getenv(ENV_RERANKER_ZEROENTROPY_API_KEY),
             reranker_zeroentropy_model=os.getenv(ENV_RERANKER_ZEROENTROPY_MODEL, DEFAULT_RERANKER_ZEROENTROPY_MODEL),
             reranker_zeroentropy_base_url=os.getenv(ENV_RERANKER_ZEROENTROPY_BASE_URL) or None,
-            # SiliconFlow reranker (Cohere-compatible /rerank endpoint)
-            reranker_siliconflow_api_key=os.getenv(ENV_RERANKER_SILICONFLOW_API_KEY),
-            reranker_siliconflow_model=os.getenv(ENV_RERANKER_SILICONFLOW_MODEL, DEFAULT_RERANKER_SILICONFLOW_MODEL),
-            reranker_siliconflow_base_url=os.getenv(
-                ENV_RERANKER_SILICONFLOW_BASE_URL, DEFAULT_RERANKER_SILICONFLOW_BASE_URL
-            ),
             # Google Discovery Engine reranker (with fallback to LLM Vertex AI keys)
             reranker_google_model=os.getenv(ENV_RERANKER_GOOGLE_MODEL, DEFAULT_RERANKER_GOOGLE_MODEL),
             reranker_google_project_id=os.getenv(ENV_RERANKER_GOOGLE_PROJECT_ID)
@@ -1404,7 +1346,6 @@ class HindsightConfig:
             mcp_stateless=os.getenv(ENV_MCP_STATELESS, str(DEFAULT_MCP_STATELESS)).lower() == "true",
             enable_bank_config_api=os.getenv(ENV_ENABLE_BANK_CONFIG_API, str(DEFAULT_ENABLE_BANK_CONFIG_API)).lower()
             == "true",
-            default_bank_template=_parse_default_bank_template(os.getenv(ENV_DEFAULT_BANK_TEMPLATE)),
             # Recall
             graph_retriever=os.getenv(ENV_GRAPH_RETRIEVER, DEFAULT_GRAPH_RETRIEVER),
             recall_max_concurrent=int(os.getenv(ENV_RECALL_MAX_CONCURRENT, str(DEFAULT_RECALL_MAX_CONCURRENT))),
@@ -1538,12 +1479,6 @@ class HindsightConfig:
             reflect_source_facts_max_tokens=int(
                 os.getenv(ENV_REFLECT_SOURCE_FACTS_MAX_TOKENS, str(DEFAULT_REFLECT_SOURCE_FACTS_MAX_TOKENS))
             ),
-            recall_include_chunks=os.getenv(ENV_RECALL_INCLUDE_CHUNKS, str(DEFAULT_RECALL_INCLUDE_CHUNKS)).lower()
-            in ("true", "1", "yes"),
-            recall_max_tokens=int(os.getenv(ENV_RECALL_MAX_TOKENS, str(DEFAULT_RECALL_MAX_TOKENS))),
-            recall_chunks_max_tokens=int(
-                os.getenv(ENV_RECALL_CHUNKS_MAX_TOKENS, str(DEFAULT_RECALL_CHUNKS_MAX_TOKENS))
-            ),
             # Disposition settings (None = fall back to DB value)
             disposition_skepticism=int(os.getenv(ENV_DISPOSITION_SKEPTICISM))
             if os.getenv(ENV_DISPOSITION_SKEPTICISM)

hindsight-api-slim/hindsight_api/engine/cross_encoder.py

@@ -30,8 +30,6 @@ from ..config import (
     DEFAULT_RERANKER_LOCAL_MODEL,
     DEFAULT_RERANKER_LOCAL_TRUST_REMOTE_CODE,
     DEFAULT_RERANKER_PROVIDER,
-    DEFAULT_RERANKER_SILICONFLOW_BASE_URL,
-    DEFAULT_RERANKER_SILICONFLOW_MODEL,
     DEFAULT_RERANKER_TEI_BATCH_SIZE,
     DEFAULT_RERANKER_TEI_MAX_CONCURRENT,
     DEFAULT_RERANKER_ZEROENTROPY_MODEL,
@@ -46,7 +44,6 @@ from ..config import (
     ENV_RERANKER_LOCAL_MODEL,
     ENV_RERANKER_LOCAL_TRUST_REMOTE_CODE,
     ENV_RERANKER_PROVIDER,
-    ENV_RERANKER_SILICONFLOW_API_KEY,
     ENV_RERANKER_TEI_BATCH_SIZE,
     ENV_RERANKER_TEI_MAX_CONCURRENT,
     ENV_RERANKER_TEI_URL,
@@ -521,84 +518,6 @@ class RemoteTEICrossEncoder(CrossEncoderModel):
         return await self._predict_async(pairs)
 
 
-class _CohereCompatibleRerankClient:
-    """
-    Internal HTTP client for Cohere-compatible /rerank endpoints.
-
-    Shared by all providers that speak the Cohere rerank wire format —
-    {model, query, documents[, top_n]} request and
-    {results: [{index, relevance_score}, ...]} response. This covers
-    SiliconFlow, ZeroEntropy, Jina, Voyage, BGE self-hosted, and Cohere
-    itself when reached via a custom base_url (e.g. Azure AI Foundry).
-
-    Not a CrossEncoderModel — providers compose it and expose their own
-    provider_name / initialization logging.
-    """
-
-    def __init__(
-        self,
-        api_key: str,
-        model: str,
-        rerank_url: str,
-        timeout: float = 60.0,
-        include_top_n: bool = True,
-    ):
-        self.api_key = api_key
-        self.model = model
-        self.rerank_url = rerank_url
-        self.timeout = timeout
-        self.include_top_n = include_top_n
-        self._async_client: httpx.AsyncClient | None = None
-
-    async def initialize(self) -> None:
-        if self._async_client is not None:
-            return
-        self._async_client = httpx.AsyncClient(
-            timeout=self.timeout,
-            headers={
-                "Authorization": f"Bearer {self.api_key}",
-                "Content-Type": "application/json",
-            },
-        )
-
-    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
-        if self._async_client is None:
-            raise RuntimeError("Reranker not initialized. Call initialize() first.")
-
-        if not pairs:
-            return []
-
-        query_groups: dict[str, list[tuple[int, str]]] = {}
-        for idx, (query, text) in enumerate(pairs):
-            query_groups.setdefault(query, []).append((idx, text))
-
-        all_scores = [0.0] * len(pairs)
-
-        for query, indexed_texts in query_groups.items():
-            texts = [text for _, text in indexed_texts]
-            indices = [idx for idx, _ in indexed_texts]
-
-            body: dict[str, object] = {
-                "model": self.model,
-                "query": query,
-                "documents": texts,
-                "return_documents": False,
-            }
-            if self.include_top_n:
-                body["top_n"] = len(texts)
-
-            response = await self._async_client.post(self.rerank_url, json=body)
-            response.raise_for_status()
-            result = response.json()
-
-            for item in result.get("results", []):
-                original_idx = item["index"]
-                score = item["relevance_score"]
-                all_scores[indices[original_idx]] = score
-
-        return all_scores
-
-
 class CohereCrossEncoder(CrossEncoderModel):
     """
     Cohere cross-encoder implementation using the Cohere Rerank API.
@@ -627,20 +546,7 @@ class CohereCrossEncoder(CrossEncoderModel):
         self.base_url = base_url
         self.timeout = timeout
         self._client = None
-        # Used when base_url is set (Azure AI Foundry and other Cohere-compatible hosts).
-        # Azure endpoints already include the full invoke path, so rerank_url == base_url
-        # and top_n is omitted to match the existing Azure contract.
-        self._http_client: _CohereCompatibleRerankClient | None = (
-            _CohereCompatibleRerankClient(
-                api_key=api_key,
-                model=model,
-                rerank_url=base_url,
-                timeout=timeout,
-                include_top_n=False,
-            )
-            if base_url
-            else None
-        )
+        self._httpx_client: httpx.Client | None = None
 
     @property
     def provider_name(self) -> str:
@@ -648,15 +554,23 @@ class CohereCrossEncoder(CrossEncoderModel):
 
     async def initialize(self) -> None:
         """Initialize the Cohere client."""
-        if self._client is not None or (self._http_client and self._http_client._async_client):
+        if self._client is not None or self._httpx_client is not None:
             return
 
         base_url_msg = f" at {self.base_url}" if self.base_url else ""
         logger.info(f"Reranker: initializing Cohere provider with model {self.model}{base_url_msg}")
 
-        if self._http_client is not None:
-            await self._http_client.initialize()
-            logger.info("Reranker: Cohere provider initialized (Cohere-compatible HTTP endpoint)")
+        if self.base_url:
+            # For custom endpoints (Azure AI Foundry), use httpx directly to avoid SDK path appending
+            # Azure endpoints already include the full path (e.g., /models/.../invoke)
+            self._httpx_client = httpx.Client(
+                timeout=self.timeout,
+                headers={
+                    "Authorization": f"Bearer {self.api_key}",
+                    "Content-Type": "application/json",
+                },
+            )
+            logger.info("Reranker: Cohere provider initialized (using httpx for custom endpoint)")
         else:
             # For native Cohere API, use the official SDK
             try:
@@ -677,24 +591,25 @@ class CohereCrossEncoder(CrossEncoderModel):
         Returns:
             List of relevance scores
         """
-        if self._client is None and self._http_client is None:
+        if self._client is None and self._httpx_client is None:
             raise RuntimeError("Reranker not initialized. Call initialize() first.")
 
         if not pairs:
             return []
 
-        if self._http_client is not None:
-            return await self._http_client.predict(pairs)
-
-        # Run sync Cohere SDK calls in thread pool
+        # Run sync Cohere API calls in thread pool
         loop = asyncio.get_event_loop()
-        return await loop.run_in_executor(None, self._predict_sync_sdk, pairs)
+        return await loop.run_in_executor(None, self._predict_sync, pairs)
 
-    def _predict_sync_sdk(self, pairs: list[tuple[str, str]]) -> list[float]:
-        """Synchronous predict using the native Cohere SDK."""
+    def _predict_sync(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """Synchronous predict implementation for Cohere API."""
+        # Group pairs by query for efficient batching
+        # Cohere rerank expects one query with multiple documents
         query_groups: dict[str, list[tuple[int, str]]] = {}
         for idx, (query, text) in enumerate(pairs):
-            query_groups.setdefault(query, []).append((idx, text))
+            if query not in query_groups:
+                query_groups[query] = []
+            query_groups[query].append((idx, text))
 
         all_scores = [0.0] * len(pairs)
 
@@ -702,17 +617,40 @@ class CohereCrossEncoder(CrossEncoderModel):
             texts = [text for _, text in indexed_texts]
             indices = [idx for idx, _ in indexed_texts]
 
-            response = self._client.rerank(
-                query=query,
-                documents=texts,
-                model=self.model,
-                return_documents=False,
-            )
+            if self._httpx_client:
+                # Direct HTTP request for custom endpoints (Azure AI Foundry)
+                response = self._httpx_client.post(
+                    self.base_url,
+                    json={
+                        "model": self.model,
+                        "query": query,
+                        "documents": texts,
+                        "return_documents": False,
+                    },
+                )
+                response.raise_for_status()
+                result = response.json()
 
-            for result in response.results:
-                original_idx = result.index
-                score = result.relevance_score
-                all_scores[indices[original_idx]] = score
+                # Map scores back to original positions
+                # Azure Cohere response format: {"results": [{"index": 0, "relevance_score": 0.9}, ...]}
+                for item in result.get("results", []):
+                    original_idx = item["index"]
+                    score = item["relevance_score"]
+                    all_scores[indices[original_idx]] = score
+            else:
+                # Native Cohere SDK for standard API
+                response = self._client.rerank(
+                    query=query,
+                    documents=texts,
+                    model=self.model,
+                    return_documents=False,
+                )
+
+                # Map scores back to original positions
+                for result in response.results:
+                    original_idx = result.index
+                    score = result.relevance_score
+                    all_scores[indices[original_idx]] = score
 
         return all_scores
 
@@ -735,70 +673,89 @@ class ZeroEntropyCrossEncoder(CrossEncoderModel):
         base_url: str | None = None,
         timeout: float = 60.0,
     ):
+        """
+        Initialize ZeroEntropy cross-encoder client.
+
+        Args:
+            api_key: ZeroEntropy API key
+            model: ZeroEntropy rerank model name (default: zerank-2)
+            base_url: Custom base URL for ZeroEntropy-compatible API (e.g., mock server or proxy)
+            timeout: Request timeout in seconds (default: 60.0)
+        """
+        self.api_key = api_key
         self.model = model
         self.base_url = base_url.rstrip("/") if base_url else self.DEFAULT_BASE_URL
-        self._client = _CohereCompatibleRerankClient(
-            api_key=api_key,
-            model=model,
-            rerank_url=f"{self.base_url}{self.RERANK_PATH}",
-            timeout=timeout,
-        )
+        self.rerank_url = f"{self.base_url}{self.RERANK_PATH}"
+        self.timeout = timeout
+        self._async_client: httpx.AsyncClient | None = None
 
     @property
     def provider_name(self) -> str:
         return "zeroentropy"
 
     async def initialize(self) -> None:
-        if self._client._async_client is not None:
+        """Initialize the async HTTP client."""
+        if self._async_client is not None:
             return
+
         logger.info(f"Reranker: initializing ZeroEntropy provider with model {self.model}")
-        await self._client.initialize()
+        self._async_client = httpx.AsyncClient(
+            timeout=self.timeout,
+            headers={
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+            },
+        )
         logger.info("Reranker: ZeroEntropy provider initialized")
 
     async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
-        return await self._client.predict(pairs)
+        """
+        Score query-document pairs using the ZeroEntropy Rerank API.
 
+        Args:
+            pairs: List of (query, document) tuples to score
 
-class SiliconFlowCrossEncoder(CrossEncoderModel):
-    """
-    SiliconFlow cross-encoder implementation.
+        Returns:
+            List of relevance scores
+        """
+        if self._async_client is None:
+            raise RuntimeError("Reranker not initialized. Call initialize() first.")
 
-    SiliconFlow (https://siliconflow.cn) exposes a Cohere-compatible /rerank
-    endpoint. Shares the HTTP client with ZeroEntropy/Cohere-custom-endpoint
-    via _CohereCompatibleRerankClient.
-    """
+        if not pairs:
+            return []
 
-    RERANK_PATH = "/rerank"
+        # Group pairs by query for efficient batching
+        query_groups: dict[str, list[tuple[int, str]]] = {}
+        for idx, (query, text) in enumerate(pairs):
+            if query not in query_groups:
+                query_groups[query] = []
+            query_groups[query].append((idx, text))
 
-    def __init__(
-        self,
-        api_key: str,
-        model: str = DEFAULT_RERANKER_SILICONFLOW_MODEL,
-        base_url: str = DEFAULT_RERANKER_SILICONFLOW_BASE_URL,
-        timeout: float = 60.0,
-    ):
-        self.model = model
-        self.base_url = base_url.rstrip("/")
-        self._client = _CohereCompatibleRerankClient(
-            api_key=api_key,
-            model=model,
-            rerank_url=f"{self.base_url}{self.RERANK_PATH}",
-            timeout=timeout,
-        )
+        all_scores = [0.0] * len(pairs)
 
-    @property
-    def provider_name(self) -> str:
-        return "siliconflow"
+        for query, indexed_texts in query_groups.items():
+            texts = [text for _, text in indexed_texts]
+            indices = [idx for idx, _ in indexed_texts]
 
-    async def initialize(self) -> None:
-        if self._client._async_client is not None:
-            return
-        logger.info(f"Reranker: initializing SiliconFlow provider at {self.base_url} with model {self.model}")
-        await self._client.initialize()
-        logger.info("Reranker: SiliconFlow provider initialized")
+            response = await self._async_client.post(
+                self.rerank_url,
+                json={
+                    "model": self.model,
+                    "query": query,
+                    "documents": texts,
+                    "top_n": len(texts),
+                },
+            )
+            response.raise_for_status()
+            result = response.json()
 
-    async def predict(self, pairs: list[tuple[str, str]]) -> list[float]:
-        return await self._client.predict(pairs)
+            # Map scores back to original positions
+            for item in result.get("results", []):
+                original_idx = item["index"]
+                score = item["relevance_score"]
+                all_scores[indices[original_idx]] = score
+
+        return all_scores
 
 
 class RRFPassthroughCrossEncoder(CrossEncoderModel):
@@ -1250,31 +1207,14 @@ class JinaMLXCrossEncoder(CrossEncoderModel):
         if self._reranker is not None:
             return
 
-        # Pre-warm transformers.AutoTokenizer to fully populate the transformers
-        # namespace before mlx_lm imports it. transformers 5.x uses _LazyModule,
-        # which has an unguarded window where `from transformers import AutoTokenizer`
-        # raises ImportError if another thread is concurrently initializing the
-        # namespace (e.g. embeddings init in an executor thread).
-        # See: https://github.com/vectorize-io/hindsight/issues/994
-        import transformers
-
-        _ = transformers.AutoTokenizer
-
         try:
             import mlx.core  # noqa: F401
             import mlx_lm  # noqa: F401
-        except ImportError as exc:
-            # Only swallow "package not installed" errors. Anything else (e.g. a
-            # transitive import failure inside mlx_lm) must surface verbatim so
-            # the real cause is debuggable instead of being masked by a generic
-            # "install mlx" message.
-            msg = str(exc)
-            if "mlx" not in msg and "mlx_lm" not in msg:
-                raise
+        except ImportError:
             raise ImportError(
                 "mlx and mlx-lm are required for JinaMLXCrossEncoder. "
                 "Install with: pip install mlx>=0.31.0 mlx-lm>=0.31.1 safetensors>=0.6.2"
-            ) from exc
+            )
 
         loop = asyncio.get_event_loop()
         await loop.run_in_executor(None, self._load_model)
@@ -1573,17 +1513,6 @@ def create_cross_encoder_from_env() -> CrossEncoderModel:
             api_key=api_key,
             model=config.reranker_zeroentropy_model,
         )
-    elif provider == "siliconflow":
-        api_key = config.reranker_siliconflow_api_key
-        if not api_key:
-            raise ValueError(
-                f"{ENV_RERANKER_SILICONFLOW_API_KEY} is required when {ENV_RERANKER_PROVIDER} is 'siliconflow'"
-            )
-        return SiliconFlowCrossEncoder(
-            api_key=api_key,
-            model=config.reranker_siliconflow_model,
-            base_url=config.reranker_siliconflow_base_url,
-        )
     elif provider == "google":
         project_id = config.reranker_google_project_id
         if not project_id:
@@ -1602,5 +1531,5 @@ def create_cross_encoder_from_env() -> CrossEncoderModel:
         return JinaMLXCrossEncoder()
     else:
         raise ValueError(
-            f"Unknown reranker provider: {provider}. Supported: 'local', 'tei', 'cohere', 'zeroentropy', 'siliconflow', 'google', 'flashrank', 'litellm', 'litellm-sdk', 'rrf', 'jina-mlx'"
+            f"Unknown reranker provider: {provider}. Supported: 'local', 'tei', 'cohere', 'zeroentropy', 'google', 'flashrank', 'litellm', 'litellm-sdk', 'rrf', 'jina-mlx'"
         )

hindsight-api-slim/hindsight_api/engine/llm_wrapper.py

@@ -536,15 +536,6 @@ class LLMProvider:
             OutputTooLongError: If output exceeds token limits.
             Exception: Re-raises API errors after retries exhausted.
         """
-        # Stage breadcrumb so the worker log shows which LLM call a task is
-        # currently inside; the stage_age field then reveals long JSON-schema
-        # retry loops (e.g. a small model that can't satisfy strict_schema).
-        # No-op outside a worker context.
-        from ..worker.stage import set_stage
-
-        structured = "+structured" if response_format is not None else ""
-        set_stage(f"llm.{self.provider}.{scope}{structured}")
-
         async with _global_llm_semaphore:
             # Delegate to provider implementation
             result = await self._provider_impl.call(
@@ -601,10 +592,6 @@ class LLMProvider:
         Returns:
             LLMToolCallResult with content and/or tool_calls.
         """
-        from ..worker.stage import set_stage
-
-        set_stage(f"llm.{self.provider}.{scope}+tools")
-
         async with _global_llm_semaphore:
             # Delegate to provider implementation
             result = await self._provider_impl.call_with_tools(

hindsight-api-slim/hindsight_api/engine/memory_engine.py

@@ -24,18 +24,11 @@ import asyncpg
 import httpx
 import tiktoken
 
-from ..config import (
-    DEFAULT_RECALL_CHUNKS_MAX_TOKENS,
-    DEFAULT_RECALL_INCLUDE_CHUNKS,
-    DEFAULT_RECALL_MAX_TOKENS,
-    DEFAULT_REFLECT_SOURCE_FACTS_MAX_TOKENS,
-    get_config,
-)
+from ..config import DEFAULT_REFLECT_SOURCE_FACTS_MAX_TOKENS, get_config
 from ..metrics import get_metrics_collector
 from ..tracing import create_operation_span
 from ..utils import mask_network_location
 from ..worker.exceptions import RetryTaskAt
-from ..worker.stage import set_stage
 from .audit import AuditLogger, audit_context
 from .db_budget import budgeted_operation
 from .operation_metadata import (
@@ -958,9 +951,6 @@ class MemoryEngine(MemoryEngineInterface):
         fact_types = trigger_data.get("fact_types")
         exclude_mental_models = trigger_data.get("exclude_mental_models", False)
         stored_exclude_ids: list[str] = trigger_data.get("exclude_mental_model_ids") or []
-        recall_include_chunks_override = trigger_data.get("include_chunks")
-        recall_max_tokens_override = trigger_data.get("recall_max_tokens")
-        recall_chunks_max_tokens_override = trigger_data.get("recall_chunks_max_tokens")
 
         tag_filtering = _resolve_refresh_tag_filtering(mental_model.get("tags"), trigger_data)
 
@@ -976,9 +966,6 @@ class MemoryEngine(MemoryEngineInterface):
             fact_types=fact_types,
             exclude_mental_models=exclude_mental_models,
             exclude_mental_model_ids=list({*stored_exclude_ids, mental_model_id}),
-            recall_include_chunks=recall_include_chunks_override,
-            recall_max_tokens_override=recall_max_tokens_override,
-            recall_chunks_max_tokens_override=recall_chunks_max_tokens_override,
         )
 
         generated_content = reflect_result.text or "No content generated"
@@ -1103,9 +1090,6 @@ class MemoryEngine(MemoryEngineInterface):
             self._audit_logger, task_type or "unknown", "system", bank_id, request=task_dict
         ) as audit_entry:
             try:
-                # Stage breadcrumb for the worker poller's WORKER_TASK log line.
-                # No-op outside a worker context.
-                set_stage(f"task.{task_type}")
                 if task_type == "batch_retain":
                     await self._handle_batch_retain(task_dict)
                 elif task_type == "file_convert_retain":
@@ -1156,26 +1140,6 @@ class MemoryEngine(MemoryEngineInterface):
                     logger.error(f"Not retrying task {task_type} (non-retryable), marking as failed")
                     if operation_id:
                         await self._mark_operation_failed(operation_id, str(e), error_traceback)
-                elif isinstance(e, asyncpg.exceptions.IntegrityConstraintViolationError):
-                    # Non-retryable: deterministic Postgres integrity violations
-                    # (UniqueViolationError, ForeignKeyViolationError, CheckViolationError,
-                    # NotNullViolationError, ExclusionViolationError) will never succeed on
-                    # retry — the offending row state is already committed. Retrying just
-                    # burns worker capacity. See vectorize-io/hindsight#980.
-                    logger.error(
-                        f"Not retrying task {task_type} (integrity violation, deterministic): {type(e).__name__}"
-                    )
-                    if task_type == "consolidation" and operation_id:
-                        await self._fire_consolidation_webhook(
-                            bank_id=task_dict.get("bank_id", ""),
-                            operation_id=operation_id,
-                            status="failed",
-                            result=None,
-                            error_message=str(e),
-                            schema=schema,
-                        )
-                    if operation_id:
-                        await self._mark_operation_failed(operation_id, str(e), error_traceback)
                 else:
                     if task_type == "consolidation" and operation_id:
                         # Fire failure webhook (non-transactional — operation not yet marked failed;
@@ -2779,11 +2743,8 @@ class MemoryEngine(MemoryEngineInterface):
         pool = await self._get_pool()
         recall_start = time.time()
 
-        # Buffer logs for clean output in concurrent scenarios.
-        # Include a uuid suffix so two recalls on the same bank within the
-        # same millisecond don't collide on the budgeted_operation key
-        # (`recall-{recall_id}`), which would raise "Operation ... already exists".
-        recall_id = f"{bank_id[:8]}-{int(time.time() * 1000) % 100000}-{uuid.uuid4().hex[:6]}"
+        # Buffer logs for clean output in concurrent scenarios
+        recall_id = f"{bank_id[:8]}-{int(time.time() * 1000) % 100000}"
         log_buffer = []
         tags_info = f", tags={tags}, tags_match={tags_match}" if tags else ""
         log_buffer.append(
@@ -2804,8 +2765,7 @@ class MemoryEngine(MemoryEngineInterface):
             embedding_span.set_attribute("hindsight.query", query[:100])
 
             try:
-                query_embeddings = await embedding_utils.generate_embeddings_batch(self.embeddings, [query])
-                query_embedding = query_embeddings[0]
+                query_embedding = embedding_utils.generate_embedding(self.embeddings, query)
                 step_duration = time.time() - step_start
                 log_buffer.append(f"  [1] Generate query embedding: {step_duration:.3f}s")
             finally:
@@ -3126,13 +3086,8 @@ class MemoryEngine(MemoryEngineInterface):
 
             # Step 4.5: Combine cross-encoder score with retrieval signals via multiplicative boosts.
             # See apply_combined_scoring for the full rationale and formula.
-            # is_passthrough_reranker tells the scoring code to seed CE scores
-            # from RRF rank — only meaningful when the configured reranker is
-            # the slim/passthrough one that returns a constant score per pair.
             if scored_results:
-                ce = reranker_instance.cross_encoder
-                is_passthrough = ce is not None and ce.provider_name == "rrf"
-                apply_combined_scoring(scored_results, now=utcnow(), is_passthrough_reranker=is_passthrough)
+                apply_combined_scoring(scored_results, now=utcnow())
                 scored_results.sort(key=lambda x: x.weight, reverse=True)
                 log_buffer.append("  [4.6] Combined scoring: ce * recency_boost(0.2) * temporal_boost(0.2)")
 
@@ -5198,13 +5153,7 @@ class MemoryEngine(MemoryEngineInterface):
             ctx = BankReadContext(bank_id=bank_id, operation="get_bank_profile", request_context=request_context)
             await self._validate_operation(self._operation_validator.validate_bank_read(ctx))
         pool = await self._get_pool()
-        profile, created = await bank_utils.get_or_create_bank_profile(pool, bank_id)
-
-        # Apply HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to freshly-created banks. Done
-        # before reading the resolved config below so the template's overrides
-        # (e.g. reflect_mission, dispositions) are visible on this very call.
-        if created:
-            await self._apply_default_bank_template(bank_id, request_context)
+        profile = await bank_utils.get_bank_profile(pool, bank_id)
 
         # reflect_mission and disposition in config take precedence over the legacy DB columns
         config_dict = await self._config_resolver.get_bank_config(bank_id, request_context)
@@ -5229,62 +5178,6 @@ class MemoryEngine(MemoryEngineInterface):
             "mission": mission,
         }
 
-    async def _apply_default_bank_template(
-        self,
-        bank_id: str,
-        request_context: "RequestContext",
-    ) -> None:
-        """Apply HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to a freshly-created bank.
-
-        No-op if the env var is unset. A malformed default template is logged
-        and swallowed here rather than raised, so a bad server-level setting
-        cannot wedge bank creation across all callers. Misconfiguration is
-        still surfaced loudly via `logger.error`.
-        """
-        from ..config import get_config
-
-        template_dict = get_config().default_bank_template
-        if not template_dict:
-            return
-
-        # Lazy import to avoid a cycle (http.py imports memory_engine).
-        from pydantic import ValidationError
-
-        from hindsight_api.api.http import (
-            BankTemplateManifest,
-            apply_bank_template_manifest,
-            validate_bank_template,
-        )
-
-        try:
-            manifest = BankTemplateManifest.model_validate(template_dict)
-        except ValidationError as e:
-            errors = [f"{'.'.join(str(loc) for loc in err['loc'])}: {err['msg']}" for err in e.errors()]
-            logger.error(
-                "HINDSIGHT_API_DEFAULT_BANK_TEMPLATE failed schema validation "
-                f"and will be ignored for bank '{bank_id}': {'; '.join(errors)}"
-            )
-            return
-
-        semantic_errors = validate_bank_template(manifest)
-        if semantic_errors:
-            logger.error(
-                "HINDSIGHT_API_DEFAULT_BANK_TEMPLATE failed semantic validation "
-                f"and will be ignored for bank '{bank_id}': {'; '.join(semantic_errors)}"
-            )
-            return
-
-        try:
-            await apply_bank_template_manifest(
-                memory=self,
-                bank_id=bank_id,
-                manifest=manifest,
-                request_context=request_context,
-            )
-            logger.info(f"Applied HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to newly-created bank '{bank_id}'")
-        except Exception as e:
-            logger.error(f"Failed to apply HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to bank '{bank_id}': {e}")
-
     async def update_bank_disposition(
         self,
         bank_id: str,
@@ -5411,9 +5304,6 @@ class MemoryEngine(MemoryEngineInterface):
         exclude_mental_model_ids: list[str] | None = None,
         fact_types: list[str] | None = None,
         exclude_mental_models: bool = False,
-        recall_include_chunks: bool | None = None,
-        recall_max_tokens_override: int | None = None,
-        recall_chunks_max_tokens_override: int | None = None,
         _skip_span: bool = False,
     ) -> ReflectResult:
         """
@@ -5536,23 +5426,6 @@ class MemoryEngine(MemoryEngineInterface):
             "reflect_source_facts_max_tokens", DEFAULT_REFLECT_SOURCE_FACTS_MAX_TOKENS
         )
 
-        # Resolve recall overrides: caller arg (e.g. mental model trigger) → bank config → env default
-        effective_recall_include_chunks = (
-            recall_include_chunks
-            if recall_include_chunks is not None
-            else config_dict.get("recall_include_chunks", DEFAULT_RECALL_INCLUDE_CHUNKS)
-        )
-        effective_recall_max_tokens = (
-            recall_max_tokens_override
-            if recall_max_tokens_override is not None
-            else config_dict.get("recall_max_tokens", DEFAULT_RECALL_MAX_TOKENS)
-        )
-        effective_recall_chunks_max_tokens = (
-            recall_chunks_max_tokens_override
-            if recall_chunks_max_tokens_override is not None
-            else config_dict.get("recall_chunks_max_tokens", DEFAULT_RECALL_CHUNKS_MAX_TOKENS)
-        )
-
         async def search_observations_fn(q: str, max_tokens: int = 5000) -> dict[str, Any]:
             return await tool_search_observations(
                 self,
@@ -5573,14 +5446,7 @@ class MemoryEngine(MemoryEngineInterface):
         recall_fact_types = [ft for ft in (fact_types or ["world", "experience"]) if ft in ("world", "experience")]
         include_recall = bool(recall_fact_types)
 
-        # Defaults are bound at closure-definition time (re-evaluated on each
-        # reflect_async call), so per-bank/per-trigger overrides apply when the
-        # agent invokes recall without explicit token args.
-        async def recall_fn(
-            q: str,
-            max_tokens: int = effective_recall_max_tokens,
-            max_chunk_tokens: int = effective_recall_chunks_max_tokens,
-        ) -> dict[str, Any]:
+        async def recall_fn(q: str, max_tokens: int = 4096, max_chunk_tokens: int = 1000) -> dict[str, Any]:
             return await tool_recall(
                 self,
                 bank_id,
@@ -5592,7 +5458,6 @@ class MemoryEngine(MemoryEngineInterface):
                 tag_groups=tag_groups,
                 max_chunk_tokens=max_chunk_tokens,
                 fact_types=recall_fact_types if fact_types is not None else None,
-                include_chunks=effective_recall_include_chunks,
             )
 
         async def expand_fn(memory_ids: list[str], depth: str) -> dict[str, Any]:
@@ -6810,9 +6675,6 @@ class MemoryEngine(MemoryEngineInterface):
             fact_types = trigger_data.get("fact_types")
             exclude_mental_models = trigger_data.get("exclude_mental_models", False)
             stored_exclude_ids: list[str] = trigger_data.get("exclude_mental_model_ids") or []
-            recall_include_chunks_override = trigger_data.get("include_chunks")
-            recall_max_tokens_override = trigger_data.get("recall_max_tokens")
-            recall_chunks_max_tokens_override = trigger_data.get("recall_chunks_max_tokens")
 
             tag_filtering = _resolve_refresh_tag_filtering(mental_model.get("tags"), trigger_data)
 
@@ -6828,9 +6690,6 @@ class MemoryEngine(MemoryEngineInterface):
                 fact_types=fact_types,
                 exclude_mental_models=exclude_mental_models,
                 exclude_mental_model_ids=list({*stored_exclude_ids, mental_model_id}),
-                recall_include_chunks=recall_include_chunks_override,
-                recall_max_tokens_override=recall_max_tokens_override,
-                recall_chunks_max_tokens_override=recall_chunks_max_tokens_override,
                 _skip_span=True,
             )
 
@@ -7479,7 +7338,6 @@ class MemoryEngine(MemoryEngineInterface):
         operation_id: str,
         *,
         request_context: "RequestContext",
-        include_payload: bool = False,
     ) -> dict[str, Any]:
         """Get the status of a specific async operation.
 
@@ -7502,10 +7360,9 @@ class MemoryEngine(MemoryEngineInterface):
         op_uuid = uuid.UUID(operation_id)
 
         async with acquire_with_retry(pool) as conn:
-            payload_column = ", task_payload" if include_payload else ""
             row = await conn.fetchrow(
                 f"""
-                SELECT operation_id, operation_type, created_at, updated_at, completed_at, status, error_message, result_metadata{payload_column}
+                SELECT operation_id, operation_type, created_at, updated_at, completed_at, status, error_message, result_metadata
                 FROM {fq_table("async_operations")}
                 WHERE operation_id = $1 AND bank_id = $2
                 """,
@@ -7517,7 +7374,6 @@ class MemoryEngine(MemoryEngineInterface):
                 # Check if this is a parent operation
                 result_metadata = json.loads(row["result_metadata"]) if row["result_metadata"] else {}
                 is_parent = result_metadata.get("is_parent", False)
-                task_payload = json.loads(row["task_payload"]) if include_payload and row["task_payload"] else None
 
                 # Use status from database (parent status is updated when all children complete/fail)
                 db_status = row["status"]
@@ -7594,7 +7450,6 @@ class MemoryEngine(MemoryEngineInterface):
                         "error_message": row["error_message"],
                         "result_metadata": result_metadata,
                         "child_operations": child_statuses,
-                        "task_payload": task_payload,
                     }
                 else:
                     # Regular operation (not a parent)
@@ -7607,7 +7462,6 @@ class MemoryEngine(MemoryEngineInterface):
                         "completed_at": row["completed_at"].isoformat() if row["completed_at"] else None,
                         "error_message": row["error_message"],
                         "result_metadata": result_metadata,
-                        "task_payload": task_payload,
                     }
             else:
                 # Operation not found
@@ -7935,9 +7789,7 @@ class MemoryEngine(MemoryEngineInterface):
 
         # Ensure the bank row exists before inserting async_operations (which now has a FK).
         # Banks are created lazily on first retain, but the FK requires the row to exist first.
-        _, created = await bank_utils.get_or_create_bank_profile(pool, bank_id)
-        if created:
-            await self._apply_default_bank_template(bank_id, request_context)
+        await bank_utils.get_bank_profile(pool, bank_id)
 
         # Create typed metadata for parent operation
         parent_metadata = BatchRetainParentMetadata(

hindsight-api-slim/hindsight_api/engine/providers/gemini_llm.py

@@ -23,7 +23,6 @@ from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError
 from hindsight_api.engine.llm_wrapper import parse_llm_json
 from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
 from hindsight_api.metrics import get_metrics_collector
-from hindsight_api.worker.stage import set_stage
 
 logger = logging.getLogger(__name__)
 
@@ -243,8 +242,6 @@ class GeminiLLM(LLMInterface):
         last_exception = None
 
         for attempt in range(max_retries + 1):
-            if attempt > 0:
-                set_stage(f"llm.gemini.{scope}.attempt={attempt + 1}/{max_retries + 1}")
             try:
                 response = await asyncio.wait_for(
                     self._client.aio.models.generate_content(
@@ -530,8 +527,6 @@ class GeminiLLM(LLMInterface):
 
         last_exception = None
         for attempt in range(max_retries + 1):
-            if attempt > 0:
-                set_stage(f"llm.gemini.tools.attempt={attempt + 1}/{max_retries + 1}")
             try:
                 response = await asyncio.wait_for(
                     self._client.aio.models.generate_content(

hindsight-api-slim/hindsight_api/engine/providers/litellm_llm.py

@@ -21,7 +21,6 @@ from typing import Any
 from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError
 from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
 from hindsight_api.metrics import get_metrics_collector
-from hindsight_api.worker.stage import set_stage
 
 logger = logging.getLogger(__name__)
 
@@ -142,8 +141,6 @@ class LiteLLMLLM(LLMInterface):
         last_exception = None
 
         for attempt in range(max_retries + 1):
-            if attempt > 0:
-                set_stage(f"llm.litellm.{scope}.attempt={attempt + 1}/{max_retries + 1}")
             try:
                 response = await self._litellm.acompletion(**call_kwargs)
 
@@ -286,8 +283,6 @@ class LiteLLMLLM(LLMInterface):
 
         last_exception = None
         for attempt in range(max_retries + 1):
-            if attempt > 0:
-                set_stage(f"llm.litellm.tools.attempt={attempt + 1}/{max_retries + 1}")
             try:
                 response = await self._litellm.acompletion(**call_kwargs)
 

hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py

@@ -33,7 +33,6 @@ from hindsight_api.config import DEFAULT_LLM_TIMEOUT, ENV_LLM_TIMEOUT
 from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError
 from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
 from hindsight_api.metrics import get_metrics_collector
-from hindsight_api.worker.stage import set_stage
 
 logger = logging.getLogger(__name__)
 
@@ -197,29 +196,16 @@ class OpenAICompatibleLLM(LLMInterface):
     def _max_tokens_param_name(self) -> str:
         """Return the correct parameter name for limiting response tokens.
 
-        Native OpenAI, Azure OpenAI, Groq, and llamacpp accept 'max_completion_tokens'.
-        Mistral and other OpenAI-compatible endpoints that haven't adopted the newer
-        parameter name require 'max_tokens', so when the openai provider is configured
-        with a non-Azure custom base_url we fall back to the widely-supported
-        'max_tokens'.
-
-        Reasoning models (GPT-5, o1, o3) only accept 'max_completion_tokens' and reject
-        'max_tokens' outright, so they always use the new parameter name regardless of
-        base_url.
+        Native OpenAI and Groq accept 'max_completion_tokens'. Mistral and other
+        OpenAI-compatible endpoints that haven't adopted the newer parameter name
+        require 'max_tokens'. Using a custom base_url with the openai provider
+        signals a third-party compatible API, so fall back to 'max_tokens'.
         """
-        # Reasoning models (GPT-5, o1, o3, ...) only accept max_completion_tokens.
-        # Azure OpenAI + GPT-5 is the canonical example: issue #978.
-        if self._supports_reasoning_model():
-            return "max_completion_tokens"
         # Native OpenAI (no custom base URL), Groq, and llamacpp use max_completion_tokens
         if self.provider in ("groq", "llamacpp"):
             return "max_completion_tokens"
         if self.provider == "openai" and not self.base_url:
             return "max_completion_tokens"
-        # Azure OpenAI is fully OpenAI-API-compatible — detect it by hostname so users
-        # can keep provider=openai + an Azure base_url (the documented setup).
-        if self.provider == "openai" and self.base_url and ".openai.azure.com" in self.base_url:
-            return "max_completion_tokens"
         # openai with custom base_url, ollama, lmstudio, minimax, volcano —
         # use the widely-supported max_tokens
         return "max_tokens"
@@ -363,11 +349,6 @@ class OpenAICompatibleLLM(LLMInterface):
         last_exception = None
 
         for attempt in range(max_retries + 1):
-            # Surface attempt count in worker stage so JSON-schema retry loops
-            # are visible from logs (small models on strict structured output
-            # often loop here). Cheap no-op outside worker context.
-            if attempt > 0:
-                set_stage(f"llm.{self.provider}.{scope}.attempt={attempt + 1}/{max_retries + 1}")
             try:
                 if response_format is not None:
                     response = await self._client.chat.completions.create(**call_params)
@@ -635,8 +616,6 @@ class OpenAICompatibleLLM(LLMInterface):
         last_exception = None
 
         for attempt in range(max_retries + 1):
-            if attempt > 0:
-                set_stage(f"llm.{self.provider}.tools.attempt={attempt + 1}/{max_retries + 1}")
             try:
                 response = await self._client.chat.completions.create(**call_params)
 
@@ -786,8 +765,6 @@ class OpenAICompatibleLLM(LLMInterface):
 
         async with httpx.AsyncClient(timeout=300.0) as client:
             for attempt in range(max_retries + 1):
-                if attempt > 0:
-                    set_stage(f"llm.ollama_native.{scope}.attempt={attempt + 1}/{max_retries + 1}")
                 try:
                     response = await client.post(native_url, json=payload)
                     response.raise_for_status()

hindsight-api-slim/hindsight_api/engine/reflect/tools.py

@@ -9,7 +9,6 @@ Implements hierarchical retrieval:
 
 import logging
 import uuid
-from dataclasses import replace
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any
 
@@ -163,18 +162,13 @@ async def tool_search_observations(
     if include_source_facts and source_facts_max_tokens > 0:
         recall_kwargs["max_source_facts_tokens"] = source_facts_max_tokens
 
-    # Use an internal request context so this recall is not billed as a
-    # user-facing operation. The reflect caller is already billed for the
-    # overall reflect operation; double-billing the sub-recalls would
-    # overcharge the customer.
-    internal_ctx = replace(request_context, internal=True)
     result = await memory_engine.recall_async(
         bank_id=bank_id,
         query=query,
         fact_type=["observation"],
         max_tokens=max_tokens,
         enable_trace=False,
-        request_context=internal_ctx,
+        request_context=request_context,
         tags=tags,
         tags_match=tags_match,
         tag_groups=tag_groups,
@@ -214,7 +208,6 @@ async def tool_recall(
     connection_budget: int = 1,
     max_chunk_tokens: int = 1000,
     fact_types: list[str] | None = None,
-    include_chunks: bool = True,
 ) -> dict[str, Any]:
     """
     Search memories using TEMPR retrieval.
@@ -231,23 +224,22 @@ async def tool_recall(
         tags: Filter by tags (includes untagged memories)
         tags_match: How to match tags - "any" (OR), "all" (AND), or "exact"
         connection_budget: Max DB connections for this recall (default 1 for internal ops)
-        max_chunk_tokens: Maximum tokens for raw source chunk text (default 1000)
+        max_chunk_tokens: Maximum tokens for raw source chunk text (default 1000, always included)
         fact_types: Optional filter for fact types to retrieve. Defaults to ["experience", "world"].
-        include_chunks: Whether to fetch raw chunk text alongside facts (default True).
 
     Returns:
-        Dict with list of matching memories including raw chunk text (when include_chunks)
+        Dict with list of matching memories including raw chunk text
     """
     # Only world/experience are valid for raw recall (observation is handled by search_observations)
     recall_fact_type = [ft for ft in (fact_types or ["experience", "world"]) if ft in ("world", "experience")]
-    internal_ctx = replace(request_context, internal=True)
+    include_chunks = True
     result = await memory_engine.recall_async(
         bank_id=bank_id,
         query=query,
         fact_type=recall_fact_type,
         max_tokens=max_tokens,
         enable_trace=False,
-        request_context=internal_ctx,
+        request_context=request_context,
         tags=tags,
         tags_match=tags_match,
         tag_groups=tag_groups,

hindsight-api-slim/hindsight_api/engine/retain/bank_utils.py

@@ -113,22 +113,6 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
     Returns:
         BankProfile with name, typed DispositionTraits, and mission
     """
-    profile, _ = await get_or_create_bank_profile(pool, bank_id)
-    return profile
-
-
-async def get_or_create_bank_profile(pool, bank_id: str) -> tuple[BankProfile, bool]:
-    """
-    Get bank profile, auto-creating with defaults if it doesn't exist.
-
-    Same as get_bank_profile, but also returns a flag indicating whether the
-    bank was freshly created on this call. Used by the memory engine to apply
-    the HINDSIGHT_API_DEFAULT_BANK_TEMPLATE hook on first bank creation.
-
-    Returns:
-        Tuple of (BankProfile, created) where created is True if the bank
-        did not exist before this call.
-    """
     async with acquire_with_retry(pool) as conn:
         # Try to get existing bank
         row = await conn.fetchrow(
@@ -145,13 +129,10 @@ async def get_or_create_bank_profile(pool, bank_id: str) -> tuple[BankProfile, b
             if isinstance(disposition_data, str):
                 disposition_data = json.loads(disposition_data)
 
-            return (
-                BankProfile(
-                    name=row["name"],
-                    disposition=DispositionTraits(**disposition_data),
-                    mission=row["mission"] or "",
-                ),
-                False,
+            return BankProfile(
+                name=row["name"],
+                disposition=DispositionTraits(**disposition_data),
+                mission=row["mission"] or "",
             )
 
         # Bank doesn't exist, create with defaults.
@@ -172,15 +153,11 @@ async def get_or_create_bank_profile(pool, bank_id: str) -> tuple[BankProfile, b
             internal_id,
         )
 
-        created = inserted is not None
-        if created:
+        if inserted:
             # Fresh insert — create per-bank vector indexes (instant on empty bank)
             await create_bank_vector_indexes(conn, bank_id, str(internal_id))
 
-        return (
-            BankProfile(name=bank_id, disposition=DispositionTraits(**DEFAULT_DISPOSITION), mission=""),
-            created,
-        )
+        return BankProfile(name=bank_id, disposition=DispositionTraits(**DEFAULT_DISPOSITION), mission="")
 
 
 async def update_bank_disposition(pool, bank_id: str, disposition: dict[str, int]) -> None:

hindsight-api-slim/hindsight_api/engine/retain/chunk_storage.py

@@ -100,20 +100,11 @@ async def store_chunks_batch(conn, bank_id: str, document_id: str, chunks: list[
         content_hashes.append(compute_chunk_hash(chunk.chunk_text))
         chunk_id_map[chunk.chunk_index] = chunk_id
 
-    # Batch upsert all chunks. ON CONFLICT makes this idempotent: re-submitting
-    # a retain under the same document_id (the pattern in vectorize-io/hindsight#977)
-    # may produce chunk_ids that already exist when upstream cascade-delete or
-    # delta-retain paths don't run (or race with a concurrent task). Overwriting
-    # is the correct behavior per the document_id grouping semantics — the caller
-    # intends this chunk to hold the latest content at that (document_id, index).
+    # Batch insert all chunks
     await conn.execute(
         f"""
         INSERT INTO {fq_table("chunks")} (chunk_id, document_id, bank_id, chunk_text, chunk_index, content_hash)
         SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::integer[], $6::text[])
-        ON CONFLICT (chunk_id) DO UPDATE SET
-            chunk_text = EXCLUDED.chunk_text,
-            chunk_index = EXCLUDED.chunk_index,
-            content_hash = EXCLUDED.content_hash
         """,
         chunk_ids,
         [document_id] * len(chunk_texts),

hindsight-api-slim/hindsight_api/engine/retain/embedding_utils.py

@@ -47,16 +47,6 @@ async def generate_embeddings_batch(embeddings_backend, texts: list[str]) -> lis
             embeddings_backend.encode,
             texts,
         )
+        return embeddings
     except Exception as e:
         raise Exception(f"Failed to generate batch embeddings: {str(e)}")
-
-    # Guarantee 1:1 alignment with input texts. A silent length mismatch here
-    # propagates downstream as zip() drops items, eventually surfacing as an
-    # IndexError in retain mapping (see issue #1037).
-    if len(embeddings) != len(texts):
-        raise RuntimeError(
-            f"Embeddings backend returned {len(embeddings)} vectors for {len(texts)} input texts; "
-            "expected exact 1:1 alignment"
-        )
-
-    return embeddings

hindsight-api-slim/hindsight_api/engine/retain/link_utils.py

@@ -739,10 +739,17 @@ async def compute_semantic_links_ann(
         return []
 
     import time as time_mod
+    import uuid as uuid_mod
 
     ann_start = time_mod.time()
     links = []
 
+    # Lower ef_search for retain ANN — default 400 is tuned for recall precision
+    # but at 164k units each HNSW probe takes 94ms. ef_search=60 gives 2.7ms/probe
+    # (35x faster) with sufficient accuracy for top-50 semantic link creation.
+    # Reset after to avoid polluting the connection pool for recall queries.
+    await conn.execute("SET hnsw.ef_search = 60")
+
     logger.debug(f"[ANN] Starting: {len(unit_ids)} seeds, top_k={top_k}")
 
     # Build per-unit fact_types (default to 'world' if not provided)
@@ -753,71 +760,54 @@ async def compute_semantic_links_ann(
     # sequential-scan every HNSW probe result against the array, destroying
     # performance (67s for 8k seeds). Self-links are harmless (ON CONFLICT DO
     # NOTHING handles duplicates in memory_links).
-    #
-    # The entire CREATE TEMP TABLE → COPY → SELECT sequence MUST run inside a
-    # single transaction. Callers may connect through pgBouncer in `transaction`
-    # pool mode, in which case the backend is only pinned to the client for the
-    # duration of a transaction. Outside a transaction, pgBouncer can rebind
-    # the client to a different backend between statements, and the temp table
-    # (which is session-scoped to its creating backend) becomes invisible.
-    # The observed failure mode was an intermittent
-    # `relation "_ann_seeds" does not exist` on the second statement.
-    #
-    # Using ON COMMIT DROP + SET LOCAL also means we don't have to remember to
-    # manually drop the temp table or reset hnsw.ef_search — the transaction
-    # end handles both.
-    rows: list = []
-    async with conn.transaction():
-        # Transaction-local ef_search. Default 400 is tuned for recall precision
-        # but at 164k units each HNSW probe takes 94ms. ef_search=60 gives 2.7ms
-        # per probe (35x faster) with sufficient accuracy for top-50 semantic
-        # link creation. SET LOCAL auto-reverts at commit, so we don't pollute
-        # the pool for subsequent recall queries.
-        await conn.execute("SET LOCAL hnsw.ef_search = 60")
+    t_setup = time_mod.time()
+    await conn.execute("CREATE TEMP TABLE IF NOT EXISTS _ann_seeds (unit_id text, emb_text text, fact_type text)")
+    await conn.execute("TRUNCATE _ann_seeds")
 
-        t_setup = time_mod.time()
-        await conn.execute("CREATE TEMP TABLE _ann_seeds (unit_id text, emb_text text, fact_type text) ON COMMIT DROP")
+    records = [
+        (uid, emb if isinstance(emb, str) else str(emb), ft) for uid, emb, ft in zip(unit_ids, embeddings, fact_types)
+    ]
+    await conn.copy_records_to_table("_ann_seeds", records=records, columns=["unit_id", "emb_text", "fact_type"])
+    logger.debug(f"[ANN] Temp table setup: {time_mod.time() - t_setup:.3f}s ({len(records)} seeds)")
 
-        records = [
-            (uid, emb if isinstance(emb, str) else str(emb), ft)
-            for uid, emb, ft in zip(unit_ids, embeddings, fact_types)
-        ]
-        await conn.copy_records_to_table("_ann_seeds", records=records, columns=["unit_id", "emb_text", "fact_type"])
-        logger.debug(f"[ANN] Temp table setup: {time_mod.time() - t_setup:.3f}s ({len(records)} seeds)")
+    # Run one ANN query per fact_type so each uses the right HNSW index.
+    rows = []
+    active_types = set(fact_types)
+    for fact_type in active_types:
+        t_query = time_mod.time()
+        seed_count = sum(1 for ft in fact_types if ft == fact_type)
+        logger.debug(f"[ANN] Querying fact_type={fact_type}: {seed_count} seeds")
+        ft_rows = await conn.fetch(
+            f"""
+            SELECT s.unit_id       AS from_id,
+                   n.id::text      AS to_id,
+                   n.similarity
+            FROM _ann_seeds s
+            CROSS JOIN LATERAL (
+                SELECT mu.id,
+                       1 - (mu.embedding <=> s.emb_text::vector) AS similarity
+                FROM {fq_table("memory_units")} mu
+                WHERE mu.bank_id = $1
+                  AND mu.fact_type = $2
+                  AND mu.embedding IS NOT NULL
+                ORDER BY mu.embedding <=> s.emb_text::vector
+                LIMIT $3
+            ) n
+            WHERE s.fact_type = $2
+            """,
+            bank_id,
+            fact_type,
+            top_k,
+            timeout=300,  # ANN on large banks can take minutes
+        )
+        logger.debug(f"[ANN] fact_type={fact_type}: {len(ft_rows)} rows in {time_mod.time() - t_query:.3f}s")
+        rows.extend(ft_rows)
 
-        # Run one ANN query per fact_type so each uses the right HNSW index.
-        active_types = set(fact_types)
-        for fact_type in active_types:
-            t_query = time_mod.time()
-            seed_count = sum(1 for ft in fact_types if ft == fact_type)
-            logger.debug(f"[ANN] Querying fact_type={fact_type}: {seed_count} seeds")
-            ft_rows = await conn.fetch(
-                f"""
-                SELECT s.unit_id       AS from_id,
-                       n.id::text      AS to_id,
-                       n.similarity
-                FROM _ann_seeds s
-                CROSS JOIN LATERAL (
-                    SELECT mu.id,
-                           1 - (mu.embedding <=> s.emb_text::vector) AS similarity
-                    FROM {fq_table("memory_units")} mu
-                    WHERE mu.bank_id = $1
-                      AND mu.fact_type = $2
-                      AND mu.embedding IS NOT NULL
-                    ORDER BY mu.embedding <=> s.emb_text::vector
-                    LIMIT $3
-                ) n
-                WHERE s.fact_type = $2
-                """,
-                bank_id,
-                fact_type,
-                top_k,
-                timeout=300,  # ANN on large banks can take minutes
-            )
-            logger.debug(f"[ANN] fact_type={fact_type}: {len(ft_rows)} rows in {time_mod.time() - t_query:.3f}s")
-            rows.extend(ft_rows)
-    # Transaction commits here. _ann_seeds is dropped (ON COMMIT DROP).
-    # hnsw.ef_search reverts (SET LOCAL).
+    # Clean up temp table (no ON COMMIT DROP since we're not in a transaction)
+    await conn.execute("DROP TABLE IF EXISTS _ann_seeds")
+
+    # Reset ef_search to default so the pooled connection doesn't affect recall queries
+    await conn.execute("RESET hnsw.ef_search")
 
     for row in rows:
         sim = float(min(1.0, max(0.0, row["similarity"])))

hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py

@@ -14,7 +14,6 @@ from collections.abc import Awaitable, Callable
 from datetime import UTC, datetime
 from typing import Any
 
-from ...worker.stage import set_stage
 from ..db_utils import acquire_with_retry
 from ..memory_engine import fq_table
 from . import bank_utils
@@ -72,6 +71,7 @@ from . import (
 from .types import (
     ChunkMetadata,
     EntityResolutionResult,
+    ExtractedFact,
     Phase1Result,
     Phase3Context,
     ProcessedFact,
@@ -133,7 +133,6 @@ async def _pre_resolve_phase1(
     Running these outside the transaction avoids holding row locks during
     slow reads, eliminating TimeoutErrors under concurrent load.
     """
-    set_stage("retain.phase1.resolve")
     from .link_utils import compute_semantic_links_ann
 
     user_entities_per_content = {idx: content.entities for idx, content in enumerate(contents) if content.entities}
@@ -239,7 +238,6 @@ async def _insert_facts_and_links(
     only the unit_entities INSERT (FK to memory_units) stays in the transaction.
     Entity link building is deferred to Phase 3 (post-transaction, best-effort).
     """
-    set_stage("retain.phase2.insert_facts")
     unit_ids = await fact_storage.insert_facts_batch(conn, bank_id, processed_facts)
     step_start = time.time()
     log_buffer.append(f"  Insert facts: {len(unit_ids)} units in {time.time() - step_start:.3f}s")
@@ -301,11 +299,8 @@ async def _insert_facts_and_links(
         causal_link_count = await link_creation.create_causal_links_batch(conn, bank_id, unit_ids, processed_facts)
         log_buffer.append(f"  Causal links: {causal_link_count} links in {time.time() - step_start:.3f}s")
 
-    # Map results back to original content items. Use processed_facts (not
-    # extracted_facts) because unit_ids has 1:1 alignment with processed_facts —
-    # any upstream drop between extraction and processing would otherwise cause
-    # an IndexError (see issue #1037).
-    result_unit_ids = _map_results_to_contents(contents, processed_facts, unit_ids if unit_ids else [])
+    # Map results back to original content items
+    result_unit_ids = _map_results_to_contents(contents, extracted_facts, unit_ids if unit_ids else [])
 
     if outbox_callback:
         await outbox_callback(conn)
@@ -327,7 +322,6 @@ async def _build_and_insert_entity_links_phase3(
     Entity links are for UI graph visualization only — retrieval uses
     the unit_entities self-join instead.
     """
-    set_stage("retain.phase3.entity_links")
     p3_unit_ids = phase3_ctx.unit_ids
     p3_resolved = phase3_ctx.resolved_entity_ids
     p3_entity_to_unit = phase3_ctx.entity_to_unit
@@ -373,7 +367,6 @@ async def _extract_and_embed(
     Returns:
         Tuple of (extracted_facts, processed_facts, chunks_metadata, usage)
     """
-    set_stage("retain.extract_and_embed")
     step_start = time.time()
     extracted_facts, chunks, usage = await fact_extraction.extract_facts_from_contents(
         contents, llm_config, agent_name, config, pool, operation_id, schema
@@ -489,8 +482,8 @@ async def retain_batch(
             return result_unit_ids, total_usage
 
     # Resolve effective document_id early so both delta and streaming paths
-    # can find existing chunks from a prior attempt. On retry, a generated
-    # document_id is recovered from operation result_metadata.document_ids[0].
+    # can find existing chunks from a prior attempt. On retry, the generated
+    # document_id is recovered from operation result_metadata.
     effective_doc_id = document_id
     if not effective_doc_id:
         doc_ids = {item.get("document_id") for item in contents_dicts if item.get("document_id")}
@@ -509,41 +502,26 @@ async def retain_batch(
                         if isinstance(row["result_metadata"], dict)
                         else json.loads(row["result_metadata"])
                     )
-                    recovered = meta.get("document_ids") or []
-                    if recovered:
-                        effective_doc_id = recovered[0]
+                    effective_doc_id = meta.get("generated_document_id")
         except Exception:
             pass
     if not effective_doc_id:
         effective_doc_id = str(uuid.uuid4())
-
-    # Record effective_doc_id on the operation (idempotent set-append). Captures
-    # both user-provided and generated ids so the operation shows every document
-    # it touched, and lets retries reuse the same generated id.
-    if operation_id:
-        try:
-            async with acquire_with_retry(pool) as conn:
-                await conn.execute(
-                    f"""
-                    UPDATE {fq_table("async_operations")}
-                    SET result_metadata = jsonb_set(
-                        COALESCE(result_metadata, '{{}}'::jsonb),
-                        '{{document_ids}}',
-                        CASE
-                            WHEN COALESCE(result_metadata->'document_ids', '[]'::jsonb) @> $1::jsonb
-                                THEN result_metadata->'document_ids'
-                            ELSE COALESCE(result_metadata->'document_ids', '[]'::jsonb) || $1::jsonb
-                        END,
-                        true
-                    ),
-                    updated_at = now()
-                    WHERE operation_id = $2
-                    """,
-                    json.dumps([effective_doc_id]),
-                    uuid.UUID(operation_id),
-                )
-        except Exception:
-            logger.warning("Failed to persist document_id", exc_info=True)
+        # Persist so retries reuse the same document_id
+        if operation_id:
+            try:
+                async with acquire_with_retry(pool) as conn:
+                    await conn.execute(
+                        f"""
+                        UPDATE {fq_table("async_operations")}
+                        SET result_metadata = result_metadata || $1::jsonb, updated_at = now()
+                        WHERE operation_id = $2
+                        """,
+                        json.dumps({"generated_document_id": effective_doc_id}),
+                        uuid.UUID(operation_id),
+                    )
+            except Exception:
+                logger.warning("Failed to persist generated document_id", exc_info=True)
 
     # --- Append mode: prepend existing document content to new content ---
     # When update_mode="append", fetch the existing document text and prepend it
@@ -1567,19 +1545,12 @@ def _build_delta_contents(
 
 def _map_results_to_contents(
     contents: list[RetainContent],
-    processed_facts: list[ProcessedFact],
+    extracted_facts: list[ExtractedFact],
     unit_ids: list[str],
 ) -> list[list[str]]:
-    """Map created unit IDs back to original content items.
-
-    `processed_facts` and `unit_ids` must have the same length: each unit_id
-    corresponds to the processed_fact at the same index.
-    """
-    if len(processed_facts) != len(unit_ids):
-        raise ValueError(f"processed_facts ({len(processed_facts)}) and unit_ids ({len(unit_ids)}) length mismatch")
-
+    """Map created unit IDs back to original content items."""
     facts_by_content: dict[int, list[int]] = {i: [] for i in range(len(contents))}
-    for i, fact in enumerate(processed_facts):
+    for i, fact in enumerate(extracted_facts):
         # Normalize content_index: some LLM providers return 1-indexed values.
         # Clamp to valid range to prevent KeyError.
         idx = fact.content_index
@@ -1588,8 +1559,12 @@ def _map_results_to_contents(
         facts_by_content[idx].append(i)
 
     result_unit_ids = []
+    unit_idx = 0
     for content_index in range(len(contents)):
-        content_unit_ids = [unit_ids[i] for i in facts_by_content[content_index]]
+        content_unit_ids = []
+        for _ in facts_by_content[content_index]:
+            content_unit_ids.append(unit_ids[unit_idx])
+            unit_idx += 1
         result_unit_ids.append(content_unit_ids)
 
     return result_unit_ids

hindsight-api-slim/hindsight_api/engine/search/reranking.py

@@ -23,7 +23,6 @@ def apply_combined_scoring(
     recency_alpha: float = _RECENCY_ALPHA,
     temporal_alpha: float = _TEMPORAL_ALPHA,
     proof_count_alpha: float = _PROOF_COUNT_ALPHA,
-    is_passthrough_reranker: bool = False,
 ) -> None:
     """Apply combined scoring to a list of ScoredResults in-place.
 
@@ -61,42 +60,6 @@ def apply_combined_scoring(
     if now.tzinfo is None:
         now = now.replace(tzinfo=UTC)
 
-    # When the configured cross-encoder is a passthrough (e.g.
-    # RRFPassthroughCrossEncoder used by slim deployments), every
-    # cross_encoder_score_normalized is identical and provides no relevance
-    # signal. In that case the multiplicative recency / temporal / proof_count
-    # boosts below become the *only* ranking signal — making the final order a
-    # pure recency sort regardless of how relevant a candidate actually is.
-    #
-    # Detect that case and seed cross_encoder_score_normalized from the RRF
-    # rank instead, so the boosts modulate a meaningful base score rather than
-    # replacing it. This is a no-op for real cross-encoders, which produce
-    # diverse scores.
-    # When the reranker is a passthrough (e.g. RRFPassthroughCrossEncoder used
-    # by slim deployments), every cross_encoder_score_normalized is identical
-    # and provides no relevance signal. The multiplicative recency / temporal /
-    # proof_count boosts below would then become the *only* ranking signal,
-    # making the final order a pure recency sort regardless of how relevant a
-    # candidate actually is.
-    #
-    # Seed cross_encoder_score_normalized from the RRF rank instead, so the
-    # boosts modulate a meaningful base score. Caller passes is_passthrough
-    # explicitly because "all scores identical" is too fragile a heuristic —
-    # a real reranker can also tie scores (especially in tests with synthetic
-    # data) and we'd corrupt legitimate single-result reranks.
-    if is_passthrough_reranker and scored_results:
-        n = len(scored_results)
-        sorted_by_rrf = sorted(
-            scored_results,
-            key=lambda s: getattr(getattr(s, "candidate", None), "rrf_score", 0.0),
-            reverse=True,
-        )
-        denom = max(1, n - 1)
-        for new_rank, sr in enumerate(sorted_by_rrf):
-            # Map rank → [0.1, 1.0] so the recency boost can still nudge
-            # ordering between adjacent candidates without overpowering RRF.
-            sr.cross_encoder_score_normalized = 1.0 - (0.9 * new_rank / denom)
-
     for sr in scored_results:
         # Recency: linear decay over 365 days → [0.1, 1.0]; neutral 0.5 if no date.
         sr.recency = 0.5

hindsight-api-slim/hindsight_api/worker/poller.py

@@ -6,17 +6,15 @@ FOR UPDATE SKIP LOCKED for safe concurrent claiming.
 """
 
 import asyncio
-import io
 import json
 import logging
 import time
 import traceback
 from collections.abc import Awaitable, Callable
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any
 
 from .exceptions import RetryTaskAt
-from .stage import StageHolder, bind_holder
 
 if TYPE_CHECKING:
     import asyncpg
@@ -28,31 +26,6 @@ logger = logging.getLogger(__name__)
 # Progress logging interval in seconds
 PROGRESS_LOG_INTERVAL = 30
 
-# Stuck-task stack-dump thresholds (seconds). Each task gets one stack dump
-# per threshold it crosses (5min, 10min, 20min, 40min, 80min...).
-STUCK_STACK_INITIAL_THRESHOLD_S = 300
-STUCK_STACK_MAX_THRESHOLD_S = 3600 * 6  # cap doubling at 6h
-
-
-@dataclass
-class ActiveTaskInfo:
-    """Tracking info for an in-flight worker task.
-
-    Carries everything the periodic stats / stuck-task logger needs
-    so it can render a useful per-task line without touching the DB.
-    """
-
-    op_type: str
-    bank_id: str
-    schema: str | None
-    bg_task: "asyncio.Task[Any]"
-    started_at: float
-    stage_holder: StageHolder
-    # Largest stuck-stack threshold (seconds) for which we've already
-    # dumped a stack trace; used to suppress repeated dumps.
-    last_stack_dump_threshold: int = 0
-    task_type: str = ""
-
 
 def fq_table(table: str, schema: str | None = None) -> str:
     """Get fully-qualified table name with optional schema prefix."""
@@ -126,8 +99,8 @@ class WorkerPoller:
         self._in_flight_lock = asyncio.Lock()
         self._last_progress_log = 0.0
         self._tasks_completed_since_log = 0
-        # Track active tasks locally: operation_id -> ActiveTaskInfo
-        self._active_tasks: dict[str, ActiveTaskInfo] = {}
+        # Track active tasks locally: operation_id -> (op_type, bank_id, schema, asyncio.Task)
+        self._active_tasks: dict[str, tuple[str, str, str | None, asyncio.Task]] = {}
         # Track in-flight tasks by operation type
         self._in_flight_by_type: dict[str, int] = {}
 
@@ -143,25 +116,17 @@ class WorkerPoller:
         """
         Calculate available slots for claiming tasks.
 
-        Consolidation has a reserved pool of ``consolidation_max_slots`` within
-        ``max_slots``. Non-consolidation tasks may use at most
-        ``max_slots - consolidation_max_slots`` slots, leaving the remainder
-        always available for consolidation. This prevents consolidation from
-        being starved when retain throughput continuously saturates the queue.
-
         Returns:
-            (non_consolidation_available, consolidation_available) tuple
+            (total_available, consolidation_available) tuple
         """
         async with self._in_flight_lock:
             total_in_flight = self._in_flight_count
             consolidation_in_flight = self._in_flight_by_type.get("consolidation", 0)
 
-        non_consolidation_in_flight = max(0, total_in_flight - consolidation_in_flight)
-        non_consolidation_max = max(0, self._max_slots - self._consolidation_max_slots)
-        non_consolidation_available = max(0, non_consolidation_max - non_consolidation_in_flight)
+        total_available = max(0, self._max_slots - total_in_flight)
         consolidation_available = max(0, self._consolidation_max_slots - consolidation_in_flight)
 
-        return non_consolidation_available, consolidation_available
+        return total_available, consolidation_available
 
     async def wait_for_active_tasks(self, timeout: float = 10.0) -> bool:
         """
@@ -199,40 +164,40 @@ class WorkerPoller:
         Returns:
             List of ClaimedTask objects containing operation_id, task_dict, and schema
         """
-        # Calculate available slots (independent pools after reservation)
-        non_consolidation_available, consolidation_available = await self._get_available_slots()
+        # Calculate available slots
+        total_available, consolidation_available = await self._get_available_slots()
 
-        if non_consolidation_available <= 0 and consolidation_available <= 0:
+        if total_available <= 0:
             return []
 
         schemas = await self._get_schemas()
         all_tasks: list[ClaimedTask] = []
-        remaining_non_consolidation = non_consolidation_available
+        remaining_total = total_available
         remaining_consolidation = consolidation_available
 
         for schema in schemas:
-            if remaining_non_consolidation <= 0 and remaining_consolidation <= 0:
+            if remaining_total <= 0:
                 break
 
-            tasks = await self._claim_batch_for_schema(schema, remaining_non_consolidation, remaining_consolidation)
+            tasks = await self._claim_batch_for_schema(schema, remaining_total, remaining_consolidation)
 
+            # Update remaining slots based on what was claimed
             for task in tasks:
                 op_type = task.task_dict.get("operation_type", "unknown")
                 if op_type == "consolidation":
                     remaining_consolidation -= 1
-                else:
-                    remaining_non_consolidation -= 1
 
             all_tasks.extend(tasks)
+            remaining_total -= len(tasks)
 
         return all_tasks
 
     async def _claim_batch_for_schema(
-        self, schema: str | None, non_consolidation_limit: int, consolidation_limit: int
+        self, schema: str | None, limit: int, consolidation_limit: int
     ) -> list[ClaimedTask]:
         """Claim tasks from a specific schema respecting slot limits."""
         try:
-            return await self._claim_batch_for_schema_inner(schema, non_consolidation_limit, consolidation_limit)
+            return await self._claim_batch_for_schema_inner(schema, limit, consolidation_limit)
         except Exception as e:
             # Format schema for logging: custom schemas in quotes, None as-is
             schema_display = f'"{schema}"' if schema else str(schema)
@@ -240,38 +205,37 @@ class WorkerPoller:
             return []
 
     async def _claim_batch_for_schema_inner(
-        self, schema: str | None, non_consolidation_limit: int, consolidation_limit: int
+        self, schema: str | None, limit: int, consolidation_limit: int
     ) -> list[ClaimedTask]:
-        """Inner implementation for claiming tasks from a specific schema with slot limits.
-
-        Non-consolidation and consolidation pools are independent: each is bounded by
-        its own limit and they do not borrow from each other.
-        """
+        """Inner implementation for claiming tasks from a specific schema with slot limits."""
         table = fq_table("async_operations", schema)
 
         async with self._pool.acquire() as conn:
             async with conn.transaction():
-                # 1. Claim non-consolidation tasks
-                non_consolidation_rows = []
-                if non_consolidation_limit > 0:
-                    non_consolidation_rows = await conn.fetch(
-                        f"""
-                        SELECT operation_id, task_payload, retry_count
-                        FROM {table}
-                        WHERE status = 'pending'
-                          AND task_payload IS NOT NULL
-                          AND operation_type != 'consolidation'
-                          AND (next_retry_at IS NULL OR next_retry_at <= NOW())
-                        ORDER BY created_at
-                        LIMIT $1
-                        FOR UPDATE SKIP LOCKED
-                        """,
-                        non_consolidation_limit,
-                    )
+                # Strategy: Claim non-consolidation tasks first, then consolidation up to limit
 
-                # 2. Claim consolidation tasks from their reserved pool
+                # 1. Claim non-consolidation tasks (up to limit)
+                non_consolidation_rows = await conn.fetch(
+                    f"""
+                    SELECT operation_id, task_payload, retry_count
+                    FROM {table}
+                    WHERE status = 'pending'
+                      AND task_payload IS NOT NULL
+                      AND operation_type != 'consolidation'
+                      AND (next_retry_at IS NULL OR next_retry_at <= NOW())
+                    ORDER BY created_at
+                    LIMIT $1
+                    FOR UPDATE SKIP LOCKED
+                    """,
+                    limit,
+                )
+
+                claimed_count = len(non_consolidation_rows)
+                remaining_limit = limit - claimed_count
+
+                # 2. Claim consolidation tasks (up to consolidation_limit and remaining_limit)
                 consolidation_rows = []
-                if consolidation_limit > 0:
+                if consolidation_limit > 0 and remaining_limit > 0:
                     consolidation_rows = await conn.fetch(
                         f"""
                         SELECT operation_id, task_payload, retry_count
@@ -290,17 +254,16 @@ class WorkerPoller:
                         LIMIT $1
                         FOR UPDATE SKIP LOCKED
                         """,
-                        consolidation_limit,
+                        min(consolidation_limit, remaining_limit),
                     )
 
-                tagged_rows = [(row, False) for row in non_consolidation_rows] + [
-                    (row, True) for row in consolidation_rows
-                ]
+                all_rows = non_consolidation_rows + consolidation_rows
 
-                if not tagged_rows:
+                if not all_rows:
                     return []
 
-                operation_ids = [row["operation_id"] for row, _ in tagged_rows]
+                # Claim the tasks by updating status and worker_id
+                operation_ids = [row["operation_id"] for row in all_rows]
                 await conn.execute(
                     f"""
                     UPDATE {table}
@@ -311,16 +274,12 @@ class WorkerPoller:
                     operation_ids,
                 )
 
+                # Parse and return task payloads with schema context
                 result = []
-                for row, is_consolidation in tagged_rows:
+                for row in all_rows:
                     task_dict = json.loads(row["task_payload"])
                     task_dict["_retry_count"] = row["retry_count"]
                     task_dict["_operation_id"] = str(row["operation_id"])
-                    # The DB row knows the operation_type, but the JSON payload may not
-                    # carry it. Inject it so in-flight tracking and slot accounting
-                    # (which key off task_dict["operation_type"]) work correctly.
-                    if is_consolidation:
-                        task_dict["operation_type"] = "consolidation"
                     result.append(
                         ClaimedTask(
                             operation_id=str(row["operation_id"]),
@@ -467,27 +426,12 @@ class WorkerPoller:
         operation_type = task.task_dict.get("operation_type", "unknown")
         bank_id = task.task_dict.get("bank_id", "unknown")
 
-        # Stage holder is updated by engine code via stage.set_stage(); the
-        # poller reads it during periodic logging to surface what each
-        # in-flight task is doing.
-        holder = StageHolder(stage=f"queued.{task_type}")
-
-        # Create background task. The holder is passed in and bound to the
-        # task's own contextvar scope inside _execute_task_inner so engine
-        # code running under that task sees it via stage.set_stage().
-        bg_task = asyncio.create_task(self._execute_task_inner(task, holder))
+        # Create background task
+        bg_task = asyncio.create_task(self._execute_task_inner(task))
 
         # Track this task as active
         async with self._in_flight_lock:
-            self._active_tasks[task.operation_id] = ActiveTaskInfo(
-                op_type=operation_type,
-                bank_id=bank_id,
-                schema=task.schema,
-                bg_task=bg_task,
-                started_at=time.monotonic(),
-                stage_holder=holder,
-                task_type=task_type,
-            )
+            self._active_tasks[task.operation_id] = (task_type, bank_id, task.schema, bg_task)
             self._in_flight_count += 1
             self._in_flight_by_type[operation_type] = self._in_flight_by_type.get(operation_type, 0) + 1
 
@@ -506,7 +450,7 @@ class WorkerPoller:
                     if self._in_flight_by_type[operation_type] == 0:
                         del self._in_flight_by_type[operation_type]
 
-    async def _execute_task_inner(self, task: ClaimedTask, holder: StageHolder | None = None):
+    async def _execute_task_inner(self, task: ClaimedTask):
         """Inner task execution with retry/fail handling.
 
         Tasks that want to be retried raise RetryTaskAt; the poller sets next_retry_at
@@ -517,14 +461,6 @@ class WorkerPoller:
         task_type = task.task_dict.get("type", "unknown")
         bank_id = task.task_dict.get("bank_id", "unknown")
 
-        # Bind the stage holder in this task's own contextvar scope so engine
-        # code running under us can update it via stage.set_stage(). If holder
-        # is None (legacy / direct invocation), set_stage becomes a no-op.
-        if holder is not None:
-            bind_holder(holder)
-            holder.stage = f"executor.{task_type}"
-            holder.updated_at = time.monotonic()
-
         try:
             schema_info = f", schema={task.schema}" if task.schema else ""
             logger.debug(f"Executing task {task.operation_id} (type={task_type}, bank={bank_id}{schema_info})")
@@ -747,7 +683,7 @@ class WorkerPoller:
         while asyncio.get_event_loop().time() - start_time < timeout:
             async with self._in_flight_lock:
                 in_flight = self._in_flight_count
-                active_task_objects = [info.bg_task for info in self._active_tasks.values()]
+                active_task_objects = [task_info[3] for task_info in self._active_tasks.values()]
 
             if in_flight == 0:
                 logger.info(f"Worker {self._worker_id} graceful shutdown complete")
@@ -765,19 +701,12 @@ class WorkerPoller:
 
         # Cancel remaining tasks
         async with self._in_flight_lock:
-            for operation_id, info in list(self._active_tasks.items()):
-                if not info.bg_task.done():
-                    info.bg_task.cancel()
+            for operation_id, (_, _, _, bg_task) in list(self._active_tasks.items()):
+                if not bg_task.done():
+                    bg_task.cancel()
 
     async def _log_progress_if_due(self):
-        """Log progress stats every PROGRESS_LOG_INTERVAL seconds.
-
-        Emits four kinds of lines:
-          * [WORKER_STATS]  - aggregate slots / pool / global pending counts
-          * [WORKER_TASK]   - one line per in-flight task with age + stage
-          * [STUCK_STACK]   - async stack trace for tasks past stuck thresholds
-          * [DB_WAITS]      - any non-idle hindsight session waiting on a lock
-        """
+        """Log progress stats every PROGRESS_LOG_INTERVAL seconds."""
         now = time.time()
         if now - self._last_progress_log < PROGRESS_LOG_INTERVAL:
             return
@@ -792,15 +721,13 @@ class WorkerPoller:
                 active_tasks = dict(self._active_tasks)
 
             consolidation_count = in_flight_by_type.get("consolidation", 0)
-            non_consolidation_in_flight = max(0, in_flight - consolidation_count)
-            non_consolidation_max = max(0, self._max_slots - self._consolidation_max_slots)
-            available_slots = max(0, non_consolidation_max - non_consolidation_in_flight)
-            available_consolidation_slots = max(0, self._consolidation_max_slots - consolidation_count)
+            available_slots = self._max_slots - in_flight
+            available_consolidation_slots = self._consolidation_max_slots - consolidation_count
 
-            # Build local processing breakdown (aggregate counts)
+            # Build local processing breakdown
             task_groups: dict[tuple[str, str], int] = {}
-            for info in active_tasks.values():
-                key = (info.op_type, info.bank_id)
+            for op_type, bank_id, _, _ in active_tasks.values():
+                key = (op_type, bank_id)
                 task_groups[key] = task_groups.get(key, 0) + 1
 
             processing_info = [f"{op}:{bank}({cnt})" for (op, bank), cnt in task_groups.items()]
@@ -812,42 +739,13 @@ class WorkerPoller:
             schemas = await self._get_schemas()
             global_pending = 0
             all_worker_counts: dict[str, int] = {}
-            # operation_type -> aggregated bucket counts across schemas
-            pending_breakdown: dict[str, dict[str, int]] = {}
 
             async with self._pool.acquire() as conn:
                 for schema in schemas:
                     table = fq_table("async_operations", schema)
 
-                    # Bucket pending rows by the same predicates the claim query
-                    # filters on, so an operator can see why pending > 0 but
-                    # nothing is being claimed (orphaned batch_retain parents,
-                    # retry backoff, etc.).
-                    breakdown_rows = await conn.fetch(
-                        f"""
-                        SELECT
-                            operation_type,
-                            COUNT(*) AS total,
-                            COUNT(*) FILTER (WHERE task_payload IS NULL) AS payload_null,
-                            COUNT(*) FILTER (
-                                WHERE next_retry_at IS NOT NULL AND next_retry_at > now()
-                            ) AS retry_blocked,
-                            COUNT(*) FILTER (WHERE worker_id IS NOT NULL) AS assigned
-                        FROM {table}
-                        WHERE status = 'pending'
-                        GROUP BY operation_type
-                        """
-                    )
-                    for br in breakdown_rows:
-                        op_type = br["operation_type"] or "unknown"
-                        bucket = pending_breakdown.setdefault(
-                            op_type, {"total": 0, "payload_null": 0, "retry_blocked": 0, "assigned": 0}
-                        )
-                        bucket["total"] += br["total"]
-                        bucket["payload_null"] += br["payload_null"]
-                        bucket["retry_blocked"] += br["retry_blocked"]
-                        bucket["assigned"] += br["assigned"]
-                        global_pending += br["total"]
+                    row = await conn.fetchrow(f"SELECT COUNT(*) as count FROM {table} WHERE status = 'pending'")
+                    global_pending += row["count"] if row else 0
 
                     worker_rows = await conn.fetch(
                         f"""
@@ -867,11 +765,6 @@ class WorkerPoller:
                     other_workers.append(f"{wid}:{cnt}")
             others_str = ", ".join(other_workers) if other_workers else "none"
 
-            # asyncpg pool stats - exhaustion presents as "everything slow",
-            # making it invisible without this line.
-            pool_str = self._format_pool_stats()
-            proc_str = self._format_proc_stats()
-
             # Display None as "default" in logs
             schemas_str = ", ".join(s if s else "default" for s in schemas)
             logger.info(
@@ -880,209 +773,12 @@ class WorkerPoller:
                 f"available={available_slots} (consolidation={available_consolidation_slots}) | "
                 f"global: pending={global_pending} (schemas: {schemas_str}) | "
                 f"others: {others_str} | "
-                f"pool: {pool_str} | "
-                f"proc: {proc_str} | "
                 f"my_active: {processing_str}"
             )
 
-            # Pending breakdown - explains why pending rows aren't being claimed
-            # (orphaned batch_retain parents have payload_null > 0, retry storms
-            # show up as retry_blocked, etc.). Skip when nothing is pending so
-            # the line doesn't add noise on idle deployments.
-            if global_pending > 0:
-                self._log_pending_breakdown(pending_breakdown)
-
-            # Per-task lines, sorted oldest-first so stuck tasks bubble to the top.
-            self._log_per_task_lines(active_tasks, now=time.monotonic())
-
-            # DB lock waits - separate from per-task lines because a single
-            # blocking session can wedge many tasks.
-            await self._log_db_waits()
-
         except Exception as e:
             logger.debug(f"Failed to log progress stats: {e}")
 
-    def _format_proc_stats(self) -> str:
-        """Render lightweight process memory stats. Returns 'unavailable' if introspection fails."""
-        try:
-            import resource
-
-            # ru_maxrss is bytes on macOS, kilobytes on Linux. Detect by checking platform.
-            import sys
-
-            usage = resource.getrusage(resource.RUSAGE_SELF)
-            rss = usage.ru_maxrss
-            if sys.platform != "darwin":
-                rss *= 1024  # Linux reports KB
-            rss_mb = rss / (1024 * 1024)
-            return f"rss_mb={rss_mb:.0f}"
-        except Exception as e:
-            logger.debug(f"Process stats unavailable: {e}")
-            return "unavailable"
-
-    def _format_pool_stats(self) -> str:
-        """Render asyncpg pool stats. Returns 'unavailable' if pool can't be introspected."""
-        pool = self._pool
-        try:
-            # asyncpg.Pool exposes _holders / _queue internally; fall back gracefully
-            # to public methods if the layout ever changes.
-            size = pool.get_size() if hasattr(pool, "get_size") else len(getattr(pool, "_holders", []))
-            free = pool.get_idle_size() if hasattr(pool, "get_idle_size") else None
-            min_size = pool.get_min_size() if hasattr(pool, "get_min_size") else None
-            max_size = pool.get_max_size() if hasattr(pool, "get_max_size") else None
-            queue = getattr(pool, "_queue", None)
-            waiters = queue.qsize() if queue is not None and hasattr(queue, "qsize") else None
-
-            parts = [f"size={size}"]
-            if min_size is not None and max_size is not None:
-                parts.append(f"limits={min_size}-{max_size}")
-            if free is not None:
-                parts.append(f"idle={free}")
-                parts.append(f"in_use={size - free}")
-            if waiters is not None:
-                parts.append(f"waiters={waiters}")
-            return " ".join(parts)
-        except Exception as e:
-            logger.debug(f"Pool stats unavailable: {e}")
-            return "unavailable"
-
-    def _log_pending_breakdown(self, breakdown: dict[str, dict[str, int]]) -> None:
-        """Emit one [PENDING_BREAKDOWN] line bucketing pending rows by claimability.
-
-        Each bucket mirrors a predicate in the claim query:
-          * payload_null   - row has no task_payload (e.g. batch_retain parent
-                             whose reconciliation never fired); claim query
-                             skips it forever
-          * retry_blocked  - next_retry_at is still in the future
-          * assigned       - worker_id already set; another worker owns it
-
-        ``claimable`` is the residual that *should* be picked up on the next
-        poll. If ``claimable > 0`` while workers report free slots, the bug is
-        somewhere else (lock contention, tenant discovery, etc.) - this line
-        narrows the search.
-        """
-        if not breakdown:
-            return
-
-        parts = []
-        for op_type in sorted(breakdown):
-            b = breakdown[op_type]
-            claimable = b["total"] - b["payload_null"] - b["retry_blocked"] - b["assigned"]
-            parts.append(
-                f"{op_type}: total={b['total']} claimable={claimable} "
-                f"payload_null={b['payload_null']} retry_blocked={b['retry_blocked']} "
-                f"assigned={b['assigned']}"
-            )
-        logger.info(f"[PENDING_BREAKDOWN] {' | '.join(parts)}")
-
-    def _log_per_task_lines(self, active_tasks: dict[str, ActiveTaskInfo], now: float) -> None:
-        """Emit one [WORKER_TASK] line per in-flight task and dump stuck stacks.
-
-        Sorted by age desc so the oldest (most likely stuck) tasks appear first.
-        """
-        if not active_tasks:
-            return
-
-        # Sort by age descending; tie-break on op_id for determinism.
-        ordered = sorted(
-            active_tasks.items(),
-            key=lambda kv: (now - kv[1].started_at, kv[0]),
-            reverse=True,
-        )
-
-        for op_id, info in ordered:
-            age_s = now - info.started_at
-            holder = info.stage_holder
-            stage = holder.stage if holder is not None else "unknown"
-            stage_age_s = (now - holder.updated_at) if holder is not None else 0.0
-            stuck_marker = "[STUCK?] " if age_s >= STUCK_STACK_INITIAL_THRESHOLD_S else ""
-            schema_part = f" schema={info.schema}" if info.schema else ""
-            logger.info(
-                f"[WORKER_TASK] {stuck_marker}op={op_id} type={info.task_type} "
-                f"op_type={info.op_type} bank={info.bank_id}{schema_part} "
-                f"age={age_s:.0f}s stage={stage} stage_age={stage_age_s:.0f}s"
-            )
-
-            self._maybe_dump_stuck_stack(op_id, info, age_s)
-
-    def _maybe_dump_stuck_stack(self, op_id: str, info: ActiveTaskInfo, age_s: float) -> None:
-        """Dump a coroutine stack for tasks that crossed a stuck threshold.
-
-        Each task gets one dump per threshold (5min, 10min, 20min, 40min...),
-        gated by `info.last_stack_dump_threshold` so logs don't flood for tasks
-        that legitimately take a long time (large LLM jobs, schema-retry loops).
-        """
-        if age_s < STUCK_STACK_INITIAL_THRESHOLD_S:
-            return
-
-        # Find the largest doubling-threshold that the task has crossed.
-        threshold = STUCK_STACK_INITIAL_THRESHOLD_S
-        crossed = STUCK_STACK_INITIAL_THRESHOLD_S
-        while threshold <= age_s and threshold <= STUCK_STACK_MAX_THRESHOLD_S:
-            crossed = threshold
-            threshold *= 2
-
-        if crossed <= info.last_stack_dump_threshold:
-            return
-
-        info.last_stack_dump_threshold = crossed
-
-        try:
-            buf = io.StringIO()
-            info.bg_task.print_stack(file=buf, limit=15)
-            stage = info.stage_holder.stage if info.stage_holder else "unknown"
-            logger.warning(
-                f"[STUCK_STACK] op={op_id} type={info.task_type} bank={info.bank_id} "
-                f"age={age_s:.0f}s threshold={crossed}s stage={stage}\n{buf.getvalue()}"
-            )
-        except Exception as e:
-            # Stack capture is best-effort - never crash the polling loop over it.
-            logger.debug(f"Failed to capture stack for {op_id}: {e}")
-
-    async def _log_db_waits(self) -> None:
-        """Log any non-idle hindsight session that's waiting on a lock or other resource.
-
-        Catches the case where a coroutine appears 'fine' from Python's perspective
-        but is blocked on a Postgres row lock - which is exactly how the 3-phase
-        retain pipeline deadlock would present.
-        """
-        try:
-            async with self._pool.acquire() as conn:
-                rows = await conn.fetch(
-                    """
-                    SELECT
-                        pid,
-                        application_name,
-                        wait_event_type,
-                        wait_event,
-                        state,
-                        EXTRACT(EPOCH FROM (now() - query_start))::int AS age_s,
-                        LEFT(query, 200) AS query
-                    FROM pg_stat_activity
-                    WHERE datname = current_database()
-                      AND state IS NOT NULL
-                      AND state != 'idle'
-                      AND wait_event IS NOT NULL
-                      AND wait_event_type NOT IN ('Activity', 'Client')
-                    ORDER BY age_s DESC NULLS LAST
-                    LIMIT 20
-                    """
-                )
-        except Exception as e:
-            # pg_stat_activity may be restricted on managed Postgres - degrade silently.
-            logger.debug(f"DB waits query failed: {e}")
-            return
-
-        if not rows:
-            return
-
-        for r in rows:
-            logger.info(
-                f"[DB_WAITS] pid={r['pid']} app={r['application_name']} "
-                f"wait={r['wait_event_type']}.{r['wait_event']} state={r['state']} "
-                f"age={r['age_s']}s query={r['query']!r}"
-            )
-
     @property
     def worker_id(self) -> str:
         """Get the worker ID."""

hindsight-api-slim/hindsight_api/worker/stage.py

@@ -1,59 +0,0 @@
-"""Stage breadcrumbs for in-flight worker tasks.
-
-The worker poller binds a `StageHolder` to each task's contextvar scope.
-Engine code calls `set_stage("retain.facts.llm")` at phase boundaries; the
-poller reads the holder periodically to surface what each in-flight task is
-currently doing in `WORKER_STATS` / `WORKER_TASK` log lines.
-
-Outside a worker context the contextvar is unset and `set_stage` is a no-op,
-so engine code is safe to call from sync HTTP requests, tests, or the CLI
-without any setup.
-"""
-
-from __future__ import annotations
-
-import time
-from contextvars import ContextVar
-from dataclasses import dataclass, field
-
-
-@dataclass
-class StageHolder:
-    """Mutable container for the current task's stage label."""
-
-    stage: str = "init"
-    updated_at: float = field(default_factory=time.monotonic)
-
-
-_current_holder: ContextVar[StageHolder | None] = ContextVar("hindsight_stage_holder", default=None)
-
-
-def bind_holder(holder: StageHolder):
-    """Bind a holder to the current async context.
-
-    Must be called from inside the task coroutine itself (not from the
-    spawning code) so the binding lives in the task's own contextvar scope.
-
-    Returns the token that can be passed to `_current_holder.reset()` if
-    the binding ever needs to be unwound.
-    """
-    return _current_holder.set(holder)
-
-
-def set_stage(name: str) -> None:
-    """Update the current task's stage label.
-
-    No-op when called outside a worker task context (e.g. from a sync HTTP
-    request, a test, or the CLI). Cheap enough to call per-phase.
-    """
-    holder = _current_holder.get()
-    if holder is None:
-        return
-    holder.stage = name
-    holder.updated_at = time.monotonic()
-
-
-def get_stage() -> str | None:
-    """Return the current stage label, or None if no holder is bound."""
-    holder = _current_holder.get()
-    return holder.stage if holder is not None else None

hindsight-api-slim/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-api-slim"
-version = "0.5.1"
+version = "0.5.0"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"

hindsight-api-slim/tests/test_async_batch_retain.py

@@ -433,130 +433,3 @@ async def test_config_retain_batch_tokens_respected(memory, request_context):
     # Even small batches use parent-child pattern now (simpler code path)
     assert "child_operations" in status
     assert status["result_metadata"]["num_sub_batches"] == 1
-
-
-async def _child_metadata(memory, bank_id: str, parent_operation_id: str, request_context):
-    """Fetch the first child operation's result_metadata for a parent batch_retain."""
-    parent = await memory.get_operation_status(
-        bank_id=bank_id,
-        operation_id=parent_operation_id,
-        request_context=request_context,
-    )
-    assert parent["status"] == "completed", parent
-    assert parent["child_operations"], "expected at least one child operation"
-    child_id = parent["child_operations"][0]["operation_id"]
-    child = await memory.get_operation_status(
-        bank_id=bank_id,
-        operation_id=child_id,
-        request_context=request_context,
-    )
-    return child["result_metadata"]
-
-
-@pytest.mark.asyncio
-async def test_retain_records_user_provided_document_ids(memory, request_context):
-    """User-supplied document_ids land in child op result_metadata.document_ids."""
-    bank_id = "test_doc_ids_user_supplied"
-    d1 = str(uuid.uuid4())
-    d2 = str(uuid.uuid4())
-    contents = [
-        {"content": "User-supplied doc one content.", "document_id": d1},
-        {"content": "User-supplied doc two content.", "document_id": d2},
-    ]
-
-    result = await memory.submit_async_retain(
-        bank_id=bank_id,
-        contents=contents,
-        request_context=request_context,
-    )
-    await asyncio.sleep(0.2)
-
-    meta = await _child_metadata(memory, bank_id, result["operation_id"], request_context)
-    assert "document_ids" in meta, meta
-    assert set(meta["document_ids"]) == {d1, d2}
-
-
-@pytest.mark.asyncio
-async def test_retain_records_generated_document_id(memory, request_context):
-    """With no document_ids supplied, retain records the single generated id."""
-    bank_id = "test_doc_ids_generated"
-    contents = [
-        {"content": "Generated doc item one."},
-        {"content": "Generated doc item two."},
-    ]
-
-    result = await memory.submit_async_retain(
-        bank_id=bank_id,
-        contents=contents,
-        request_context=request_context,
-    )
-    await asyncio.sleep(0.2)
-
-    meta = await _child_metadata(memory, bank_id, result["operation_id"], request_context)
-    assert "document_ids" in meta, meta
-    assert isinstance(meta["document_ids"], list)
-    assert len(meta["document_ids"]) == 1
-    # Must be a valid UUID string (generated by the orchestrator)
-    uuid.UUID(meta["document_ids"][0])
-
-
-@pytest.mark.asyncio
-async def test_retain_records_shared_document_id_once(memory, request_context):
-    """Items sharing one document_id record it exactly once (idempotent set-append)."""
-    bank_id = "test_doc_ids_shared"
-    shared = str(uuid.uuid4())
-    # Duplicate per-item doc_ids are rejected up front, so shared-doc mode
-    # is exercised by a single item carrying the id.
-    contents = [{"content": "Shared doc, chunk A.", "document_id": shared}]
-
-    result = await memory.submit_async_retain(
-        bank_id=bank_id,
-        contents=contents,
-        request_context=request_context,
-    )
-    await asyncio.sleep(0.2)
-
-    meta = await _child_metadata(memory, bank_id, result["operation_id"], request_context)
-    assert meta.get("document_ids") == [shared]
-
-
-@pytest.mark.asyncio
-async def test_get_operation_status_include_payload(memory, request_context):
-    """include_payload=True returns the original submission payload; default omits it."""
-    bank_id = "test_include_payload"
-    contents = [{"content": "Payload roundtrip test item."}]
-
-    result = await memory.submit_async_retain(
-        bank_id=bank_id,
-        contents=contents,
-        request_context=request_context,
-    )
-    await asyncio.sleep(0.2)
-
-    parent = await memory.get_operation_status(
-        bank_id=bank_id,
-        operation_id=result["operation_id"],
-        request_context=request_context,
-    )
-    child_id = parent["child_operations"][0]["operation_id"]
-
-    # Default: no payload
-    without = await memory.get_operation_status(
-        bank_id=bank_id,
-        operation_id=child_id,
-        request_context=request_context,
-    )
-    assert without.get("task_payload") is None
-
-    # With flag: payload populated
-    with_payload = await memory.get_operation_status(
-        bank_id=bank_id,
-        operation_id=child_id,
-        request_context=request_context,
-        include_payload=True,
-    )
-    payload = with_payload.get("task_payload")
-    assert payload is not None, with_payload
-    assert payload.get("bank_id") == bank_id
-    assert payload.get("contents")
-    assert payload["contents"][0]["content"] == "Payload roundtrip test item."

hindsight-api-slim/tests/test_async_retain_tags.py

@@ -34,12 +34,7 @@ async def test_submit_async_retain_includes_document_tags_in_task_payload():
     contents = [{"content": "Async retain payload test."}]
     document_tags = ["scope:tools", "user:alice"]
 
-    # Return (profile, created=False) so the default-template-on-create hook is skipped.
-    with patch(
-        "hindsight_api.engine.memory_engine.bank_utils.get_or_create_bank_profile",
-        new_callable=AsyncMock,
-        return_value=(MagicMock(), False),
-    ):
+    with patch("hindsight_api.engine.memory_engine.bank_utils.get_bank_profile", new_callable=AsyncMock):
         result = await MemoryEngine.submit_async_retain(
             engine,
             bank_id="bank-1",

hindsight-api-slim/tests/test_bank_template_configurable_fields.py

@@ -1,106 +0,0 @@
-"""Verify that BankTemplateConfig exposes every hierarchical field that
-_CONFIGURABLE_FIELDS already accepts at the engine layer.
-
-This test guards the fix for the gap described in the upstream PR title
-"fix(bank-template): align BankTemplateConfig with _CONFIGURABLE_FIELDS".
-Each new field is POSTed through /v1/default/banks/{id}/import and then
-read back via the bank-config endpoint; assertion is that the applied
-value round-trips through the engine.
-
-Runs via: uv run pytest tests/test_bank_template_configurable_fields.py -v
-
-The api_client fixture (shared with tests/test_bank_templates.py) wraps
-create_app(memory, initialize_memory=False) in an httpx.ASGITransport
-with base_url http://test — in-process, no network, no tenant extension.
-Copy the fixture inline here so the test file does not depend on a
-conftest we do not ship in the patch.
-"""
-
-from __future__ import annotations
-
-from datetime import datetime
-
-import httpx
-import pytest
-import pytest_asyncio
-
-from hindsight_api.api import create_app
-from hindsight_api.api.http import BankTemplateConfig
-
-# Each tuple is (field_name, applied_value). Values chosen to differ
-# visibly from defaults so round-trip bugs surface.
-NEW_FIELDS: list[tuple[str, object]] = [
-    ("retain_default_strategy", "strategy-a"),
-    ("retain_strategies", {"strategy-a": {"mode": "concise", "max_tokens": 512}}),
-    ("retain_chunk_batch_size", 7),
-    ("mcp_enabled_tools", ["list_banks", "get_bank_profile"]),
-    ("consolidation_llm_batch_size", 11),
-    ("consolidation_source_facts_max_tokens", 2048),
-    ("consolidation_source_facts_max_tokens_per_observation", 256),
-    ("max_observations_per_scope", 13),
-    ("reflect_source_facts_max_tokens", 4096),
-    ("llm_gemini_safety_settings", [{"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"}]),
-]
-
-
-@pytest_asyncio.fixture
-async def api_client(memory):
-    """Matches the fixture in tests/test_bank_templates.py — in-process
-    ASGI test client, no tenant extension, no auth."""
-    app = create_app(memory, initialize_memory=False)
-    transport = httpx.ASGITransport(app=app)
-    async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
-        yield client
-
-
-@pytest.fixture
-def bank_id():
-    return f"tmpl_config_{datetime.now().timestamp()}"
-
-
-def test_bank_template_config_declares_every_configurable_field():
-    """Pydantic-level guard: every field in NEW_FIELDS must be a declared
-    attribute of BankTemplateConfig so get_config_updates() picks it up."""
-    declared = set(BankTemplateConfig.model_fields.keys())
-    missing = [name for name, _ in NEW_FIELDS if name not in declared]
-    assert not missing, f"BankTemplateConfig missing fields: {missing}"
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize("field_name,applied_value", NEW_FIELDS, ids=[n for n, _ in NEW_FIELDS])
-async def test_new_field_round_trips_through_import(
-    api_client: httpx.AsyncClient,
-    bank_id: str,
-    field_name: str,
-    applied_value: object,
-):
-    """POST a minimal manifest with one new field set, then read bank
-    config back and assert the value made it through.
-
-    Bank config response shape per upstream's test_import_applies_config:
-    top-level keys are resolved hierarchical config; per-bank overrides
-    live under config["overrides"][<field>]. Assert on the override slot.
-    """
-    unique_bank_id = f"{bank_id}_{field_name}"
-    manifest = {
-        "version": "1",
-        "bank": {field_name: applied_value},
-    }
-
-    resp = await api_client.post(
-        f"/v1/default/banks/{unique_bank_id}/import",
-        json=manifest,
-    )
-    assert resp.status_code == 200, resp.text
-
-    # Read bank config back — field must reflect the applied value
-    # under the "overrides" slot, matching upstream's own test shape.
-    read = await api_client.get(f"/v1/default/banks/{unique_bank_id}/config")
-    assert read.status_code == 200, read.text
-    config = read.json()
-    overrides = config.get("overrides", {})
-    assert overrides.get(field_name) == applied_value, (
-        f"round-trip mismatch for {field_name}: "
-        f"sent {applied_value!r}, got {overrides.get(field_name)!r} "
-        f"(full overrides: {overrides!r})"
-    )

hindsight-api-slim/tests/test_bank_templates.py

@@ -598,182 +598,3 @@ class TestExport:
         assert resp.status_code == 200
         data = resp.json()
         assert data["version"] == "1"
-
-
-class TestDefaultBankTemplateEnvVar:
-    """Tests for HINDSIGHT_API_DEFAULT_BANK_TEMPLATE — a server-level env var
-    whose manifest is applied automatically to every newly-created bank."""
-
-    @pytest.fixture
-    def default_template(self):
-        return {
-            "version": "1",
-            "bank": {
-                "reflect_mission": "default-env-mission",
-                "retain_extraction_mode": "verbose",
-                "disposition_empathy": 5,
-                "disposition_skepticism": 1,
-            },
-            "mental_models": [
-                {
-                    "id": "default-env-model",
-                    "name": "Default Env Model",
-                    "source_query": "What is the default?",
-                },
-            ],
-            "directives": [
-                {
-                    "name": "Default Env Directive",
-                    "content": "Follow the default behavior.",
-                    "priority": 7,
-                },
-            ],
-        }
-
-    @pytest.fixture
-    def _patched_default_template(self, monkeypatch, default_template):
-        """Install the default template on the already-initialized global config.
-
-        We can't rely on env-var resolution here: MemoryEngine (and its
-        ConfigResolver) snapshot the global config at fixture init time.
-        Patching the field directly keeps the test deterministic while still
-        exercising the same code path that reads `get_config().default_bank_template`.
-        """
-        from hindsight_api.config import _get_raw_config
-
-        raw = _get_raw_config()
-        monkeypatch.setattr(raw, "default_bank_template", default_template)
-        yield default_template
-
-    @pytest.mark.asyncio
-    async def test_default_template_applied_on_new_bank(
-        self, api_client, bank_id, _patched_default_template
-    ):
-        """Creating a new bank applies the default template (config + mental models + directives)."""
-        # Trigger bank auto-creation via GET profile
-        resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
-        assert resp.status_code == 200
-
-        # Config from template should be present as bank overrides
-        config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config")
-        assert config_resp.status_code == 200
-        overrides = config_resp.json()["overrides"]
-        assert overrides["reflect_mission"] == "default-env-mission"
-        assert overrides["retain_extraction_mode"] == "verbose"
-        assert overrides["disposition_empathy"] == 5
-        assert overrides["disposition_skepticism"] == 1
-
-        # Mental model from template should exist
-        mm_resp = await api_client.get(f"/v1/default/banks/{bank_id}/mental-models/default-env-model")
-        assert mm_resp.status_code == 200
-        assert mm_resp.json()["name"] == "Default Env Model"
-
-        # Directive from template should exist
-        dir_resp = await api_client.get(f"/v1/default/banks/{bank_id}/directives")
-        assert dir_resp.status_code == 200
-        names = [d["name"] for d in dir_resp.json()["items"]]
-        assert "Default Env Directive" in names
-
-    @pytest.mark.asyncio
-    async def test_default_template_overrides_env_config_defaults(
-        self, api_client, bank_id, monkeypatch, default_template
-    ):
-        """Fields set by the default template override server-level env-var defaults.
-
-        We point both HINDSIGHT_API_RETAIN_EXTRACTION_MODE (env) and the
-        default template at different values, then confirm the template wins
-        via the per-bank config overrides layer (highest precedence).
-        """
-        from hindsight_api.config import _get_raw_config
-
-        raw = _get_raw_config()
-        # Simulate an env-level default of "concise", overridden by a template that sets "verbose".
-        monkeypatch.setattr(raw, "retain_extraction_mode", "concise")
-        monkeypatch.setattr(raw, "default_bank_template", default_template)
-
-        resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
-        assert resp.status_code == 200
-
-        config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config")
-        overrides = config_resp.json()["overrides"]
-        # Template value wins at the bank-override layer.
-        assert overrides["retain_extraction_mode"] == "verbose"
-
-    @pytest.mark.asyncio
-    async def test_default_template_not_reapplied_on_existing_bank(
-        self, api_client, bank_id, _patched_default_template
-    ):
-        """Template only applies on FIRST creation; subsequent puts are no-ops."""
-        # First hit creates the bank and applies the template
-        resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
-        assert resp.status_code == 200
-
-        # User explicitly overrides a template-set field
-        patch_resp = await api_client.patch(
-            f"/v1/default/banks/{bank_id}/config",
-            json={"updates": {"reflect_mission": "user-override"}},
-        )
-        assert patch_resp.status_code == 200
-
-        # Second put — template must NOT be reapplied (would clobber the override)
-        resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
-        assert resp.status_code == 200
-
-        config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config")
-        assert config_resp.json()["overrides"]["reflect_mission"] == "user-override"
-
-    @pytest.mark.asyncio
-    async def test_default_template_unset_is_noop(self, api_client, bank_id):
-        """With the env var unset (fixture default), bank creation behaves as before."""
-        resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
-        assert resp.status_code == 200
-
-        # No template = no overrides
-        config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config")
-        assert config_resp.json()["overrides"] == {}
-
-    @pytest.mark.asyncio
-    async def test_default_template_malformed_is_swallowed(
-        self, api_client, bank_id, monkeypatch
-    ):
-        """A malformed default template is logged and ignored — bank creation still succeeds."""
-        from hindsight_api.config import _get_raw_config
-
-        raw = _get_raw_config()
-        # Wrong version number fails Pydantic validation.
-        monkeypatch.setattr(raw, "default_bank_template", {"version": "999"})
-
-        resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
-        # Bank creation must not fail even though the template is broken.
-        assert resp.status_code == 200
-
-    def test_parse_default_bank_template_valid_json(self, monkeypatch):
-        """_parse_default_bank_template parses a valid JSON object env var."""
-        from hindsight_api.config import _parse_default_bank_template
-
-        parsed = _parse_default_bank_template('{"version": "1", "bank": {"disposition_empathy": 4}}')
-        assert parsed == {"version": "1", "bank": {"disposition_empathy": 4}}
-
-    def test_parse_default_bank_template_none_or_empty(self):
-        """Unset / empty env var resolves to None."""
-        from hindsight_api.config import _parse_default_bank_template
-
-        assert _parse_default_bank_template(None) is None
-        assert _parse_default_bank_template("") is None
-        assert _parse_default_bank_template("   ") is None
-
-    def test_parse_default_bank_template_invalid_json_raises(self):
-        """Invalid JSON fails fast with a clear error."""
-        from hindsight_api.config import _parse_default_bank_template
-
-        with pytest.raises(ValueError, match="HINDSIGHT_API_DEFAULT_BANK_TEMPLATE"):
-            _parse_default_bank_template("not-json")
-
-    def test_parse_default_bank_template_non_object_raises(self):
-        """Non-object JSON (e.g. array, string) fails fast."""
-        from hindsight_api.config import _parse_default_bank_template
-
-        with pytest.raises(ValueError, match="expected a JSON object"):
-            _parse_default_bank_template("[1, 2, 3]")
-        with pytest.raises(ValueError, match="expected a JSON object"):
-            _parse_default_bank_template('"just a string"')

hindsight-api-slim/tests/test_chunk_storage_upsert.py

@@ -1,149 +0,0 @@
-"""
-Regression tests for chunk_storage.store_chunks_batch idempotency.
-
-Covers vectorize-io/hindsight#977: re-submitting a retain under the same
-document_id must not fail with ``UniqueViolationError`` on ``pk_chunks``.
-The upstream retain paths (cascade delete on first batch, delta retain)
-should usually prevent a chunk_id collision, but any bug in those paths
-used to surface as a raw Postgres constraint violation. ``store_chunks_batch``
-is now idempotent: inserting the same ``chunk_id`` twice overwrites the
-existing row rather than raising.
-"""
-
-from datetime import datetime, timezone
-
-import pytest
-
-from hindsight_api.engine.retain import chunk_storage
-from hindsight_api.engine.retain.types import ChunkMetadata
-
-
-def _ts() -> float:
-    return datetime.now(timezone.utc).timestamp()
-
-
-async def _seed_bank_and_document(conn, bank_id: str, document_id: str) -> None:
-    """Insert the minimum rows required for the chunks FK to pass."""
-    await conn.execute(
-        "INSERT INTO banks (bank_id, name) VALUES ($1, $2) ON CONFLICT DO NOTHING",
-        bank_id,
-        bank_id,
-    )
-    await conn.execute(
-        """
-        INSERT INTO documents (id, bank_id, original_text, content_hash)
-        VALUES ($1, $2, $3, $4)
-        ON CONFLICT (id, bank_id) DO NOTHING
-        """,
-        document_id,
-        bank_id,
-        "seed",
-        "seed-hash",
-    )
-
-
-@pytest.mark.asyncio
-async def test_store_chunks_batch_is_idempotent_for_same_chunk_id(memory):
-    """
-    Regression for #977.
-
-    Directly exercises the chunk insert path: inserting a ChunkMetadata with
-    a chunk_index that already exists (i.e., the same chunk_id) must not
-    raise. The new content should overwrite the old one.
-    """
-    bank_id = f"test_chunk_upsert_{_ts()}"
-    document_id = "doc-upsert-regression"
-
-    pool = await memory._get_pool()
-    try:
-        async with pool.acquire() as conn:
-            await _seed_bank_and_document(conn, bank_id, document_id)
-
-            # First insert — fresh chunks at indices 0, 1, 2.
-            v1 = [
-                ChunkMetadata(chunk_text="alpha", fact_count=1, content_index=0, chunk_index=0),
-                ChunkMetadata(chunk_text="beta", fact_count=1, content_index=0, chunk_index=1),
-                ChunkMetadata(chunk_text="gamma", fact_count=1, content_index=0, chunk_index=2),
-            ]
-            v1_map = await chunk_storage.store_chunks_batch(conn, bank_id, document_id, v1)
-            assert set(v1_map.keys()) == {0, 1, 2}
-
-            # Second insert — overlapping chunk_index (1 and 2) with new text,
-            # plus a fresh chunk at index 3. Before the fix this raised
-            # asyncpg.exceptions.UniqueViolationError on pk_chunks; after the
-            # fix the conflicting rows are overwritten and the new one is
-            # inserted.
-            v2 = [
-                ChunkMetadata(chunk_text="beta-updated", fact_count=1, content_index=0, chunk_index=1),
-                ChunkMetadata(chunk_text="gamma-updated", fact_count=1, content_index=0, chunk_index=2),
-                ChunkMetadata(chunk_text="delta", fact_count=1, content_index=0, chunk_index=3),
-            ]
-            v2_map = await chunk_storage.store_chunks_batch(conn, bank_id, document_id, v2)
-            assert set(v2_map.keys()) == {1, 2, 3}
-
-            # Verify the stored state matches the upserted content.
-            rows = await conn.fetch(
-                """
-                SELECT chunk_index, chunk_text, content_hash
-                FROM chunks
-                WHERE document_id = $1 AND bank_id = $2
-                ORDER BY chunk_index
-                """,
-                document_id,
-                bank_id,
-            )
-            by_index = {row["chunk_index"]: row for row in rows}
-
-            assert set(by_index.keys()) == {0, 1, 2, 3}, (
-                "Expected four chunks total after upsert (0 untouched, 1-2 overwritten, 3 new)"
-            )
-            assert by_index[0]["chunk_text"] == "alpha", "Untouched chunk must be preserved"
-            assert by_index[1]["chunk_text"] == "beta-updated", "Conflicting chunk must be overwritten"
-            assert by_index[2]["chunk_text"] == "gamma-updated", "Conflicting chunk must be overwritten"
-            assert by_index[3]["chunk_text"] == "delta", "New chunk must be inserted"
-
-            # content_hash should reflect the new text, not the original.
-            assert by_index[1]["content_hash"] == chunk_storage.compute_chunk_hash("beta-updated")
-            assert by_index[2]["content_hash"] == chunk_storage.compute_chunk_hash("gamma-updated")
-    finally:
-        async with pool.acquire() as conn:
-            await conn.execute("DELETE FROM chunks WHERE bank_id = $1", bank_id)
-            await conn.execute("DELETE FROM documents WHERE bank_id = $1", bank_id)
-            await conn.execute("DELETE FROM banks WHERE bank_id = $1", bank_id)
-
-
-@pytest.mark.asyncio
-async def test_store_chunks_batch_second_call_with_identical_payload(memory):
-    """
-    The exact #977 shape: ``store_chunks_batch`` called twice with the same
-    chunks must succeed both times (the second call is a no-op in terms of
-    stored content, but must not raise).
-    """
-    bank_id = f"test_chunk_upsert_identical_{_ts()}"
-    document_id = "doc-upsert-identical"
-
-    pool = await memory._get_pool()
-    try:
-        async with pool.acquire() as conn:
-            await _seed_bank_and_document(conn, bank_id, document_id)
-
-            chunks = [
-                ChunkMetadata(chunk_text=f"chunk-{i}", fact_count=1, content_index=0, chunk_index=i)
-                for i in range(5)
-            ]
-
-            await chunk_storage.store_chunks_batch(conn, bank_id, document_id, chunks)
-            # Second call with identical chunks — must not raise.
-            await chunk_storage.store_chunks_batch(conn, bank_id, document_id, chunks)
-
-            count = await conn.fetchval(
-                "SELECT COUNT(*) FROM chunks WHERE document_id = $1 AND bank_id = $2",
-                document_id,
-                bank_id,
-            )
-            assert count == 5, "Second identical insert should not duplicate rows"
-    finally:
-        async with pool.acquire() as conn:
-            await conn.execute("DELETE FROM chunks WHERE bank_id = $1", bank_id)
-            await conn.execute("DELETE FROM documents WHERE bank_id = $1", bank_id)
-            await conn.execute("DELETE FROM banks WHERE bank_id = $1", bank_id)

hindsight-api-slim/tests/test_cohere_cross_encoder.py

@@ -5,7 +5,7 @@ Tests the Cohere cross-encoder implementation, including Azure AI Foundry endpoi
 """
 
 import os
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import MagicMock, patch
 
 import httpx
 import pytest
@@ -28,7 +28,7 @@ class TestCohereCrossEncoder:
         assert encoder.api_key == "test_key"
         assert encoder.model == "rerank-english-v3.0"
         assert encoder._client is None
-        assert encoder._http_client is None
+        assert encoder._httpx_client is None
 
         # Mock the cohere import
         mock_cohere = MagicMock()
@@ -36,7 +36,7 @@ class TestCohereCrossEncoder:
         with patch.dict("sys.modules", {"cohere": mock_cohere}):
             await encoder.initialize()
             assert encoder._client is not None
-            assert encoder._http_client is None
+            assert encoder._httpx_client is None
             mock_cohere.Client.assert_called_once_with(api_key="test_key", timeout=60.0)
 
     @pytest.mark.asyncio
@@ -52,14 +52,9 @@ class TestCohereCrossEncoder:
 
         await encoder.initialize()
 
-        assert encoder._http_client is not None
+        assert encoder._httpx_client is not None
         assert encoder._client is None
-        assert isinstance(encoder._http_client._async_client, httpx.AsyncClient)
-        assert encoder._http_client.include_top_n is False
-        assert (
-            encoder._http_client.rerank_url
-            == "https://my-endpoint.inference.ai.azure.com/models/cohere-rerank-v3-english/invoke"
-        )
+        assert isinstance(encoder._httpx_client, httpx.Client)
 
     @pytest.mark.asyncio
     async def test_initialization_missing_package(self):
@@ -155,7 +150,7 @@ class TestCohereCrossEncoder:
 
         await encoder.initialize()
 
-        # Mock async httpx response
+        # Mock httpx response
         mock_response = MagicMock()
         mock_response.json.return_value = {
             "results": [
@@ -164,9 +159,8 @@ class TestCohereCrossEncoder:
                 {"index": 2, "relevance_score": 0.5},
             ]
         }
-        mock_response.raise_for_status = MagicMock()
 
-        encoder._http_client._async_client.post = AsyncMock(return_value=mock_response)
+        encoder._httpx_client.post = MagicMock(return_value=mock_response)
 
         pairs = [
             ("What is Python?", "Python is a programming language"),
@@ -180,15 +174,13 @@ class TestCohereCrossEncoder:
         assert scores == [0.9, 0.7, 0.5]
 
         # Verify httpx.post was called with correct URL and payload
-        encoder._http_client._async_client.post.assert_called_once()
-        call_args = encoder._http_client._async_client.post.call_args
+        encoder._httpx_client.post.assert_called_once()
+        call_args = encoder._httpx_client.post.call_args
         assert call_args[0][0] == "https://my-endpoint.inference.ai.azure.com/models/cohere-rerank-v3-english/invoke"
         assert call_args.kwargs["json"]["model"] == "cohere-rerank-v3-english"
         assert call_args.kwargs["json"]["query"] == "What is Python?"
         assert len(call_args.kwargs["json"]["documents"]) == 3
         assert call_args.kwargs["json"]["return_documents"] is False
-        # Azure endpoints expect no top_n in the body
-        assert "top_n" not in call_args.kwargs["json"]
 
     @pytest.mark.asyncio
     async def test_predict_multiple_queries(self):
@@ -289,7 +281,7 @@ class TestCohereCrossEncoder:
             request=MagicMock(),
             response=MagicMock(status_code=404),
         )
-        encoder._http_client._async_client.post = AsyncMock(return_value=mock_response)
+        encoder._httpx_client.post = MagicMock(return_value=mock_response)
 
         pairs = [("What is Python?", "Python is a programming language")]
 

hindsight-api-slim/tests/test_custom_embedding_dimension.py

@@ -15,12 +15,7 @@ import pytest
 from sqlalchemy import create_engine, text
 
 from hindsight_api import MemoryEngine, RequestContext
-from hindsight_api.engine.cross_encoder import (
-    CohereCrossEncoder,
-    LocalSTCrossEncoder,
-    SiliconFlowCrossEncoder,
-    ZeroEntropyCrossEncoder,
-)
+from hindsight_api.engine.cross_encoder import CohereCrossEncoder, LocalSTCrossEncoder, ZeroEntropyCrossEncoder
 from hindsight_api.engine.embeddings import CohereEmbeddings, LocalSTEmbeddings, OpenAIEmbeddings
 from hindsight_api.engine.query_analyzer import DateparserQueryAnalyzer
 from hindsight_api.engine.task_backend import SyncTaskBackend
@@ -744,58 +739,3 @@ class TestZeroEntropyCrossEncoder:
         assert all(isinstance(s, float) for s in scores)
         # The first result should be most relevant
         assert scores[0] > scores[2], "Direct answer should score higher than unrelated text"
-
-
-# =============================================================================
-# SiliconFlow Reranker Tests
-# =============================================================================
-
-
-def has_siliconflow_api_key() -> bool:
-    """Check if SiliconFlow API key is available."""
-    return bool(os.environ.get("SILICONFLOW_API_KEY"))
-
-
-def get_siliconflow_api_key() -> str:
-    """Get SiliconFlow API key from environment."""
-    return os.environ.get("SILICONFLOW_API_KEY", "")
-
-
-@pytest.fixture(scope="module")
-def siliconflow_cross_encoder():
-    """Create SiliconFlow cross-encoder instance."""
-    if not has_siliconflow_api_key():
-        pytest.skip("SiliconFlow API key not available (set SILICONFLOW_API_KEY)")
-
-    cross_encoder = SiliconFlowCrossEncoder(
-        api_key=get_siliconflow_api_key(),
-        model="BAAI/bge-reranker-v2-m3",
-    )
-    loop = asyncio.new_event_loop()
-    try:
-        loop.run_until_complete(cross_encoder.initialize())
-    finally:
-        loop.close()
-    return cross_encoder
-
-
-class TestSiliconFlowCrossEncoder:
-    """Tests for SiliconFlow cross-encoder/reranker."""
-
-    def test_siliconflow_cross_encoder_initialization(self, siliconflow_cross_encoder):
-        """Test that SiliconFlow cross-encoder initializes correctly."""
-        assert siliconflow_cross_encoder.provider_name == "siliconflow"
-
-    @pytest.mark.asyncio
-    async def test_siliconflow_cross_encoder_predict(self, siliconflow_cross_encoder):
-        """Test that SiliconFlow cross-encoder can score pairs."""
-        pairs = [
-            ("What is the capital of France?", "Paris is the capital of France."),
-            ("What is the capital of France?", "The Eiffel Tower is in Paris."),
-            ("What is the capital of France?", "Python is a programming language."),
-        ]
-        scores = await siliconflow_cross_encoder.predict(pairs)
-
-        assert len(scores) == 3
-        assert all(isinstance(s, float) for s in scores)
-        assert scores[0] > scores[2], "Direct answer should score higher than unrelated text"

hindsight-api-slim/tests/test_hierarchical_config.py

@@ -98,7 +98,7 @@ async def test_hierarchical_fields_categorization():
     assert "retain_chunk_batch_size" in configurable
 
     # Verify count is correct
-    assert len(configurable) == 25
+    assert len(configurable) == 22
 
     # Verify credential fields (NEVER exposed)
     assert "llm_api_key" in credentials
@@ -209,47 +209,6 @@ async def test_config_validation_rejects_static_fields(memory, request_context):
         await memory.delete_bank(bank_id, request_context=request_context)
 
 
-@pytest.mark.asyncio
-async def test_config_validation_rejects_malformed_entity_labels(memory, request_context):
-    """Test that passing strings instead of LabelGroup dicts to entity_labels raises ValueError.
-
-    Regression test for the fix in PR #902: entity_labels PATCH must validate the
-    format before saving to prevent silent corruption that previously caused 500s on
-    subsequent retain calls (reported in issue #946).
-    """
-    bank_id = "test-entity-labels-validation"
-
-    try:
-        await memory.get_bank_profile(bank_id, request_context=request_context)
-
-        resolver = ConfigResolver(pool=memory._pool)
-
-        # String list instead of LabelGroup dicts must raise ValueError, not silently accept.
-        # Previously this produced HTTP 200, then 500 on the next retain call (issue #946).
-        with pytest.raises(ValueError, match="Invalid entity_labels format"):
-            await resolver.update_bank_config(
-                bank_id,
-                {"entity_labels": ["person", "client", "tool"]},
-            )
-
-        # The correct LabelGroup format must succeed
-        await resolver.update_bank_config(
-            bank_id,
-            {
-                "entity_labels": [
-                    {
-                        "key": "kind",
-                        "type": "value",
-                        "values": [{"value": "person"}, {"value": "client"}],
-                    }
-                ]
-            },
-        )
-
-    finally:
-        await memory.delete_bank(bank_id, request_context=request_context)
-
-
 @pytest.mark.asyncio
 async def test_config_freshness_across_updates(memory, request_context):
     """Test that config changes are immediately visible (no stale cache)."""
@@ -435,16 +394,10 @@ async def test_config_get_bank_config_no_static_or_credential_fields_leak(memory
 
         # SECURITY: Verify specific sensitive fields are NOT present
         sensitive_fields = [
-            "database_url",
-            "api_port",
-            "host",
-            "worker_count",  # Infrastructure
-            "llm_api_key",
-            "llm_base_url",  # Credentials
-            "retain_llm_api_key",
-            "reflect_llm_api_key",  # More credentials
-            "llm_provider",
-            "llm_model",  # Not configurable (need presets)
+            "database_url", "api_port", "host", "worker_count",  # Infrastructure
+            "llm_api_key", "llm_base_url",  # Credentials
+            "retain_llm_api_key", "reflect_llm_api_key",  # More credentials
+            "llm_provider", "llm_model",  # Not configurable (need presets)
         ]
         for field in sensitive_fields:
             assert field not in config, (
@@ -458,7 +411,7 @@ async def test_config_get_bank_config_no_static_or_credential_fields_leak(memory
             assert field in config, f"Expected configurable field '{field}' missing from config"
 
         # Should have a small number of configurable fields (not hundreds)
-        assert len(config) < 30, f"Too many fields returned: {len(config)}"
+        assert len(config) < 25, f"Too many fields returned: {len(config)}"
 
     finally:
         await memory.delete_bank(bank_id, request_context=request_context)

hindsight-api-slim/tests/test_integrity_violation_not_retried.py

@@ -1,184 +0,0 @@
-"""
-Regression tests for vectorize-io/hindsight#980.
-
-Deterministic Postgres integrity-constraint violations (UniqueViolationError,
-ForeignKeyViolationError, CheckViolationError, NotNullViolationError,
-ExclusionViolationError) must NOT be retried by the worker — they will never
-succeed on retry, and retrying just burns worker capacity for ~3 minutes
-(3 retries × 60s) before finally giving up.
-
-These tests verify that ``MemoryEngine.execute_task`` classifies
-``asyncpg.exceptions.IntegrityConstraintViolationError`` as non-retryable
-and marks the operation as failed on the first occurrence.
-"""
-
-import json
-import uuid
-from unittest.mock import AsyncMock, patch
-
-import asyncpg
-import pytest
-
-from hindsight_api.worker.exceptions import RetryTaskAt
-
-
-async def _ensure_bank(pool, bank_id: str) -> None:
-    """Upsert a minimal bank row so FK on async_operations passes."""
-    await pool.execute(
-        "INSERT INTO banks (bank_id, name) VALUES ($1, $2) ON CONFLICT DO NOTHING",
-        bank_id,
-        bank_id,
-    )
-
-
-async def _create_pending_operation(pool, bank_id: str, operation_id: uuid.UUID) -> None:
-    """Insert a pending batch_retain operation row for the test."""
-    payload = json.dumps(
-        {
-            "type": "batch_retain",
-            "operation_id": str(operation_id),
-            "bank_id": bank_id,
-            "contents": [{"content": "test", "document_id": "doc-1"}],
-        }
-    )
-    await pool.execute(
-        """
-        INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
-        VALUES ($1, $2, 'retain', 'pending', $3::jsonb)
-        """,
-        operation_id,
-        bank_id,
-        payload,
-    )
-
-
-@pytest.mark.asyncio
-async def test_unique_violation_marks_failed_without_retry(memory):
-    """
-    UniqueViolationError must mark the operation as failed immediately, not
-    raise RetryTaskAt. This is the primary symptom from #977: re-submitting
-    retain caused PK collisions that the poller retried ~3 times before
-    giving up. With #980's fix, the first collision fails the task.
-    """
-    bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-    operation_id = uuid.uuid4()
-
-    pool = await memory._get_pool()
-    await _ensure_bank(pool, bank_id)
-    await _create_pending_operation(pool, bank_id, operation_id)
-
-    # Synthesize a real asyncpg UniqueViolationError the way the server would
-    # raise it (matches the error observed in the bug report's logs).
-    unique_violation = asyncpg.exceptions.UniqueViolationError(
-        'duplicate key value violates unique constraint "pk_chunks"'
-    )
-
-    task_dict = {
-        "type": "batch_retain",
-        "operation_id": str(operation_id),
-        "bank_id": bank_id,
-        "contents": [{"content": "test", "document_id": "doc-1"}],
-    }
-
-    # Force _handle_batch_retain to raise the integrity error, isolating the
-    # execute_task exception-classification path.
-    with patch.object(memory, "_handle_batch_retain", side_effect=unique_violation):
-        # Must not raise RetryTaskAt — the whole point of the fix.
-        try:
-            await memory.execute_task(task_dict)
-        except RetryTaskAt as exc:
-            pytest.fail(
-                f"IntegrityConstraintViolationError must not be retried, but execute_task raised {exc!r}"
-            )
-
-    # The operation must be marked 'failed' (not left pending / retrying).
-    row = await pool.fetchrow(
-        "SELECT status, error_message FROM async_operations WHERE operation_id = $1",
-        operation_id,
-    )
-    assert row is not None, "Operation row disappeared"
-    assert row["status"] == "failed", (
-        f"Expected status='failed' after integrity violation, got {row['status']!r}"
-    )
-    assert row["error_message"] is not None
-    assert "pk_chunks" in row["error_message"]
-
-    # Cleanup
-    await pool.execute("DELETE FROM async_operations WHERE operation_id = $1", operation_id)
-    await pool.execute("DELETE FROM banks WHERE bank_id = $1", bank_id)
-
-
-@pytest.mark.asyncio
-async def test_foreign_key_violation_also_not_retried(memory):
-    """
-    All subclasses of IntegrityConstraintViolationError are non-retryable —
-    verify ForeignKeyViolationError is classified the same way as
-    UniqueViolationError.
-    """
-    bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-    operation_id = uuid.uuid4()
-
-    pool = await memory._get_pool()
-    await _ensure_bank(pool, bank_id)
-    await _create_pending_operation(pool, bank_id, operation_id)
-
-    fk_violation = asyncpg.exceptions.ForeignKeyViolationError(
-        "insert or update on table \"memory_units\" violates foreign key constraint \"fk_bank\""
-    )
-
-    task_dict = {
-        "type": "batch_retain",
-        "operation_id": str(operation_id),
-        "bank_id": bank_id,
-        "contents": [{"content": "test", "document_id": "doc-1"}],
-    }
-
-    with patch.object(memory, "_handle_batch_retain", side_effect=fk_violation):
-        try:
-            await memory.execute_task(task_dict)
-        except RetryTaskAt as exc:
-            pytest.fail(
-                f"ForeignKeyViolationError must not be retried, but execute_task raised {exc!r}"
-            )
-
-    row = await pool.fetchrow(
-        "SELECT status FROM async_operations WHERE operation_id = $1",
-        operation_id,
-    )
-    assert row["status"] == "failed"
-
-    await pool.execute("DELETE FROM async_operations WHERE operation_id = $1", operation_id)
-    await pool.execute("DELETE FROM banks WHERE bank_id = $1", bank_id)
-
-
-@pytest.mark.asyncio
-async def test_non_integrity_error_still_retried(memory):
-    """
-    Sanity check: non-integrity errors (network errors, timeouts, value errors)
-    should STILL use the existing retry path — i.e., raise RetryTaskAt when
-    ``_retry_count < 3``. Only integrity violations are the new non-retryable
-    class.
-    """
-    bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-    operation_id = uuid.uuid4()
-
-    pool = await memory._get_pool()
-    await _ensure_bank(pool, bank_id)
-    await _create_pending_operation(pool, bank_id, operation_id)
-
-    task_dict = {
-        "type": "batch_retain",
-        "operation_id": str(operation_id),
-        "bank_id": bank_id,
-        "contents": [{"content": "test", "document_id": "doc-1"}],
-        # _retry_count = 0 (first attempt), so the existing retry path should fire.
-    }
-
-    transient_error = RuntimeError("transient connection blip")
-
-    with patch.object(memory, "_handle_batch_retain", side_effect=transient_error):
-        with pytest.raises(RetryTaskAt):
-            await memory.execute_task(task_dict)
-
-    await pool.execute("DELETE FROM async_operations WHERE operation_id = $1", operation_id)
-    await pool.execute("DELETE FROM banks WHERE bank_id = $1", bank_id)

hindsight-api-slim/tests/test_jina_mlx_import_error.py

@@ -1,78 +0,0 @@
-"""
-Regression test for the JinaMLXCrossEncoder import-error handling.
-
-See: https://github.com/vectorize-io/hindsight/issues/994
-
-Before the fix, the bare `except ImportError` around `import mlx_lm` masked
-*any* ImportError raised transitively during mlx_lm's own initialization
-(e.g. transformers 5.x's _LazyModule race producing
-`ImportError: cannot import name 'AutoTokenizer' from 'transformers'`),
-replacing it with a misleading "install mlx" message.
-
-These tests verify:
-1. A transitive ImportError raised from inside mlx_lm surfaces verbatim.
-2. A genuine "package not installed" ImportError still produces the install hint.
-"""
-
-import sys
-import types
-from unittest.mock import patch
-
-import pytest
-
-from hindsight_api.engine.cross_encoder import JinaMLXCrossEncoder
-
-
-def _stub_mlx_modules() -> dict[str, types.ModuleType]:
-    """Stub mlx + mlx.core so `import mlx.core` succeeds even without mlx installed."""
-    import importlib.machinery
-
-    mlx = types.ModuleType("mlx")
-    mlx.__spec__ = importlib.machinery.ModuleSpec("mlx", loader=None)
-    mlx_core = types.ModuleType("mlx.core")
-    mlx_core.__spec__ = importlib.machinery.ModuleSpec("mlx.core", loader=None)
-    mlx.core = mlx_core
-    return {"mlx": mlx, "mlx.core": mlx_core}
-
-
-@pytest.mark.asyncio
-async def test_initialize_surfaces_transitive_import_error():
-    """A transformers-lazy-load-style failure must propagate, not be masked."""
-    encoder = JinaMLXCrossEncoder()
-
-    real_import = __import__
-
-    def fake_import(name, *args, **kwargs):
-        if name == "mlx_lm" or name.startswith("mlx_lm."):
-            raise ImportError("cannot import name 'AutoTokenizer' from 'transformers'")
-        return real_import(name, *args, **kwargs)
-
-    sys.modules.pop("mlx_lm", None)
-
-    with patch.dict(sys.modules, _stub_mlx_modules()):
-        with patch("builtins.__import__", side_effect=fake_import):
-            with pytest.raises(ImportError, match="AutoTokenizer"):
-                await encoder.initialize()
-
-
-@pytest.mark.asyncio
-async def test_initialize_reports_install_hint_when_mlx_missing():
-    """A genuine 'package not installed' error still gets the friendly install hint."""
-    encoder = JinaMLXCrossEncoder()
-
-    real_import = __import__
-
-    def fake_import(name, *args, **kwargs):
-        if name == "mlx_lm" or name.startswith("mlx_lm."):
-            raise ImportError("No module named 'mlx_lm'")
-        if name == "mlx" or name.startswith("mlx."):
-            raise ImportError("No module named 'mlx'")
-        return real_import(name, *args, **kwargs)
-
-    sys.modules.pop("mlx_lm", None)
-    sys.modules.pop("mlx", None)
-    sys.modules.pop("mlx.core", None)
-
-    with patch("builtins.__import__", side_effect=fake_import):
-        with pytest.raises(ImportError, match="mlx and mlx-lm are required"):
-            await encoder.initialize()

hindsight-api-slim/tests/test_link_utils.py

@@ -2,14 +2,12 @@
 import numpy as np
 import pytest
 from datetime import datetime, timezone, timedelta
-from unittest.mock import AsyncMock, MagicMock
 
 from hindsight_api.engine.retain.link_utils import (
     _normalize_datetime,
     _cap_links_per_unit,
     compute_temporal_links,
     compute_temporal_query_bounds,
-    compute_semantic_links_ann,
     compute_semantic_links_within_batch,
     MAX_TEMPORAL_LINKS_PER_UNIT,
 )
@@ -390,145 +388,3 @@ class TestComputeSemanticLinksWithinBatch:
             assert link_type == "semantic"
             assert 0.0 <= weight <= 1.0
             assert entity_id is None
-
-
-class TestComputeSemanticLinksAnnPgBouncerSafety:
-    """Regression tests ensuring compute_semantic_links_ann stays in a single
-    transaction so that the `_ann_seeds` temp table remains visible when the
-    caller's connection goes through pgBouncer in `transaction` pool mode.
-
-    In pgBouncer transaction mode, the backend is only pinned to the client
-    for the duration of an actual PostgreSQL transaction. Outside a
-    transaction, consecutive statements can land on different backends, and
-    session-scoped temp tables (which are bound to the backend that created
-    them) become invisible. The observed failure mode was an intermittent
-    `relation "_ann_seeds" does not exist` on the statement immediately
-    following the CREATE TEMP TABLE.
-    """
-
-    @pytest.fixture
-    def mock_conn(self):
-        """An asyncpg-like connection mock with an async `transaction()`
-        context manager and awaitable execute/fetch/copy helpers."""
-        conn = MagicMock()
-
-        txn_cm = MagicMock()
-        txn_cm.__aenter__ = AsyncMock(return_value=None)
-        txn_cm.__aexit__ = AsyncMock(return_value=None)
-        conn.transaction = MagicMock(return_value=txn_cm)
-
-        conn.execute = AsyncMock()
-        conn.copy_records_to_table = AsyncMock()
-        conn.fetch = AsyncMock(return_value=[])
-        return conn
-
-    @pytest.mark.asyncio
-    async def test_empty_inputs_skip_transaction(self, mock_conn):
-        """No seeds -> no work, no transaction, no temp-table churn."""
-        result = await compute_semantic_links_ann(
-            conn=mock_conn,
-            bank_id="bank-1",
-            unit_ids=[],
-            embeddings=[],
-        )
-        assert result == []
-        mock_conn.transaction.assert_not_called()
-        mock_conn.execute.assert_not_called()
-
-    @pytest.mark.asyncio
-    async def test_runs_inside_a_transaction(self, mock_conn):
-        """The full CREATE TEMP TABLE -> COPY -> SELECT sequence must happen
-        inside a single `async with conn.transaction():` block."""
-        emb = [0.1] * 384
-        await compute_semantic_links_ann(
-            conn=mock_conn,
-            bank_id="bank-1",
-            unit_ids=["u1", "u2"],
-            embeddings=[emb, emb],
-            fact_types=["world", "world"],
-        )
-
-        # Transaction context manager was entered.
-        mock_conn.transaction.assert_called_once()
-        txn_cm = mock_conn.transaction.return_value
-        txn_cm.__aenter__.assert_awaited_once()
-        txn_cm.__aexit__.assert_awaited_once()
-
-    @pytest.mark.asyncio
-    async def test_temp_table_uses_on_commit_drop(self, mock_conn):
-        """The CREATE TEMP TABLE statement must use ON COMMIT DROP so the
-        table is transaction-scoped. Without ON COMMIT DROP the table would
-        be session-scoped and would not survive pgBouncer backend rebinding
-        between transactions."""
-        emb = [0.1] * 384
-        await compute_semantic_links_ann(
-            conn=mock_conn,
-            bank_id="bank-1",
-            unit_ids=["u1"],
-            embeddings=[emb],
-            fact_types=["world"],
-        )
-
-        executed_sql = [call.args[0] for call in mock_conn.execute.call_args_list]
-        create_statements = [s for s in executed_sql if "CREATE TEMP TABLE" in s]
-        assert len(create_statements) == 1, "Should create _ann_seeds exactly once"
-        assert "_ann_seeds" in create_statements[0]
-        assert "ON COMMIT DROP" in create_statements[0], (
-            "CREATE TEMP TABLE must use ON COMMIT DROP so the table is cleaned "
-            "up at transaction end and is transaction-scoped"
-        )
-
-        # Must not use IF NOT EXISTS — the table is fresh each transaction.
-        assert "IF NOT EXISTS" not in create_statements[0], (
-            "With ON COMMIT DROP the table is always fresh at transaction start, "
-            "so IF NOT EXISTS is both unnecessary and misleading (suggests the "
-            "table might persist across transactions)"
-        )
-
-    @pytest.mark.asyncio
-    async def test_no_manual_drop_or_truncate(self, mock_conn):
-        """With ON COMMIT DROP we must not re-add manual TRUNCATE or DROP
-        statements — they were the source of the original pgBouncer bug."""
-        emb = [0.1] * 384
-        await compute_semantic_links_ann(
-            conn=mock_conn,
-            bank_id="bank-1",
-            unit_ids=["u1"],
-            embeddings=[emb],
-            fact_types=["world"],
-        )
-
-        executed_sql = [call.args[0] for call in mock_conn.execute.call_args_list]
-        assert not any("TRUNCATE _ann_seeds" in s for s in executed_sql), (
-            "TRUNCATE is unnecessary with ON COMMIT DROP and was previously "
-            "the statement that failed with 'relation does not exist' when "
-            "pgBouncer rebound the backend"
-        )
-        assert not any("DROP TABLE" in s and "_ann_seeds" in s for s in executed_sql), (
-            "Explicit DROP is unnecessary with ON COMMIT DROP"
-        )
-
-    @pytest.mark.asyncio
-    async def test_uses_set_local_for_ef_search(self, mock_conn):
-        """hnsw.ef_search must be set with SET LOCAL so the change is scoped
-        to the transaction. Without SET LOCAL, the setting would leak onto
-        the pooled backend and affect subsequent recall queries that land
-        on the same backend."""
-        emb = [0.1] * 384
-        await compute_semantic_links_ann(
-            conn=mock_conn,
-            bank_id="bank-1",
-            unit_ids=["u1"],
-            embeddings=[emb],
-            fact_types=["world"],
-        )
-
-        executed_sql = [call.args[0] for call in mock_conn.execute.call_args_list]
-        ef_statements = [s for s in executed_sql if "hnsw.ef_search" in s]
-        assert ef_statements, "ef_search must be tuned down for retain ANN"
-        for stmt in ef_statements:
-            assert stmt.strip().startswith("SET LOCAL"), (
-                f"hnsw.ef_search must use SET LOCAL, got: {stmt}"
-            )
-        # And there must not be a RESET — SET LOCAL handles it at commit.
-        assert not any("RESET hnsw.ef_search" in s for s in executed_sql)

hindsight-api-slim/tests/test_openai_max_tokens_param.py

@@ -1,72 +0,0 @@
-"""
-Tests for OpenAICompatibleLLM._max_tokens_param_name.
-
-Regression coverage for issue #978: Azure OpenAI + GPT-5 models were failing with
-"'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead."
-because PR #858 started sending 'max_tokens' whenever the openai provider had a
-custom base_url. Reasoning models only accept 'max_completion_tokens', and Azure
-OpenAI is fully OpenAI-API-compatible, so both cases must keep using the new
-parameter name.
-"""
-
-from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
-
-
-def _make(provider: str, model: str, base_url: str = "") -> OpenAICompatibleLLM:
-    return OpenAICompatibleLLM(
-        provider=provider,
-        api_key="test-key",
-        base_url=base_url,
-        model=model,
-    )
-
-
-class TestMaxTokensParamName:
-    def test_native_openai_uses_max_completion_tokens(self):
-        llm = _make("openai", "gpt-4o-mini")
-        assert llm._max_tokens_param_name() == "max_completion_tokens"
-
-    def test_openai_custom_base_url_falls_back_to_max_tokens(self):
-        """Mistral/Together-style OpenAI-compatible endpoints need max_tokens (PR #858)."""
-        llm = _make("openai", "mistral-large-latest", base_url="https://api.mistral.ai/v1")
-        assert llm._max_tokens_param_name() == "max_tokens"
-
-    def test_azure_openai_uses_max_completion_tokens(self):
-        """Regression for #978: Azure is fully OpenAI-API-compatible, not a third-party clone."""
-        llm = _make(
-            "openai",
-            "gpt-4o-mini",
-            base_url="https://my-resource.openai.azure.com/openai/v1/",
-        )
-        assert llm._max_tokens_param_name() == "max_completion_tokens"
-
-    def test_reasoning_model_always_uses_max_completion_tokens(self):
-        """Regression for #978: GPT-5/o1/o3 reject max_tokens outright, base_url must not matter."""
-        # Azure + GPT-5 (exact reporter setup)
-        azure_gpt5 = _make(
-            "openai",
-            "gpt-5.4-nano",
-            base_url="https://my-resource.openai.azure.com/openai/v1/",
-        )
-        assert azure_gpt5._max_tokens_param_name() == "max_completion_tokens"
-
-        # Even a Mistral-style custom base_url must not downgrade a reasoning model
-        for model in ("gpt-5", "gpt-5-mini", "o1-mini", "o3", "deepseek-r1"):
-            llm = _make("openai", model, base_url="https://some-proxy.example.com/v1")
-            assert llm._max_tokens_param_name() == "max_completion_tokens", model
-
-    def test_groq_uses_max_completion_tokens(self):
-        llm = _make("groq", "openai/gpt-oss-120b", base_url="https://api.groq.com/openai/v1")
-        assert llm._max_tokens_param_name() == "max_completion_tokens"
-
-    def test_llamacpp_uses_max_completion_tokens(self):
-        llm = _make("llamacpp", "some-model", base_url="http://localhost:8080/v1")
-        assert llm._max_tokens_param_name() == "max_completion_tokens"
-
-    def test_ollama_uses_max_tokens(self):
-        llm = _make("ollama", "gemma3:12b", base_url="http://localhost:11434/v1")
-        assert llm._max_tokens_param_name() == "max_tokens"
-
-    def test_lmstudio_uses_max_tokens(self):
-        llm = _make("lmstudio", "openai/gpt-oss-20b", base_url="http://localhost:1234/v1")
-        assert llm._max_tokens_param_name() == "max_tokens"

hindsight-api-slim/tests/test_recall_config.py

@@ -1,231 +0,0 @@
-"""
-Tests for the internal recall configuration knobs used during mental model
-refresh: recall_include_chunks, recall_max_tokens, recall_chunks_max_tokens.
-
-These are exposed both as hierarchical config fields (env → tenant → bank)
-and as overrides on a mental model's `trigger` JSONB field.
-"""
-
-import dataclasses
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-from hindsight_api.engine.reflect.tools import tool_recall
-from hindsight_api.engine.response_models import RecallResult as RecallResultModel
-from hindsight_api.models import RequestContext
-
-
-def _make_mock_engine():
-    engine = MagicMock()
-    engine.recall_async = AsyncMock(return_value=RecallResultModel(results=[], entities={}, chunks={}))
-    return engine
-
-
-@pytest.fixture
-def mock_request_context():
-    # internal=True bypasses the tenant extension, letting these unit tests
-    # exercise engine methods without standing up auth.
-    return RequestContext(internal=True)
-
-
-class TestToolRecallIncludeChunks:
-    """tool_recall must honor the include_chunks parameter (was hardcoded True)."""
-
-    @pytest.mark.asyncio
-    async def test_default_includes_chunks(self, mock_request_context):
-        engine = _make_mock_engine()
-
-        await tool_recall(engine, "bank-1", "q", mock_request_context)
-
-        kwargs = engine.recall_async.call_args.kwargs
-        assert kwargs["include_chunks"] is True
-
-    @pytest.mark.asyncio
-    async def test_include_chunks_false_propagates(self, mock_request_context):
-        engine = _make_mock_engine()
-
-        await tool_recall(engine, "bank-1", "q", mock_request_context, include_chunks=False)
-
-        kwargs = engine.recall_async.call_args.kwargs
-        assert kwargs["include_chunks"] is False
-
-    @pytest.mark.asyncio
-    async def test_max_chunk_tokens_propagates(self, mock_request_context):
-        engine = _make_mock_engine()
-
-        await tool_recall(
-            engine, "bank-1", "q", mock_request_context, max_chunk_tokens=2500, max_tokens=512
-        )
-
-        kwargs = engine.recall_async.call_args.kwargs
-        assert kwargs["max_chunk_tokens"] == 2500
-        assert kwargs["max_tokens"] == 512
-
-
-class TestRecallConfigFields:
-    """Hierarchical config fields for internal recall."""
-
-    def test_fields_exist_on_dataclass(self):
-        from hindsight_api.config import HindsightConfig
-
-        names = {f.name for f in dataclasses.fields(HindsightConfig)}
-        assert "recall_include_chunks" in names
-        assert "recall_max_tokens" in names
-        assert "recall_chunks_max_tokens" in names
-
-    def test_fields_are_configurable(self):
-        from hindsight_api.config import HindsightConfig
-
-        configurable = HindsightConfig.get_configurable_fields()
-        assert "recall_include_chunks" in configurable
-        assert "recall_max_tokens" in configurable
-        assert "recall_chunks_max_tokens" in configurable
-
-    def test_default_values(self):
-        from hindsight_api.config import (
-            DEFAULT_RECALL_CHUNKS_MAX_TOKENS,
-            DEFAULT_RECALL_INCLUDE_CHUNKS,
-            DEFAULT_RECALL_MAX_TOKENS,
-        )
-
-        assert DEFAULT_RECALL_INCLUDE_CHUNKS is True
-        assert DEFAULT_RECALL_MAX_TOKENS == 2048
-        assert DEFAULT_RECALL_CHUNKS_MAX_TOKENS == 1000
-
-    def test_env_var_constants(self):
-        from hindsight_api.config import (
-            ENV_RECALL_CHUNKS_MAX_TOKENS,
-            ENV_RECALL_INCLUDE_CHUNKS,
-            ENV_RECALL_MAX_TOKENS,
-        )
-
-        assert ENV_RECALL_INCLUDE_CHUNKS == "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS"
-        assert ENV_RECALL_MAX_TOKENS == "HINDSIGHT_API_RECALL_MAX_TOKENS"
-        assert ENV_RECALL_CHUNKS_MAX_TOKENS == "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS"
-
-    @patch.dict(
-        "os.environ",
-        {
-            "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS": "false",
-            "HINDSIGHT_API_RECALL_MAX_TOKENS": "777",
-            "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS": "333",
-        },
-    )
-    def test_from_env_reads_overrides(self):
-        from hindsight_api.config import HindsightConfig
-
-        config = HindsightConfig.from_env()
-        assert config.recall_include_chunks is False
-        assert config.recall_max_tokens == 777
-        assert config.recall_chunks_max_tokens == 333
-
-
-class TestMentalModelTriggerRecallFields:
-    """MentalModelTrigger Pydantic model accepts the new override fields."""
-
-    def test_trigger_accepts_new_fields(self):
-        from hindsight_api.api.http import MentalModelTrigger
-
-        trigger = MentalModelTrigger(
-            include_chunks=False,
-            recall_max_tokens=512,
-            recall_chunks_max_tokens=0,
-        )
-        assert trigger.include_chunks is False
-        assert trigger.recall_max_tokens == 512
-        assert trigger.recall_chunks_max_tokens == 0
-
-    def test_trigger_defaults_are_none(self):
-        from hindsight_api.api.http import MentalModelTrigger
-
-        trigger = MentalModelTrigger()
-        assert trigger.include_chunks is None
-        assert trigger.recall_max_tokens is None
-        assert trigger.recall_chunks_max_tokens is None
-
-
-class TestRefreshTriggerWiring:
-    """Verify mental-model refresh forwards trigger overrides into reflect_async kwargs."""
-
-    @pytest.mark.asyncio
-    async def test_trigger_overrides_passed_to_reflect_async(self, mock_request_context):
-        from hindsight_api.engine.memory_engine import MemoryEngine
-        from hindsight_api.engine.response_models import ReflectResult
-
-        engine = MemoryEngine.__new__(MemoryEngine)
-
-        async def fake_get_mental_model(bank_id, mental_model_id, request_context):
-            return {
-                "id": mental_model_id,
-                "source_query": "What do we know?",
-                "tags": [],
-                "trigger": {
-                    "include_chunks": False,
-                    "recall_max_tokens": 512,
-                    "recall_chunks_max_tokens": 0,
-                    "fact_types": ["world"],
-                },
-            }
-
-        captured = {}
-
-        async def fake_reflect_async(**kwargs):
-            captured.update(kwargs)
-            return ReflectResult(text="ok", based_on={})
-
-        async def fake_update_mental_model(*args, **kwargs):
-            return None
-
-        engine.get_mental_model = fake_get_mental_model
-        engine.reflect_async = fake_reflect_async
-        engine.update_mental_model = fake_update_mental_model
-        engine._operation_validator = None
-        engine._tenant_extension = None
-
-        await engine.refresh_mental_model(
-            bank_id="bank-1",
-            mental_model_id="mm-1",
-            request_context=mock_request_context,
-        )
-
-        assert captured["recall_include_chunks"] is False
-        assert captured["recall_max_tokens_override"] == 512
-        assert captured["recall_chunks_max_tokens_override"] == 0
-        assert captured["fact_types"] == ["world"]
-
-    @pytest.mark.asyncio
-    async def test_missing_trigger_fields_pass_none(self, mock_request_context):
-        from hindsight_api.engine.memory_engine import MemoryEngine
-        from hindsight_api.engine.response_models import ReflectResult
-
-        engine = MemoryEngine.__new__(MemoryEngine)
-
-        async def fake_get_mental_model(bank_id, mental_model_id, request_context):
-            return {"id": mental_model_id, "source_query": "q", "tags": [], "trigger": {}}
-
-        captured = {}
-
-        async def fake_reflect_async(**kwargs):
-            captured.update(kwargs)
-            return ReflectResult(text="ok", based_on={})
-
-        async def fake_update_mental_model(*args, **kwargs):
-            return None
-
-        engine.get_mental_model = fake_get_mental_model
-        engine.reflect_async = fake_reflect_async
-        engine.update_mental_model = fake_update_mental_model
-        engine._operation_validator = None
-        engine._tenant_extension = None
-
-        await engine.refresh_mental_model(
-            bank_id="bank-1",
-            mental_model_id="mm-1",
-            request_context=mock_request_context,
-        )
-
-        # When trigger fields are absent, None is forwarded so reflect_async falls back to bank/global config.
-        assert captured["recall_include_chunks"] is None
-        assert captured["recall_max_tokens_override"] is None
-        assert captured["recall_chunks_max_tokens_override"] is None

hindsight-api-slim/tests/test_reflect_internal_billing.py

@@ -1,80 +0,0 @@
-"""Regression test for #972: reflect sub-recalls must be marked internal.
-
-When reflect calls search_observations or recall, the sub-recalls must use
-``request_context.internal=True`` to avoid double-billing.  The reflect caller
-is already billed for the overall operation; sub-recalls are implementation
-details that should not generate additional billing events.
-"""
-
-from dataclasses import dataclass, field
-from unittest.mock import AsyncMock, MagicMock
-
-import pytest
-
-from hindsight_api.engine.reflect.tools import tool_recall, tool_search_observations
-from hindsight_api.engine.response_models import RecallResult
-
-
-@dataclass
-class _FakeRequestContext:
-    """Dataclass stand-in matching the fields used by ``dataclasses.replace``."""
-
-    api_key: str | None = None
-    api_key_id: str | None = None
-    tenant_id: str | None = None
-    internal: bool = False
-    mcp_authenticated: bool = False
-    user_initiated: bool = False
-    allowed_bank_ids: list[str] | None = None
-
-
-def _mock_engine():
-    engine = MagicMock()
-    engine.recall_async = AsyncMock(
-        return_value=RecallResult(results=[], source_facts={})
-    )
-    return engine
-
-
-class TestReflectInternalBilling:
-    """Verify that reflect sub-recalls are marked internal (#972)."""
-
-    @pytest.mark.asyncio
-    async def test_search_observations_marks_recall_internal(self):
-        engine = _mock_engine()
-        ctx = _FakeRequestContext(api_key="k", internal=False)
-
-        await tool_search_observations(engine, "bank-1", "query", ctx)
-
-        engine.recall_async.assert_called_once()
-        passed_ctx = engine.recall_async.call_args.kwargs["request_context"]
-        assert passed_ctx.internal is True, "sub-recall must be internal"
-
-    @pytest.mark.asyncio
-    async def test_search_observations_preserves_original_context(self):
-        engine = _mock_engine()
-        ctx = _FakeRequestContext(api_key="k", internal=False)
-
-        await tool_search_observations(engine, "bank-1", "query", ctx)
-
-        assert ctx.internal is False, "original context must not be mutated"
-
-    @pytest.mark.asyncio
-    async def test_recall_marks_recall_internal(self):
-        engine = _mock_engine()
-        ctx = _FakeRequestContext(api_key="k", internal=False)
-
-        await tool_recall(engine, "bank-1", "query", ctx)
-
-        engine.recall_async.assert_called_once()
-        passed_ctx = engine.recall_async.call_args.kwargs["request_context"]
-        assert passed_ctx.internal is True, "sub-recall must be internal"
-
-    @pytest.mark.asyncio
-    async def test_recall_preserves_original_context(self):
-        engine = _mock_engine()
-        ctx = _FakeRequestContext(api_key="k", internal=False)
-
-        await tool_recall(engine, "bank-1", "query", ctx)
-
-        assert ctx.internal is False, "original context must not be mutated"

hindsight-api-slim/tests/test_reflect_source_facts_config.py

@@ -11,7 +11,6 @@ import pytest
 
 from hindsight_api.engine.reflect.tools import tool_search_observations
 from hindsight_api.engine.response_models import RecallResult
-from hindsight_api.models import RequestContext
 
 
 def _make_mock_engine(recall_result=None):
@@ -25,11 +24,7 @@ def _make_mock_engine(recall_result=None):
 
 @pytest.fixture
 def mock_request_context():
-    # Use a real dataclass instance — tool_search_observations calls
-    # dataclasses.replace(request_context, internal=True), which fails on
-    # MagicMock. The fields don't matter for these tests; we only inspect
-    # the kwargs passed to the mocked recall_async.
-    return RequestContext()
+    return MagicMock()
 
 
 class TestSearchObservationsSourceFacts:

hindsight-api-slim/tests/test_retain_orchestrator_mapping.py

@@ -1,117 +0,0 @@
-"""Unit tests for retain orchestrator mapping and embeddings length guarantee.
-
-Regression coverage for issue #1037: a silent length mismatch between the
-extracted facts and the generated embeddings caused
-`_map_results_to_contents` to raise IndexError during batch_retain.
-"""
-
-from __future__ import annotations
-
-import asyncio
-from datetime import datetime
-from unittest.mock import MagicMock
-
-import pytest
-
-from hindsight_api.engine.retain import embedding_utils
-from hindsight_api.engine.retain.orchestrator import _map_results_to_contents
-from hindsight_api.engine.retain.types import ProcessedFact, RetainContent
-
-
-def _make_processed_fact(content_index: int, text: str = "fact") -> ProcessedFact:
-    return ProcessedFact(
-        fact_text=text,
-        fact_type="world",
-        embedding=[0.0, 0.0, 0.0],
-        occurred_start=None,
-        occurred_end=None,
-        mentioned_at=datetime(2026, 1, 1),
-        context="",
-        metadata={},
-        content_index=content_index,
-    )
-
-
-def _make_content(text: str = "x") -> RetainContent:
-    return RetainContent(content=text)
-
-
-class TestMapResultsToContents:
-    def test_groups_unit_ids_by_content_index(self):
-        contents = [_make_content("a"), _make_content("b"), _make_content("c")]
-        processed = [
-            _make_processed_fact(0, "a1"),
-            _make_processed_fact(0, "a2"),
-            _make_processed_fact(2, "c1"),
-        ]
-        unit_ids = ["u-a1", "u-a2", "u-c1"]
-
-        result = _map_results_to_contents(contents, processed, unit_ids)
-
-        assert result == [["u-a1", "u-a2"], [], ["u-c1"]]
-
-    def test_handles_out_of_range_content_index(self):
-        contents = [_make_content("a"), _make_content("b")]
-        processed = [
-            _make_processed_fact(-1, "f1"),
-            _make_processed_fact(99, "f2"),
-        ]
-        unit_ids = ["u1", "u2"]
-
-        result = _map_results_to_contents(contents, processed, unit_ids)
-
-        assert result == [["u1"], ["u2"]]
-
-    def test_empty_inputs(self):
-        assert _map_results_to_contents([], [], []) == []
-
-    def test_length_mismatch_raises(self):
-        # Regression for #1037: previously the function silently overran unit_ids.
-        contents = [_make_content("a")]
-        processed = [_make_processed_fact(0), _make_processed_fact(0)]
-        unit_ids = ["u1"]  # one fewer than processed_facts
-
-        with pytest.raises(ValueError, match="length mismatch"):
-            _map_results_to_contents(contents, processed, unit_ids)
-
-    def test_unit_ids_assigned_by_processed_fact_position(self):
-        # Even if processed_facts are interleaved across contents, each unit_id
-        # must follow its corresponding processed_fact (positional alignment).
-        contents = [_make_content("a"), _make_content("b")]
-        processed = [
-            _make_processed_fact(1, "b1"),
-            _make_processed_fact(0, "a1"),
-            _make_processed_fact(1, "b2"),
-        ]
-        unit_ids = ["u-b1", "u-a1", "u-b2"]
-
-        result = _map_results_to_contents(contents, processed, unit_ids)
-
-        assert result == [["u-a1"], ["u-b1", "u-b2"]]
-
-
-class TestEmbeddingsBatchLengthGuarantee:
-    def test_raises_when_backend_returns_fewer_embeddings(self):
-        # Regression for #1037: backends that silently truncate must not pass
-        # through — `zip(extracted_facts, embeddings)` would otherwise drop
-        # facts and break unit_id alignment downstream.
-        backend = MagicMock()
-        backend.encode.return_value = [[0.1, 0.2]]  # only 1 vector for 3 inputs
-
-        with pytest.raises(RuntimeError, match="returned 1 vectors for 3 input texts"):
-            asyncio.run(embedding_utils.generate_embeddings_batch(backend, ["a", "b", "c"]))
-
-    def test_raises_when_backend_returns_more_embeddings(self):
-        backend = MagicMock()
-        backend.encode.return_value = [[0.1], [0.2], [0.3]]
-
-        with pytest.raises(RuntimeError, match="returned 3 vectors for 2 input texts"):
-            asyncio.run(embedding_utils.generate_embeddings_batch(backend, ["a", "b"]))
-
-    def test_passes_through_aligned_embeddings(self):
-        backend = MagicMock()
-        backend.encode.return_value = [[0.1], [0.2]]
-
-        result = asyncio.run(embedding_utils.generate_embeddings_batch(backend, ["a", "b"]))
-
-        assert result == [[0.1], [0.2]]

hindsight-api-slim/tests/test_stage.py

@@ -1,71 +0,0 @@
-"""Unit tests for the worker stage breadcrumb module."""
-
-import asyncio
-
-import pytest
-
-from hindsight_api.worker.stage import StageHolder, bind_holder, get_stage, set_stage
-
-
-def test_set_stage_is_noop_without_holder():
-    # No holder bound in this context: must not raise, and get_stage returns None.
-    set_stage("anything")
-    assert get_stage() is None
-
-
-@pytest.mark.asyncio
-async def test_holder_bound_inside_task_is_visible_to_called_code():
-    holder = StageHolder()
-
-    async def inner():
-        # The poller binds the holder from inside the task coroutine so it
-        # lives in that task's contextvar scope; mirror that here.
-        bind_holder(holder)
-        set_stage("phase1")
-        # Engine code further down the call stack reads via set_stage.
-        set_stage("phase2")
-        assert get_stage() == "phase2"
-
-    await asyncio.create_task(inner())
-
-    # Holder is mutable: the spawning context sees the latest stage written
-    # by the child task without needing access to the contextvar.
-    assert holder.stage == "phase2"
-
-
-@pytest.mark.asyncio
-async def test_holder_does_not_leak_across_tasks():
-    # Each asyncio.create_task copies the parent's context. Binding inside
-    # one task must not affect a sibling task's view.
-    holder_a = StageHolder()
-    holder_b = StageHolder()
-
-    async def task_a():
-        bind_holder(holder_a)
-        set_stage("a")
-
-    async def task_b():
-        bind_holder(holder_b)
-        set_stage("b")
-
-    await asyncio.gather(asyncio.create_task(task_a()), asyncio.create_task(task_b()))
-
-    assert holder_a.stage == "a"
-    assert holder_b.stage == "b"
-    # Outside both tasks, no holder is bound.
-    assert get_stage() is None
-
-
-@pytest.mark.asyncio
-async def test_set_stage_updates_timestamp():
-    holder = StageHolder()
-
-    async def inner():
-        bind_holder(holder)
-        first = holder.updated_at
-        # asyncio.sleep guarantees monotonic clock advances on next set.
-        await asyncio.sleep(0.01)
-        set_stage("next")
-        assert holder.updated_at > first
-
-    await asyncio.create_task(inner())

hindsight-api-slim/tests/test_worker.py

@@ -217,7 +217,6 @@ class TestWorkerPoller:
             worker_id="test-worker-1",
             executor=lambda x: None,
             max_slots=3,  # Limit to 3 concurrent tasks
-            consolidation_max_slots=0,  # No reservation; all 3 slots available for non-consolidation
         )
 
         claimed = await poller.claim_batch()
@@ -1364,7 +1363,7 @@ async def test_worker_slot_limits_enforced(pool, clean_operations):
         executor=controlled_executor,
         poll_interval_ms=50,
         max_slots=3,  # Only allow 3 concurrent tasks
-        consolidation_max_slots=0,  # No consolidation reservation; all 3 slots available for retain
+        consolidation_max_slots=1,
     )
 
     # Submit 10 tasks
@@ -1429,195 +1428,6 @@ async def test_worker_slot_limits_enforced(pool, clean_operations):
             pass
 
 
-async def test_consolidation_slots_reserved_when_retain_saturates(pool, clean_operations):
-    """Regression: consolidation must not be starved when retain saturates the queue.
-
-    With ``max_slots=5`` and ``consolidation_max_slots=2``, retain tasks may use at
-    most 3 concurrent slots, leaving 2 slots reserved for consolidation. Without
-    the reservation (issue #1006), a continuous stream of retain tasks would fill
-    every slot and consolidation would never run.
-    """
-    from hindsight_api.worker.poller import WorkerPoller
-
-    started: dict[str, str] = {}  # op_id -> op_type
-    finish_events: dict[str, asyncio.Event] = {}
-
-    async def blocking_executor(task_dict: dict):
-        op_id = task_dict["operation_id"]
-        started[op_id] = task_dict.get("operation_type", "unknown")
-        event = asyncio.Event()
-        finish_events[op_id] = event
-        await event.wait()
-
-    poller = WorkerPoller(
-        pool=pool,
-        worker_id="test-worker-consolidation-reservation",
-        executor=blocking_executor,
-        poll_interval_ms=50,
-        max_slots=5,
-        consolidation_max_slots=2,
-    )
-
-    bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-    await _ensure_bank(pool, bank_id)
-
-    # Submit 10 retain tasks first — these should be claimed up to the
-    # non-consolidation cap (max_slots - consolidation_max_slots = 3).
-    for _ in range(10):
-        op_id = uuid.uuid4()
-        payload = json.dumps(
-            {"type": "test", "operation_type": "retain", "operation_id": str(op_id), "bank_id": bank_id}
-        )
-        await pool.execute(
-            """
-            INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
-            VALUES ($1, $2, 'retain', 'pending', $3::jsonb)
-            """,
-            op_id,
-            bank_id,
-            payload,
-        )
-
-    # Submit 1 consolidation task. Note the payload deliberately omits operation_type
-    # to verify the poller injects it from the DB column.
-    consolidation_op_id = uuid.uuid4()
-    consolidation_payload = json.dumps({"type": "test", "operation_id": str(consolidation_op_id), "bank_id": bank_id})
-    await pool.execute(
-        """
-        INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload)
-        VALUES ($1, $2, 'consolidation', 'pending', $3::jsonb)
-        """,
-        consolidation_op_id,
-        bank_id,
-        consolidation_payload,
-    )
-
-    poll_task = asyncio.create_task(poller.run())
-
-    try:
-        # Wait for the worker to fill its slots: 3 retain + 1 consolidation = 4 active.
-        for _ in range(200):
-            if len(started) >= 4:
-                break
-            await asyncio.sleep(0.01)
-
-        retain_started = [op for op, t in started.items() if t == "retain"]
-        consolidation_started = [op for op, t in started.items() if t == "consolidation"]
-
-        assert len(retain_started) == 3, (
-            f"Retain should be capped at max_slots - consolidation_max_slots = 3, got {len(retain_started)}"
-        )
-        assert len(consolidation_started) == 1, (
-            f"Consolidation should claim its reserved slot even while retain saturates, "
-            f"got {len(consolidation_started)}"
-        )
-        assert str(consolidation_op_id) in consolidation_started
-
-        # In-flight tracking must record the consolidation task under the right key,
-        # otherwise the consolidation pool accounting drifts on subsequent claims.
-        async with poller._in_flight_lock:
-            assert poller._in_flight_by_type.get("consolidation", 0) == 1
-
-    finally:
-        for event in finish_events.values():
-            event.set()
-        await poller.shutdown_graceful(timeout=2.0)
-        try:
-            await asyncio.wait_for(poll_task, timeout=1.0)
-        except asyncio.CancelledError:
-            pass
-
-
-@pytest.mark.asyncio
-async def test_pending_breakdown_explains_unclaimable_rows(pool, clean_operations, caplog):
-    """Pending rows that the claim query filters out must be visible in logs.
-
-    Background: production incident where a 'pending' retain sat in the queue for
-    hours while workers had free slots. With only the global pending count in
-    [WORKER_STATS] there's no way to tell whether the rows are claimable-but-not-
-    being-claimed (real bug) vs filtered out by the claim WHERE clause (data
-    state). This test verifies [PENDING_BREAKDOWN] surfaces each filter bucket
-    so operators can diagnose without DB access.
-    """
-    import logging
-
-    from hindsight_api.worker.poller import WorkerPoller
-
-    poller = WorkerPoller(
-        pool=pool,
-        worker_id="test-worker-pending-breakdown",
-        executor=lambda _t: asyncio.sleep(0),
-        poll_interval_ms=50,
-        max_slots=5,
-    )
-
-    bank_id = f"test-worker-{uuid.uuid4().hex[:8]}"
-    await _ensure_bank(pool, bank_id)
-
-    # Mix of pending rows that the claim query treats differently:
-    #   * payload_null  - batch_retain parent (orphan candidate)
-    #   * retry_blocked - failed once, scheduled an hour out
-    #   * assigned      - worker_id stamped (e.g. left over from a prior crash
-    #                     that re-queued without clearing worker_id)
-    #   * claimable     - normal retain ready to go
-    #   * consolidation - normal consolidation, also claimable
-    rows = [
-        ("batch_retain", None, None, None),  # payload_null
-        ("retain", json.dumps({"type": "test"}), "future", None),  # retry_blocked
-        ("retain", json.dumps({"type": "test"}), None, "ghost-worker"),  # assigned
-        ("retain", json.dumps({"type": "test"}), None, None),  # claimable
-        ("consolidation", json.dumps({"type": "test"}), None, None),  # claimable
-    ]
-    for op_type, payload, retry_marker, worker_id in rows:
-        op_id = uuid.uuid4()
-        await pool.execute(
-            """
-            INSERT INTO async_operations
-                (operation_id, bank_id, operation_type, status, task_payload,
-                 next_retry_at, worker_id)
-            VALUES ($1, $2, $3, 'pending', $4::jsonb,
-                    CASE WHEN $5::text = 'future' THEN now() + interval '1 hour' ELSE NULL END,
-                    $6)
-            """,
-            op_id,
-            bank_id,
-            op_type,
-            payload,
-            retry_marker,
-            worker_id,
-        )
-
-    # Trigger one stats emit. _last_progress_log starts at 0, so the first call
-    # always logs.
-    with caplog.at_level(logging.INFO, logger="hindsight_api.worker.poller"):
-        await poller._log_progress_if_due()
-
-    breakdown_lines = [r.message for r in caplog.records if r.message.startswith("[PENDING_BREAKDOWN]")]
-    assert len(breakdown_lines) == 1, f"Expected exactly one breakdown line, got: {breakdown_lines}"
-
-    # The breakdown is global (not bank-scoped), so other rows in the table may
-    # contribute. Parse the per-op_type buckets from the line and assert that
-    # our additions appear (>= 1 for each bucket we populated).
-    line = breakdown_lines[0]
-    buckets: dict[str, dict[str, int]] = {}
-    for section in line.removeprefix("[PENDING_BREAKDOWN]").split("|"):
-        section = section.strip()
-        if ":" not in section:
-            continue
-        op_type, fields = section.split(":", 1)
-        kv = {}
-        for token in fields.strip().split():
-            k, _, v = token.partition("=")
-            kv[k] = int(v)
-        buckets[op_type.strip()] = kv
-
-    assert buckets["batch_retain"]["payload_null"] >= 1
-    assert buckets["retain"]["retry_blocked"] >= 1
-    assert buckets["retain"]["assigned"] >= 1
-    assert buckets["retain"]["claimable"] >= 1
-    assert buckets["consolidation"]["claimable"] >= 1
-
-
 class TestMarkFailedParentPropagation:
     """Tests for _mark_failed parent propagation in WorkerPoller.
 

hindsight-api/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hindsight-api"
-version = "0.5.1"
+version = "0.5.0"
 description = "Hindsight: Agent Memory That Works Like Human Memory"
 readme = "README.md"
 requires-python = ">=3.11"

hindsight-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hindsight-cli"
-version = "0.5.1"
+version = "0.5.0"
 edition = "2021"
 authors = ["Hindsight Team"]
 description = "A beautiful CLI for Hindsight - semantic memory system"

hindsight-cli/src/api.rs

@@ -5,43 +5,11 @@
 
 use anyhow::Result;
 pub use hindsight_client::types;
-use hindsight_client::{Client as AsyncClient, Error as ClientError};
+use hindsight_client::Client as AsyncClient;
 use serde::{Deserialize, Serialize};
 use serde_json;
 use std::collections::HashMap;
 
-/// Convert a progenitor client error into an anyhow error that includes the
-/// HTTP response body. Without this, errors render as
-/// "Unexpected Response: Response { ... }" with no body, hiding validation
-/// details (see issue #1007).
-async fn humanize_client_error<E>(err: ClientError<E>) -> anyhow::Error
-where
-    E: serde::Serialize + std::fmt::Debug + Send + Sync + 'static,
-{
-    match err {
-        ClientError::ErrorResponse(rv) => {
-            let status = rv.status();
-            let body = rv.into_inner();
-            let body_str = serde_json::to_string(&body).unwrap_or_else(|_| format!("{:?}", body));
-            anyhow::anyhow!("API request failed ({}): {}", status, body_str)
-        }
-        ClientError::UnexpectedResponse(response) => {
-            let status = response.status();
-            let body = response.text().await.unwrap_or_default();
-            if body.is_empty() {
-                anyhow::anyhow!("API request failed ({})", status)
-            } else {
-                anyhow::anyhow!("API request failed ({}): {}", status, body)
-            }
-        }
-        ClientError::InvalidResponsePayload(bytes, src) => {
-            let body = String::from_utf8_lossy(&bytes);
-            anyhow::anyhow!("Invalid response payload ({}): {}", src, body)
-        }
-        other => anyhow::anyhow!("{}", other),
-    }
-}
-
 // Types not defined in OpenAPI spec (TODO: add to openapi.json)
 #[derive(Debug, Serialize, Deserialize)]
 pub struct AgentStats {
@@ -216,10 +184,7 @@ impl ApiClient {
             );
         }
         self.runtime.block_on(async {
-            let response = match self.client.recall_memories(agent_id, None, request).await {
-                Ok(r) => r,
-                Err(e) => return Err(humanize_client_error(e).await),
-            };
+            let response = self.client.recall_memories(agent_id, None, request).await?;
             Ok(response.into_inner())
         })
     }
@@ -231,10 +196,7 @@ impl ApiClient {
         _verbose: bool,
     ) -> Result<types::ReflectResponse> {
         self.runtime.block_on(async {
-            let response = match self.client.reflect(agent_id, None, request).await {
-                Ok(r) => r,
-                Err(e) => return Err(humanize_client_error(e).await),
-            };
+            let response = self.client.reflect(agent_id, None, request).await?;
             Ok(response.into_inner())
         })
     }
@@ -247,10 +209,7 @@ impl ApiClient {
         _verbose: bool,
     ) -> Result<MemoryPutResult> {
         self.runtime.block_on(async {
-            let response = match self.client.retain_memories(agent_id, None, request).await {
-                Ok(r) => r,
-                Err(e) => return Err(humanize_client_error(e).await),
-            };
+            let response = self.client.retain_memories(agent_id, None, request).await?;
             let result = response.into_inner();
             Ok(MemoryPutResult {
                 success: result.success,
@@ -735,7 +694,7 @@ impl ApiClient {
         self.runtime.block_on(async {
             let response = self
                 .client
-                .get_operation_status(bank_id, operation_id, None, None)
+                .get_operation_status(bank_id, operation_id, None)
                 .await?;
             Ok(response.into_inner())
         })

hindsight-cli/src/commands/memory.rs

@@ -21,7 +21,7 @@ use serde_json;
 struct MemoryUnitDetail {
     id: String,
     text: String,
-    #[serde(rename = "fact_type")]
+    #[serde(rename = "type")]
     type_: Option<String>,
     document_id: Option<String>,
     context: Option<String>,
@@ -106,14 +106,13 @@ pub fn list(
                 } else {
                     for item in &result.items {
                         let fact_type = item
-                            .get("fact_type")
+                            .get("type")
                             .and_then(|v| v.as_str())
                             .unwrap_or("unknown");
                         let type_t = match fact_type {
                             "world" => 0.0,
                             "experience" => 0.5,
                             "opinion" => 1.0,
-                            "observation" => 0.25,
                             _ => 0.5,
                         };
 
@@ -180,7 +179,6 @@ pub fn get(
                     "world" => 0.0,
                     "experience" => 0.5,
                     "opinion" => 1.0,
-                    "observation" => 0.25,
                     _ => 0.5,
                 };
 

hindsight-cli/src/commands/mental_model.rs

@@ -124,9 +124,6 @@ pub fn create(
             fact_types: None,
             tag_groups: None,
             tags_match: None,
-            include_chunks: None,
-            recall_max_tokens: None,
-            recall_chunks_max_tokens: None,
         })
     } else {
         None
@@ -201,9 +198,6 @@ pub fn update(
         fact_types: None,
         tag_groups: None,
         tags_match: None,
-        include_chunks: None,
-        recall_max_tokens: None,
-        recall_chunks_max_tokens: None,
     });
 
     let request = types::UpdateMentalModelRequest {

hindsight-cli/src/ui.rs

@@ -84,8 +84,6 @@ pub fn print_fact(fact: &RecallResult, _show_activation: bool) {
     let type_t = match fact_type {
         "world" => 0.0,
         "agent" => 0.5,
-        "experience" => 0.5,
-        "observation" => 0.25,
         "opinion" => 1.0,
         _ => 0.5,
     };

hindsight-clients/go/api/openapi.yaml

@@ -7,7 +7,7 @@ info:
     name: Apache 2.0
     url: https://www.apache.org/licenses/LICENSE-2.0.html
   title: Hindsight HTTP API
-  version: 0.5.1
+  version: 0.5.0
 servers:
 - url: /
 paths:
@@ -1781,19 +1781,6 @@ paths:
           title: Operation Id
           type: string
         style: simple
-      - description: Include the raw task payload (submission params) in the response.
-          May be large.
-        explode: true
-        in: query
-        name: include_payload
-        required: false
-        schema:
-          default: false
-          description: Include the raw task payload (submission params) in the response.
-            May be large.
-          title: Include Payload
-          type: boolean
-        style: form
       - explode: false
         in: header
         name: authorization
@@ -3589,39 +3576,6 @@ components:
         entities_allow_free_form:
           nullable: true
           type: boolean
-        retain_default_strategy:
-          nullable: true
-          type: string
-        retain_strategies:
-          additionalProperties: {}
-          nullable: true
-        retain_chunk_batch_size:
-          nullable: true
-          type: integer
-        mcp_enabled_tools:
-          items:
-            type: string
-          nullable: true
-          type: array
-        consolidation_llm_batch_size:
-          nullable: true
-          type: integer
-        consolidation_source_facts_max_tokens:
-          nullable: true
-          type: integer
-        consolidation_source_facts_max_tokens_per_observation:
-          nullable: true
-          type: integer
-        max_observations_per_scope:
-          nullable: true
-          type: integer
-        reflect_source_facts_max_tokens:
-          nullable: true
-          type: integer
-        llm_gemini_safety_settings:
-          items: {}
-          nullable: true
-          type: array
       title: BankTemplateConfig
     BankTemplateDirective:
       description: |-
@@ -4853,7 +4807,6 @@ components:
           id: id
           trigger:
             refresh_after_consolidation: false
-            recall_chunks_max_tokens: 1
             tag_groups:
             - match: any_strict
               tags:
@@ -4869,10 +4822,8 @@ components:
             exclude_mental_model_ids:
             - exclude_mental_model_ids
             - exclude_mental_model_ids
-            include_chunks: true
             tags_match: any
             exclude_mental_models: false
-            recall_max_tokens: 6
           last_refreshed_at: last_refreshed_at
           content: content
           tags:
@@ -4888,7 +4839,6 @@ components:
           id: id
           trigger:
             refresh_after_consolidation: false
-            recall_chunks_max_tokens: 1
             tag_groups:
             - match: any_strict
               tags:
@@ -4904,10 +4854,8 @@ components:
             exclude_mental_model_ids:
             - exclude_mental_model_ids
             - exclude_mental_model_ids
-            include_chunks: true
             tags_match: any
             exclude_mental_models: false
-            recall_max_tokens: 6
           last_refreshed_at: last_refreshed_at
           content: content
           tags:
@@ -4934,7 +4882,6 @@ components:
         id: id
         trigger:
           refresh_after_consolidation: false
-          recall_chunks_max_tokens: 1
           tag_groups:
           - match: any_strict
             tags:
@@ -4950,10 +4897,8 @@ components:
           exclude_mental_model_ids:
           - exclude_mental_model_ids
           - exclude_mental_model_ids
-          include_chunks: true
           tags_match: any
           exclude_mental_models: false
-          recall_max_tokens: 6
         last_refreshed_at: last_refreshed_at
         content: content
         tags:
@@ -5041,21 +4986,11 @@ components:
             $ref: '#/components/schemas/MentalModelTrigger_Input_tag_groups_inner'
           nullable: true
           type: array
-        include_chunks:
-          nullable: true
-          type: boolean
-        recall_max_tokens:
-          nullable: true
-          type: integer
-        recall_chunks_max_tokens:
-          nullable: true
-          type: integer
       title: MentalModelTrigger
     MentalModelTrigger-Output:
       description: Trigger settings for a mental model.
       example:
         refresh_after_consolidation: false
-        recall_chunks_max_tokens: 1
         tag_groups:
         - match: any_strict
           tags:
@@ -5071,10 +5006,8 @@ components:
         exclude_mental_model_ids:
         - exclude_mental_model_ids
         - exclude_mental_model_ids
-        include_chunks: true
         tags_match: any
         exclude_mental_models: false
-        recall_max_tokens: 6
       properties:
         refresh_after_consolidation:
           default: false
@@ -5115,15 +5048,6 @@ components:
             $ref: '#/components/schemas/MentalModelTrigger_Output_tag_groups_inner'
           nullable: true
           type: array
-        include_chunks:
-          nullable: true
-          type: boolean
-        recall_max_tokens:
-          nullable: true
-          type: integer
-        recall_chunks_max_tokens:
-          nullable: true
-          type: integer
       title: MentalModelTrigger
     OperationResponse:
       description: Response model for a single async operation.
@@ -5207,9 +5131,6 @@ components:
             $ref: '#/components/schemas/ChildOperationStatus'
           nullable: true
           type: array
-        task_payload:
-          additionalProperties: {}
-          nullable: true
       required:
       - operation_id
       - status

hindsight-clients/go/api_audit.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_bank_templates.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_banks.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_directives.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_documents.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_entities.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_files.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_memory.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_mental_models.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_monitoring.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/api_operations.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -154,16 +154,9 @@ type ApiGetOperationStatusRequest struct {
 	ApiService *OperationsAPIService
 	bankId string
 	operationId string
-	includePayload *bool
 	authorization *string
 }
 
-// Include the raw task payload (submission params) in the response. May be large.
-func (r ApiGetOperationStatusRequest) IncludePayload(includePayload bool) ApiGetOperationStatusRequest {
-	r.includePayload = &includePayload
-	return r
-}
-
 func (r ApiGetOperationStatusRequest) Authorization(authorization string) ApiGetOperationStatusRequest {
 	r.authorization = &authorization
 	return r
@@ -215,12 +208,6 @@ func (a *OperationsAPIService) GetOperationStatusExecute(r ApiGetOperationStatus
 	localVarQueryParams := url.Values{}
 	localVarFormParams := url.Values{}
 
-	if r.includePayload != nil {
-		parameterAddToHeaderOrQuery(localVarQueryParams, "include_payload", r.includePayload, "form", "")
-	} else {
-		var defaultValue bool = false
-		r.includePayload = &defaultValue
-	}
 	// to determine the Content-Type header
 	localVarHTTPContentTypes := []string{}
 

hindsight-clients/go/api_webhooks.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/client.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -41,7 +41,7 @@ var (
 	queryDescape    = strings.NewReplacer( "%5B", "[", "%5D", "]" )
 )
 
-// APIClient manages communication with the Hindsight HTTP API API v0.5.1
+// APIClient manages communication with the Hindsight HTTP API API v0.5.0
 // In most cases there should be only one, shared, APIClient.
 type APIClient struct {
 	cfg    *Configuration

hindsight-clients/go/configuration.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/hindsight_client.go

@@ -2,31 +2,9 @@ package hindsight
 
 import (
 	"net/http"
-	"runtime/debug"
 	"time"
 )
 
-// defaultUserAgent returns the User-Agent string sent on every request unless
-// the caller overrides cfg.UserAgent. The version is read from build info so
-// it stays in sync with the module version automatically; falls back to
-// "devel" when running from an unpinned local checkout.
-func defaultUserAgent() string {
-	version := "devel"
-	if info, ok := debug.ReadBuildInfo(); ok {
-		for _, dep := range info.Deps {
-			if dep.Path == "github.com/vectorize-io/hindsight/hindsight-clients/go" {
-				version = dep.Version
-				break
-			}
-		}
-	}
-	return "hindsight-client-go/" + version
-}
-
-// DefaultUserAgent is the User-Agent string sent on every request unless the
-// caller overrides cfg.UserAgent (e.g. for integrations identifying themselves).
-var DefaultUserAgent = defaultUserAgent()
-
 // NewAPIClientWithToken creates a new API client configured with a base URL and API token.
 // The token is sent as a Bearer token in the Authorization header for all requests.
 // Note: this uses http.DefaultClient which has no timeout. Use NewAPIClientWithTimeout
@@ -38,7 +16,6 @@ var DefaultUserAgent = defaultUserAgent()
 //	resp, _, err := client.MemoryAPI.RetainMemories(ctx, bankID).RetainRequest(req).Execute()
 func NewAPIClientWithToken(baseURL, token string) *APIClient {
 	cfg := NewConfiguration()
-	cfg.UserAgent = DefaultUserAgent
 	cfg.Servers = ServerConfigurations{
 		{URL: baseURL},
 	}
@@ -55,7 +32,6 @@ func NewAPIClientWithToken(baseURL, token string) *APIClient {
 //	resp, _, err := client.MemoryAPI.RetainMemories(ctx, bankID).RetainRequest(req).Execute()
 func NewAPIClientWithTimeout(baseURL, token string, timeout time.Duration) *APIClient {
 	cfg := NewConfiguration()
-	cfg.UserAgent = DefaultUserAgent
 	cfg.Servers = ServerConfigurations{
 		{URL: baseURL},
 	}

hindsight-clients/go/model_add_background_request.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_async_operation_submit_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_audit_log_entry.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_audit_log_list_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_audit_log_stats_bucket.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_audit_log_stats_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_background_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_config_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_config_update.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_list_item.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_list_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_profile_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_stats_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_template_config.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
@@ -31,16 +31,6 @@ type BankTemplateConfig struct {
 	DispositionEmpathy NullableInt32 `json:"disposition_empathy,omitempty"`
 	EntityLabels []map[string]interface{} `json:"entity_labels,omitempty"`
 	EntitiesAllowFreeForm NullableBool `json:"entities_allow_free_form,omitempty"`
-	RetainDefaultStrategy NullableString `json:"retain_default_strategy,omitempty"`
-	RetainStrategies map[string]interface{} `json:"retain_strategies,omitempty"`
-	RetainChunkBatchSize NullableInt32 `json:"retain_chunk_batch_size,omitempty"`
-	McpEnabledTools []string `json:"mcp_enabled_tools,omitempty"`
-	ConsolidationLlmBatchSize NullableInt32 `json:"consolidation_llm_batch_size,omitempty"`
-	ConsolidationSourceFactsMaxTokens NullableInt32 `json:"consolidation_source_facts_max_tokens,omitempty"`
-	ConsolidationSourceFactsMaxTokensPerObservation NullableInt32 `json:"consolidation_source_facts_max_tokens_per_observation,omitempty"`
-	MaxObservationsPerScope NullableInt32 `json:"max_observations_per_scope,omitempty"`
-	ReflectSourceFactsMaxTokens NullableInt32 `json:"reflect_source_facts_max_tokens,omitempty"`
-	LlmGeminiSafetySettings []interface{} `json:"llm_gemini_safety_settings,omitempty"`
 }
 
 // NewBankTemplateConfig instantiates a new BankTemplateConfig object
@@ -555,399 +545,6 @@ func (o *BankTemplateConfig) UnsetEntitiesAllowFreeForm() {
 	o.EntitiesAllowFreeForm.Unset()
 }
 
-// GetRetainDefaultStrategy returns the RetainDefaultStrategy field value if set, zero value otherwise (both if not set or set to explicit null).
-func (o *BankTemplateConfig) GetRetainDefaultStrategy() string {
-	if o == nil || IsNil(o.RetainDefaultStrategy.Get()) {
-		var ret string
-		return ret
-	}
-	return *o.RetainDefaultStrategy.Get()
-}
-
-// GetRetainDefaultStrategyOk returns a tuple with the RetainDefaultStrategy field value if set, nil otherwise
-// and a boolean to check if the value has been set.
-// NOTE: If the value is an explicit nil, `nil, true` will be returned
-func (o *BankTemplateConfig) GetRetainDefaultStrategyOk() (*string, bool) {
-	if o == nil {
-		return nil, false
-	}
-	return o.RetainDefaultStrategy.Get(), o.RetainDefaultStrategy.IsSet()
-}
-
-// HasRetainDefaultStrategy returns a boolean if a field has been set.
-func (o *BankTemplateConfig) HasRetainDefaultStrategy() bool {
-	if o != nil && o.RetainDefaultStrategy.IsSet() {
-		return true
-	}
-
-	return false
-}
-
-// SetRetainDefaultStrategy gets a reference to the given NullableString and assigns it to the RetainDefaultStrategy field.
-func (o *BankTemplateConfig) SetRetainDefaultStrategy(v string) {
-	o.RetainDefaultStrategy.Set(&v)
-}
-// SetRetainDefaultStrategyNil sets the value for RetainDefaultStrategy to be an explicit nil
-func (o *BankTemplateConfig) SetRetainDefaultStrategyNil() {
-	o.RetainDefaultStrategy.Set(nil)
-}
-
-// UnsetRetainDefaultStrategy ensures that no value is present for RetainDefaultStrategy, not even an explicit nil
-func (o *BankTemplateConfig) UnsetRetainDefaultStrategy() {
-	o.RetainDefaultStrategy.Unset()
-}
-
-// GetRetainStrategies returns the RetainStrategies field value if set, zero value otherwise (both if not set or set to explicit null).
-func (o *BankTemplateConfig) GetRetainStrategies() map[string]interface{} {
-	if o == nil {
-		var ret map[string]interface{}
-		return ret
-	}
-	return o.RetainStrategies
-}
-
-// GetRetainStrategiesOk returns a tuple with the RetainStrategies field value if set, nil otherwise
-// and a boolean to check if the value has been set.
-// NOTE: If the value is an explicit nil, `nil, true` will be returned
-func (o *BankTemplateConfig) GetRetainStrategiesOk() (map[string]interface{}, bool) {
-	if o == nil || IsNil(o.RetainStrategies) {
-		return map[string]interface{}{}, false
-	}
-	return o.RetainStrategies, true
-}
-
-// HasRetainStrategies returns a boolean if a field has been set.
-func (o *BankTemplateConfig) HasRetainStrategies() bool {
-	if o != nil && !IsNil(o.RetainStrategies) {
-		return true
-	}
-
-	return false
-}
-
-// SetRetainStrategies gets a reference to the given map[string]interface{} and assigns it to the RetainStrategies field.
-func (o *BankTemplateConfig) SetRetainStrategies(v map[string]interface{}) {
-	o.RetainStrategies = v
-}
-
-// GetRetainChunkBatchSize returns the RetainChunkBatchSize field value if set, zero value otherwise (both if not set or set to explicit null).
-func (o *BankTemplateConfig) GetRetainChunkBatchSize() int32 {
-	if o == nil || IsNil(o.RetainChunkBatchSize.Get()) {
-		var ret int32
-		return ret
-	}
-	return *o.RetainChunkBatchSize.Get()
-}
-
-// GetRetainChunkBatchSizeOk returns a tuple with the RetainChunkBatchSize field value if set, nil otherwise
-// and a boolean to check if the value has been set.
-// NOTE: If the value is an explicit nil, `nil, true` will be returned
-func (o *BankTemplateConfig) GetRetainChunkBatchSizeOk() (*int32, bool) {
-	if o == nil {
-		return nil, false
-	}
-	return o.RetainChunkBatchSize.Get(), o.RetainChunkBatchSize.IsSet()
-}
-
-// HasRetainChunkBatchSize returns a boolean if a field has been set.
-func (o *BankTemplateConfig) HasRetainChunkBatchSize() bool {
-	if o != nil && o.RetainChunkBatchSize.IsSet() {
-		return true
-	}
-
-	return false
-}
-
-// SetRetainChunkBatchSize gets a reference to the given NullableInt32 and assigns it to the RetainChunkBatchSize field.
-func (o *BankTemplateConfig) SetRetainChunkBatchSize(v int32) {
-	o.RetainChunkBatchSize.Set(&v)
-}
-// SetRetainChunkBatchSizeNil sets the value for RetainChunkBatchSize to be an explicit nil
-func (o *BankTemplateConfig) SetRetainChunkBatchSizeNil() {
-	o.RetainChunkBatchSize.Set(nil)
-}
-
-// UnsetRetainChunkBatchSize ensures that no value is present for RetainChunkBatchSize, not even an explicit nil
-func (o *BankTemplateConfig) UnsetRetainChunkBatchSize() {
-	o.RetainChunkBatchSize.Unset()
-}
-
-// GetMcpEnabledTools returns the McpEnabledTools field value if set, zero value otherwise (both if not set or set to explicit null).
-func (o *BankTemplateConfig) GetMcpEnabledTools() []string {
-	if o == nil {
-		var ret []string
-		return ret
-	}
-	return o.McpEnabledTools
-}
-
-// GetMcpEnabledToolsOk returns a tuple with the McpEnabledTools field value if set, nil otherwise
-// and a boolean to check if the value has been set.
-// NOTE: If the value is an explicit nil, `nil, true` will be returned
-func (o *BankTemplateConfig) GetMcpEnabledToolsOk() ([]string, bool) {
-	if o == nil || IsNil(o.McpEnabledTools) {
-		return nil, false
-	}
-	return o.McpEnabledTools, true
-}
-
-// HasMcpEnabledTools returns a boolean if a field has been set.
-func (o *BankTemplateConfig) HasMcpEnabledTools() bool {
-	if o != nil && !IsNil(o.McpEnabledTools) {
-		return true
-	}
-
-	return false
-}
-
-// SetMcpEnabledTools gets a reference to the given []string and assigns it to the McpEnabledTools field.
-func (o *BankTemplateConfig) SetMcpEnabledTools(v []string) {
-	o.McpEnabledTools = v
-}
-
-// GetConsolidationLlmBatchSize returns the ConsolidationLlmBatchSize field value if set, zero value otherwise (both if not set or set to explicit null).
-func (o *BankTemplateConfig) GetConsolidationLlmBatchSize() int32 {
-	if o == nil || IsNil(o.ConsolidationLlmBatchSize.Get()) {
-		var ret int32
-		return ret
-	}
-	return *o.ConsolidationLlmBatchSize.Get()
-}
-
-// GetConsolidationLlmBatchSizeOk returns a tuple with the ConsolidationLlmBatchSize field value if set, nil otherwise
-// and a boolean to check if the value has been set.
-// NOTE: If the value is an explicit nil, `nil, true` will be returned
-func (o *BankTemplateConfig) GetConsolidationLlmBatchSizeOk() (*int32, bool) {
-	if o == nil {
-		return nil, false
-	}
-	return o.ConsolidationLlmBatchSize.Get(), o.ConsolidationLlmBatchSize.IsSet()
-}
-
-// HasConsolidationLlmBatchSize returns a boolean if a field has been set.
-func (o *BankTemplateConfig) HasConsolidationLlmBatchSize() bool {
-	if o != nil && o.ConsolidationLlmBatchSize.IsSet() {
-		return true
-	}
-
-	return false
-}
-
-// SetConsolidationLlmBatchSize gets a reference to the given NullableInt32 and assigns it to the ConsolidationLlmBatchSize field.
-func (o *BankTemplateConfig) SetConsolidationLlmBatchSize(v int32) {
-	o.ConsolidationLlmBatchSize.Set(&v)
-}
-// SetConsolidationLlmBatchSizeNil sets the value for ConsolidationLlmBatchSize to be an explicit nil
-func (o *BankTemplateConfig) SetConsolidationLlmBatchSizeNil() {
-	o.ConsolidationLlmBatchSize.Set(nil)
-}
-
-// UnsetConsolidationLlmBatchSize ensures that no value is present for ConsolidationLlmBatchSize, not even an explicit nil
-func (o *BankTemplateConfig) UnsetConsolidationLlmBatchSize() {
-	o.ConsolidationLlmBatchSize.Unset()
-}
-
-// GetConsolidationSourceFactsMaxTokens returns the ConsolidationSourceFactsMaxTokens field value if set, zero value otherwise (both if not set or set to explicit null).
-func (o *BankTemplateConfig) GetConsolidationSourceFactsMaxTokens() int32 {
-	if o == nil || IsNil(o.ConsolidationSourceFactsMaxTokens.Get()) {
-		var ret int32
-		return ret
-	}
-	return *o.ConsolidationSourceFactsMaxTokens.Get()
-}
-
-// GetConsolidationSourceFactsMaxTokensOk returns a tuple with the ConsolidationSourceFactsMaxTokens field value if set, nil otherwise
-// and a boolean to check if the value has been set.
-// NOTE: If the value is an explicit nil, `nil, true` will be returned
-func (o *BankTemplateConfig) GetConsolidationSourceFactsMaxTokensOk() (*int32, bool) {
-	if o == nil {
-		return nil, false
-	}
-	return o.ConsolidationSourceFactsMaxTokens.Get(), o.ConsolidationSourceFactsMaxTokens.IsSet()
-}
-
-// HasConsolidationSourceFactsMaxTokens returns a boolean if a field has been set.
-func (o *BankTemplateConfig) HasConsolidationSourceFactsMaxTokens() bool {
-	if o != nil && o.ConsolidationSourceFactsMaxTokens.IsSet() {
-		return true
-	}
-
-	return false
-}
-
-// SetConsolidationSourceFactsMaxTokens gets a reference to the given NullableInt32 and assigns it to the ConsolidationSourceFactsMaxTokens field.
-func (o *BankTemplateConfig) SetConsolidationSourceFactsMaxTokens(v int32) {
-	o.ConsolidationSourceFactsMaxTokens.Set(&v)
-}
-// SetConsolidationSourceFactsMaxTokensNil sets the value for ConsolidationSourceFactsMaxTokens to be an explicit nil
-func (o *BankTemplateConfig) SetConsolidationSourceFactsMaxTokensNil() {
-	o.ConsolidationSourceFactsMaxTokens.Set(nil)
-}
-
-// UnsetConsolidationSourceFactsMaxTokens ensures that no value is present for ConsolidationSourceFactsMaxTokens, not even an explicit nil
-func (o *BankTemplateConfig) UnsetConsolidationSourceFactsMaxTokens() {
-	o.ConsolidationSourceFactsMaxTokens.Unset()
-}
-
-// GetConsolidationSourceFactsMaxTokensPerObservation returns the ConsolidationSourceFactsMaxTokensPerObservation field value if set, zero value otherwise (both if not set or set to explicit null).
-func (o *BankTemplateConfig) GetConsolidationSourceFactsMaxTokensPerObservation() int32 {
-	if o == nil || IsNil(o.ConsolidationSourceFactsMaxTokensPerObservation.Get()) {
-		var ret int32
-		return ret
-	}
-	return *o.ConsolidationSourceFactsMaxTokensPerObservation.Get()
-}
-
-// GetConsolidationSourceFactsMaxTokensPerObservationOk returns a tuple with the ConsolidationSourceFactsMaxTokensPerObservation field value if set, nil otherwise
-// and a boolean to check if the value has been set.
-// NOTE: If the value is an explicit nil, `nil, true` will be returned
-func (o *BankTemplateConfig) GetConsolidationSourceFactsMaxTokensPerObservationOk() (*int32, bool) {
-	if o == nil {
-		return nil, false
-	}
-	return o.ConsolidationSourceFactsMaxTokensPerObservation.Get(), o.ConsolidationSourceFactsMaxTokensPerObservation.IsSet()
-}
-
-// HasConsolidationSourceFactsMaxTokensPerObservation returns a boolean if a field has been set.
-func (o *BankTemplateConfig) HasConsolidationSourceFactsMaxTokensPerObservation() bool {
-	if o != nil && o.ConsolidationSourceFactsMaxTokensPerObservation.IsSet() {
-		return true
-	}
-
-	return false
-}
-
-// SetConsolidationSourceFactsMaxTokensPerObservation gets a reference to the given NullableInt32 and assigns it to the ConsolidationSourceFactsMaxTokensPerObservation field.
-func (o *BankTemplateConfig) SetConsolidationSourceFactsMaxTokensPerObservation(v int32) {
-	o.ConsolidationSourceFactsMaxTokensPerObservation.Set(&v)
-}
-// SetConsolidationSourceFactsMaxTokensPerObservationNil sets the value for ConsolidationSourceFactsMaxTokensPerObservation to be an explicit nil
-func (o *BankTemplateConfig) SetConsolidationSourceFactsMaxTokensPerObservationNil() {
-	o.ConsolidationSourceFactsMaxTokensPerObservation.Set(nil)
-}
-
-// UnsetConsolidationSourceFactsMaxTokensPerObservation ensures that no value is present for ConsolidationSourceFactsMaxTokensPerObservation, not even an explicit nil
-func (o *BankTemplateConfig) UnsetConsolidationSourceFactsMaxTokensPerObservation() {
-	o.ConsolidationSourceFactsMaxTokensPerObservation.Unset()
-}
-
-// GetMaxObservationsPerScope returns the MaxObservationsPerScope field value if set, zero value otherwise (both if not set or set to explicit null).
-func (o *BankTemplateConfig) GetMaxObservationsPerScope() int32 {
-	if o == nil || IsNil(o.MaxObservationsPerScope.Get()) {
-		var ret int32
-		return ret
-	}
-	return *o.MaxObservationsPerScope.Get()
-}
-
-// GetMaxObservationsPerScopeOk returns a tuple with the MaxObservationsPerScope field value if set, nil otherwise
-// and a boolean to check if the value has been set.
-// NOTE: If the value is an explicit nil, `nil, true` will be returned
-func (o *BankTemplateConfig) GetMaxObservationsPerScopeOk() (*int32, bool) {
-	if o == nil {
-		return nil, false
-	}
-	return o.MaxObservationsPerScope.Get(), o.MaxObservationsPerScope.IsSet()
-}
-
-// HasMaxObservationsPerScope returns a boolean if a field has been set.
-func (o *BankTemplateConfig) HasMaxObservationsPerScope() bool {
-	if o != nil && o.MaxObservationsPerScope.IsSet() {
-		return true
-	}
-
-	return false
-}
-
-// SetMaxObservationsPerScope gets a reference to the given NullableInt32 and assigns it to the MaxObservationsPerScope field.
-func (o *BankTemplateConfig) SetMaxObservationsPerScope(v int32) {
-	o.MaxObservationsPerScope.Set(&v)
-}
-// SetMaxObservationsPerScopeNil sets the value for MaxObservationsPerScope to be an explicit nil
-func (o *BankTemplateConfig) SetMaxObservationsPerScopeNil() {
-	o.MaxObservationsPerScope.Set(nil)
-}
-
-// UnsetMaxObservationsPerScope ensures that no value is present for MaxObservationsPerScope, not even an explicit nil
-func (o *BankTemplateConfig) UnsetMaxObservationsPerScope() {
-	o.MaxObservationsPerScope.Unset()
-}
-
-// GetReflectSourceFactsMaxTokens returns the ReflectSourceFactsMaxTokens field value if set, zero value otherwise (both if not set or set to explicit null).
-func (o *BankTemplateConfig) GetReflectSourceFactsMaxTokens() int32 {
-	if o == nil || IsNil(o.ReflectSourceFactsMaxTokens.Get()) {
-		var ret int32
-		return ret
-	}
-	return *o.ReflectSourceFactsMaxTokens.Get()
-}
-
-// GetReflectSourceFactsMaxTokensOk returns a tuple with the ReflectSourceFactsMaxTokens field value if set, nil otherwise
-// and a boolean to check if the value has been set.
-// NOTE: If the value is an explicit nil, `nil, true` will be returned
-func (o *BankTemplateConfig) GetReflectSourceFactsMaxTokensOk() (*int32, bool) {
-	if o == nil {
-		return nil, false
-	}
-	return o.ReflectSourceFactsMaxTokens.Get(), o.ReflectSourceFactsMaxTokens.IsSet()
-}
-
-// HasReflectSourceFactsMaxTokens returns a boolean if a field has been set.
-func (o *BankTemplateConfig) HasReflectSourceFactsMaxTokens() bool {
-	if o != nil && o.ReflectSourceFactsMaxTokens.IsSet() {
-		return true
-	}
-
-	return false
-}
-
-// SetReflectSourceFactsMaxTokens gets a reference to the given NullableInt32 and assigns it to the ReflectSourceFactsMaxTokens field.
-func (o *BankTemplateConfig) SetReflectSourceFactsMaxTokens(v int32) {
-	o.ReflectSourceFactsMaxTokens.Set(&v)
-}
-// SetReflectSourceFactsMaxTokensNil sets the value for ReflectSourceFactsMaxTokens to be an explicit nil
-func (o *BankTemplateConfig) SetReflectSourceFactsMaxTokensNil() {
-	o.ReflectSourceFactsMaxTokens.Set(nil)
-}
-
-// UnsetReflectSourceFactsMaxTokens ensures that no value is present for ReflectSourceFactsMaxTokens, not even an explicit nil
-func (o *BankTemplateConfig) UnsetReflectSourceFactsMaxTokens() {
-	o.ReflectSourceFactsMaxTokens.Unset()
-}
-
-// GetLlmGeminiSafetySettings returns the LlmGeminiSafetySettings field value if set, zero value otherwise (both if not set or set to explicit null).
-func (o *BankTemplateConfig) GetLlmGeminiSafetySettings() []interface{} {
-	if o == nil {
-		var ret []interface{}
-		return ret
-	}
-	return o.LlmGeminiSafetySettings
-}
-
-// GetLlmGeminiSafetySettingsOk returns a tuple with the LlmGeminiSafetySettings field value if set, nil otherwise
-// and a boolean to check if the value has been set.
-// NOTE: If the value is an explicit nil, `nil, true` will be returned
-func (o *BankTemplateConfig) GetLlmGeminiSafetySettingsOk() ([]interface{}, bool) {
-	if o == nil || IsNil(o.LlmGeminiSafetySettings) {
-		return nil, false
-	}
-	return o.LlmGeminiSafetySettings, true
-}
-
-// HasLlmGeminiSafetySettings returns a boolean if a field has been set.
-func (o *BankTemplateConfig) HasLlmGeminiSafetySettings() bool {
-	if o != nil && !IsNil(o.LlmGeminiSafetySettings) {
-		return true
-	}
-
-	return false
-}
-
-// SetLlmGeminiSafetySettings gets a reference to the given []interface{} and assigns it to the LlmGeminiSafetySettings field.
-func (o *BankTemplateConfig) SetLlmGeminiSafetySettings(v []interface{}) {
-	o.LlmGeminiSafetySettings = v
-}
-
 func (o BankTemplateConfig) MarshalJSON() ([]byte, error) {
 	toSerialize,err := o.ToMap()
 	if err != nil {
@@ -994,36 +591,6 @@ func (o BankTemplateConfig) ToMap() (map[string]interface{}, error) {
 	if o.EntitiesAllowFreeForm.IsSet() {
 		toSerialize["entities_allow_free_form"] = o.EntitiesAllowFreeForm.Get()
 	}
-	if o.RetainDefaultStrategy.IsSet() {
-		toSerialize["retain_default_strategy"] = o.RetainDefaultStrategy.Get()
-	}
-	if o.RetainStrategies != nil {
-		toSerialize["retain_strategies"] = o.RetainStrategies
-	}
-	if o.RetainChunkBatchSize.IsSet() {
-		toSerialize["retain_chunk_batch_size"] = o.RetainChunkBatchSize.Get()
-	}
-	if o.McpEnabledTools != nil {
-		toSerialize["mcp_enabled_tools"] = o.McpEnabledTools
-	}
-	if o.ConsolidationLlmBatchSize.IsSet() {
-		toSerialize["consolidation_llm_batch_size"] = o.ConsolidationLlmBatchSize.Get()
-	}
-	if o.ConsolidationSourceFactsMaxTokens.IsSet() {
-		toSerialize["consolidation_source_facts_max_tokens"] = o.ConsolidationSourceFactsMaxTokens.Get()
-	}
-	if o.ConsolidationSourceFactsMaxTokensPerObservation.IsSet() {
-		toSerialize["consolidation_source_facts_max_tokens_per_observation"] = o.ConsolidationSourceFactsMaxTokensPerObservation.Get()
-	}
-	if o.MaxObservationsPerScope.IsSet() {
-		toSerialize["max_observations_per_scope"] = o.MaxObservationsPerScope.Get()
-	}
-	if o.ReflectSourceFactsMaxTokens.IsSet() {
-		toSerialize["reflect_source_facts_max_tokens"] = o.ReflectSourceFactsMaxTokens.Get()
-	}
-	if o.LlmGeminiSafetySettings != nil {
-		toSerialize["llm_gemini_safety_settings"] = o.LlmGeminiSafetySettings
-	}
 	return toSerialize, nil
 }
 

hindsight-clients/go/model_bank_template_directive.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_template_import_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_template_manifest.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_bank_template_mental_model.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_budget.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_cancel_operation_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_child_operation_status.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_chunk_data.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_chunk_include_options.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_chunk_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_clear_memory_observations_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_consolidation_response.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_create_bank_request.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_create_directive_request.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.

hindsight-clients/go/model_create_mental_model_request.go

@@ -3,7 +3,7 @@ Hindsight HTTP API
 
 HTTP API for Hindsight
 
-API version: 0.5.1
+API version: 0.5.0
 */
 
 // Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.